diff --git a/.Rbuildignore b/.Rbuildignore index 2bc8698b2..7109ad47f 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -1,14 +1,21 @@ ^.*\.Rproj$ -^\.Rproj\.user$ -^\.Rprofile$ -^src/tbb/build/lib_.*$ -^tests/testthat/pkg/RcppParallelTest/src/.*\.s?o$ -^tests/testthat/pkg/RcppParallelTest/src/.*\.dll$ -^inst/lib$ ^.travis.yml ^README.md -^doc$ -^src/.*\.o$ -^gen/ +^\.Rprofile$ +^\.Rproj\.user$ ^appveyor\.yml$ +^check$ +^doc$ +^gen$ +^libs$ +^inst/lib$ +^inst/libs$ ^revdep$ +^src/.*\.o$ +^src/tbb/build$ +^tags$ +^tests/testthat/pkg/RcppParallelTest/src/.*\.dll$ +^tests/testthat/pkg/RcppParallelTest/src/.*\.s?o$ +^tools/tbb$ +^\.github$ +^patches \ No newline at end of file diff --git a/.github/.gitignore b/.github/.gitignore new file mode 100644 index 000000000..2d19fc766 --- /dev/null +++ b/.github/.gitignore @@ -0,0 +1 @@ +*.html diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/R-CMD-check.yaml new file mode 100644 index 000000000..897585d50 --- /dev/null +++ b/.github/workflows/R-CMD-check.yaml @@ -0,0 +1,39 @@ +# Workflow derived from https://github.com/r-lib/actions/tree/v2/examples +# Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help +on: + push: + branches: [main, master] + pull_request: + branches: [main, master] + +name: R-CMD-check + +jobs: + R-CMD-check: + runs-on: ${{ matrix.config.os }} + + name: ${{ matrix.config.os }} (${{ matrix.config.r }}) + + strategy: + fail-fast: false + matrix: + config: + - {os: macOS-latest, r: 'release'} + - {os: ubuntu-latest, r: 'release'} + - {os: windows-latest, r: 'release'} + env: + GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + R_KEEP_PKG_SOURCE: yes + steps: + - uses: actions/checkout@v3 + + - uses: r-lib/actions/setup-r@v2 + with: + use-public-rspm: true + + - uses: r-lib/actions/setup-r-dependencies@v2 + with: + extra-packages: any::rcmdcheck + needs: check + + - uses: r-lib/actions/check-r-package@v2 diff --git a/.gitignore b/.gitignore index 4efd25bce..240c3f21c 100644 --- a/.gitignore +++ b/.gitignore @@ -3,9 +3,18 @@ .Rhistory .RData .DS_Store -config.log -config.status -autom4te.cache/ -src-i386/ -src-x64/ +check inst/doc +inst/lib +inst/libs +libs +revdep +src-i386 +src-x64 +tbb.log + +src/tbb/build +src/tbb/build-tbb + +R/tbb-autodetected.R + diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 13c0af011..000000000 --- a/.travis.yml +++ /dev/null @@ -1,23 +0,0 @@ -# Run Travis CI for R using https://eddelbuettel.github.io/r-travis/ - -language: c -sudo: required -group: edge - -before_install: - - curl -OLs https://eddelbuettel.github.io/r-travis/run.sh && chmod 0755 run.sh - - ./run.sh bootstrap - -install: - - ./run.sh install_aptget r-cran-rcpp r-cran-bh r-cran-runit r-cran-knitr r-cran-rmarkdown - -script: - - ./run.sh run_tests - -after_failure: - - ./run.sh dump_logs - -notifications: - email: - on_success: change - on_failure: change \ No newline at end of file diff --git a/DESCRIPTION b/DESCRIPTION index baf8f0d24..d51de8a09 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,35 +1,39 @@ Package: RcppParallel Type: Package Title: Parallel Programming Tools for 'Rcpp' -Version: 4.4.4 +Version: 5.1.10.9000 Authors@R: c( + person("Kevin", "Ushey", role = c("aut", "cre"), email = "kevin@rstudio.com", + comment = c(ORCID = "0000-0003-2880-7407")), person("JJ", "Allaire", role = c("aut"), email = "jj@rstudio.com"), person("Romain", "Francois", role = c("aut", "cph")), - person("Kevin", "Ushey", role = c("aut", "cre"), email = "kevin@rstudio.com"), person("Gregory", "Vandenbrouck", role = "aut"), person("Marcus", "Geelnard", role = c("aut", "cph"), - comment = "TinyThread library, http://tinythreadpp.bitsnbites.eu/"), - person(family = "RStudio", role = "cph"), - person(family = "Intel", role = c("aut", "cph"), - comment = "Intel TBB library, https://www.threadingbuildingblocks.org/"), - person(family = "Microsoft", role = "cph") + comment = "TinyThread library, https://tinythreadpp.bitsnbites.eu/"), + person("Hamada S.", "Badr", email = "badr@jhu.edu", role = c("ctb"), + comment = c(ORCID = "0000-0002-9808-2344")), + person("Dirk", "Eddelbuettel", role = c("aut"), email = "edd@debian.org", + comment = c(ORCID = "0000-0001-6419-907X")), + person(family = "Intel", role = c("aut", "cph"), comment = "oneTBB library"), + person(family = "UXL Foundation", role = c("aut", "cph"), comment = "oneTBB library"), + person(family = "Microsoft", role = "cph"), + person(family = "Posit, PBC", role = "cph") ) Description: High level functions for parallel programming with 'Rcpp'. For example, the 'parallelFor()' function can be used to convert the work of a standard serial "for" loop into a parallel one and the 'parallelReduce()' function can be used for accumulating aggregate or other values. -Depends: R (>= 3.0.2) +Depends: R (>= 3.6.0) Suggests: Rcpp, RUnit, knitr, rmarkdown -SystemRequirements: GNU make, Windows: cmd.exe and cscript.exe, Solaris: g++ is required -License: GPL-2 -URL: http://rcppcore.github.io/RcppParallel, https://github.com/RcppCore/RcppParallel +Roxygen: list(markdown = TRUE) +SystemRequirements: CMake (>= 3.5) +License: GPL (>= 3) +URL: https://rcppcore.github.io/RcppParallel/, https://github.com/RcppCore/RcppParallel +BugReports: https://github.com/RcppCore/RcppParallel/issues Biarch: TRUE -Collate: - 'build.R' - 'hooks.R' - 'options.R' - 'skeleton.R' +RoxygenNote: 7.3.2 +Encoding: UTF-8 diff --git a/NAMESPACE b/NAMESPACE index 2cd799681..94da28d51 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -1,6 +1,9 @@ -export(setThreadOptions) -export(defaultNumThreads) -export(RcppParallelLibs) +# Generated by roxygen2: do not edit by hand + export(CxxFlags) export(LdFlags) export(RcppParallel.package.skeleton) +export(RcppParallelLibs) +export(defaultNumThreads) +export(setThreadOptions) +export(tbbLibraryPath) diff --git a/NEWS.md b/NEWS.md new file mode 100644 index 000000000..80cdc71e8 --- /dev/null +++ b/NEWS.md @@ -0,0 +1,203 @@ + +## RcppParallel 6.0.0 (UNRELEASED) + +* RcppParallel no longer includes tbb headers as part of the RcppParallel/TBB.h + header, and instead only exposes its TBB-specific APIs for parallel work. + +* RcppParallel now bundles oneTBB 2022.0.0. Note that the TBB ABI has changed; + packages which depend on RcppParallel may need to be rebuilt. + +* On Windows, RcppParallel now uses the copy of TBB provided by Rtools. + If TBB is not available, RcppParallel will use only the fallback 'tinythread' + implementation. In practice, this implies that RcppParallel will now only + provide a TBB backend with R (>= 4.2.0). + +## RcppParallel 5.1.11 + +* Compatibility fixes for LLVM 21. + +## RcppParallel 5.1.10 + +* Fixed an issue where packages linking to RcppParallel could inadverently + depend on internals of the TBB library available during compilation, even + if the package did not explicitly use TBB itself. + +## RcppParallel 5.1.9 + +* RcppParallel no longer passes `-rpath` when building / linking on Windows. + This fixes build issues when building RcppParallel when using the LLVM + linker on Windows. (@kalibera) + +## RcppParallel 5.1.8 + +* RcppParallel now explicitly links to the bundled copy of TBB on macOS. (#206; @jeroen) + +## RcppParallel 5.1.7 + +* Remove deprecated `std::iterator`. (#192; @Enchufa2) + +## RcppParallel 5.1.6 + +* Patch for TBB to allow compilation with gcc-13. + +* Fixed a memory leak that could occur when using TinyThread on POSIX systems. + (#185; @dipertix and and @kevinushey) + +## RcppParallel 5.1.5 + +* Patches to ensure compatibility with the R 4.2.0 UCRT toolchain on Windows, + adapted from patches contributed by Tomas Kalibera. + +* Fixed an issue where setting `TBB_ROOT` (or `TBB_INC` / `TBB_LIB`) would + copy rather than symlink the associated libraries. (#161) + +## RcppParallel 5.1.4 + +* Fixed an issue causing client packages of RcppParallel to fail to compile + on Solaris. + +## RcppParallel 5.1.3 + +* Fixed an issue that prevented compilation of RcppParallel with R (< 4.0.0) + of R on Windows. + +* The `RCPP_PARALLEL_USE_TBBMALLOC_PROXY` environment variable can now be used + to control whether RcppParallel loads the `tbbmalloc_proxy` library on load. + See https://www.threadingbuildingblocks.org/docs/help/tbb_userguide/Automically_Replacing_malloc.html + for more information. + +## RcppParallel 5.1.2 + +* `RcppParallel` gains the `tbbLibraryPath()` function, to be used when attempting + to query the location of the TBB libraries that `RcppParallel` has been + configured to use. This may be useful for R packages which wish to explicitly + use, or link to, these libraries. + +## RcppParallel 5.1.1 + +* Updated bundled version of TBB (Intel TBB 2019 Update 8). + +* RcppParallel can now be configured to use an external copy of TBB, via the + `TBB_LIB` and `TBB_INC` environment variables. These should be set to the + directories containing the TBB libraries and headers, respectively. + +* Added support for the latest versions of Intel oneAPI TBB / oneTBB. + +* Updated TBB functionality for the new interface. + +* Falling back to building TBB from local source code. + +* Backward TBB compatibility based on `__TBB_tbb_stddef_H`. + +* Resolved conflicts between system and local TBB headers. + +* Fixed URLs, used HTTPS, and minor cleanups. + +* Updated package DESCRIPTION and bumped version. + +* `setThreadOptions(...)` can again be called multiple times per session. + The requested number of threads will be used for invocations to `parallelFor()` + and `parallelReduce()` that don't explicitly request a specific number of threads. + +* The `parallelFor()` and `parallelReduce()` functions gain the `numThreads` + argument, allowing one to limit the number of threads used for a + particular computation. + +## RcppParallel 5.0.3 + +* Fixed compilation on macOS M1 machines. + +## RcppParallel 5.0.2 + +* `setThreadOptions(...)` can now only be called once per session, to avoid + segfaults when compiling RcppParallel / TBB with gcc 10.1. Subsequent + calls to `setThreadOptions(...)` are ignored. + +## RcppParallel 5.0.1 + +* Fixed compilation issue on OpenSUSE Tumbleweed with -flto=auto + +* Fixed compilation when CPPFLAGS = -I/usr/local/include and a version + of libtbb is installed there + +## RcppParallel 5.0.0 + +* RcppParallel backend can now be customized with RCPP_PARALLEL_BACKEND + environment variable (supported values are 'tbb' and 'tinythread') + +* Fixed issue when compiling RcppParallel on macOS Catalina + +* Fixed issue when compiling RcppParallel with Rtools40 + +## RcppParallel 4.4.4 + +* Fixed an issue when compiling RcppParallel with clang-9 on Fedora + +## RcppParallel 4.4.3 + +* Suppress gcc-9 warnings related -Wclass-memaccess + +* Added TBB headers for serial TBB operations (#90, @mikejiang) + +* Fixed row iterator constructor (#87, @wtianyi) + +* Fixed compilation on FreeBSD + +## RcppParallel 4.4.2 + +* Suppress gcc-8 warnings related to -Wclass-memaccess + +* Use PKG_CXXFLAGS rather than PKG_CPPFLAGS + +* Remove unused dependency on the BH package + +## RcppParallel 4.4.1 + +* Ensure user-specified R configuration passed to TBB + +* Work around warnings emitted by gcc 8 + +## RcppParallel 4.4.0 + +* Respect user-defined compiler settings (e.g. from ~/.R/Makevars). + +* Remove TBB's attempts to suppress compiler diagnostics. + +* Allow setting the number of threads to use via RCPP_PARALLEL_NUM_THREADS + environment variable. + +* Update to TBB 2018 Update 1. + +* Add native registration of compiled functions. + +## RcppParallel 4.3.20 + +* Add support for Rtools 3.3 w/ GCC 4.9 + +## RcppParallel 4.3.14 + +* Add support for TBB on Solaris + +* Fix failure to compile on OS X Snow Leopard R toolchain + +* Add const and non-const operator[] for RMatrix class + +## RcppParallel 4.3.8 + +* Add tbbmalloc library + +* Correctly pass clang to TBB configure when R is using clang + +## RcppParallel 4.3.6 + +* Support for TBB on Windows + +## RcppParallel 4.3.3 + +* Update to TBB 4.3 (fixes clang compilation error in platform.h) + +* Forward CXX to TBB Makefile + +## RcppParallel 4.2.5 + +* Initial release diff --git a/R/RcppParallel-package.R b/R/RcppParallel-package.R new file mode 100644 index 000000000..79081f9fb --- /dev/null +++ b/R/RcppParallel-package.R @@ -0,0 +1,24 @@ + +#' Parallel programming tools for Rcpp +#' +#' High level functions for doing parallel programming with Rcpp. For example, +#' the `parallelFor()` function can be used to convert the work of a +#' standard serial "for" loop into a parallel one, and the `parallelReduce()` +#' function can be used for accumulating aggregate or other values. +#' +#' The high level interface enables safe and robust parallel programming +#' without direct manipulation of operating system threads. On Windows, macOS, +#' and Linux systems the underlying implementation is based on Intel TBB +#' (Threading Building Blocks). On other platforms, a less-performant fallback +#' implementation based on the TinyThread library is used. +#' +#' For additional documentation, see the package website at: +#' +#' +#' +#' +#' @name RcppParallel-package +#' @docType package +#' @aliases RcppParallel RcppParallel-package +#' @keywords package parallel +NULL diff --git a/R/aaa.R b/R/aaa.R new file mode 100644 index 000000000..568a2aac9 --- /dev/null +++ b/R/aaa.R @@ -0,0 +1,8 @@ + +# stubs that get overridden via configure script +TBB_ENABLED <- TRUE +TBB_LIB <- "" +TBB_INC <- "" + +TBB_NAME <- "tbb" +TBB_MALLOC_NAME <- "tbbmalloc" \ No newline at end of file diff --git a/R/build.R b/R/build.R deleted file mode 100644 index 95cbf5e8b..000000000 --- a/R/build.R +++ /dev/null @@ -1,99 +0,0 @@ - - -# Output the CXX flags. These flags are propagated to sourceCpp via the -# inlineCxxPlugin (defined below) and to packages via a line in Makevars[.win] -# like this: -# -# PKG_CXXFLAGS += $(shell "${R_HOME}/bin${R_ARCH_BIN}/Rscript.exe" -e "RcppParallel::CxxFlags()") -# -CxxFlags <- function() { - cat(tbbCxxFlags()) -} - - -# Output the LD flags for building against TBB. These flags are propagated -# to sourceCpp via the inlineCxxPlugin (defined below) and to packages -# via a line in Makevars[.win] like this: -# -# PKG_LIBS += $(shell "${R_HOME}/bin${R_ARCH_BIN}/Rscript.exe" -e "RcppParallel::LdFlags()") -# -LdFlags <- function() { - cat(tbbLdFlags()) -} - -# alias for backward compatibility -RcppParallelLibs <- function() { - LdFlags() -} - -# Inline plugin used by sourceCpp. -inlineCxxPlugin <- function() { - list( - env = list( - PKG_CXXFLAGS = tbbCxxFlags(), - PKG_LIBS = tbbLdFlags() - ), - includes = "#include ", - LinkingTo = "RcppParallel", - body = function(x) x, - Depends = "RcppParallel" - ) -} - -tbbCxxFlags <- function() { - - flags <- c() - - # opt-in to TBB on Windows - if (Sys.info()['sysname'] == "Windows") - flags <- paste(flags, "-DRCPP_PARALLEL_USE_TBB=1") - - flags -} - -# Return the linker flags requried for TBB on this platform -tbbLdFlags <- function() { - # on Windows and Solaris we need to explicitly link against tbb.dll - if ((Sys.info()['sysname'] %in% c("Windows", "SunOS")) && !isSparc()) { - tbb <- tbbLibPath() - paste("-L", asBuildPath(dirname(tbb)), " -ltbb -ltbbmalloc", sep = "") - } else { - "" - } -} - -# Determine the platform-specific path to the TBB library -tbbLibPath <- function(suffix = "") { - sysname <- Sys.info()['sysname'] - tbbSupported <- list( - "Darwin" = paste("libtbb", suffix, ".dylib", sep = ""), - "Linux" = paste("libtbb", suffix, ".so.2", sep = ""), - "Windows" = paste("tbb", suffix, ".dll", sep = ""), - "SunOS" = paste("libtbb", suffix, ".so", sep = "") - ) - if ((sysname %in% names(tbbSupported)) && !isSparc()) { - libDir <- "lib/" - if (sysname == "Windows") - libDir <- paste(libDir, .Platform$r_arch, "/", sep="") - system.file(paste(libDir, tbbSupported[[sysname]], sep = ""), - package = "RcppParallel") - } else { - NULL - } -} - -isSparc <- function() { - Sys.info()['sysname'] == "SunOS" && Sys.info()[["machine"]] != "i86pc" -} - -# Helper function to ape the behavior of the R build system -# when providing paths to libraries -asBuildPath <- function(path) { - if (.Platform$OS.type == "windows") { - path <- normalizePath(path) - if (grepl(' ', path, fixed=TRUE)) - path <- utils::shortPathName(path) - path <- gsub("\\\\", "/", path) - } - return(path) -} diff --git a/R/flags.R b/R/flags.R new file mode 100644 index 000000000..6ae1f8280 --- /dev/null +++ b/R/flags.R @@ -0,0 +1,53 @@ + +#' Compilation flags for RcppParallel +#' +#' Output the compiler or linker flags required to build against RcppParallel. +#' +#' These functions are typically called from `Makevars` as follows: +#' +#' ``` +#' PKG_LIBS += $(shell "${R_HOME}/bin/Rscript" -e "RcppParallel::LdFlags()") +#' ``` +#' +#' On Windows, the flags ensure that the package links with the built-in TBB +#' library. On Linux and macOS, the output is empty, because TBB is loaded +#' dynamically on load by `RcppParallel`. +#' +#' \R packages using RcppParallel should also add the following to their +#' `NAMESPACE` file: +#' +#' ``` +#' importFrom(RcppParallel, RcppParallelLibs) +#' ``` +#' +#' This is necessary to ensure that \pkg{RcppParallel} (and so, TBB) is loaded +#' and available. +#' +#' @name flags +#' @rdname flags +#' @aliases RcppParallelLibs LdFlags CxxFlags +#' +#' @return Returns \code{NULL}, invisibly. These functions are called for +#' their side effects (writing the associated flags to stdout). +#' +NULL + + +#' @name flags +#' @export +CxxFlags <- function() { + cat(tbbCxxFlags()) +} + +#' @name flags +#' @export +LdFlags <- function() { + cat(tbbLdFlags()) +} + +#' @name flags +#' @export +RcppParallelLibs <- function() { + LdFlags() +} + diff --git a/R/hooks.R b/R/hooks.R deleted file mode 100644 index be993c8f0..000000000 --- a/R/hooks.R +++ /dev/null @@ -1,48 +0,0 @@ - -dllInfo <- NULL -mallocDllInfo <- NULL - -.onLoad <- function(libname, pkgname) { - - # load tbb and tbbmalloc on supported platforms - tbb <- tbbLibPath() - if (!is.null(tbb)) { - if (!file.exists(tbb)) { - warning(paste("TBB library", tbb, "not found.")) - } else { - dllInfo <<- dyn.load(tbb, local = FALSE, now = TRUE) - } - } - tbbMalloc <- tbbLibPath("malloc") - if (!is.null(tbbMalloc)) { - if (!file.exists(tbbMalloc)) { - warning(paste("TBB malloc library", tbbMalloc, "not found.")) - } else { - mallocDllInfo <<- dyn.load(tbbMalloc, local = FALSE, now = TRUE) - } - } - - # load the package library - library.dynam("RcppParallel", pkgname, libname) - - # set default thread options - numThreads <- "auto" - numThreadsEnv <- Sys.getenv("RCPP_PARALLEL_NUM_THREADS", unset = NA) - if (!is.na(numThreadsEnv)) - setThreadOptions(numThreads = as.integer(numThreadsEnv)) - else - setThreadOptions(numThreads = "auto") -} - -.onUnload <- function(libpath) { - - # unload the package library - library.dynam.unload("RcppParallel", libpath) - - # unload tbb if we loaded it - if (!is.null(dllInfo)) - dyn.unload(dllInfo[["path"]]) - # unload tbbmalloc if we loaded it - if (!is.null(mallocDllInfo)) - dyn.unload(mallocDllInfo[["path"]]) -} diff --git a/R/options.R b/R/options.R index eac85482c..9b54a37c5 100644 --- a/R/options.R +++ b/R/options.R @@ -1,7 +1,48 @@ - -setThreadOptions <- function(numThreads = "auto", stackSize = "auto") { - +#' Thread options for RcppParallel +#' +#' Set thread options (number of threads to use for task scheduling and stack +#' size per-thread) for RcppParallel. +#' +#' RcppParallel is automatically initialized with the default number of threads +#' and thread stack size when it loads. You can call `setThreadOptions()` at +#' any time to change the defaults. +#' +#' The `parallelFor()` and `parallelReduce()` also accept `numThreads` as +#' an argument, if you'd like to control the number of threads specifically +#' to be made available for a particular parallel function call. Note that +#' this value is advisory, and TBB may choose a smaller number of threads +#' if the number of requested threads cannot be honored on the system. +#' +#' @aliases setThreadOptions defaultNumThreads +#' +#' @param numThreads +#' Number of threads to use for task scheduling. Call `defaultNumThreads()` +#' to determine the the default value used for "auto". +#' +#' @param stackSize +#' Stack size (in bytes) to use for worker threads. The +#' default used for "auto" is 2MB on 32-bit systems and 4MB on 64-bit systems +#' (note that this parameter has no effect on Windows). +#' +#' @return +#' `defaultNumThreads()` returns the default number of threads used by +#' RcppParallel, if another value isn't specified either via +#' `setThreadOptions()` or explicitly in calls to `parallelFor()` and +#' `parallelReduce()`. +#' +#' @examples +#' +#' \dontrun{ +#' library(RcppParallel) +#' setThreadOptions(numThreads = 4) +#' defaultNumThreads() +#' } +#' +#' @export setThreadOptions +setThreadOptions <- function(numThreads = "auto", + stackSize = "auto") +{ # validate and resolve numThreads if (identical(numThreads, "auto")) numThreads <- -1L @@ -18,19 +59,27 @@ setThreadOptions <- function(numThreads = "auto", stackSize = "auto") { else stackSize <- as.integer(stackSize) - # Call setThreadOptions if using tbb - if (!is.null(dllInfo)) { - invisible(.Call("setThreadOptions", numThreads, stackSize, - PACKAGE = "RcppParallel")) - } - + # set RCPP_PARALLEL_NUM_THREADS if (numThreads == -1L) Sys.unsetenv("RCPP_PARALLEL_NUM_THREADS") else Sys.setenv(RCPP_PARALLEL_NUM_THREADS = numThreads) + + # set RCPP_PARALLEL_STACK_SIZE + if (stackSize == 0L) + Sys.unsetenv("RCPP_PARALLEL_STACK_SIZE") + else + Sys.setenv(RCPP_PARALLEL_STACK_SIZE = stackSize) } +#' @rdname setThreadOptions +#' @export defaultNumThreads <- function() { .Call("defaultNumThreads", PACKAGE = "RcppParallel") } +isUsingTbb <- function() { + backend <- Sys.getenv("RCPP_PARALLEL_BACKEND", "tbb") + identical(backend, "tbb") +} + diff --git a/R/platform.R b/R/platform.R new file mode 100644 index 000000000..fe7569ff1 --- /dev/null +++ b/R/platform.R @@ -0,0 +1,26 @@ + +is_windows <- function() { + .Platform$OS.type == "windows" +} + +is_mac <- function() { + Sys.info()[["sysname"]] == "Darwin" +} + +is_unix <- function() { + .Platform$OS.type == "unix" +} + +is_solaris <- function() { + Sys.info()[["sysname"]] == "SunOS" +} + +is_sparc <- function() { + info <- Sys.info() + all( + info[["sysname"]] == "SunOS", + info[["machine"]] != "i86pc" + ) +} + + diff --git a/R/plugin.R b/R/plugin.R new file mode 100644 index 000000000..78dc2aab2 --- /dev/null +++ b/R/plugin.R @@ -0,0 +1,15 @@ + +# Inline plugin used by sourceCpp. +inlineCxxPlugin <- function() { + + list( + env = list( + PKG_CXXFLAGS = tbbCxxFlags(), + PKG_LIBS = tbbLdFlags() + ), + includes = "#include ", + LinkingTo = "RcppParallel", + body = identity, + Depends = "RcppParallel" + ) +} diff --git a/R/skeleton.R b/R/skeleton.R index 88cbcdbcb..4b12b711d 100644 --- a/R/skeleton.R +++ b/R/skeleton.R @@ -1,3 +1,53 @@ + +#' Create a skeleton for a new package depending on RcppParallel +#' +#' \code{RcppParallel.package.skeleton} automates the creation of a new source +#' package that intends to use features of RcppParallel. +#' +#' It is based on the \link[utils]{package.skeleton} function which it executes +#' first. +#' +#' In addition to \link[Rcpp]{Rcpp.package.skeleton} : +#' +#' The \samp{DESCRIPTION} file gains an Imports line requesting that the +#' package depends on RcppParallel and a LinkingTo line so that the package +#' finds RcppParallel header files. +#' +#' The \samp{NAMESPACE} gains a \code{useDynLib} directive as well as an +#' \code{importFrom(RcppParallel, evalCpp} to ensure instantiation of +#' RcppParallel. +#' +#' The \samp{src} directory is created if it does not exists and a +#' \samp{Makevars} file is added setting the environment variables +#' \samp{PKG_LIBS} to accomodate the necessary flags to link with the +#' RcppParallel library. +#' +#' If the \code{example_code} argument is set to \code{TRUE}, example files +#' \samp{vector-sum.cpp} is created in the \samp{src} directory. +#' \code{Rcpp::compileAttributes()} is then called to generate +#' \code{src/RcppExports.cpp} and \code{R/RcppExports.R}. These files are given +#' as an example and should eventually by removed from the generated package. +#' +#' @param name The name of your R package. +#' @param example_code If \code{TRUE}, example C++ code using RcppParallel is +#' added to the package. +#' @param ... Optional arguments passed to \link[Rcpp]{Rcpp.package.skeleton}. +#' @return Nothing, used for its side effects +#' @seealso \link[utils]{package.skeleton} +#' @references Read the \emph{Writing R Extensions} manual for more details. +#' +#' Once you have created a \emph{source} package you need to install it: see +#' the \emph{R Installation and Administration} manual, \code{\link{INSTALL}} +#' and \code{\link{install.packages}}. +#' @keywords programming +#' @examples +#' +#' \dontrun{ +#' # simple package +#' RcppParallel.package.skeleton("foobar") +#' } +#' +#' @export RcppParallel.package.skeleton RcppParallel.package.skeleton <- function(name = "anRpackage", example_code = TRUE, ...) @@ -21,9 +71,12 @@ RcppParallel.package.skeleton <- function(name = "anRpackage", on.exit(setwd(owd), add = TRUE) # remove dummy stuff - unlink("data/dummy.Rda") + unlink("data", recursive=TRUE) unlink("man/dummy.Rd") unlink("Read-and-delete-me") + lns <- readLines("NAMESPACE") + writeLines(lns[!grepl("dummy", lns)], "NAMESPACE") + unlink("src/init.c") message("\nAdding RcppParallel settings") @@ -56,7 +109,7 @@ RcppParallel.package.skeleton <- function(name = "anRpackage", message(" >> added src/Makevars") cat( c( - 'CXX_STD = CXX11', + '# We also need importFrom(RcppParallel,RcppParallelLibs) in NAMESPACE', 'PKG_LIBS += $(shell ${R_HOME}/bin/Rscript -e "RcppParallel::RcppParallelLibs()")' ), file = "src/Makevars", @@ -67,7 +120,6 @@ RcppParallel.package.skeleton <- function(name = "anRpackage", message(" >> added src/Makevars.win") cat( c( - 'CXX_STD = CXX11', 'PKG_CXXFLAGS += -DRCPP_PARALLEL_USE_TBB=1', 'PKG_LIBS += $(shell "${R_HOME}/bin${R_ARCH_BIN}/Rscript.exe" -e "RcppParallel::RcppParallelLibs()")' ), diff --git a/R/tbb-autodetected.R.in b/R/tbb-autodetected.R.in new file mode 100644 index 000000000..7cc750e09 --- /dev/null +++ b/R/tbb-autodetected.R.in @@ -0,0 +1,7 @@ + +TBB_ENABLED <- @TBB_ENABLED@ +TBB_LIB <- "@TBB_LIB@" +TBB_INC <- "@TBB_INC@" + +TBB_NAME <- "@TBB_NAME@" +TBB_MALLOC_NAME <- "@TBB_MALLOC_NAME@" \ No newline at end of file diff --git a/R/tbb.R b/R/tbb.R new file mode 100644 index 000000000..3d8fbb328 --- /dev/null +++ b/R/tbb.R @@ -0,0 +1,135 @@ + +#' Get the Path to a TBB Library +#' +#' Retrieve the path to a TBB library. This can be useful for \R packages +#' using RcppParallel that wish to use, or re-use, the version of TBB that +#' RcppParallel has been configured to use. +#' +#' @param name +#' The name of the TBB library to be resolved. Normally, this is one of +#' `tbb`, `tbbmalloc`, or `tbbmalloc_proxy`. When `NULL`, the library +#' path containing the TBB libraries is returned instead. +#' +#' @export +tbbLibraryPath <- function(name = NULL) { + + # library paths for different OSes + sysname <- Sys.info()[["sysname"]] + + # find root for TBB install + tbbRoot <- Sys.getenv("TBB_LIB", unset = tbbRoot()) + if (is.null(name)) + return(tbbRoot) + + # form library names + tbbLibNames <- list( + "Darwin" = paste0("lib", name, ".dylib"), + "Windows" = paste0("lib", name, c("12", ""), ".a"), + "SunOS" = paste0("lib", name, ".so"), + "Linux" = paste0("lib", name, c(".so.2", ".so")) + ) + + # skip systems that we know not to be compatible + isCompatible <- !is_sparc() && !is.null(tbbLibNames[[sysname]]) + if (!isCompatible) + return(NULL) + + # find the request library (if any) + libNames <- tbbLibNames[[sysname]] + for (libName in libNames) { + + tbbName <- file.path(tbbRoot, libName) + if (file.exists(tbbName)) + return(tbbName) + + arch <- if (nzchar(.Platform$r_arch)) .Platform$r_arch + suffix <- paste(c("lib", arch, libName), collapse = "/") + tbbName <- system.file(suffix, package = "RcppParallel") + if (file.exists(tbbName)) + return(tbbName) + + } + +} + +tbbCxxFlags <- function() { + + if (!TBB_ENABLED) + return("-DRCPP_PARALLEL_USE_TBB=0") + + flags <- c("-DRCPP_PARALLEL_USE_TBB=1") + + # if TBB_INC is set, apply those library paths + tbbInc <- Sys.getenv("TBB_INC", unset = TBB_INC) + if (!file.exists(tbbInc)) { + tbbInc <- system.file("include", package = "RcppParallel") + } + + # add include path + if (nzchar(tbbInc) && file.exists(tbbInc)) { + + # prefer new interface if version.h exists -- we keep this + # for compatibility with packages like StanHeaders, rstan + versionPath <- file.path(tbbInc, "tbb/version.h") + if (file.exists(versionPath)) + flags <- c(flags, "-DTBB_INTERFACE_NEW") + + # now add the include path + flags <- c(flags, paste0("-I", asBuildPath(tbbInc))) + + } + + # return flags as string + paste(flags, collapse = " ") + +} + +# Return the linker flags required for TBB on this platform +tbbLdFlags <- function() { + + # on Windows, we statically link to oneTBB + if (is_windows()) { + + libPath <- system.file("libs", package = "RcppParallel") + if (nzchar(.Platform$r_arch)) + libPath <- file.path(libPath, .Platform$r_arch) + + ldFlags <- sprintf("-L%s -lRcppParallel", asBuildPath(libPath)) + return(ldFlags) + + } + + # shortcut if TBB_LIB defined + tbbLib <- Sys.getenv("TBB_LINK_LIB", Sys.getenv("TBB_LIB", unset = TBB_LIB)) + if (nzchar(tbbLib)) { + if (R.version$os == "emscripten") { + fmt <- "-L%1$s -l%2$s" + return(sprintf(fmt, asBuildPath(tbbLib), TBB_NAME)) + } + fmt <- "-L%1$s -Wl,-rpath,%1$s -l%2$s -l%3$s" + return(sprintf(fmt, asBuildPath(tbbLib), TBB_NAME, TBB_MALLOC_NAME)) + } + + # explicitly link on macOS + # https://github.com/RcppCore/RcppParallel/issues/206 + if (is_mac()) { + fmt <- "-L%s -l%s -l%s" + return(sprintf(fmt, asBuildPath(tbbLibraryPath()), TBB_NAME, TBB_MALLOC_NAME)) + } + + # nothing required on other platforms + "" + +} + +tbbRoot <- function() { + + if (nzchar(TBB_LIB)) + return(TBB_LIB) + + rArch <- .Platform$r_arch + parts <- c("lib", if (nzchar(rArch)) rArch) + libDir <- paste(parts, collapse = "/") + system.file(libDir, package = "RcppParallel") + +} diff --git a/R/utils.R b/R/utils.R new file mode 100644 index 000000000..53ced6389 --- /dev/null +++ b/R/utils.R @@ -0,0 +1,24 @@ + +# generate paths consumable by the compilers and linkers +# in particular, on Windows and Solaris, this means the path _cannot_ be quoted !! +asBuildPath <- function(path) { + + # normalize paths using forward slashes + path <- normalizePath(path, winslash = "/", mustWork = FALSE) + + # prefer short path names if the path has spaces + if (is_windows() && grepl(" ", path, fixed = TRUE)) + path <- utils::shortPathName(path) + + # if we still have spaces, and we're not Windows or Solaris, try quoting + if (grepl(" ", path, fixed = TRUE) && !is_solaris()) + path <- shQuote(path) + + # ensure we use forward slashes, even on Windows + path <- chartr("\\", "/", path) + + # return path + path + +} + diff --git a/R/zzz.R b/R/zzz.R new file mode 100644 index 000000000..54374e097 --- /dev/null +++ b/R/zzz.R @@ -0,0 +1,71 @@ + +# !diagnostics suppress=.dllInfo,.tbbDllInfo,.tbbMallocDllInfo,.tbbMallocProxyDllInfo + +# NOTE: we intentionally do _not_ load tbbmalloc_proxy by default, as its +# intended use is to replace the default allocator, something that may be +# dangerous to do by default. in addition, TBB's documentation recommends +# only loading explicitly via e.g. LD_PRELOAD +.dllInfo <- NULL +.tbbDllInfo <- NULL +.tbbMallocDllInfo <- NULL +.tbbMallocProxyDllInfo <- NULL + +loadTbbLibrary <- function(name) { + # TBB is statically linked on Windows + if (is_windows()) { + return(NULL) + } + path <- tbbLibraryPath(name) + if (is.null(path)) + return(NULL) + + if (!file.exists(path)) { + warning("TBB library ", shQuote(name), " not found.") + return(NULL) + } + + dyn.load(path, local = FALSE, now = TRUE) + +} + +.onLoad <- function(libname, pkgname) { + + # on Windows, load RcppParallel first + if (.Platform$OS.type == "windows") { + .dllInfo <<- library.dynam("RcppParallel", pkgname, libname) + } + + # load tbb, tbbmalloc + .tbbDllInfo <<- loadTbbLibrary("tbb") + .tbbMallocDllInfo <<- loadTbbLibrary("tbbmalloc") + + # load tbbmalloc_proxy, but only if requested + useTbbMallocProxy <- Sys.getenv("RCPP_PARALLEL_USE_TBBMALLOC_PROXY", unset = "FALSE") + if (useTbbMallocProxy %in% c("TRUE", "True", "true", "1")) + .tbbMallocProxyDllInfo <<- loadTbbLibrary("tbbmalloc_proxy") + + # load RcppParallel library if available + if (.Platform$OS.type != "windows") { + .dllInfo <<- library.dynam("RcppParallel", pkgname, libname, local = FALSE) + } + +} + +.onUnload <- function(libpath) { + + # unload the package library + if (!is.null(.dllInfo)) + library.dynam.unload("RcppParallel", libpath) + + # NOTE: we do not explicitly unload tbbmalloc_proxy as switching + # the allocator at runtime can cause issues + + # unload tbbmalloc if we loaded it + if (!is.null(.tbbMallocDllInfo)) + dyn.unload(.tbbMallocDllInfo[["path"]]) + + # unload tbb if we loaded it + if (!is.null(.tbbDllInfo)) + dyn.unload(.tbbDllInfo[["path"]]) + +} diff --git a/README.md b/README.md index f0c68b576..620e6d21b 100644 --- a/README.md +++ b/README.md @@ -1,19 +1,63 @@ -## RcppParallel Package -[![Travis-CI Build Status](https://travis-ci.org/RcppCore/RcppParallel.svg?branch=master)](https://travis-ci.org/RcppCore/RcppParallel) -[![AppVeyor Build Status](https://ci.appveyor.com/api/projects/status/github/RcppCore/RcppParallel?branch=master&svg=true)](https://ci.appveyor.com/project/RcppCore/RcppParallel) -[![CRAN_Status_Badge](http://www.r-pkg.org/badges/version/RcppParallel)](https://cran.r-project.org/package=RcppParallel) +## RcppParallel -High level functions for doing parallel programming with Rcpp. For example, the `parallelFor` function can be used to convert the work of a standard serial "for" loop into a parallel one and the `parallelReduce` function can be used for accumulating aggregate or other values. + +[![CRAN](https://www.r-pkg.org/badges/version/RcppParallel)](https://cran.r-project.org/package=RcppParallel) +[![R-CMD-check](https://github.com/RcppCore/RcppParallel/actions/workflows/R-CMD-check.yaml/badge.svg)](https://github.com/RcppCore/RcppParallel/actions/workflows/R-CMD-check.yaml) + -The high level interface enables safe and robust parallel programming without direct manipulation of operating system threads. On Windows, OS X, and Linux systems the underlying implementation is based on [Intel TBB](https://www.threadingbuildingblocks.org/) (Threading Building Blocks). On other platforms a less-performant fallback implementation based on the [TinyThread](http://tinythreadpp.bitsnbites.eu/) library is used. +High level functions for parallel programming with Rcpp. The `parallelFor()` function can be used to convert the work of a standard serial "for" loop into a parallel one, and the `parallelReduce()` function can be used for accumulating aggregate or other values. + +The high level interface enables safe and robust parallel programming without direct manipulation of operating system threads. On Windows, macOS, and Linux systems, the underlying implementation is based on [Intel TBB](https://github.com/oneapi-src/oneTBB) (Threading Building Blocks). On other platforms, a less-performant fallback implementation based on the [TinyThread](https://tinythreadpp.bitsnbites.eu/) library is used. For additional documentation on using RcppParallel see the package website at http://rcppcore.github.io/RcppParallel/. + +### Intel TBB + +`RcppParallel` supports the new interface of Intel TBB, and can be configured to use an external copy of TBB (e.g., with [`oneTBB`](https://github.com/oneapi-src/oneTBB) or the system TBB library), using the `TBB_LIB` and `TBB_INC` environment variables. + +To build the development version of `RcppParallel` with [`oneTBB`](https://github.com/oneapi-src/oneTBB): + +- Install [`oneTBB`](https://github.com/oneapi-src/oneTBB). + +For example, installing [`oneTBB`](https://github.com/oneapi-src/oneTBB) on Linux 64-bit (`x86_64`) to `$HOME` directory (change if needed!): + +```bash +TBB_RELEASE="https://api.github.com/repos/oneapi-src/oneTBB/releases/latest" +TBB_TAG=$(curl --silent $TBB_RELEASE | grep -Po '"tag_name": "\K.*?(?=")') +TBB_VERSION=${TBB_TAG#?} + +wget https://github.com/oneapi-src/oneTBB/releases/download/v$TBB_VERSION/oneapi-tbb-$TBB_VERSION-lin.tgz +tar zxvf oneapi-tbb-$TBB_VERSION-lin.tgz -C $HOME + +export TBB="$HOME/oneapi-tbb-$TBB_VERSION" +``` +Note that you may replace `TBB_VERSION=${TBB_TAG#?}` with a custom version number if needed ( check available releases [here](https://github.com/oneapi-src/oneTBB/releases) ). + +- Set the TBB environment variables (specifically: `TBB` for the installation prefix, `TBB_INC` for the directory that includes the header files, and `TBB_LIB` for the libraries directory). + +For example, installing [`oneTBB`](https://github.com/oneapi-src/oneTBB) on Linux 64-bit (`x86_64`) to `$HOME` directory (change if needed!): + +```bash +source $TBB/env/vars.sh intel64 + +export TBB_INC="$TBB/include" +export TBB_LIB="$TBB/lib/intel64/gcc4.8" +``` + +- Build the development version of `RcppParallel`: + +```r +install.packages("remotes") +remotes::install_github("RcppCore/RcppParallel") +``` + + ### License -The RcppParallel package is made available under the [GPLv2](http://www.gnu.org/licenses/old-licenses/gpl-2.0.html). +The RcppParallel package is made available under the [GPLv2](http://www.gnu.org/licenses/old-licenses/gpl-2.0.html) license. -The TinyThread library is licensed under the [zlib/libpng](http://www.opensource.org/licenses/zlib-license.php) license as described [here](https://gitorious.org/tinythread/tinythreadpp/source/master:README.txt). +The [TinyThread library](https://tinythreadpp.bitsnbites.eu/) is licensed under the [zlib/libpng](https://opensource.org/licenses/zlib-license.php) license. -The Intel TBB Library is licensed under the Apache 2.0 (as described at https://www.threadingbuildingblocks.org/faq/10). +The Intel TBB Library is licensed under the Apache 2.0 license, as described at https://github.com/oneapi-src/oneTBB/blob/master/LICENSE.txt. diff --git a/RcppParallel.Rproj b/RcppParallel.Rproj index aa82b53fb..2970ea06c 100644 --- a/RcppParallel.Rproj +++ b/RcppParallel.Rproj @@ -1,4 +1,5 @@ Version: 1.0 +ProjectId: 8e3d73b0-404c-42f5-b2ef-46f759f65dd4 RestoreWorkspace: No SaveWorkspace: No @@ -12,6 +13,10 @@ Encoding: UTF-8 RnwWeave: Sweave LaTeX: pdfLaTeX +AutoAppendNewline: Yes + BuildType: Package -PackageInstallArgs: --with-keep.source --clean +PackageCleanBeforeInstall: No +PackageInstallArgs: --with-keep.source PackageCheckArgs: --as-cran +PackageRoxygenize: rd,collate,namespace diff --git a/appveyor.yml b/appveyor.yml deleted file mode 100644 index c6c14384e..000000000 --- a/appveyor.yml +++ /dev/null @@ -1,45 +0,0 @@ -# DO NOT CHANGE the "init" and "install" sections below - -# Download script file from GitHub -init: - ps: | - $ErrorActionPreference = "Stop" - Invoke-WebRequest http://raw.github.com/krlmlr/r-appveyor/master/scripts/appveyor-tool.ps1 -OutFile "..\appveyor-tool.ps1" - Import-Module '..\appveyor-tool.ps1' - -install: - ps: Bootstrap - -cache: - - C:\RLibrary - -# Adapt as necessary starting from here - -build_script: - - travis-tool.sh install_deps - -test_script: - - travis-tool.sh run_tests - -on_failure: - - 7z a failure.zip *.Rcheck\* - - appveyor PushArtifact failure.zip - -artifacts: - - path: '*.Rcheck\**\*.log' - name: Logs - - - path: '*.Rcheck\**\*.out' - name: Logs - - - path: '*.Rcheck\**\*.fail' - name: Logs - - - path: '*.Rcheck\**\*.Rout' - name: Logs - - - path: '\*_*.tar.gz' - name: Bits - - - path: '\*_*.zip' - name: Bits diff --git a/cleanup b/cleanup index 1a49d2249..ddbda5cf6 100755 --- a/cleanup +++ b/cleanup @@ -1,3 +1,3 @@ #!/usr/bin/env sh : "${R_HOME=`R RHOME`}" -"${R_HOME}/bin/Rscript" tools/config.R cleanup +"${R_HOME}/bin/Rscript" tools/config.R cleanup "$@" diff --git a/cleanup.win b/cleanup.win index 7128030cc..918a2f0c8 100755 --- a/cleanup.win +++ b/cleanup.win @@ -1,2 +1,2 @@ #!/usr/bin/env sh -"${R_HOME}/bin${R_ARCH_BIN}/Rscript.exe" tools/config.R cleanup +"${R_HOME}/bin${R_ARCH_BIN}/Rscript.exe" tools/config.R cleanup "$@" diff --git a/configure b/configure index 0cad1066c..b08715386 100755 --- a/configure +++ b/configure @@ -1,3 +1,3 @@ #!/usr/bin/env sh : "${R_HOME=`R RHOME`}" -"${R_HOME}/bin/Rscript" tools/config.R configure +"${R_HOME}/bin/Rscript" tools/config.R configure "$@" diff --git a/configure.win b/configure.win index c83ab78b5..a47d1205c 100755 --- a/configure.win +++ b/configure.win @@ -1,2 +1,2 @@ #!/usr/bin/env sh -"${R_HOME}/bin${R_ARCH_BIN}/Rscript.exe" tools/config.R configure +"${R_HOME}/bin${R_ARCH_BIN}/Rscript.exe" tools/config.R configure "$@" diff --git a/inst/.gitignore b/inst/.gitignore index c3af85790..93f665e4d 100644 --- a/inst/.gitignore +++ b/inst/.gitignore @@ -1 +1,2 @@ lib/ +libs/ diff --git a/inst/NEWS b/inst/NEWS deleted file mode 100644 index 0d375c434..000000000 --- a/inst/NEWS +++ /dev/null @@ -1,68 +0,0 @@ -RcppParallel 4.4.4 ------------------------------------------------------------------------- -* Fixed an issue when compiling RcppParallel with clang-9 on Fedora - -RcppParallel 4.4.3 ------------------------------------------------------------------------- - -* Suppress gcc-9 warnings related -Wclass-memaccess -* Added TBB headers for serial TBB operations (#90, @mikejiang) -* Fixed row iterator constructor (#87, @wtianyi) -* Fixed compilation on FreeBSD - -RcppParallel 4.4.2 ------------------------------------------------------------------------- - -* Suppress gcc-8 warnings related to -Wclass-memaccess -* Use PKG_CXXFLAGS rather than PKG_CPPFLAGS -* Remove unused dependency on the BH package - -RcppParallel 4.4.1 ------------------------------------------------------------------------- - -* Ensure user-specified R configuration passed to TBB -* Work around warnings emitted by gcc 8 - -RcppParallel 4.4.0 ------------------------------------------------------------------------- - -* Respect user-defined compiler settings (e.g. from ~/.R/Makevars). -* Remove TBB's attempts to suppress compiler diagnostics. -* Allow setting the number of threads to use via RCPP_PARALLEL_NUM_THREADS - environment variable. -* Update to TBB 2018 Update 1. -* Add native registration of compiled functions. - -RcppParallel 4.3.20 ------------------------------------------------------------------------- - -* Add support for Rtools 3.3 w/ GCC 4.9 - -RcppParallel 4.3.14 ------------------------------------------------------------------------- - -* Add support for TBB on Solaris -* Fix failure to compile on OS X Snow Leopard R toolchain -* Add const and non-const operator[] for RMatrix class - -RcppParallel 4.3.8 ------------------------------------------------------------------------- - -* Add tbbmalloc library -* Correctly pass clang to TBB configure when R is using clang - -RcppParallel 4.3.6 ------------------------------------------------------------------------- - -* Support for TBB on Windows - -RcppParallel 4.3.3 ------------------------------------------------------------------------- - -* Update to TBB 4.3 (fixes clang compilation error in platform.h) -* Forward CXX to TBB Makefile - -RcppParallel 4.2.5 ------------------------------------------------------------------------- - -* Initial release diff --git a/inst/include/.gitignore b/inst/include/.gitignore new file mode 100644 index 000000000..4c130be3e --- /dev/null +++ b/inst/include/.gitignore @@ -0,0 +1,7 @@ + +# These TBB libraries are copied in at configure time. +/index.html +/oneapi +/serial +/tbb + diff --git a/inst/include/RcppParallel.h b/inst/include/RcppParallel.h index 9b7da04dd..87d6eb5ca 100644 --- a/inst/include/RcppParallel.h +++ b/inst/include/RcppParallel.h @@ -6,50 +6,76 @@ #include "RcppParallel/TinyThread.h" // Use TBB only where it's known to compile and work correctly -// (NOTE: Windows TBB is temporarily opt-in for packages for +// (NOTE: Windows TBB is temporarily opt-in for packages for // compatibility with CRAN packages not previously configured // to link to TBB in Makevars.win) #ifndef RCPP_PARALLEL_USE_TBB -#if defined(__APPLE__) || defined(__gnu_linux__) || ((defined(__sun) && defined(__SVR4) && !defined(__sparc))) -#define RCPP_PARALLEL_USE_TBB 1 -#include "RcppParallel/TBB.h" -#else -#define RCPP_PARALLEL_USE_TBB 0 -#endif +# if defined(__APPLE__) || defined(__gnu_linux__) || (defined(__sun) && defined(__SVR4) && !defined(__sparc)) +# define RCPP_PARALLEL_USE_TBB 1 +# else +# define RCPP_PARALLEL_USE_TBB 0 +# endif #endif #if RCPP_PARALLEL_USE_TBB - #include "RcppParallel/TBB.h" +# include "RcppParallel/TBB.h" #endif +#include "RcppParallel/Backend.h" #include "RcppParallel/RVector.h" #include "RcppParallel/RMatrix.h" namespace RcppParallel { -inline void parallelFor(std::size_t begin, std::size_t end, - Worker& worker, std::size_t grainSize = 1) { - +inline void parallelFor(std::size_t begin, + std::size_t end, + Worker& worker, + std::size_t grainSize = 1, + int numThreads = -1) +{ + grainSize = resolveValue("RCPP_PARALLEL_GRAIN_SIZE", grainSize, std::size_t(1)); + numThreads = resolveValue("RCPP_PARALLEL_NUM_THREADS", numThreads, -1); + #if RCPP_PARALLEL_USE_TBB - tbbParallelFor(begin, end, worker, grainSize); + if (internal::backend() == internal::BACKEND_TBB) + tbbParallelFor(begin, end, worker, grainSize, numThreads); + else + ttParallelFor(begin, end, worker, grainSize); #else ttParallelFor(begin, end, worker, grainSize); #endif - } template -inline void parallelReduce(std::size_t begin, std::size_t end, - Reducer& reducer, std::size_t grainSize = 1) { - +inline void parallelReduce(std::size_t begin, + std::size_t end, + Reducer& reducer, + std::size_t grainSize = 1, + int numThreads = -1) +{ + grainSize = resolveValue("RCPP_PARALLEL_GRAIN_SIZE", grainSize, std::size_t(1)); + numThreads = resolveValue("RCPP_PARALLEL_NUM_THREADS", numThreads, -1); + #if RCPP_PARALLEL_USE_TBB - tbbParallelReduce(begin, end, reducer, grainSize); + if (internal::backend() == internal::BACKEND_TBB) + tbbParallelReduce(begin, end, reducer, grainSize, numThreads); + else + ttParallelReduce(begin, end, reducer, grainSize); #else ttParallelReduce(begin, end, reducer, grainSize); #endif - } -} // namespace RcppParallel +} // end namespace RcppParallel + +// TRUE and FALSE macros that may come with system headers on some systems +// But conflict with R.h (R_ext/Boolean.h) +// TRUE and FALSE macros should be undef in RcppParallel.h +#ifdef TRUE + #undef TRUE +#endif +#ifdef FALSE + #undef FALSE +#endif #endif // __RCPP_PARALLEL__ diff --git a/inst/include/RcppParallel/Backend.h b/inst/include/RcppParallel/Backend.h new file mode 100644 index 000000000..13971b9e0 --- /dev/null +++ b/inst/include/RcppParallel/Backend.h @@ -0,0 +1,84 @@ + +#ifndef __RCPP_PARALLEL_BACKEND__ +#define __RCPP_PARALLEL_BACKEND__ + +#include +#include + +extern "C" { +void REprintf(const char*, ...); +} + +namespace RcppParallel { +namespace internal { + +enum backend_type { + BACKEND_TBB, + BACKEND_TINYTHREAD +}; + +#if RCPP_PARALLEL_USE_TBB + +inline backend_type defaultBackend() +{ + return BACKEND_TBB; +} + +#else + +inline backend_type defaultBackend() +{ + return BACKEND_TINYTHREAD; +} + +#endif + +inline const char* backendToString(backend_type backend) +{ + switch (backend) + { + case BACKEND_TBB: + return "tbb"; + case BACKEND_TINYTHREAD: + return "tinythread"; + } + + // shouldn't be reached but need to silence compiler warnings + return "tbb"; +} + +inline backend_type backend() +{ + const char* requestedBackend = std::getenv("RCPP_PARALLEL_BACKEND"); + if (requestedBackend == NULL) + { + return defaultBackend(); + } + else if (std::strcmp(requestedBackend, "tbb") == 0) + { +#if RCPP_PARALLEL_USE_TBB + return BACKEND_TBB; +#else + const char* msg = + "tbb backend is not available; using tinythread instead"; + + REprintf("%s\n", msg); + return BACKEND_TINYTHREAD; +#endif + } + else if (strcmp(requestedBackend, "tinythread") == 0) + { + return BACKEND_TINYTHREAD; + } + else + { + const char* fmt = "unknown parallel backend '%s'; using '%s' instead\n"; + REprintf(fmt, requestedBackend, backendToString(defaultBackend())); + return defaultBackend(); + } +} + +} // namespace internal +} // namespace RcppParallel + +#endif /* __RCPP_PARALLEL_BACKEND__ */ diff --git a/inst/include/RcppParallel/Common.h b/inst/include/RcppParallel/Common.h index b14d3f0cb..ae2b1327d 100644 --- a/inst/include/RcppParallel/Common.h +++ b/inst/include/RcppParallel/Common.h @@ -1,34 +1,86 @@ #ifndef __RCPP_PARALLEL_COMMON__ #define __RCPP_PARALLEL_COMMON__ +#include #include +#include + +#include +#include +#include +#include namespace RcppParallel { +template +inline int resolveValue(const char* envvar, + T requestedValue, + U defaultValue) +{ + // if the requested value is non-zero and not the default, we can use it + bool useRequestedValue = + requestedValue != static_cast(defaultValue) && + requestedValue > 0; + + if (useRequestedValue) + return requestedValue; + + // otherwise, try reading the default from associated envvar + // if the environment variable is unset, use the default + const char* var = getenv(envvar); + if (var == NULL) + return defaultValue; + + // try to convert the string to a number + // if an error occurs during conversion, just use default + errno = 0; + char* end; + long value = strtol(var, &end, 10); + + // check for conversion failure + if (end == var || *end != '\0' || errno == ERANGE) + return defaultValue; + + // okay, return the parsed environment variable value + return value; +} + +// Tag type used for disambiguating splitting constructors +struct Split {}; + // Work executed within a background thread. We implement dynamic // dispatch using vtables so we can have a stable type to cast // to from the void* passed to the worker thread (required because // the tinythreads interface allows to pass only a void* to the // thread main rather than a generic type / template) - -struct Worker -{ +struct Worker +{ // construct and destruct (delete virtually) Worker() {} virtual ~Worker() {} - + // dispatch work over a range of values - virtual void operator()(std::size_t begin, std::size_t end) = 0; - - // disable copying and assignment + virtual void operator()(std::size_t begin, std::size_t end) = 0; + private: + // disable copying and assignment Worker(const Worker&); void operator=(const Worker&); }; -// Tag type used for disambiguating splitting constructors +// Used for controlling the stack size for threads / tasks within a scope. +class ThreadStackSizeControl +{ +public: + ThreadStackSizeControl(); + ~ThreadStackSizeControl(); + +private: + // COPYING: not copyable + ThreadStackSizeControl(const ThreadStackSizeControl&); + ThreadStackSizeControl& operator=(const ThreadStackSizeControl&); +}; -struct Split {}; } // namespace RcppParallel diff --git a/inst/include/RcppParallel/RMatrix.h b/inst/include/RcppParallel/RMatrix.h index ca391b79a..576be720f 100644 --- a/inst/include/RcppParallel/RMatrix.h +++ b/inst/include/RcppParallel/RMatrix.h @@ -14,16 +14,21 @@ class RMatrix { public: template - class row_iterator - : public std::iterator { + class row_iterator { public: - inline row_iterator(Row& row, std::size_t i) + using iterator_category = std::random_access_iterator_tag; + using value_type = V; + using difference_type = std::size_t; + using pointer = value_type*; + using reference = value_type&; + + inline row_iterator(Row& row, difference_type i) : start_(row.start_), parentNrow_(row.parent_.nrow()), index_(i) { } - inline row_iterator(V* start, std::size_t parentNrow, std::size_t index) + inline row_iterator(pointer start, difference_type parentNrow, difference_type index) : start_(start), parentNrow_(parentNrow), index_(index) { } @@ -57,23 +62,23 @@ class RMatrix { return tmp ; } - row_iterator operator+(std::size_t n) const { + row_iterator operator+(difference_type n) const { return row_iterator(start_, parentNrow_ ,index_ + n ) ; } - row_iterator operator-(std::size_t n) const { + row_iterator operator-(difference_type n) const { return row_iterator(start_, parentNrow_, index_ - n ) ; } - std::size_t operator+(const row_iterator& other) const { + difference_type operator+(const row_iterator& other) const { return index_ + other.index_; } - std::size_t operator-(const row_iterator& other) const { + difference_type operator-(const row_iterator& other) const { return index_ - other.index_ ; } - row_iterator& operator+=(std::size_t n) { index_ += n ; return *this; } - row_iterator& operator-=(std::size_t n) { index_ -= n ; return *this; } + row_iterator& operator+=(difference_type n) { index_ += n ; return *this; } + row_iterator& operator-=(difference_type n) { index_ -= n ; return *this; } bool operator==(const row_iterator& other) const { return index_ == other.index_; } bool operator!=(const row_iterator& other) const { return index_ != other.index_; } @@ -83,16 +88,16 @@ class RMatrix { bool operator>=(const row_iterator& other) const { return index_ >= other.index_; } - inline V& operator*() { return start_[index_ * parentNrow_]; } + inline reference operator*() { return start_[index_ * parentNrow_]; } - inline V* operator->() { return &(start_[index_ * parentNrow_]); } + inline pointer operator->() { return &(start_[index_ * parentNrow_]); } - inline V& operator[](int i) { return start_[(index_+i) * parentNrow_]; } + inline reference operator[](int i) { return start_[(index_+i) * parentNrow_]; } private: - V* start_; - std::size_t parentNrow_; - std::size_t index_; + pointer start_; + difference_type parentNrow_; + difference_type index_; }; typedef row_iterator iterator; @@ -204,7 +209,7 @@ class RMatrix { { } - inline RMatrix(const T* data, std::size_t nrow, std::size_t ncol) + inline RMatrix(T* data, std::size_t nrow, std::size_t ncol) : data_(data), nrow_(nrow), ncol_(ncol) { } diff --git a/inst/include/RcppParallel/TBB.h b/inst/include/RcppParallel/TBB.h index 0d72178bb..5ac5722d8 100644 --- a/inst/include/RcppParallel/TBB.h +++ b/inst/include/RcppParallel/TBB.h @@ -3,74 +3,162 @@ #include "Common.h" -#include -#include +#ifndef TBB_PREVIEW_GLOBAL_CONTROL +# define TBB_PREVIEW_GLOBAL_CONTROL 1 +#endif -namespace RcppParallel { +// For compatibility with existing packages on CRAN. +#include "tbb/blocked_range.h" +#include "tbb/concurrent_unordered_set.h" +#include "tbb/concurrent_unordered_map.h" +#include "tbb/global_control.h" +#include "tbb/mutex.h" +#include "tbb/parallel_for.h" +#include "tbb/parallel_for_each.h" +#include "tbb/parallel_reduce.h" +#include "tbb/parallel_sort.h" +#include "tbb/spin_mutex.h" -namespace { +// For compatibility with older R packages. +namespace tbb { -struct TBBWorker -{ - explicit TBBWorker(Worker& worker) : worker_(worker) {} +#ifndef __TBB_task_scheduler_init_H +#define __TBB_task_scheduler_init_H + +class task_scheduler_init { - void operator()(const tbb::blocked_range& r) const { - worker_(r.begin(), r.end()); +public: + task_scheduler_init( + int number_of_threads = -1, + std::size_t stack_size = 0) + { } - -private: - Worker& worker_; + + static int default_num_threads() + { + return 2; + } + + static const int automatic = -1; + static const int deferred = -2; + }; -template -struct TBBReducer -{ - explicit TBBReducer(Reducer& reducer) - : pSplitReducer_(NULL), reducer_(reducer) +#endif + +} // end namespace tbb + + +namespace RcppParallel { + +// This class is primarily used to implement type erasure. The goals here were: +// +// 1. Hide the tbb symbols / implementation details from client R packages. +// That is, they should get the tools they need only via RcppParallel. +// +// 2. Do this in a way that preserves binary compatibility with pre-existing +// classes that make use of parallelReduce(). +// +// 3. Ensure that those packages, when re-compiled without source changes, +// can still function as expected. +// +// The downside here is that all the indirection through std::function<> +// and the requirement for RTTI is probably expensive, but I couldn't find +// a better way forward that could also preserve binary compatibility with +// existing pre-built pacakges. +// +// Hopefully, in a future release, we can do away with this wrapper, once +// packages have been rebuilt and no longer implicitly depend on TBB internals. +struct ReducerWrapper { + + template + ReducerWrapper(T* reducer) + { + self_ = reinterpret_cast(reducer); + owned_ = false; + + work_ = [&](void* self, std::size_t begin, std::size_t end) + { + (*reinterpret_cast(self))(begin, end); + }; + + split_ = [&](void* object, Split split) + { + return new T(*reinterpret_cast(object), split); + }; + + join_ = [&](void* self, void* other) + { + (*reinterpret_cast(self)).join(*reinterpret_cast(other)); + }; + + deleter_ = [&](void* object) + { + delete (T*) object; + }; + } + + ~ReducerWrapper() { + if (owned_) + { + deleter_(self_); + self_ = nullptr; + } } - - TBBReducer(TBBReducer& tbbReducer, tbb::split) - : pSplitReducer_(new Reducer(tbbReducer.reducer_, RcppParallel::Split())), - reducer_(*pSplitReducer_) + + void operator()(std::size_t begin, std::size_t end) const { + work_(self_, begin, end); } - - virtual ~TBBReducer() { delete pSplitReducer_; } - void operator()(const tbb::blocked_range& r) { - reducer_(r.begin(), r.end()); + ReducerWrapper(const ReducerWrapper& rhs, Split split) + { + self_ = rhs.split_(rhs.self_, split); + owned_ = true; + + work_ = rhs.work_; + split_ = rhs.split_; + join_ = rhs.join_; + deleter_ = rhs.deleter_; } - - void join(const TBBReducer& tbbReducer) { - reducer_.join(tbbReducer.reducer_); + + void join(const ReducerWrapper& rhs) const + { + join_(self_, rhs.self_); } - + private: - Reducer* pSplitReducer_; - Reducer& reducer_; + void* self_ = nullptr; + bool owned_ = false; + + std::function work_; + std::function split_; + std::function join_; + std::function deleter_; }; - -} // anonymous namespace +void tbbParallelFor(std::size_t begin, + std::size_t end, + Worker& worker, + std::size_t grainSize = 1, + int numThreads = -1); -inline void tbbParallelFor(std::size_t begin, std::size_t end, - Worker& worker, std::size_t grainSize = 1) { - - TBBWorker tbbWorker(worker); - - tbb::parallel_for(tbb::blocked_range(begin, end, grainSize), - tbbWorker); -} +void tbbParallelReduceImpl(std::size_t begin, + std::size_t end, + ReducerWrapper& wrapper, + std::size_t grainSize = 1, + int numThreads = -1); template -inline void tbbParallelReduce(std::size_t begin, std::size_t end, - Reducer& reducer, std::size_t grainSize = 1) { - - TBBReducer tbbReducer(reducer); - - tbb::parallel_reduce(tbb::blocked_range(begin, end, grainSize), - tbbReducer); +void tbbParallelReduce(std::size_t begin, + std::size_t end, + Reducer& reducer, + std::size_t grainSize = 1, + int numThreads = -1) +{ + ReducerWrapper wrapper(&reducer); + tbbParallelReduceImpl(begin, end, wrapper, grainSize, numThreads); } } // namespace RcppParallel diff --git a/inst/include/RcppParallel/TinyThread.h b/inst/include/RcppParallel/TinyThread.h index 809f34bc2..dfa3dcb73 100644 --- a/inst/include/RcppParallel/TinyThread.h +++ b/inst/include/RcppParallel/TinyThread.h @@ -2,12 +2,12 @@ #define __RCPP_PARALLEL_TINYTHREAD__ #include +#include #include "Common.h" #include -#include #include @@ -89,10 +89,15 @@ std::vector splitInputRange(const IndexRange& range, // allocate ranges std::vector ranges; std::size_t begin = range.begin(); + std::size_t end = begin; while (begin < range.end()) { - std::size_t end = std::min(begin + grainSize, range.end()); - ranges.push_back(IndexRange(begin, end)); - begin = end; + if ((range.end() - (begin + grainSize)) < grainSize) + end = range.end(); + else + end = std::min(begin + grainSize, range.end()); + + ranges.push_back(IndexRange(begin, end)); + begin = end; } // return ranges @@ -102,9 +107,11 @@ std::vector splitInputRange(const IndexRange& range, } // anonymous namespace // Execute the Worker over the IndexRange in parallel -inline void ttParallelFor(std::size_t begin, std::size_t end, - Worker& worker, std::size_t grainSize = 1) { - +inline void ttParallelFor(std::size_t begin, + std::size_t end, + Worker& worker, + std::size_t grainSize = 1) +{ // split the work IndexRange inputRange(begin, end); std::vector ranges = splitInputRange(inputRange, grainSize); @@ -124,9 +131,11 @@ inline void ttParallelFor(std::size_t begin, std::size_t end, // Execute the IWorker over the range in parallel then join results template -inline void ttParallelReduce(std::size_t begin, std::size_t end, - Reducer& reducer, std::size_t grainSize = 1) { - +inline void ttParallelReduce(std::size_t begin, + std::size_t end, + Reducer& reducer, + std::size_t grainSize = 1) +{ // split the work IndexRange inputRange(begin, end); std::vector ranges = splitInputRange(inputRange, grainSize); diff --git a/inst/include/index.html b/inst/include/index.html deleted file mode 100644 index b0962e012..000000000 --- a/inst/include/index.html +++ /dev/null @@ -1,25 +0,0 @@ - - - -

Overview

-Include files for Intel® Threading Building Blocks (Intel® TBB). - -

Directories

-
-
tbb -
Include files for Intel TBB classes and functions. -
serial/tbb -
Include files for a sequential implementation of the parallel_for algorithm. -
- -
-Up to parent directory -

-Copyright © 2005-2017 Intel Corporation. All Rights Reserved. -

-Intel is a registered trademark or trademark of Intel Corporation -or its subsidiaries in the United States and other countries. -

-* Other names and brands may be claimed as the property of others. - - diff --git a/inst/include/serial/tbb/parallel_for.h b/inst/include/serial/tbb/parallel_for.h deleted file mode 100644 index e5959c222..000000000 --- a/inst/include/serial/tbb/parallel_for.h +++ /dev/null @@ -1,219 +0,0 @@ -/* - Copyright (c) 2005-2018 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_SERIAL_parallel_for_H -#define __TBB_SERIAL_parallel_for_H - -#include "tbb_annotate.h" - -#ifndef __TBB_NORMAL_EXECUTION -#include "tbb/blocked_range.h" -#include "tbb/partitioner.h" -#endif - -#if TBB_USE_EXCEPTIONS -#include -#include // required to construct std exception classes -#else -#include -#include -#endif - -namespace tbb { -namespace serial { -namespace interface9 { - -// parallel_for serial annotated implementation - -template< typename Range, typename Body, typename Partitioner > -class start_for : tbb::internal::no_copy { - Range my_range; - const Body my_body; - typename Partitioner::task_partition_type my_partition; - void execute(); - - //! Constructor for root task. - start_for( const Range& range, const Body& body, Partitioner& partitioner ) : - my_range( range ), - my_body( body ), - my_partition( partitioner ) - { - } - - //! Splitting constructor used to generate children. - /** this becomes left child. Newly constructed object is right child. */ - start_for( start_for& parent_, typename Partitioner::split_type& split_obj ) : - my_range( parent_.my_range, split_obj ), - my_body( parent_.my_body ), - my_partition( parent_.my_partition, split_obj ) - { - } - -public: - static void run( const Range& range, const Body& body, Partitioner& partitioner ) { - if( !range.empty() ) { - ANNOTATE_SITE_BEGIN( tbb_parallel_for ); - { - start_for a( range, body, partitioner ); - a.execute(); - } - ANNOTATE_SITE_END( tbb_parallel_for ); - } - } -}; - -template< typename Range, typename Body, typename Partitioner > -void start_for< Range, Body, Partitioner >::execute() { - if( !my_range.is_divisible() || !my_partition.is_divisible() ) { - ANNOTATE_TASK_BEGIN( tbb_parallel_for_range ); - { - my_body( my_range ); - } - ANNOTATE_TASK_END( tbb_parallel_for_range ); - } else { - typename Partitioner::split_type split_obj; - start_for b( *this, split_obj ); - this->execute(); // Execute the left interval first to keep the serial order. - b.execute(); // Execute the right interval then. - } -} - -//! Parallel iteration over range with default partitioner. -/** @ingroup algorithms **/ -template -void parallel_for( const Range& range, const Body& body ) { - serial::interface9::start_for::run(range,body,__TBB_DEFAULT_PARTITIONER()); -} - -//! Parallel iteration over range with simple partitioner. -/** @ingroup algorithms **/ -template -void parallel_for( const Range& range, const Body& body, const simple_partitioner& partitioner ) { - serial::interface9::start_for::run(range,body,partitioner); -} - -//! Parallel iteration over range with auto_partitioner. -/** @ingroup algorithms **/ -template -void parallel_for( const Range& range, const Body& body, const auto_partitioner& partitioner ) { - serial::interface9::start_for::run(range,body,partitioner); -} - -//! Parallel iteration over range with static_partitioner. -/** @ingroup algorithms **/ -template -void parallel_for( const Range& range, const Body& body, const static_partitioner& partitioner ) { - serial::interface9::start_for::run(range,body,partitioner); -} - -//! Parallel iteration over range with affinity_partitioner. -/** @ingroup algorithms **/ -template -void parallel_for( const Range& range, const Body& body, affinity_partitioner& partitioner ) { - serial::interface9::start_for::run(range,body,partitioner); -} - -//! Implementation of parallel iteration over stepped range of integers with explicit step and partitioner (ignored) -template -void parallel_for_impl(Index first, Index last, Index step, const Function& f, Partitioner& ) { - if (step <= 0 ) { -#if TBB_USE_EXCEPTIONS - throw std::invalid_argument( "nonpositive_step" ); -#else - std::cerr << "nonpositive step in a call to parallel_for" << std::endl; - std::abort(); -#endif - } else if (last > first) { - // Above "else" avoids "potential divide by zero" warning on some platforms - ANNOTATE_SITE_BEGIN( tbb_parallel_for ); - for( Index i = first; i < last; i = i + step ) { - ANNOTATE_TASK_BEGIN( tbb_parallel_for_iteration ); - { f( i ); } - ANNOTATE_TASK_END( tbb_parallel_for_iteration ); - } - ANNOTATE_SITE_END( tbb_parallel_for ); - } -} - -//! Parallel iteration over a range of integers with explicit step and default partitioner -template -void parallel_for(Index first, Index last, Index step, const Function& f) { - parallel_for_impl(first, last, step, f, auto_partitioner()); -} -//! Parallel iteration over a range of integers with explicit step and simple partitioner -template -void parallel_for(Index first, Index last, Index step, const Function& f, const simple_partitioner& p) { - parallel_for_impl(first, last, step, f, p); -} -//! Parallel iteration over a range of integers with explicit step and auto partitioner -template -void parallel_for(Index first, Index last, Index step, const Function& f, const auto_partitioner& p) { - parallel_for_impl(first, last, step, f, p); -} -//! Parallel iteration over a range of integers with explicit step and static partitioner -template -void parallel_for(Index first, Index last, Index step, const Function& f, const static_partitioner& p) { - parallel_for_impl(first, last, step, f, p); -} -//! Parallel iteration over a range of integers with explicit step and affinity partitioner -template -void parallel_for(Index first, Index last, Index step, const Function& f, affinity_partitioner& p) { - parallel_for_impl(first, last, step, f, p); -} - -//! Parallel iteration over a range of integers with default step and default partitioner -template -void parallel_for(Index first, Index last, const Function& f) { - parallel_for_impl(first, last, static_cast(1), f, auto_partitioner()); -} -//! Parallel iteration over a range of integers with default step and simple partitioner -template -void parallel_for(Index first, Index last, const Function& f, const simple_partitioner& p) { - parallel_for_impl(first, last, static_cast(1), f, p); -} -//! Parallel iteration over a range of integers with default step and auto partitioner -template - void parallel_for(Index first, Index last, const Function& f, const auto_partitioner& p) { - parallel_for_impl(first, last, static_cast(1), f, p); -} -//! Parallel iteration over a range of integers with default step and static partitioner -template -void parallel_for(Index first, Index last, const Function& f, const static_partitioner& p) { - parallel_for_impl(first, last, static_cast(1), f, p); -} -//! Parallel iteration over a range of integers with default step and affinity_partitioner -template -void parallel_for(Index first, Index last, const Function& f, affinity_partitioner& p) { - parallel_for_impl(first, last, static_cast(1), f, p); -} - -} // namespace interfaceX - -using interface9::parallel_for; - -} // namespace serial - -#ifndef __TBB_NORMAL_EXECUTION -using serial::interface9::parallel_for; -#endif - -} // namespace tbb - -#endif /* __TBB_SERIAL_parallel_for_H */ diff --git a/inst/include/tbb/aggregator.h b/inst/include/tbb/aggregator.h deleted file mode 100644 index 6aecbb748..000000000 --- a/inst/include/tbb/aggregator.h +++ /dev/null @@ -1,202 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB__aggregator_H -#define __TBB__aggregator_H - -#if !TBB_PREVIEW_AGGREGATOR -#error Set TBB_PREVIEW_AGGREGATOR before including aggregator.h -#endif - -#include "atomic.h" -#include "tbb_profiling.h" - -namespace tbb { -namespace interface6 { - -using namespace tbb::internal; - -class aggregator_operation { - template friend class aggregator_ext; - uintptr_t status; - aggregator_operation* my_next; -public: - enum aggregator_operation_status { agg_waiting=0, agg_finished }; - aggregator_operation() : status(agg_waiting), my_next(NULL) {} - /// Call start before handling this operation - void start() { call_itt_notify(acquired, &status); } - /// Call finish when done handling this operation - /** The operation will be released to its originating thread, and possibly deleted. */ - void finish() { itt_store_word_with_release(status, uintptr_t(agg_finished)); } - aggregator_operation* next() { return itt_hide_load_word(my_next);} - void set_next(aggregator_operation* n) { itt_hide_store_word(my_next, n); } -}; - -namespace internal { - -class basic_operation_base : public aggregator_operation { - friend class basic_handler; - virtual void apply_body() = 0; -public: - basic_operation_base() : aggregator_operation() {} - virtual ~basic_operation_base() {} -}; - -template -class basic_operation : public basic_operation_base, no_assign { - const Body& my_body; - void apply_body() __TBB_override { my_body(); } -public: - basic_operation(const Body& b) : basic_operation_base(), my_body(b) {} -}; - -class basic_handler { -public: - basic_handler() {} - void operator()(aggregator_operation* op_list) const { - while (op_list) { - // ITT note: &(op_list->status) tag is used to cover accesses to the operation data. - // The executing thread "acquires" the tag (see start()) and then performs - // the associated operation w/o triggering a race condition diagnostics. - // A thread that created the operation is waiting for its status (see execute_impl()), - // so when this thread is done with the operation, it will "release" the tag - // and update the status (see finish()) to give control back to the waiting thread. - basic_operation_base& request = static_cast(*op_list); - // IMPORTANT: need to advance op_list to op_list->next() before calling request.finish() - op_list = op_list->next(); - request.start(); - request.apply_body(); - request.finish(); - } - } -}; - -} // namespace internal - -//! Aggregator base class and expert interface -/** An aggregator for collecting operations coming from multiple sources and executing - them serially on a single thread. */ -template -class aggregator_ext : tbb::internal::no_copy { -public: - aggregator_ext(const handler_type& h) : handler_busy(0), handle_operations(h) { mailbox = NULL; } - - //! EXPERT INTERFACE: Enter a user-made operation into the aggregator's mailbox. - /** Details of user-made operations must be handled by user-provided handler */ - void process(aggregator_operation *op) { execute_impl(*op); } - -protected: - /** Place operation in mailbox, then either handle mailbox or wait for the operation - to be completed by a different thread. */ - void execute_impl(aggregator_operation& op) { - aggregator_operation* res; - - // ITT note: &(op.status) tag is used to cover accesses to this operation. This - // thread has created the operation, and now releases it so that the handler - // thread may handle the associated operation w/o triggering a race condition; - // thus this tag will be acquired just before the operation is handled in the - // handle_operations functor. - call_itt_notify(releasing, &(op.status)); - // insert the operation into the list - do { - // ITT may flag the following line as a race; it is a false positive: - // This is an atomic read; we don't provide itt_hide_load_word for atomics - op.my_next = res = mailbox; // NOT A RACE - } while (mailbox.compare_and_swap(&op, res) != res); - if (!res) { // first in the list; handle the operations - // ITT note: &mailbox tag covers access to the handler_busy flag, which this - // waiting handler thread will try to set before entering handle_operations. - call_itt_notify(acquired, &mailbox); - start_handle_operations(); - __TBB_ASSERT(op.status, NULL); - } - else { // not first; wait for op to be ready - call_itt_notify(prepare, &(op.status)); - spin_wait_while_eq(op.status, uintptr_t(aggregator_operation::agg_waiting)); - itt_load_word_with_acquire(op.status); - } - } - - -private: - //! An atomically updated list (aka mailbox) of aggregator_operations - atomic mailbox; - - //! Controls thread access to handle_operations - /** Behaves as boolean flag where 0=false, 1=true */ - uintptr_t handler_busy; - - handler_type handle_operations; - - //! Trigger the handling of operations when the handler is free - void start_handle_operations() { - aggregator_operation *pending_operations; - - // ITT note: &handler_busy tag covers access to mailbox as it is passed - // between active and waiting handlers. Below, the waiting handler waits until - // the active handler releases, and the waiting handler acquires &handler_busy as - // it becomes the active_handler. The release point is at the end of this - // function, when all operations in mailbox have been handled by the - // owner of this aggregator. - call_itt_notify(prepare, &handler_busy); - // get handler_busy: only one thread can possibly spin here at a time - spin_wait_until_eq(handler_busy, uintptr_t(0)); - call_itt_notify(acquired, &handler_busy); - // acquire fence not necessary here due to causality rule and surrounding atomics - __TBB_store_with_release(handler_busy, uintptr_t(1)); - - // ITT note: &mailbox tag covers access to the handler_busy flag itself. - // Capturing the state of the mailbox signifies that handler_busy has been - // set and a new active handler will now process that list's operations. - call_itt_notify(releasing, &mailbox); - // grab pending_operations - pending_operations = mailbox.fetch_and_store(NULL); - - // handle all the operations - handle_operations(pending_operations); - - // release the handler - itt_store_word_with_release(handler_busy, uintptr_t(0)); - } -}; - -//! Basic aggregator interface -class aggregator : private aggregator_ext { -public: - aggregator() : aggregator_ext(internal::basic_handler()) {} - //! BASIC INTERFACE: Enter a function for exclusive execution by the aggregator. - /** The calling thread stores the function object in a basic_operation and - places the operation in the aggregator's mailbox */ - template - void execute(const Body& b) { - internal::basic_operation op(b); - this->execute_impl(op); - } -}; - -} // namespace interface6 - -using interface6::aggregator; -using interface6::aggregator_ext; -using interface6::aggregator_operation; - -} // namespace tbb - -#endif // __TBB__aggregator_H diff --git a/inst/include/tbb/aligned_space.h b/inst/include/tbb/aligned_space.h deleted file mode 100644 index 56fd85f34..000000000 --- a/inst/include/tbb/aligned_space.h +++ /dev/null @@ -1,47 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_aligned_space_H -#define __TBB_aligned_space_H - -#include "tbb_stddef.h" -#include "tbb_machine.h" - -namespace tbb { - -//! Block of space aligned sufficiently to construct an array T with N elements. -/** The elements are not constructed or destroyed by this class. - @ingroup memory_allocation */ -template -class aligned_space { -private: - typedef __TBB_TypeWithAlignmentAtLeastAsStrict(T) element_type; - element_type array[(sizeof(T)*N+sizeof(element_type)-1)/sizeof(element_type)]; -public: - //! Pointer to beginning of array - T* begin() {return internal::punned_cast(this);} - - //! Pointer to one past last element in array. - T* end() {return begin()+N;} -}; - -} // namespace tbb - -#endif /* __TBB_aligned_space_H */ diff --git a/inst/include/tbb/atomic.h b/inst/include/tbb/atomic.h deleted file mode 100644 index 72ec534e3..000000000 --- a/inst/include/tbb/atomic.h +++ /dev/null @@ -1,558 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_atomic_H -#define __TBB_atomic_H - -#include - -#if _MSC_VER -#define __TBB_LONG_LONG __int64 -#else -#define __TBB_LONG_LONG long long -#endif /* _MSC_VER */ - -#include "tbb_machine.h" - -#if _MSC_VER && !__INTEL_COMPILER - // Suppress overzealous compiler warnings till the end of the file - #pragma warning (push) - #pragma warning (disable: 4244 4267 4512) -#endif - -namespace tbb { - -//! Specifies memory semantics. -enum memory_semantics { - //! Sequential consistency - full_fence, - //! Acquire - acquire, - //! Release - release, - //! No ordering - relaxed -}; - -//! @cond INTERNAL -namespace internal { - -#if __TBB_ALIGNAS_PRESENT - #define __TBB_DECL_ATOMIC_FIELD(t,f,a) alignas(a) t f; -#elif __TBB_ATTRIBUTE_ALIGNED_PRESENT - #define __TBB_DECL_ATOMIC_FIELD(t,f,a) t f __attribute__ ((aligned(a))); -#elif __TBB_DECLSPEC_ALIGN_PRESENT - #define __TBB_DECL_ATOMIC_FIELD(t,f,a) __declspec(align(a)) t f; -#else - #error Do not know syntax for forcing alignment. -#endif - -template -struct atomic_rep; // Primary template declared, but never defined. - -template<> -struct atomic_rep<1> { // Specialization - typedef int8_t word; -}; -template<> -struct atomic_rep<2> { // Specialization - typedef int16_t word; -}; -template<> -struct atomic_rep<4> { // Specialization -#if _MSC_VER && !_WIN64 - // Work-around that avoids spurious /Wp64 warnings - typedef intptr_t word; -#else - typedef int32_t word; -#endif -}; -#if __TBB_64BIT_ATOMICS -template<> -struct atomic_rep<8> { // Specialization - typedef int64_t word; -}; -#endif - -template -struct aligned_storage; - -//the specializations are needed to please MSVC syntax of __declspec(align()) which accept _literal_ constants only -#if __TBB_ATOMIC_CTORS - #define ATOMIC_STORAGE_PARTIAL_SPECIALIZATION(S) \ - template \ - struct aligned_storage { \ - __TBB_DECL_ATOMIC_FIELD(value_type,my_value,S) \ - aligned_storage() = default ; \ - constexpr aligned_storage(value_type value):my_value(value){} \ - }; \ - -#else - #define ATOMIC_STORAGE_PARTIAL_SPECIALIZATION(S) \ - template \ - struct aligned_storage { \ - __TBB_DECL_ATOMIC_FIELD(value_type,my_value,S) \ - }; \ - -#endif - -template -struct aligned_storage { - value_type my_value; -#if __TBB_ATOMIC_CTORS - aligned_storage() = default ; - constexpr aligned_storage(value_type value):my_value(value){} -#endif -}; - -ATOMIC_STORAGE_PARTIAL_SPECIALIZATION(2) -ATOMIC_STORAGE_PARTIAL_SPECIALIZATION(4) -#if __TBB_64BIT_ATOMICS -ATOMIC_STORAGE_PARTIAL_SPECIALIZATION(8) -#endif - -template -struct atomic_traits; // Primary template declared, but not defined. - -#define __TBB_DECL_FENCED_ATOMIC_PRIMITIVES(S,M) \ - template<> struct atomic_traits { \ - typedef atomic_rep::word word; \ - inline static word compare_and_swap( volatile void* location, word new_value, word comparand ) { \ - return __TBB_machine_cmpswp##S##M(location,new_value,comparand); \ - } \ - inline static word fetch_and_add( volatile void* location, word addend ) { \ - return __TBB_machine_fetchadd##S##M(location,addend); \ - } \ - inline static word fetch_and_store( volatile void* location, word value ) { \ - return __TBB_machine_fetchstore##S##M(location,value); \ - } \ - }; - -#define __TBB_DECL_ATOMIC_PRIMITIVES(S) \ - template \ - struct atomic_traits { \ - typedef atomic_rep::word word; \ - inline static word compare_and_swap( volatile void* location, word new_value, word comparand ) { \ - return __TBB_machine_cmpswp##S(location,new_value,comparand); \ - } \ - inline static word fetch_and_add( volatile void* location, word addend ) { \ - return __TBB_machine_fetchadd##S(location,addend); \ - } \ - inline static word fetch_and_store( volatile void* location, word value ) { \ - return __TBB_machine_fetchstore##S(location,value); \ - } \ - }; - -template -struct atomic_load_store_traits; // Primary template declaration - -#define __TBB_DECL_ATOMIC_LOAD_STORE_PRIMITIVES(M) \ - template<> struct atomic_load_store_traits { \ - template \ - inline static T load( const volatile T& location ) { \ - return __TBB_load_##M( location ); \ - } \ - template \ - inline static void store( volatile T& location, T value ) { \ - __TBB_store_##M( location, value ); \ - } \ - } - -#if __TBB_USE_FENCED_ATOMICS -__TBB_DECL_FENCED_ATOMIC_PRIMITIVES(1,full_fence) -__TBB_DECL_FENCED_ATOMIC_PRIMITIVES(2,full_fence) -__TBB_DECL_FENCED_ATOMIC_PRIMITIVES(4,full_fence) -__TBB_DECL_FENCED_ATOMIC_PRIMITIVES(1,acquire) -__TBB_DECL_FENCED_ATOMIC_PRIMITIVES(2,acquire) -__TBB_DECL_FENCED_ATOMIC_PRIMITIVES(4,acquire) -__TBB_DECL_FENCED_ATOMIC_PRIMITIVES(1,release) -__TBB_DECL_FENCED_ATOMIC_PRIMITIVES(2,release) -__TBB_DECL_FENCED_ATOMIC_PRIMITIVES(4,release) -__TBB_DECL_FENCED_ATOMIC_PRIMITIVES(1,relaxed) -__TBB_DECL_FENCED_ATOMIC_PRIMITIVES(2,relaxed) -__TBB_DECL_FENCED_ATOMIC_PRIMITIVES(4,relaxed) -#if __TBB_64BIT_ATOMICS -__TBB_DECL_FENCED_ATOMIC_PRIMITIVES(8,full_fence) -__TBB_DECL_FENCED_ATOMIC_PRIMITIVES(8,acquire) -__TBB_DECL_FENCED_ATOMIC_PRIMITIVES(8,release) -__TBB_DECL_FENCED_ATOMIC_PRIMITIVES(8,relaxed) -#endif -#else /* !__TBB_USE_FENCED_ATOMICS */ -__TBB_DECL_ATOMIC_PRIMITIVES(1) -__TBB_DECL_ATOMIC_PRIMITIVES(2) -__TBB_DECL_ATOMIC_PRIMITIVES(4) -#if __TBB_64BIT_ATOMICS -__TBB_DECL_ATOMIC_PRIMITIVES(8) -#endif -#endif /* !__TBB_USE_FENCED_ATOMICS */ - -__TBB_DECL_ATOMIC_LOAD_STORE_PRIMITIVES(full_fence); -__TBB_DECL_ATOMIC_LOAD_STORE_PRIMITIVES(acquire); -__TBB_DECL_ATOMIC_LOAD_STORE_PRIMITIVES(release); -__TBB_DECL_ATOMIC_LOAD_STORE_PRIMITIVES(relaxed); - -//! Additive inverse of 1 for type T. -/** Various compilers issue various warnings if -1 is used with various integer types. - The baroque expression below avoids all the warnings (we hope). */ -#define __TBB_MINUS_ONE(T) (T(T(0)-T(1))) - -//! Base class that provides basic functionality for atomic without fetch_and_add. -/** Works for any type T that has the same size as an integral type, has a trivial constructor/destructor, - and can be copied/compared by memcpy/memcmp. */ -template -struct atomic_impl { -protected: - aligned_storage my_storage; -private: - //TODO: rechecks on recent versions of gcc if union is still the _only_ way to do a conversion without warnings - //! Union type used to convert type T to underlying integral type. - template - union converter { - typedef typename atomic_rep::word bits_type; - converter(){} - converter(value_type a_value) : value(a_value) {} - value_type value; - bits_type bits; - }; - - template - static typename converter::bits_type to_bits(value_t value){ - return converter(value).bits; - } - template - static value_t to_value(typename converter::bits_type bits){ - converter u; - u.bits = bits; - return u.value; - } - - template - union ptr_converter; //Primary template declared, but never defined. - - template - union ptr_converter { - ptr_converter(){} - ptr_converter(value_t* a_value) : value(a_value) {} - value_t* value; - uintptr_t bits; - }; - //TODO: check if making to_bits accepting reference (thus unifying it with to_bits_ref) - //does not hurt performance - template - static typename converter::bits_type & to_bits_ref(value_t& value){ - //TODO: this #ifdef is temporary workaround, as union conversion seems to fail - //on suncc for 64 bit types for 32 bit target - #if !__SUNPRO_CC - return *(typename converter::bits_type*)ptr_converter(&value).bits; - #else - return *(typename converter::bits_type*)(&value); - #endif - } - - -public: - typedef T value_type; - -#if __TBB_ATOMIC_CTORS - atomic_impl() = default ; - constexpr atomic_impl(value_type value):my_storage(value){} -#endif - template - value_type fetch_and_store( value_type value ) { - return to_value( - internal::atomic_traits::fetch_and_store( &my_storage.my_value, to_bits(value) ) - ); - } - - value_type fetch_and_store( value_type value ) { - return fetch_and_store(value); - } - - template - value_type compare_and_swap( value_type value, value_type comparand ) { - return to_value( - internal::atomic_traits::compare_and_swap( &my_storage.my_value, to_bits(value), to_bits(comparand) ) - ); - } - - value_type compare_and_swap( value_type value, value_type comparand ) { - return compare_and_swap(value,comparand); - } - - operator value_type() const volatile { // volatile qualifier here for backwards compatibility - return to_value( - __TBB_load_with_acquire( to_bits_ref(my_storage.my_value) ) - ); - } - - template - value_type load () const { - return to_value( - internal::atomic_load_store_traits::load( to_bits_ref(my_storage.my_value) ) - ); - } - - value_type load () const { - return load(); - } - - template - void store ( value_type value ) { - internal::atomic_load_store_traits::store( to_bits_ref(my_storage.my_value), to_bits(value)); - } - - void store ( value_type value ) { - store( value ); - } - -protected: - value_type store_with_release( value_type rhs ) { - //TODO: unify with store - __TBB_store_with_release( to_bits_ref(my_storage.my_value), to_bits(rhs) ); - return rhs; - } -}; - -//! Base class that provides basic functionality for atomic with fetch_and_add. -/** I is the underlying type. - D is the difference type. - StepType should be char if I is an integral type, and T if I is a T*. */ -template -struct atomic_impl_with_arithmetic: atomic_impl { -public: - typedef I value_type; -#if __TBB_ATOMIC_CTORS - atomic_impl_with_arithmetic() = default ; - constexpr atomic_impl_with_arithmetic(value_type value): atomic_impl(value){} -#endif - template - value_type fetch_and_add( D addend ) { - return value_type(internal::atomic_traits::fetch_and_add( &this->my_storage.my_value, addend*sizeof(StepType) )); - } - - value_type fetch_and_add( D addend ) { - return fetch_and_add(addend); - } - - template - value_type fetch_and_increment() { - return fetch_and_add(1); - } - - value_type fetch_and_increment() { - return fetch_and_add(1); - } - - template - value_type fetch_and_decrement() { - return fetch_and_add(__TBB_MINUS_ONE(D)); - } - - value_type fetch_and_decrement() { - return fetch_and_add(__TBB_MINUS_ONE(D)); - } - -public: - value_type operator+=( D value ) { - return fetch_and_add(value)+value; - } - - value_type operator-=( D value ) { - // Additive inverse of value computed using binary minus, - // instead of unary minus, for sake of avoiding compiler warnings. - return operator+=(D(0)-value); - } - - value_type operator++() { - return fetch_and_add(1)+1; - } - - value_type operator--() { - return fetch_and_add(__TBB_MINUS_ONE(D))-1; - } - - value_type operator++(int) { - return fetch_and_add(1); - } - - value_type operator--(int) { - return fetch_and_add(__TBB_MINUS_ONE(D)); - } -}; - -} /* Internal */ -//! @endcond - -//! Primary template for atomic. -/** See the Reference for details. - @ingroup synchronization */ -template -struct atomic: internal::atomic_impl { -#if __TBB_ATOMIC_CTORS - atomic() = default; - constexpr atomic(T arg): internal::atomic_impl(arg) {} -#endif - T operator=( T rhs ) { - // "this" required here in strict ISO C++ because store_with_release is a dependent name - return this->store_with_release(rhs); - } - atomic& operator=( const atomic& rhs ) {this->store_with_release(rhs); return *this;} -}; - -#if __TBB_ATOMIC_CTORS - #define __TBB_DECL_ATOMIC(T) \ - template<> struct atomic: internal::atomic_impl_with_arithmetic { \ - atomic() = default; \ - constexpr atomic(T arg): internal::atomic_impl_with_arithmetic(arg) {} \ - \ - T operator=( T rhs ) {return store_with_release(rhs);} \ - atomic& operator=( const atomic& rhs ) {store_with_release(rhs); return *this;} \ - }; -#else - #define __TBB_DECL_ATOMIC(T) \ - template<> struct atomic: internal::atomic_impl_with_arithmetic { \ - T operator=( T rhs ) {return store_with_release(rhs);} \ - atomic& operator=( const atomic& rhs ) {store_with_release(rhs); return *this;} \ - }; -#endif - -#if __TBB_64BIT_ATOMICS -//TODO: consider adding non-default (and atomic) copy constructor for 32bit platform -__TBB_DECL_ATOMIC(__TBB_LONG_LONG) -__TBB_DECL_ATOMIC(unsigned __TBB_LONG_LONG) -#else -// test_atomic will verify that sizeof(long long)==8 -#endif -__TBB_DECL_ATOMIC(long) -__TBB_DECL_ATOMIC(unsigned long) - -#if _MSC_VER && !_WIN64 -#if __TBB_ATOMIC_CTORS -/* Special version of __TBB_DECL_ATOMIC that avoids gratuitous warnings from cl /Wp64 option. - It is identical to __TBB_DECL_ATOMIC(unsigned) except that it replaces operator=(T) - with an operator=(U) that explicitly converts the U to a T. Types T and U should be - type synonyms on the platform. Type U should be the wider variant of T from the - perspective of /Wp64. */ -#define __TBB_DECL_ATOMIC_ALT(T,U) \ - template<> struct atomic: internal::atomic_impl_with_arithmetic { \ - atomic() = default ; \ - constexpr atomic(T arg): internal::atomic_impl_with_arithmetic(arg) {} \ - T operator=( U rhs ) {return store_with_release(T(rhs));} \ - atomic& operator=( const atomic& rhs ) {store_with_release(rhs); return *this;} \ - }; -#else -#define __TBB_DECL_ATOMIC_ALT(T,U) \ - template<> struct atomic: internal::atomic_impl_with_arithmetic { \ - T operator=( U rhs ) {return store_with_release(T(rhs));} \ - atomic& operator=( const atomic& rhs ) {store_with_release(rhs); return *this;} \ - }; -#endif -__TBB_DECL_ATOMIC_ALT(unsigned,size_t) -__TBB_DECL_ATOMIC_ALT(int,ptrdiff_t) -#else -__TBB_DECL_ATOMIC(unsigned) -__TBB_DECL_ATOMIC(int) -#endif /* _MSC_VER && !_WIN64 */ - -__TBB_DECL_ATOMIC(unsigned short) -__TBB_DECL_ATOMIC(short) -__TBB_DECL_ATOMIC(char) -__TBB_DECL_ATOMIC(signed char) -__TBB_DECL_ATOMIC(unsigned char) - -#if !_MSC_VER || defined(_NATIVE_WCHAR_T_DEFINED) -__TBB_DECL_ATOMIC(wchar_t) -#endif /* _MSC_VER||!defined(_NATIVE_WCHAR_T_DEFINED) */ - -//! Specialization for atomic with arithmetic and operator->. -template struct atomic: internal::atomic_impl_with_arithmetic { -#if __TBB_ATOMIC_CTORS - atomic() = default ; - constexpr atomic(T* arg): internal::atomic_impl_with_arithmetic(arg) {} -#endif - T* operator=( T* rhs ) { - // "this" required here in strict ISO C++ because store_with_release is a dependent name - return this->store_with_release(rhs); - } - atomic& operator=( const atomic& rhs ) { - this->store_with_release(rhs); return *this; - } - T* operator->() const { - return (*this); - } -}; - -//! Specialization for atomic, for sake of not allowing arithmetic or operator->. -template<> struct atomic: internal::atomic_impl { -#if __TBB_ATOMIC_CTORS - atomic() = default ; - constexpr atomic(void* arg): internal::atomic_impl(arg) {} -#endif - void* operator=( void* rhs ) { - // "this" required here in strict ISO C++ because store_with_release is a dependent name - return this->store_with_release(rhs); - } - atomic& operator=( const atomic& rhs ) { - this->store_with_release(rhs); return *this; - } -}; - -// Helpers to workaround ugly syntax of calling template member function of a -// template class with template argument dependent on template parameters. - -template -T load ( const atomic& a ) { return a.template load(); } - -template -void store ( atomic& a, T value ) { a.template store(value); } - -namespace interface6{ -//! Make an atomic for use in an initialization (list), as an alternative to zero-initialization or normal assignment. -template -atomic make_atomic(T t) { - atomic a; - store(a,t); - return a; -} -} -using interface6::make_atomic; - -namespace internal { -template -void swap(atomic & lhs, atomic & rhs){ - T tmp = load(lhs); - store(lhs,load(rhs)); - store(rhs,tmp); -} - -// only to aid in the gradual conversion of ordinary variables to proper atomics -template -inline atomic& as_atomic( T& t ) { - return (atomic&)t; -} -} // namespace tbb::internal - -} // namespace tbb - -#if _MSC_VER && !__INTEL_COMPILER - #pragma warning (pop) -#endif // warnings are restored - -#endif /* __TBB_atomic_H */ diff --git a/inst/include/tbb/cache_aligned_allocator.h b/inst/include/tbb/cache_aligned_allocator.h deleted file mode 100644 index d435e785c..000000000 --- a/inst/include/tbb/cache_aligned_allocator.h +++ /dev/null @@ -1,137 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_cache_aligned_allocator_H -#define __TBB_cache_aligned_allocator_H - -#include -#include "tbb_stddef.h" -#if __TBB_ALLOCATOR_CONSTRUCT_VARIADIC - #include // std::forward -#endif - -namespace tbb { - -//! @cond INTERNAL -namespace internal { - //! Cache/sector line size. - /** @ingroup memory_allocation */ - size_t __TBB_EXPORTED_FUNC NFS_GetLineSize(); - - //! Allocate memory on cache/sector line boundary. - /** @ingroup memory_allocation */ - void* __TBB_EXPORTED_FUNC NFS_Allocate( size_t n_element, size_t element_size, void* hint ); - - //! Free memory allocated by NFS_Allocate. - /** Freeing a NULL pointer is allowed, but has no effect. - @ingroup memory_allocation */ - void __TBB_EXPORTED_FUNC NFS_Free( void* ); -} -//! @endcond - -#if _MSC_VER && !defined(__INTEL_COMPILER) - // Workaround for erroneous "unreferenced parameter" warning in method destroy. - #pragma warning (push) - #pragma warning (disable: 4100) -#endif - -//! Meets "allocator" requirements of ISO C++ Standard, Section 20.1.5 -/** The members are ordered the same way they are in section 20.4.1 - of the ISO C++ standard. - @ingroup memory_allocation */ -template -class cache_aligned_allocator { -public: - typedef typename internal::allocator_type::value_type value_type; - typedef value_type* pointer; - typedef const value_type* const_pointer; - typedef value_type& reference; - typedef const value_type& const_reference; - typedef size_t size_type; - typedef ptrdiff_t difference_type; - template struct rebind { - typedef cache_aligned_allocator other; - }; - - cache_aligned_allocator() throw() {} - cache_aligned_allocator( const cache_aligned_allocator& ) throw() {} - template cache_aligned_allocator(const cache_aligned_allocator&) throw() {} - - pointer address(reference x) const {return &x;} - const_pointer address(const_reference x) const {return &x;} - - //! Allocate space for n objects, starting on a cache/sector line. - pointer allocate( size_type n, const void* hint=0 ) { - // The "hint" argument is always ignored in NFS_Allocate thus const_cast shouldn't hurt - return pointer(internal::NFS_Allocate( n, sizeof(value_type), const_cast(hint) )); - } - - //! Free block of memory that starts on a cache line - void deallocate( pointer p, size_type ) { - internal::NFS_Free(p); - } - - //! Largest value for which method allocate might succeed. - size_type max_size() const throw() { - return (~size_t(0)-internal::NFS_MaxLineSize)/sizeof(value_type); - } - - //! Copy-construct value at location pointed to by p. -#if __TBB_ALLOCATOR_CONSTRUCT_VARIADIC - template - void construct(U *p, Args&&... args) - { ::new((void *)p) U(std::forward(args)...); } -#else // __TBB_ALLOCATOR_CONSTRUCT_VARIADIC -#if __TBB_CPP11_RVALUE_REF_PRESENT - void construct( pointer p, value_type&& value ) {::new((void*)(p)) value_type(std::move(value));} -#endif - void construct( pointer p, const value_type& value ) {::new((void*)(p)) value_type(value);} -#endif // __TBB_ALLOCATOR_CONSTRUCT_VARIADIC - - //! Destroy value at location pointed to by p. - void destroy( pointer p ) {p->~value_type();} -}; - -#if _MSC_VER && !defined(__INTEL_COMPILER) - #pragma warning (pop) -#endif // warning 4100 is back - -//! Analogous to std::allocator, as defined in ISO C++ Standard, Section 20.4.1 -/** @ingroup memory_allocation */ -template<> -class cache_aligned_allocator { -public: - typedef void* pointer; - typedef const void* const_pointer; - typedef void value_type; - template struct rebind { - typedef cache_aligned_allocator other; - }; -}; - -template -inline bool operator==( const cache_aligned_allocator&, const cache_aligned_allocator& ) {return true;} - -template -inline bool operator!=( const cache_aligned_allocator&, const cache_aligned_allocator& ) {return false;} - -} // namespace tbb - -#endif /* __TBB_cache_aligned_allocator_H */ diff --git a/inst/include/tbb/combinable.h b/inst/include/tbb/combinable.h deleted file mode 100644 index 0063dbb46..000000000 --- a/inst/include/tbb/combinable.h +++ /dev/null @@ -1,85 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_combinable_H -#define __TBB_combinable_H - -#include "enumerable_thread_specific.h" -#include "cache_aligned_allocator.h" - -namespace tbb { -/** \name combinable - **/ -//@{ -//! Thread-local storage with optional reduction -/** @ingroup containers */ - template - class combinable { - - private: - typedef typename tbb::cache_aligned_allocator my_alloc; - typedef typename tbb::enumerable_thread_specific my_ets_type; - my_ets_type my_ets; - - public: - - combinable() { } - - template - explicit combinable( finit _finit) : my_ets(_finit) { } - - //! destructor - ~combinable() { } - - combinable( const combinable& other) : my_ets(other.my_ets) { } - -#if __TBB_ETS_USE_CPP11 - combinable( combinable&& other) : my_ets( std::move(other.my_ets)) { } -#endif - - combinable & operator=( const combinable & other) { - my_ets = other.my_ets; - return *this; - } - -#if __TBB_ETS_USE_CPP11 - combinable & operator=( combinable && other) { - my_ets=std::move(other.my_ets); - return *this; - } -#endif - - void clear() { my_ets.clear(); } - - T& local() { return my_ets.local(); } - - T& local(bool & exists) { return my_ets.local(exists); } - - // combine_func_t has signature T(T,T) or T(const T&, const T&) - template - T combine(combine_func_t f_combine) { return my_ets.combine(f_combine); } - - // combine_func_t has signature void(T) or void(const T&) - template - void combine_each(combine_func_t f_combine) { my_ets.combine_each(f_combine); } - - }; -} // namespace tbb -#endif /* __TBB_combinable_H */ diff --git a/inst/include/tbb/compat/condition_variable b/inst/include/tbb/compat/condition_variable deleted file mode 100644 index 43edfc032..000000000 --- a/inst/include/tbb/compat/condition_variable +++ /dev/null @@ -1,476 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_condition_variable_H -#define __TBB_condition_variable_H - -#if _WIN32||_WIN64 -#include "../machine/windows_api.h" - -namespace tbb { -namespace interface5 { -namespace internal { -struct condition_variable_using_event -{ - //! Event for blocking waiting threads. - HANDLE event; - //! Protects invariants involving n_waiters, release_count, and epoch. - CRITICAL_SECTION mutex; - //! Number of threads waiting on this condition variable - int n_waiters; - //! Number of threads remaining that should no longer wait on this condition variable. - int release_count; - //! To keep threads from waking up prematurely with earlier signals. - unsigned epoch; -}; -}}} // namespace tbb::interface5::internal - -#ifndef CONDITION_VARIABLE_INIT -typedef void* CONDITION_VARIABLE; -typedef CONDITION_VARIABLE* PCONDITION_VARIABLE; -#endif - -#else /* if not _WIN32||_WIN64 */ -#include // some systems need it for ETIMEDOUT -#include -#if __linux__ -#include -#else /* generic Unix */ -#include -#endif -#endif /* _WIN32||_WIN64 */ - -#include "../tbb_stddef.h" -#include "../mutex.h" -#include "../tbb_thread.h" -#include "../tbb_exception.h" -#include "../tbb_profiling.h" - -namespace tbb { - -namespace interface5 { - -// C++0x standard working draft 30.4.3 -// Lock tag types -struct defer_lock_t { }; //! do not acquire ownership of the mutex -struct try_to_lock_t { }; //! try to acquire ownership of the mutex without blocking -struct adopt_lock_t { }; //! assume the calling thread has already -const defer_lock_t defer_lock = {}; -const try_to_lock_t try_to_lock = {}; -const adopt_lock_t adopt_lock = {}; - -// C++0x standard working draft 30.4.3.1 -//! lock_guard -template -class lock_guard : tbb::internal::no_copy { -public: - //! mutex type - typedef M mutex_type; - - //! Constructor - /** precondition: If mutex_type is not a recursive mutex, the calling thread - does not own the mutex m. */ - explicit lock_guard(mutex_type& m) : pm(m) {m.lock();} - - //! Adopt_lock constructor - /** precondition: the calling thread owns the mutex m. */ - lock_guard(mutex_type& m, adopt_lock_t) : pm(m) {} - - //! Destructor - ~lock_guard() { pm.unlock(); } -private: - mutex_type& pm; -}; - -// C++0x standard working draft 30.4.3.2 -//! unique_lock -template -class unique_lock : tbb::internal::no_copy { - friend class condition_variable; -public: - typedef M mutex_type; - - // 30.4.3.2.1 construct/copy/destroy - // NB: Without constructors that take an r-value reference to a unique_lock, the following constructor is of little use. - //! Constructor - /** postcondition: pm==0 && owns==false */ - unique_lock() : pm(NULL), owns(false) {} - - //! Constructor - /** precondition: if mutex_type is not a recursive mutex, the calling thread - does not own the mutex m. If the precondition is not met, a deadlock occurs. - postcondition: pm==&m and owns==true */ - explicit unique_lock(mutex_type& m) : pm(&m) {m.lock(); owns=true;} - - //! Defer_lock constructor - /** postcondition: pm==&m and owns==false */ - unique_lock(mutex_type& m, defer_lock_t) : pm(&m), owns(false) {} - - //! Try_to_lock constructor - /** precondition: if mutex_type is not a recursive mutex, the calling thread - does not own the mutex m. If the precondition is not met, a deadlock occurs. - postcondition: pm==&m and owns==res where res is the value returned by - the call to m.try_lock(). */ - unique_lock(mutex_type& m, try_to_lock_t) : pm(&m) {owns = m.try_lock();} - - //! Adopt_lock constructor - /** precondition: the calling thread owns the mutex. If it does not, mutex->unlock() would fail. - postcondition: pm==&m and owns==true */ - unique_lock(mutex_type& m, adopt_lock_t) : pm(&m), owns(true) {} - - //! Timed unique_lock acquisition. - /** To avoid requiring support for namespace chrono, this method deviates from the working draft in that - it uses tbb::tick_count::interval_t to specify the time duration. */ - unique_lock(mutex_type& m, const tick_count::interval_t &i) : pm(&m) {owns = try_lock_for( i );} - -#if __TBB_CPP11_RVALUE_REF_PRESENT - //! Move constructor - /** postconditions: pm == src_p.pm and owns == src_p.owns (where src_p is the state of src just prior to this - construction), src.pm == 0 and src.owns == false. */ - unique_lock(unique_lock && src): pm(NULL), owns(false) {this->swap(src);} - - //! Move assignment - /** effects: If owns calls pm->unlock(). - Postconditions: pm == src_p.pm and owns == src_p.owns (where src_p is the state of src just prior to this - assignment), src.pm == 0 and src.owns == false. */ - unique_lock& operator=(unique_lock && src) { - if (owns) - this->unlock(); - pm = NULL; - this->swap(src); - return *this; - } -#endif // __TBB_CPP11_RVALUE_REF_PRESENT - - //! Destructor - ~unique_lock() { if( owns ) pm->unlock(); } - - // 30.4.3.2.2 locking - //! Lock the mutex and own it. - void lock() { - if( pm ) { - if( !owns ) { - pm->lock(); - owns = true; - } else - throw_exception_v4( tbb::internal::eid_possible_deadlock ); - } else - throw_exception_v4( tbb::internal::eid_operation_not_permitted ); - __TBB_ASSERT( owns, NULL ); - } - - //! Try to lock the mutex. - /** If successful, note that this lock owns it. Otherwise, set it false. */ - bool try_lock() { - if( pm ) { - if( !owns ) - owns = pm->try_lock(); - else - throw_exception_v4( tbb::internal::eid_possible_deadlock ); - } else - throw_exception_v4( tbb::internal::eid_operation_not_permitted ); - return owns; - } - - //! Try to lock the mutex. - bool try_lock_for( const tick_count::interval_t &i ); - - //! Unlock the mutex - /** And note that this lock no longer owns it. */ - void unlock() { - if( owns ) { - pm->unlock(); - owns = false; - } else - throw_exception_v4( tbb::internal::eid_operation_not_permitted ); - __TBB_ASSERT( !owns, NULL ); - } - - // 30.4.3.2.3 modifiers - //! Swap the two unique locks - void swap(unique_lock& u) { - mutex_type* t_pm = u.pm; u.pm = pm; pm = t_pm; - bool t_owns = u.owns; u.owns = owns; owns = t_owns; - } - - //! Release control over the mutex. - mutex_type* release() { - mutex_type* o_pm = pm; - pm = NULL; - owns = false; - return o_pm; - } - - // 30.4.3.2.4 observers - //! Does this lock own the mutex? - bool owns_lock() const { return owns; } - - // TODO: Un-comment 'explicit' when the last non-C++0x compiler support is dropped - //! Does this lock own the mutex? - /*explicit*/ operator bool() const { return owns; } - - //! Return the mutex that this lock currently has. - mutex_type* mutex() const { return pm; } - -private: - mutex_type* pm; - bool owns; -}; - -template -bool unique_lock::try_lock_for( const tick_count::interval_t &i) -{ - const int unique_lock_tick = 100; /* microseconds; 0.1 milliseconds */ - // the smallest wait-time is 0.1 milliseconds. - bool res = pm->try_lock(); - int duration_in_micro; - if( !res && (duration_in_micro=int(i.seconds()*1e6))>unique_lock_tick ) { - tick_count::interval_t i_100( double(unique_lock_tick)/1e6 /* seconds */); // 100 microseconds = 0.1*10E-3 - do { - this_tbb_thread::sleep(i_100); // sleep for 100 micro seconds - duration_in_micro -= unique_lock_tick; - res = pm->try_lock(); - } while( !res && duration_in_micro>unique_lock_tick ); - } - return (owns=res); -} - -//! Swap the two unique locks that have the mutexes of same type -template -void swap(unique_lock& x, unique_lock& y) { x.swap( y ); } - -namespace internal { - -#if _WIN32||_WIN64 -union condvar_impl_t { - condition_variable_using_event cv_event; - CONDITION_VARIABLE cv_native; -}; -void __TBB_EXPORTED_FUNC internal_initialize_condition_variable( condvar_impl_t& cv ); -void __TBB_EXPORTED_FUNC internal_destroy_condition_variable( condvar_impl_t& cv ); -void __TBB_EXPORTED_FUNC internal_condition_variable_notify_one( condvar_impl_t& cv ); -void __TBB_EXPORTED_FUNC internal_condition_variable_notify_all( condvar_impl_t& cv ); -bool __TBB_EXPORTED_FUNC internal_condition_variable_wait( condvar_impl_t& cv, mutex* mtx, const tick_count::interval_t* i = NULL ); - -#else /* if !(_WIN32||_WIN64), i.e., POSIX threads */ -typedef pthread_cond_t condvar_impl_t; -#endif - -} // namespace internal - -//! cv_status -/** C++0x standard working draft 30.5 */ -enum cv_status { no_timeout, timeout }; - -//! condition variable -/** C++0x standard working draft 30.5.1 - @ingroup synchronization */ -class condition_variable : tbb::internal::no_copy { -public: - //! Constructor - condition_variable() { -#if _WIN32||_WIN64 - internal_initialize_condition_variable( my_cv ); -#else - pthread_cond_init( &my_cv, NULL ); -#endif - } - - //! Destructor - ~condition_variable() { - //precondition: There shall be no thread blocked on *this. -#if _WIN32||_WIN64 - internal_destroy_condition_variable( my_cv ); -#else - pthread_cond_destroy( &my_cv ); -#endif - } - - //! Notify one thread and wake it up - void notify_one() { -#if _WIN32||_WIN64 - internal_condition_variable_notify_one( my_cv ); -#else - pthread_cond_signal( &my_cv ); -#endif - } - - //! Notify all threads - void notify_all() { -#if _WIN32||_WIN64 - internal_condition_variable_notify_all( my_cv ); -#else - pthread_cond_broadcast( &my_cv ); -#endif - } - - //! Release the mutex associated with the lock and wait on this condition variable - void wait(unique_lock& lock); - - //! Wait on this condition variable while pred is false - template - void wait(unique_lock& lock, Predicate pred) { - while( !pred() ) - wait( lock ); - } - - //! Timed version of wait() - cv_status wait_for(unique_lock& lock, const tick_count::interval_t &i ); - - //! Timed version of the predicated wait - /** The loop terminates when pred() returns true or when the time duration specified by rel_time (i) has elapsed. */ - template - bool wait_for(unique_lock& lock, const tick_count::interval_t &i, Predicate pred) - { - while( !pred() ) { - cv_status st = wait_for( lock, i ); - if( st==timeout ) - return pred(); - } - return true; - } - - // C++0x standard working draft. 30.2.3 - typedef internal::condvar_impl_t* native_handle_type; - - native_handle_type native_handle() { return (native_handle_type) &my_cv; } - -private: - internal::condvar_impl_t my_cv; -}; - - -#if _WIN32||_WIN64 -inline void condition_variable::wait( unique_lock& lock ) -{ - __TBB_ASSERT( lock.owns, NULL ); - lock.owns = false; - if( !internal_condition_variable_wait( my_cv, lock.mutex() ) ) { - int ec = GetLastError(); - // on Windows 7, SleepConditionVariableCS() may return ERROR_TIMEOUT while the doc says it returns WAIT_TIMEOUT - __TBB_ASSERT_EX( ec!=WAIT_TIMEOUT&&ec!=ERROR_TIMEOUT, NULL ); - lock.owns = true; - throw_exception_v4( tbb::internal::eid_condvar_wait_failed ); - } - lock.owns = true; -} - -inline cv_status condition_variable::wait_for( unique_lock& lock, const tick_count::interval_t& i ) -{ - cv_status rc = no_timeout; - __TBB_ASSERT( lock.owns, NULL ); - lock.owns = false; - // condvar_wait could be SleepConditionVariableCS (or SleepConditionVariableSRW) or our own pre-vista cond_var_wait() - if( !internal_condition_variable_wait( my_cv, lock.mutex(), &i ) ) { - int ec = GetLastError(); - if( ec==WAIT_TIMEOUT || ec==ERROR_TIMEOUT ) - rc = timeout; - else { - lock.owns = true; - throw_exception_v4( tbb::internal::eid_condvar_wait_failed ); - } - } - lock.owns = true; - return rc; -} - -#else /* !(_WIN32||_WIN64) */ -inline void condition_variable::wait( unique_lock& lock ) -{ - __TBB_ASSERT( lock.owns, NULL ); - lock.owns = false; - if( pthread_cond_wait( &my_cv, lock.mutex()->native_handle() ) ) { - lock.owns = true; - throw_exception_v4( tbb::internal::eid_condvar_wait_failed ); - } - // upon successful return, the mutex has been locked and is owned by the calling thread. - lock.owns = true; -} - -inline cv_status condition_variable::wait_for( unique_lock& lock, const tick_count::interval_t& i ) -{ -#if __linux__ - struct timespec req; - double sec = i.seconds(); - clock_gettime( CLOCK_REALTIME, &req ); - req.tv_sec += static_cast(sec); - req.tv_nsec += static_cast( (sec - static_cast(sec))*1e9 ); -#else /* generic Unix */ - struct timeval tv; - struct timespec req; - double sec = i.seconds(); - int status = gettimeofday(&tv, NULL); - __TBB_ASSERT_EX( status==0, "gettimeofday failed" ); - req.tv_sec = tv.tv_sec + static_cast(sec); - req.tv_nsec = tv.tv_usec*1000 + static_cast( (sec - static_cast(sec))*1e9 ); -#endif /*(choice of OS) */ - if( req.tv_nsec>=1e9 ) { - req.tv_sec += 1; - req.tv_nsec -= static_cast(1e9); - } - __TBB_ASSERT( 0<=req.tv_nsec && req.tv_nsec<1e9, NULL ); - - int ec; - cv_status rc = no_timeout; - __TBB_ASSERT( lock.owns, NULL ); - lock.owns = false; - if( ( ec=pthread_cond_timedwait( &my_cv, lock.mutex()->native_handle(), &req ) ) ) { - if( ec==ETIMEDOUT ) - rc = timeout; - else { - __TBB_ASSERT( lock.try_lock()==false, NULL ); - lock.owns = true; - throw_exception_v4( tbb::internal::eid_condvar_wait_failed ); - } - } - lock.owns = true; - return rc; -} -#endif /* !(_WIN32||_WIN64) */ - -} // namespace interface5 - -__TBB_DEFINE_PROFILING_SET_NAME(interface5::condition_variable) - -} // namespace tbb - -#if TBB_IMPLEMENT_CPP0X - -namespace std { - -using tbb::interface5::defer_lock_t; -using tbb::interface5::try_to_lock_t; -using tbb::interface5::adopt_lock_t; -using tbb::interface5::defer_lock; -using tbb::interface5::try_to_lock; -using tbb::interface5::adopt_lock; -using tbb::interface5::lock_guard; -using tbb::interface5::unique_lock; -using tbb::interface5::swap; /* this is for void std::swap(unique_lock&,unique_lock&) */ -using tbb::interface5::condition_variable; -using tbb::interface5::cv_status; -using tbb::interface5::timeout; -using tbb::interface5::no_timeout; - -} // namespace std - -#endif /* TBB_IMPLEMENT_CPP0X */ - -#endif /* __TBB_condition_variable_H */ diff --git a/inst/include/tbb/compat/ppl.h b/inst/include/tbb/compat/ppl.h deleted file mode 100644 index 840dfb22f..000000000 --- a/inst/include/tbb/compat/ppl.h +++ /dev/null @@ -1,62 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_compat_ppl_H -#define __TBB_compat_ppl_H - -#include "../task_group.h" -#include "../parallel_invoke.h" -#include "../parallel_for_each.h" -#include "../parallel_for.h" -#include "../tbb_exception.h" -#include "../critical_section.h" -#include "../reader_writer_lock.h" -#include "../combinable.h" - -namespace Concurrency { - -#if __TBB_TASK_GROUP_CONTEXT - using tbb::task_handle; - using tbb::task_group_status; - using tbb::task_group; - using tbb::structured_task_group; - using tbb::invalid_multiple_scheduling; - using tbb::missing_wait; - using tbb::make_task; - - using tbb::not_complete; - using tbb::complete; - using tbb::canceled; - - using tbb::is_current_task_group_canceling; -#endif /* __TBB_TASK_GROUP_CONTEXT */ - - using tbb::parallel_invoke; - using tbb::strict_ppl::parallel_for; - using tbb::parallel_for_each; - using tbb::critical_section; - using tbb::reader_writer_lock; - using tbb::combinable; - - using tbb::improper_lock; - -} // namespace Concurrency - -#endif /* __TBB_compat_ppl_H */ diff --git a/inst/include/tbb/compat/thread b/inst/include/tbb/compat/thread deleted file mode 100644 index 0edd92899..000000000 --- a/inst/include/tbb/compat/thread +++ /dev/null @@ -1,60 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_thread_H -#define __TBB_thread_H - -#include "../tbb_config.h" - -#if TBB_IMPLEMENT_CPP0X - -#include "../tbb_thread.h" - -namespace std { - -typedef tbb::tbb_thread thread; - -namespace this_thread { - using tbb::this_tbb_thread::get_id; - using tbb::this_tbb_thread::yield; - - inline void sleep_for(const tbb::tick_count::interval_t& rel_time) { - tbb::internal::thread_sleep_v3( rel_time ); - } -} - -} // namespace std - -#else /* TBB_IMPLEMENT_CPP0X */ - -#define __TBB_COMPAT_THREAD_RECURSION_PROTECTOR 1 -#include -#undef __TBB_COMPAT_THREAD_RECURSION_PROTECTOR - -#endif /* TBB_IMPLEMENT_CPP0X */ - -#else /* __TBB_thread_H */ - -#if __TBB_COMPAT_THREAD_RECURSION_PROTECTOR -#error The tbb/compat/thread header attempts to include itself. \ - Please make sure that {TBBROOT}/include/tbb/compat is NOT in include paths. -#endif - -#endif /* __TBB_thread_H */ diff --git a/inst/include/tbb/compat/tuple b/inst/include/tbb/compat/tuple deleted file mode 100644 index 5767c49e7..000000000 --- a/inst/include/tbb/compat/tuple +++ /dev/null @@ -1,488 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_tuple_H -#define __TBB_tuple_H - -#include -#include "../tbb_stddef.h" - -// build preprocessor variables for varying number of arguments -// Need the leading comma so the empty __TBB_T_PACK will not cause a syntax error. -#if __TBB_VARIADIC_MAX <= 5 -#define __TBB_T_PACK -#define __TBB_U_PACK -#define __TBB_TYPENAME_T_PACK -#define __TBB_TYPENAME_U_PACK -#define __TBB_NULL_TYPE_PACK -#define __TBB_REF_T_PARAM_PACK -#define __TBB_CONST_REF_T_PARAM_PACK -#define __TBB_T_PARAM_LIST_PACK -#define __TBB_CONST_NULL_REF_PACK -// -#elif __TBB_VARIADIC_MAX == 6 -#define __TBB_T_PACK ,__T5 -#define __TBB_U_PACK ,__U5 -#define __TBB_TYPENAME_T_PACK , typename __T5 -#define __TBB_TYPENAME_U_PACK , typename __U5 -#define __TBB_NULL_TYPE_PACK , null_type -#define __TBB_REF_T_PARAM_PACK ,__T5& t5 -#define __TBB_CONST_REF_T_PARAM_PACK ,const __T5& t5 -#define __TBB_T_PARAM_LIST_PACK ,t5 -#define __TBB_CONST_NULL_REF_PACK , const null_type& -// -#elif __TBB_VARIADIC_MAX == 7 -#define __TBB_T_PACK ,__T5, __T6 -#define __TBB_U_PACK ,__U5, __U6 -#define __TBB_TYPENAME_T_PACK , typename __T5 , typename __T6 -#define __TBB_TYPENAME_U_PACK , typename __U5 , typename __U6 -#define __TBB_NULL_TYPE_PACK , null_type, null_type -#define __TBB_REF_T_PARAM_PACK ,__T5& t5, __T6& t6 -#define __TBB_CONST_REF_T_PARAM_PACK ,const __T5& t5, const __T6& t6 -#define __TBB_T_PARAM_LIST_PACK ,t5 ,t6 -#define __TBB_CONST_NULL_REF_PACK , const null_type&, const null_type& -// -#elif __TBB_VARIADIC_MAX == 8 -#define __TBB_T_PACK ,__T5, __T6, __T7 -#define __TBB_U_PACK ,__U5, __U6, __U7 -#define __TBB_TYPENAME_T_PACK , typename __T5 , typename __T6, typename __T7 -#define __TBB_TYPENAME_U_PACK , typename __U5 , typename __U6, typename __U7 -#define __TBB_NULL_TYPE_PACK , null_type, null_type, null_type -#define __TBB_REF_T_PARAM_PACK ,__T5& t5, __T6& t6, __T7& t7 -#define __TBB_CONST_REF_T_PARAM_PACK , const __T5& t5, const __T6& t6, const __T7& t7 -#define __TBB_T_PARAM_LIST_PACK ,t5 ,t6 ,t7 -#define __TBB_CONST_NULL_REF_PACK , const null_type&, const null_type&, const null_type& -// -#elif __TBB_VARIADIC_MAX == 9 -#define __TBB_T_PACK ,__T5, __T6, __T7, __T8 -#define __TBB_U_PACK ,__U5, __U6, __U7, __U8 -#define __TBB_TYPENAME_T_PACK , typename __T5, typename __T6, typename __T7, typename __T8 -#define __TBB_TYPENAME_U_PACK , typename __U5, typename __U6, typename __U7, typename __U8 -#define __TBB_NULL_TYPE_PACK , null_type, null_type, null_type, null_type -#define __TBB_REF_T_PARAM_PACK ,__T5& t5, __T6& t6, __T7& t7, __T8& t8 -#define __TBB_CONST_REF_T_PARAM_PACK , const __T5& t5, const __T6& t6, const __T7& t7, const __T8& t8 -#define __TBB_T_PARAM_LIST_PACK ,t5 ,t6 ,t7 ,t8 -#define __TBB_CONST_NULL_REF_PACK , const null_type&, const null_type&, const null_type&, const null_type& -// -#elif __TBB_VARIADIC_MAX >= 10 -#define __TBB_T_PACK ,__T5, __T6, __T7, __T8, __T9 -#define __TBB_U_PACK ,__U5, __U6, __U7, __U8, __U9 -#define __TBB_TYPENAME_T_PACK , typename __T5, typename __T6, typename __T7, typename __T8, typename __T9 -#define __TBB_TYPENAME_U_PACK , typename __U5, typename __U6, typename __U7, typename __U8, typename __U9 -#define __TBB_NULL_TYPE_PACK , null_type, null_type, null_type, null_type, null_type -#define __TBB_REF_T_PARAM_PACK ,__T5& t5, __T6& t6, __T7& t7, __T8& t8, __T9& t9 -#define __TBB_CONST_REF_T_PARAM_PACK , const __T5& t5, const __T6& t6, const __T7& t7, const __T8& t8, const __T9& t9 -#define __TBB_T_PARAM_LIST_PACK ,t5 ,t6 ,t7 ,t8 ,t9 -#define __TBB_CONST_NULL_REF_PACK , const null_type&, const null_type&, const null_type&, const null_type&, const null_type& -#endif - - - -namespace tbb { -namespace interface5 { - -namespace internal { -struct null_type { }; -} -using internal::null_type; - -// tuple forward declaration -template = 6 -, typename __T5=null_type -#if __TBB_VARIADIC_MAX >= 7 -, typename __T6=null_type -#if __TBB_VARIADIC_MAX >= 8 -, typename __T7=null_type -#if __TBB_VARIADIC_MAX >= 9 -, typename __T8=null_type -#if __TBB_VARIADIC_MAX >= 10 -, typename __T9=null_type -#endif -#endif -#endif -#endif -#endif -> -class tuple; - -namespace internal { - -// const null_type temp -inline const null_type cnull() { return null_type(); } - -// cons forward declaration -template struct cons; - -// type of a component of the cons -template -struct component { - typedef typename __T::tail_type next; - typedef typename component<__N-1,next>::type type; -}; - -template -struct component<0,__T> { - typedef typename __T::head_type type; -}; - -template<> -struct component<0,null_type> { - typedef null_type type; -}; - -// const version of component - -template -struct component<__N, const __T> -{ - typedef typename __T::tail_type next; - typedef const typename component<__N-1,next>::type type; -}; - -template -struct component<0, const __T> -{ - typedef const typename __T::head_type type; -}; - - -// helper class for getting components of cons -template< int __N> -struct get_helper { -template -inline static typename component<__N, cons<__HT,__TT> >::type& get(cons<__HT,__TT>& ti) { - return get_helper<__N-1>::get(ti.tail); -} -template -inline static typename component<__N, cons<__HT,__TT> >::type const& get(const cons<__HT,__TT>& ti) { - return get_helper<__N-1>::get(ti.tail); -} -}; - -template<> -struct get_helper<0> { -template -inline static typename component<0, cons<__HT,__TT> >::type& get(cons<__HT,__TT>& ti) { - return ti.head; -} -template -inline static typename component<0, cons<__HT,__TT> >::type const& get(const cons<__HT,__TT>& ti) { - return ti.head; -} -}; - -// traits adaptor -template -struct tuple_traits { - typedef cons <__T0, typename tuple_traits<__T1, __T2, __T3, __T4 __TBB_T_PACK , null_type>::U > U; -}; - -template -struct tuple_traits<__T0, null_type, null_type, null_type, null_type __TBB_NULL_TYPE_PACK > { - typedef cons<__T0, null_type> U; -}; - -template<> -struct tuple_traits { - typedef null_type U; -}; - - -// core cons defs -template -struct cons{ - - typedef __HT head_type; - typedef __TT tail_type; - - head_type head; - tail_type tail; - - static const int length = 1 + tail_type::length; - - // default constructors - explicit cons() : head(), tail() { } - - // non-default constructors - cons(head_type& h, const tail_type& t) : head(h), tail(t) { } - - template - cons(const __T0& t0, const __T1& t1, const __T2& t2, const __T3& t3, const __T4& t4 __TBB_CONST_REF_T_PARAM_PACK) : - head(t0), tail(t1, t2, t3, t4 __TBB_T_PARAM_LIST_PACK, cnull()) { } - - template - cons(__T0& t0, __T1& t1, __T2& t2, __T3& t3, __T4& t4 __TBB_REF_T_PARAM_PACK) : - head(t0), tail(t1, t2, t3, t4 __TBB_T_PARAM_LIST_PACK , cnull()) { } - - template - cons(const cons<__HT1,__TT1>& other) : head(other.head), tail(other.tail) { } - - cons& operator=(const cons& other) { head = other.head; tail = other.tail; return *this; } - - friend bool operator==(const cons& me, const cons& other) { - return me.head == other.head && me.tail == other.tail; - } - friend bool operator<(const cons& me, const cons& other) { - return me.head < other.head || (!(other.head < me.head) && me.tail < other.tail); - } - friend bool operator>(const cons& me, const cons& other) { return other=(const cons& me, const cons& other) { return !(meother); } - - template - friend bool operator==(const cons<__HT,__TT>& me, const cons<__HT1,__TT1>& other) { - return me.head == other.head && me.tail == other.tail; - } - - template - friend bool operator<(const cons<__HT,__TT>& me, const cons<__HT1,__TT1>& other) { - return me.head < other.head || (!(other.head < me.head) && me.tail < other.tail); - } - - template - friend bool operator>(const cons<__HT,__TT>& me, const cons<__HT1,__TT1>& other) { return other - friend bool operator!=(const cons<__HT,__TT>& me, const cons<__HT1,__TT1>& other) { return !(me==other); } - - template - friend bool operator>=(const cons<__HT,__TT>& me, const cons<__HT1,__TT1>& other) { return !(me - friend bool operator<=(const cons<__HT,__TT>& me, const cons<__HT1,__TT1>& other) { return !(me>other); } - - -}; // cons - - -template -struct cons<__HT,null_type> { - - typedef __HT head_type; - typedef null_type tail_type; - - head_type head; - - static const int length = 1; - - // default constructor - cons() : head() { /*std::cout << "default constructor 1\n";*/ } - - cons(const null_type&, const null_type&, const null_type&, const null_type&, const null_type& __TBB_CONST_NULL_REF_PACK) : head() { /*std::cout << "default constructor 2\n";*/ } - - // non-default constructor - template - cons(__T1& t1, const null_type&, const null_type&, const null_type&, const null_type& __TBB_CONST_NULL_REF_PACK) : head(t1) { /*std::cout << "non-default a1, t1== " << t1 << "\n";*/} - - cons(head_type& h, const null_type& = null_type() ) : head(h) { } - cons(const head_type& t0, const null_type&, const null_type&, const null_type&, const null_type& __TBB_CONST_NULL_REF_PACK) : head(t0) { } - - // converting constructor - template - cons(__HT1 h1, const null_type&, const null_type&, const null_type&, const null_type& __TBB_CONST_NULL_REF_PACK) : head(h1) { } - - // copy constructor - template - cons( const cons<__HT1, null_type>& other) : head(other.head) { } - - // assignment operator - cons& operator=(const cons& other) { head = other.head; return *this; } - - friend bool operator==(const cons& me, const cons& other) { return me.head == other.head; } - friend bool operator<(const cons& me, const cons& other) { return me.head < other.head; } - friend bool operator>(const cons& me, const cons& other) { return otherother); } - friend bool operator>=(const cons& me, const cons& other) {return !(me - friend bool operator==(const cons<__HT,null_type>& me, const cons<__HT1,null_type>& other) { - return me.head == other.head; - } - - template - friend bool operator<(const cons<__HT,null_type>& me, const cons<__HT1,null_type>& other) { - return me.head < other.head; - } - - template - friend bool operator>(const cons<__HT,null_type>& me, const cons<__HT1,null_type>& other) { return other - friend bool operator!=(const cons<__HT,null_type>& me, const cons<__HT1,null_type>& other) { return !(me==other); } - - template - friend bool operator<=(const cons<__HT,null_type>& me, const cons<__HT1,null_type>& other) { return !(me>other); } - - template - friend bool operator>=(const cons<__HT,null_type>& me, const cons<__HT1,null_type>& other) { return !(me -struct cons { typedef null_type tail_type; static const int length = 0; }; - -// wrapper for default constructor -template -inline const __T wrap_dcons(__T*) { return __T(); } - -} // namespace internal - -// tuple definition -template -class tuple : public internal::tuple_traits<__T0, __T1, __T2, __T3, __T4 __TBB_T_PACK >::U { - // friends - template friend class tuple_size; - template friend struct tuple_element; - - // stl components - typedef tuple<__T0,__T1,__T2,__T3,__T4 __TBB_T_PACK > value_type; - typedef value_type *pointer; - typedef const value_type *const_pointer; - typedef value_type &reference; - typedef const value_type &const_reference; - typedef size_t size_type; - - typedef typename internal::tuple_traits<__T0,__T1,__T2,__T3, __T4 __TBB_T_PACK >::U my_cons; - -public: - tuple(const __T0& t0=internal::wrap_dcons((__T0*)NULL) - ,const __T1& t1=internal::wrap_dcons((__T1*)NULL) - ,const __T2& t2=internal::wrap_dcons((__T2*)NULL) - ,const __T3& t3=internal::wrap_dcons((__T3*)NULL) - ,const __T4& t4=internal::wrap_dcons((__T4*)NULL) -#if __TBB_VARIADIC_MAX >= 6 - ,const __T5& t5=internal::wrap_dcons((__T5*)NULL) -#if __TBB_VARIADIC_MAX >= 7 - ,const __T6& t6=internal::wrap_dcons((__T6*)NULL) -#if __TBB_VARIADIC_MAX >= 8 - ,const __T7& t7=internal::wrap_dcons((__T7*)NULL) -#if __TBB_VARIADIC_MAX >= 9 - ,const __T8& t8=internal::wrap_dcons((__T8*)NULL) -#if __TBB_VARIADIC_MAX >= 10 - ,const __T9& t9=internal::wrap_dcons((__T9*)NULL) -#endif -#endif -#endif -#endif -#endif - ) : - my_cons(t0,t1,t2,t3,t4 __TBB_T_PARAM_LIST_PACK) { } - - template - struct internal_tuple_element { - typedef typename internal::component<__N,my_cons>::type type; - }; - - template - typename internal_tuple_element<__N>::type& get() { return internal::get_helper<__N>::get(*this); } - - template - typename internal_tuple_element<__N>::type const& get() const { return internal::get_helper<__N>::get(*this); } - - template - tuple& operator=(const internal::cons<__U1,__U2>& other) { - my_cons::operator=(other); - return *this; - } - - template - tuple& operator=(const std::pair<__U1,__U2>& other) { - // __TBB_ASSERT(tuple_size::value == 2, "Invalid size for pair to tuple assignment"); - this->head = other.first; - this->tail.head = other.second; - return *this; - } - - friend bool operator==(const tuple& me, const tuple& other) {return static_cast(me)==(other);} - friend bool operator<(const tuple& me, const tuple& other) {return static_cast(me)<(other);} - friend bool operator>(const tuple& me, const tuple& other) {return static_cast(me)>(other);} - friend bool operator!=(const tuple& me, const tuple& other) {return static_cast(me)!=(other);} - friend bool operator>=(const tuple& me, const tuple& other) {return static_cast(me)>=(other);} - friend bool operator<=(const tuple& me, const tuple& other) {return static_cast(me)<=(other);} - -}; // tuple - -// empty tuple -template<> -class tuple : public null_type { -}; - -// helper classes - -template < typename __T> -class tuple_size { -public: - static const size_t value = 1 + tuple_size::value; -}; - -template <> -class tuple_size > { -public: - static const size_t value = 0; -}; - -template <> -class tuple_size { -public: - static const size_t value = 0; -}; - -template -struct tuple_element { - typedef typename internal::component<__N, typename __T::my_cons>::type type; -}; - -template -inline static typename tuple_element<__N,tuple<__T0,__T1,__T2,__T3,__T4 __TBB_T_PACK > >::type& - get(tuple<__T0,__T1,__T2,__T3,__T4 __TBB_T_PACK >& t) { return internal::get_helper<__N>::get(t); } - -template -inline static typename tuple_element<__N,tuple<__T0,__T1,__T2,__T3,__T4 __TBB_T_PACK > >::type const& - get(const tuple<__T0,__T1,__T2,__T3,__T4 __TBB_T_PACK >& t) { return internal::get_helper<__N>::get(t); } - -} // interface5 -} // tbb - -#if !__TBB_CPP11_TUPLE_PRESENT -namespace tbb { - namespace flow { - using tbb::interface5::tuple; - using tbb::interface5::tuple_size; - using tbb::interface5::tuple_element; - using tbb::interface5::get; - } -} -#endif - -#undef __TBB_T_PACK -#undef __TBB_U_PACK -#undef __TBB_TYPENAME_T_PACK -#undef __TBB_TYPENAME_U_PACK -#undef __TBB_NULL_TYPE_PACK -#undef __TBB_REF_T_PARAM_PACK -#undef __TBB_CONST_REF_T_PARAM_PACK -#undef __TBB_T_PARAM_LIST_PACK -#undef __TBB_CONST_NULL_REF_PACK - -#endif /* __TBB_tuple_H */ diff --git a/inst/include/tbb/concurrent_hash_map.h b/inst/include/tbb/concurrent_hash_map.h deleted file mode 100644 index 8497c8388..000000000 --- a/inst/include/tbb/concurrent_hash_map.h +++ /dev/null @@ -1,1510 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_concurrent_hash_map_H -#define __TBB_concurrent_hash_map_H - -#include "tbb_stddef.h" - -#if !TBB_USE_EXCEPTIONS && _MSC_VER - // Suppress "C++ exception handler used, but unwind semantics are not enabled" warning in STL headers - #pragma warning (push) - #pragma warning (disable: 4530) -#endif - -#include -#include // Need std::pair -#include // Need std::memset -#include __TBB_STD_SWAP_HEADER - -#if !TBB_USE_EXCEPTIONS && _MSC_VER - #pragma warning (pop) -#endif - -#include "cache_aligned_allocator.h" -#include "tbb_allocator.h" -#include "spin_rw_mutex.h" -#include "atomic.h" -#include "tbb_exception.h" -#include "tbb_profiling.h" -#include "internal/_tbb_hash_compare_impl.h" -#if __TBB_INITIALIZER_LISTS_PRESENT -#include -#endif -#if TBB_USE_PERFORMANCE_WARNINGS || __TBB_STATISTICS -#include -#endif -#if __TBB_STATISTICS -#include -#endif - -namespace tbb { - -namespace interface5 { - - template, typename A = tbb_allocator > > - class concurrent_hash_map; - - //! @cond INTERNAL - namespace internal { - using namespace tbb::internal; - - - //! Type of a hash code. - typedef size_t hashcode_t; - //! Node base type - struct hash_map_node_base : tbb::internal::no_copy { - //! Mutex type - typedef spin_rw_mutex mutex_t; - //! Scoped lock type for mutex - typedef mutex_t::scoped_lock scoped_t; - //! Next node in chain - hash_map_node_base *next; - mutex_t mutex; - }; - //! Incompleteness flag value - static hash_map_node_base *const rehash_req = reinterpret_cast(size_t(3)); - //! Rehashed empty bucket flag - static hash_map_node_base *const empty_rehashed = reinterpret_cast(size_t(0)); - //! base class of concurrent_hash_map - class hash_map_base { - public: - //! Size type - typedef size_t size_type; - //! Type of a hash code. - typedef size_t hashcode_t; - //! Segment index type - typedef size_t segment_index_t; - //! Node base type - typedef hash_map_node_base node_base; - //! Bucket type - struct bucket : tbb::internal::no_copy { - //! Mutex type for buckets - typedef spin_rw_mutex mutex_t; - //! Scoped lock type for mutex - typedef mutex_t::scoped_lock scoped_t; - mutex_t mutex; - node_base *node_list; - }; - //! Count of segments in the first block - static size_type const embedded_block = 1; - //! Count of segments in the first block - static size_type const embedded_buckets = 1< my_mask; - //! Segment pointers table. Also prevents false sharing between my_mask and my_size - segments_table_t my_table; - //! Size of container in stored items - atomic my_size; // It must be in separate cache line from my_mask due to performance effects - //! Zero segment - bucket my_embedded_segment[embedded_buckets]; -#if __TBB_STATISTICS - atomic my_info_resizes; // concurrent ones - mutable atomic my_info_restarts; // race collisions - atomic my_info_rehashes; // invocations of rehash_bucket -#endif - //! Constructor - hash_map_base() { - std::memset( (void*) this, 0, pointers_per_table*sizeof(segment_ptr_t) // 32*4=128 or 64*8=512 - + sizeof(my_size) + sizeof(my_mask) // 4+4 or 8+8 - + embedded_buckets*sizeof(bucket) ); // n*8 or n*16 - for( size_type i = 0; i < embedded_block; i++ ) // fill the table - my_table[i] = my_embedded_segment + segment_base(i); - my_mask = embedded_buckets - 1; - __TBB_ASSERT( embedded_block <= first_block, "The first block number must include embedded blocks"); -#if __TBB_STATISTICS - my_info_resizes = 0; // concurrent ones - my_info_restarts = 0; // race collisions - my_info_rehashes = 0; // invocations of rehash_bucket -#endif - } - - //! @return segment index of given index in the array - static segment_index_t segment_index_of( size_type index ) { - return segment_index_t( __TBB_Log2( index|1 ) ); - } - - //! @return the first array index of given segment - static segment_index_t segment_base( segment_index_t k ) { - return (segment_index_t(1)<(ptr) > uintptr_t(63); - } - - //! Initialize buckets - static void init_buckets( segment_ptr_t ptr, size_type sz, bool is_initial ) { - if( is_initial ) std::memset((void*) ptr, 0, sz*sizeof(bucket) ); - else for(size_type i = 0; i < sz; i++, ptr++) { - *reinterpret_cast(&ptr->mutex) = 0; - ptr->node_list = rehash_req; - } - } - - //! Add node @arg n to bucket @arg b - static void add_to_bucket( bucket *b, node_base *n ) { - __TBB_ASSERT(b->node_list != rehash_req, NULL); - n->next = b->node_list; - b->node_list = n; // its under lock and flag is set - } - - //! Exception safety helper - struct enable_segment_failsafe : tbb::internal::no_copy { - segment_ptr_t *my_segment_ptr; - enable_segment_failsafe(segments_table_t &table, segment_index_t k) : my_segment_ptr(&table[k]) {} - ~enable_segment_failsafe() { - if( my_segment_ptr ) *my_segment_ptr = 0; // indicate no allocation in progress - } - }; - - //! Enable segment - void enable_segment( segment_index_t k, bool is_initial = false ) { - __TBB_ASSERT( k, "Zero segment must be embedded" ); - enable_segment_failsafe watchdog( my_table, k ); - cache_aligned_allocator alloc; - size_type sz; - __TBB_ASSERT( !is_valid(my_table[k]), "Wrong concurrent assignment"); - if( k >= first_block ) { - sz = segment_size( k ); - segment_ptr_t ptr = alloc.allocate( sz ); - init_buckets( ptr, sz, is_initial ); - itt_hide_store_word( my_table[k], ptr ); - sz <<= 1;// double it to get entire capacity of the container - } else { // the first block - __TBB_ASSERT( k == embedded_block, "Wrong segment index" ); - sz = segment_size( first_block ); - segment_ptr_t ptr = alloc.allocate( sz - embedded_buckets ); - init_buckets( ptr, sz - embedded_buckets, is_initial ); - ptr -= segment_base(embedded_block); - for(segment_index_t i = embedded_block; i < first_block; i++) // calc the offsets - itt_hide_store_word( my_table[i], ptr + segment_base(i) ); - } - itt_store_word_with_release( my_mask, sz-1 ); - watchdog.my_segment_ptr = 0; - } - - //! Get bucket by (masked) hashcode - bucket *get_bucket( hashcode_t h ) const throw() { // TODO: add throw() everywhere? - segment_index_t s = segment_index_of( h ); - h -= segment_base(s); - segment_ptr_t seg = my_table[s]; - __TBB_ASSERT( is_valid(seg), "hashcode must be cut by valid mask for allocated segments" ); - return &seg[h]; - } - - // internal serial rehashing helper - void mark_rehashed_levels( hashcode_t h ) throw () { - segment_index_t s = segment_index_of( h ); - while( segment_ptr_t seg = my_table[++s] ) - if( seg[h].node_list == rehash_req ) { - seg[h].node_list = empty_rehashed; - mark_rehashed_levels( h + ((hashcode_t)1<node_list) != rehash_req ) - { -#if __TBB_STATISTICS - my_info_restarts++; // race collisions -#endif - return true; - } - } - return false; - } - - //! Insert a node and check for load factor. @return segment index to enable. - segment_index_t insert_new_node( bucket *b, node_base *n, hashcode_t mask ) { - size_type sz = ++my_size; // prefix form is to enforce allocation after the first item inserted - add_to_bucket( b, n ); - // check load factor - if( sz >= mask ) { // TODO: add custom load_factor - segment_index_t new_seg = __TBB_Log2( mask+1 ); //optimized segment_index_of - __TBB_ASSERT( is_valid(my_table[new_seg-1]), "new allocations must not publish new mask until segment has allocated"); - static const segment_ptr_t is_allocating = (segment_ptr_t)2; - if( !itt_hide_load_word(my_table[new_seg]) - && as_atomic(my_table[new_seg]).compare_and_swap(is_allocating, NULL) == NULL ) - return new_seg; // The value must be processed - } - return 0; - } - - //! Prepare enough segments for number of buckets - void reserve(size_type buckets) { - if( !buckets-- ) return; - bool is_initial = !my_size; - for( size_type m = my_mask; buckets > m; m = my_mask ) - enable_segment( segment_index_of( m+1 ), is_initial ); - } - //! Swap hash_map_bases - void internal_swap(hash_map_base &table) { - using std::swap; - swap(this->my_mask, table.my_mask); - swap(this->my_size, table.my_size); - for(size_type i = 0; i < embedded_buckets; i++) - swap(this->my_embedded_segment[i].node_list, table.my_embedded_segment[i].node_list); - for(size_type i = embedded_block; i < pointers_per_table; i++) - swap(this->my_table[i], table.my_table[i]); - } - }; - - template - class hash_map_range; - - //! Meets requirements of a forward iterator for STL */ - /** Value is either the T or const T type of the container. - @ingroup containers */ - template - class hash_map_iterator - : public std::iterator - { - typedef Container map_type; - typedef typename Container::node node; - typedef hash_map_base::node_base node_base; - typedef hash_map_base::bucket bucket; - - template - friend bool operator==( const hash_map_iterator& i, const hash_map_iterator& j ); - - template - friend bool operator!=( const hash_map_iterator& i, const hash_map_iterator& j ); - - template - friend ptrdiff_t operator-( const hash_map_iterator& i, const hash_map_iterator& j ); - - template - friend class hash_map_iterator; - - template - friend class hash_map_range; - - void advance_to_next_bucket() { // TODO?: refactor to iterator_base class - size_t k = my_index+1; - __TBB_ASSERT( my_bucket, "advancing an invalid iterator?"); - while( k <= my_map->my_mask ) { - // Following test uses 2's-complement wizardry - if( k&(k-2) ) // not the beginning of a segment - ++my_bucket; - else my_bucket = my_map->get_bucket( k ); - my_node = static_cast( my_bucket->node_list ); - if( hash_map_base::is_valid(my_node) ) { - my_index = k; return; - } - ++k; - } - my_bucket = 0; my_node = 0; my_index = k; // the end - } -#if !defined(_MSC_VER) || defined(__INTEL_COMPILER) - template - friend class interface5::concurrent_hash_map; -#else - public: // workaround -#endif - //! concurrent_hash_map over which we are iterating. - const Container *my_map; - - //! Index in hash table for current item - size_t my_index; - - //! Pointer to bucket - const bucket *my_bucket; - - //! Pointer to node that has current item - node *my_node; - - hash_map_iterator( const Container &map, size_t index, const bucket *b, node_base *n ); - - public: - //! Construct undefined iterator - hash_map_iterator(): my_map(), my_index(), my_bucket(), my_node() {} - hash_map_iterator( const hash_map_iterator &other ) : - my_map(other.my_map), - my_index(other.my_index), - my_bucket(other.my_bucket), - my_node(other.my_node) - {} - Value& operator*() const { - __TBB_ASSERT( hash_map_base::is_valid(my_node), "iterator uninitialized or at end of container?" ); - return my_node->item; - } - Value* operator->() const {return &operator*();} - hash_map_iterator& operator++(); - - //! Post increment - hash_map_iterator operator++(int) { - hash_map_iterator old(*this); - operator++(); - return old; - } - }; - - template - hash_map_iterator::hash_map_iterator( const Container &map, size_t index, const bucket *b, node_base *n ) : - my_map(&map), - my_index(index), - my_bucket(b), - my_node( static_cast(n) ) - { - if( b && !hash_map_base::is_valid(n) ) - advance_to_next_bucket(); - } - - template - hash_map_iterator& hash_map_iterator::operator++() { - my_node = static_cast( my_node->next ); - if( !my_node ) advance_to_next_bucket(); - return *this; - } - - template - bool operator==( const hash_map_iterator& i, const hash_map_iterator& j ) { - return i.my_node == j.my_node && i.my_map == j.my_map; - } - - template - bool operator!=( const hash_map_iterator& i, const hash_map_iterator& j ) { - return i.my_node != j.my_node || i.my_map != j.my_map; - } - - //! Range class used with concurrent_hash_map - /** @ingroup containers */ - template - class hash_map_range { - typedef typename Iterator::map_type map_type; - Iterator my_begin; - Iterator my_end; - mutable Iterator my_midpoint; - size_t my_grainsize; - //! Set my_midpoint to point approximately half way between my_begin and my_end. - void set_midpoint() const; - template friend class hash_map_range; - public: - //! Type for size of a range - typedef std::size_t size_type; - typedef typename Iterator::value_type value_type; - typedef typename Iterator::reference reference; - typedef typename Iterator::difference_type difference_type; - typedef Iterator iterator; - - //! True if range is empty. - bool empty() const {return my_begin==my_end;} - - //! True if range can be partitioned into two subranges. - bool is_divisible() const { - return my_midpoint!=my_end; - } - //! Split range. - hash_map_range( hash_map_range& r, split ) : - my_end(r.my_end), - my_grainsize(r.my_grainsize) - { - r.my_end = my_begin = r.my_midpoint; - __TBB_ASSERT( !empty(), "Splitting despite the range is not divisible" ); - __TBB_ASSERT( !r.empty(), "Splitting despite the range is not divisible" ); - set_midpoint(); - r.set_midpoint(); - } - //! type conversion - template - hash_map_range( hash_map_range& r) : - my_begin(r.my_begin), - my_end(r.my_end), - my_midpoint(r.my_midpoint), - my_grainsize(r.my_grainsize) - {} - //! Init range with container and grainsize specified - hash_map_range( const map_type &map, size_type grainsize_ = 1 ) : - my_begin( Iterator( map, 0, map.my_embedded_segment, map.my_embedded_segment->node_list ) ), - my_end( Iterator( map, map.my_mask + 1, 0, 0 ) ), - my_grainsize( grainsize_ ) - { - __TBB_ASSERT( grainsize_>0, "grainsize must be positive" ); - set_midpoint(); - } - const Iterator& begin() const {return my_begin;} - const Iterator& end() const {return my_end;} - //! The grain size for this range. - size_type grainsize() const {return my_grainsize;} - }; - - template - void hash_map_range::set_midpoint() const { - // Split by groups of nodes - size_t m = my_end.my_index-my_begin.my_index; - if( m > my_grainsize ) { - m = my_begin.my_index + m/2u; - hash_map_base::bucket *b = my_begin.my_map->get_bucket(m); - my_midpoint = Iterator(*my_begin.my_map,m,b,b->node_list); - } else { - my_midpoint = my_end; - } - __TBB_ASSERT( my_begin.my_index <= my_midpoint.my_index, - "my_begin is after my_midpoint" ); - __TBB_ASSERT( my_midpoint.my_index <= my_end.my_index, - "my_midpoint is after my_end" ); - __TBB_ASSERT( my_begin != my_midpoint || my_begin == my_end, - "[my_begin, my_midpoint) range should not be empty" ); - } - - } // internal -//! @endcond - -#if _MSC_VER && !defined(__INTEL_COMPILER) - // Suppress "conditional expression is constant" warning. - #pragma warning( push ) - #pragma warning( disable: 4127 ) -#endif - -//! Unordered map from Key to T. -/** concurrent_hash_map is associative container with concurrent access. - -@par Compatibility - The class meets all Container Requirements from C++ Standard (See ISO/IEC 14882:2003(E), clause 23.1). - -@par Exception Safety - - Hash function is not permitted to throw an exception. User-defined types Key and T are forbidden from throwing an exception in destructors. - - If exception happens during insert() operations, it has no effect (unless exception raised by HashCompare::hash() function during grow_segment). - - If exception happens during operator=() operation, the container can have a part of source items, and methods size() and empty() can return wrong results. - -@par Changes since TBB 2.1 - - Replaced internal algorithm and data structure. Patent is pending. - - Added buckets number argument for constructor - -@par Changes since TBB 2.0 - - Fixed exception-safety - - Added template argument for allocator - - Added allocator argument in constructors - - Added constructor from a range of iterators - - Added several new overloaded insert() methods - - Added get_allocator() - - Added swap() - - Added count() - - Added overloaded erase(accessor &) and erase(const_accessor&) - - Added equal_range() [const] - - Added [const_]pointer, [const_]reference, and allocator_type types - - Added global functions: operator==(), operator!=(), and swap() - - @ingroup containers */ -template -class concurrent_hash_map : protected internal::hash_map_base { - template - friend class internal::hash_map_iterator; - - template - friend class internal::hash_map_range; - -public: - typedef Key key_type; - typedef T mapped_type; - typedef std::pair value_type; - typedef hash_map_base::size_type size_type; - typedef ptrdiff_t difference_type; - typedef value_type *pointer; - typedef const value_type *const_pointer; - typedef value_type &reference; - typedef const value_type &const_reference; - typedef internal::hash_map_iterator iterator; - typedef internal::hash_map_iterator const_iterator; - typedef internal::hash_map_range range_type; - typedef internal::hash_map_range const_range_type; - typedef Allocator allocator_type; - -protected: - friend class const_accessor; - struct node; - typedef typename Allocator::template rebind::other node_allocator_type; - node_allocator_type my_allocator; - HashCompare my_hash_compare; - - struct node : public node_base { - value_type item; - node( const Key &key ) : item(key, T()) {} - node( const Key &key, const T &t ) : item(key, t) {} -#if __TBB_CPP11_RVALUE_REF_PRESENT - node( const Key &key, T &&t ) : item(key, std::move(t)) {} - node( value_type&& i ) : item(std::move(i)){} -#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT - template - node( Args&&... args ) : item(std::forward(args)...) {} -#if __TBB_COPY_FROM_NON_CONST_REF_BROKEN - node( value_type& i ) : item(const_cast(i)) {} -#endif //__TBB_COPY_FROM_NON_CONST_REF_BROKEN -#endif //__TBB_CPP11_VARIADIC_TEMPLATES_PRESENT -#endif //__TBB_CPP11_RVALUE_REF_PRESENT - node( const value_type& i ) : item(i) {} - - // exception-safe allocation, see C++ Standard 2003, clause 5.3.4p17 - void *operator new( size_t /*size*/, node_allocator_type &a ) { - void *ptr = a.allocate(1); - if(!ptr) - tbb::internal::throw_exception(tbb::internal::eid_bad_alloc); - return ptr; - } - // match placement-new form above to be called if exception thrown in constructor - void operator delete( void *ptr, node_allocator_type &a ) { a.deallocate(static_cast(ptr),1); } - }; - - void delete_node( node_base *n ) { - my_allocator.destroy( static_cast(n) ); - my_allocator.deallocate( static_cast(n), 1); - } - - static node* allocate_node_copy_construct(node_allocator_type& allocator, const Key &key, const T * t){ - return new( allocator ) node(key, *t); - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - static node* allocate_node_move_construct(node_allocator_type& allocator, const Key &key, const T * t){ - return new( allocator ) node(key, std::move(*const_cast(t))); - } -#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT - template - static node* allocate_node_emplace_construct(node_allocator_type& allocator, Args&&... args){ - return new( allocator ) node(std::forward(args)...); - } -#endif //#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT -#endif - - static node* allocate_node_default_construct(node_allocator_type& allocator, const Key &key, const T * ){ - return new( allocator ) node(key); - } - - static node* do_not_allocate_node(node_allocator_type& , const Key &, const T * ){ - __TBB_ASSERT(false,"this dummy function should not be called"); - return NULL; - } - - node *search_bucket( const key_type &key, bucket *b ) const { - node *n = static_cast( b->node_list ); - while( is_valid(n) && !my_hash_compare.equal(key, n->item.first) ) - n = static_cast( n->next ); - __TBB_ASSERT(n != internal::rehash_req, "Search can be executed only for rehashed bucket"); - return n; - } - - //! bucket accessor is to find, rehash, acquire a lock, and access a bucket - class bucket_accessor : public bucket::scoped_t { - bucket *my_b; - public: - bucket_accessor( concurrent_hash_map *base, const hashcode_t h, bool writer = false ) { acquire( base, h, writer ); } - //! find a bucket by masked hashcode, optionally rehash, and acquire the lock - inline void acquire( concurrent_hash_map *base, const hashcode_t h, bool writer = false ) { - my_b = base->get_bucket( h ); - // TODO: actually, notification is unnecessary here, just hiding double-check - if( itt_load_word_with_acquire(my_b->node_list) == internal::rehash_req - && try_acquire( my_b->mutex, /*write=*/true ) ) - { - if( my_b->node_list == internal::rehash_req ) base->rehash_bucket( my_b, h ); //recursive rehashing - } - else bucket::scoped_t::acquire( my_b->mutex, writer ); - __TBB_ASSERT( my_b->node_list != internal::rehash_req, NULL); - } - //! check whether bucket is locked for write - bool is_writer() { return bucket::scoped_t::is_writer; } - //! get bucket pointer - bucket *operator() () { return my_b; } - }; - - // TODO refactor to hash_base - void rehash_bucket( bucket *b_new, const hashcode_t h ) { - __TBB_ASSERT( *(intptr_t*)(&b_new->mutex), "b_new must be locked (for write)"); - __TBB_ASSERT( h > 1, "The lowermost buckets can't be rehashed" ); - __TBB_store_with_release(b_new->node_list, internal::empty_rehashed); // mark rehashed - hashcode_t mask = ( 1u<<__TBB_Log2( h ) ) - 1; // get parent mask from the topmost bit -#if __TBB_STATISTICS - my_info_rehashes++; // invocations of rehash_bucket -#endif - - bucket_accessor b_old( this, h & mask ); - - mask = (mask<<1) | 1; // get full mask for new bucket - __TBB_ASSERT( (mask&(mask+1))==0 && (h & mask) == h, NULL ); - restart: - for( node_base **p = &b_old()->node_list, *n = __TBB_load_with_acquire(*p); is_valid(n); n = *p ) { - hashcode_t c = my_hash_compare.hash( static_cast(n)->item.first ); -#if TBB_USE_ASSERT - hashcode_t bmask = h & (mask>>1); - bmask = bmask==0? 1 : ( 1u<<(__TBB_Log2( bmask )+1 ) ) - 1; // minimal mask of parent bucket - __TBB_ASSERT( (c & bmask) == (h & bmask), "hash() function changed for key in table" ); -#endif - if( (c & mask) == h ) { - if( !b_old.is_writer() ) - if( !b_old.upgrade_to_writer() ) { - goto restart; // node ptr can be invalid due to concurrent erase - } - *p = n->next; // exclude from b_old - add_to_bucket( b_new, n ); - } else p = &n->next; // iterate to next item - } - } - - struct call_clear_on_leave { - concurrent_hash_map* my_ch_map; - call_clear_on_leave( concurrent_hash_map* a_ch_map ) : my_ch_map(a_ch_map) {} - void dismiss() {my_ch_map = 0;} - ~call_clear_on_leave(){ - if (my_ch_map){ - my_ch_map->clear(); - } - } - }; -public: - - class accessor; - //! Combines data access, locking, and garbage collection. - class const_accessor : private node::scoped_t /*which derived from no_copy*/ { - friend class concurrent_hash_map; - friend class accessor; - public: - //! Type of value - typedef const typename concurrent_hash_map::value_type value_type; - - //! True if result is empty. - bool empty() const { return !my_node; } - - //! Set to null - void release() { - if( my_node ) { - node::scoped_t::release(); - my_node = 0; - } - } - - //! Return reference to associated value in hash table. - const_reference operator*() const { - __TBB_ASSERT( my_node, "attempt to dereference empty accessor" ); - return my_node->item; - } - - //! Return pointer to associated value in hash table. - const_pointer operator->() const { - return &operator*(); - } - - //! Create empty result - const_accessor() : my_node(NULL) {} - - //! Destroy result after releasing the underlying reference. - ~const_accessor() { - my_node = NULL; // scoped lock's release() is called in its destructor - } - protected: - bool is_writer() { return node::scoped_t::is_writer; } - node *my_node; - hashcode_t my_hash; - }; - - //! Allows write access to elements and combines data access, locking, and garbage collection. - class accessor: public const_accessor { - public: - //! Type of value - typedef typename concurrent_hash_map::value_type value_type; - - //! Return reference to associated value in hash table. - reference operator*() const { - __TBB_ASSERT( this->my_node, "attempt to dereference empty accessor" ); - return this->my_node->item; - } - - //! Return pointer to associated value in hash table. - pointer operator->() const { - return &operator*(); - } - }; - - //! Construct empty table. - explicit concurrent_hash_map( const allocator_type &a = allocator_type() ) - : internal::hash_map_base(), my_allocator(a) - {} - - //! Construct empty table with n preallocated buckets. This number serves also as initial concurrency level. - concurrent_hash_map( size_type n, const allocator_type &a = allocator_type() ) - : my_allocator(a) - { - reserve( n ); - } - - //! Copy constructor - concurrent_hash_map( const concurrent_hash_map &table, const allocator_type &a = allocator_type() ) - : internal::hash_map_base(), my_allocator(a) - { - internal_copy(table); - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - //! Move constructor - concurrent_hash_map( concurrent_hash_map &&table ) - : internal::hash_map_base(), my_allocator(std::move(table.get_allocator())) - { - swap(table); - } - - //! Move constructor - concurrent_hash_map( concurrent_hash_map &&table, const allocator_type &a ) - : internal::hash_map_base(), my_allocator(a) - { - if (a == table.get_allocator()){ - this->swap(table); - }else{ - call_clear_on_leave scope_guard(this); - internal_copy(std::make_move_iterator(table.begin()), std::make_move_iterator(table.end())); - scope_guard.dismiss(); - } - } -#endif //__TBB_CPP11_RVALUE_REF_PRESENT - - //! Construction with copying iteration range and given allocator instance - template - concurrent_hash_map( I first, I last, const allocator_type &a = allocator_type() ) - : my_allocator(a) - { - reserve( std::distance(first, last) ); // TODO: load_factor? - internal_copy(first, last); - } - -#if __TBB_INITIALIZER_LISTS_PRESENT - //! Construct empty table with n preallocated buckets. This number serves also as initial concurrency level. - concurrent_hash_map( std::initializer_list il, const allocator_type &a = allocator_type() ) - : my_allocator(a) - { - reserve(il.size()); - internal_copy(il.begin(), il.end()); - } - -#endif //__TBB_INITIALIZER_LISTS_PRESENT - - //! Assignment - concurrent_hash_map& operator=( const concurrent_hash_map &table ) { - if( this!=&table ) { - clear(); - internal_copy(table); - } - return *this; - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - //! Move Assignment - concurrent_hash_map& operator=( concurrent_hash_map &&table ) { - if(this != &table){ - typedef typename tbb::internal::allocator_traits::propagate_on_container_move_assignment pocma_t; - if(pocma_t::value || this->my_allocator == table.my_allocator) { - concurrent_hash_map trash (std::move(*this)); - //TODO: swapping allocators here may be a problem, replace with single direction moving iff pocma is set - this->swap(table); - } else { - //do per element move - concurrent_hash_map moved_copy(std::move(table), this->my_allocator); - this->swap(moved_copy); - } - } - return *this; - } -#endif //__TBB_CPP11_RVALUE_REF_PRESENT - -#if __TBB_INITIALIZER_LISTS_PRESENT - //! Assignment - concurrent_hash_map& operator=( std::initializer_list il ) { - clear(); - reserve(il.size()); - internal_copy(il.begin(), il.end()); - return *this; - } -#endif //__TBB_INITIALIZER_LISTS_PRESENT - - - //! Rehashes and optionally resizes the whole table. - /** Useful to optimize performance before or after concurrent operations. - Also enables using of find() and count() concurrent methods in serial context. */ - void rehash(size_type n = 0); - - //! Clear table - void clear(); - - //! Clear table and destroy it. - ~concurrent_hash_map() { clear(); } - - //------------------------------------------------------------------------ - // Parallel algorithm support - //------------------------------------------------------------------------ - range_type range( size_type grainsize=1 ) { - return range_type( *this, grainsize ); - } - const_range_type range( size_type grainsize=1 ) const { - return const_range_type( *this, grainsize ); - } - - //------------------------------------------------------------------------ - // STL support - not thread-safe methods - //------------------------------------------------------------------------ - iterator begin() { return iterator( *this, 0, my_embedded_segment, my_embedded_segment->node_list ); } - iterator end() { return iterator( *this, 0, 0, 0 ); } - const_iterator begin() const { return const_iterator( *this, 0, my_embedded_segment, my_embedded_segment->node_list ); } - const_iterator end() const { return const_iterator( *this, 0, 0, 0 ); } - std::pair equal_range( const Key& key ) { return internal_equal_range( key, end() ); } - std::pair equal_range( const Key& key ) const { return internal_equal_range( key, end() ); } - - //! Number of items in table. - size_type size() const { return my_size; } - - //! True if size()==0. - bool empty() const { return my_size == 0; } - - //! Upper bound on size. - size_type max_size() const {return (~size_type(0))/sizeof(node);} - - //! Returns the current number of buckets - size_type bucket_count() const { return my_mask+1; } - - //! return allocator object - allocator_type get_allocator() const { return this->my_allocator; } - - //! swap two instances. Iterators are invalidated - void swap( concurrent_hash_map &table ); - - //------------------------------------------------------------------------ - // concurrent map operations - //------------------------------------------------------------------------ - - //! Return count of items (0 or 1) - size_type count( const Key &key ) const { - return const_cast(this)->lookup(/*insert*/false, key, NULL, NULL, /*write=*/false, &do_not_allocate_node ); - } - - //! Find item and acquire a read lock on the item. - /** Return true if item is found, false otherwise. */ - bool find( const_accessor &result, const Key &key ) const { - result.release(); - return const_cast(this)->lookup(/*insert*/false, key, NULL, &result, /*write=*/false, &do_not_allocate_node ); - } - - //! Find item and acquire a write lock on the item. - /** Return true if item is found, false otherwise. */ - bool find( accessor &result, const Key &key ) { - result.release(); - return lookup(/*insert*/false, key, NULL, &result, /*write=*/true, &do_not_allocate_node ); - } - - //! Insert item (if not already present) and acquire a read lock on the item. - /** Returns true if item is new. */ - bool insert( const_accessor &result, const Key &key ) { - result.release(); - return lookup(/*insert*/true, key, NULL, &result, /*write=*/false, &allocate_node_default_construct ); - } - - //! Insert item (if not already present) and acquire a write lock on the item. - /** Returns true if item is new. */ - bool insert( accessor &result, const Key &key ) { - result.release(); - return lookup(/*insert*/true, key, NULL, &result, /*write=*/true, &allocate_node_default_construct ); - } - - //! Insert item by copying if there is no such key present already and acquire a read lock on the item. - /** Returns true if item is new. */ - bool insert( const_accessor &result, const value_type &value ) { - result.release(); - return lookup(/*insert*/true, value.first, &value.second, &result, /*write=*/false, &allocate_node_copy_construct ); - } - - //! Insert item by copying if there is no such key present already and acquire a write lock on the item. - /** Returns true if item is new. */ - bool insert( accessor &result, const value_type &value ) { - result.release(); - return lookup(/*insert*/true, value.first, &value.second, &result, /*write=*/true, &allocate_node_copy_construct ); - } - - //! Insert item by copying if there is no such key present already - /** Returns true if item is inserted. */ - bool insert( const value_type &value ) { - return lookup(/*insert*/true, value.first, &value.second, NULL, /*write=*/false, &allocate_node_copy_construct ); - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - //! Insert item by copying if there is no such key present already and acquire a read lock on the item. - /** Returns true if item is new. */ - bool insert( const_accessor &result, value_type && value ) { - return generic_move_insert(result, std::move(value)); - } - - //! Insert item by copying if there is no such key present already and acquire a write lock on the item. - /** Returns true if item is new. */ - bool insert( accessor &result, value_type && value ) { - return generic_move_insert(result, std::move(value)); - } - - //! Insert item by copying if there is no such key present already - /** Returns true if item is inserted. */ - bool insert( value_type && value ) { - return generic_move_insert(accessor_not_used(), std::move(value)); - } - -#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT - //! Insert item by copying if there is no such key present already and acquire a read lock on the item. - /** Returns true if item is new. */ - template - bool emplace( const_accessor &result, Args&&... args ) { - return generic_emplace(result, std::forward(args)...); - } - - //! Insert item by copying if there is no such key present already and acquire a write lock on the item. - /** Returns true if item is new. */ - template - bool emplace( accessor &result, Args&&... args ) { - return generic_emplace(result, std::forward(args)...); - } - - //! Insert item by copying if there is no such key present already - /** Returns true if item is inserted. */ - template - bool emplace( Args&&... args ) { - return generic_emplace(accessor_not_used(), std::forward(args)...); - } -#endif //__TBB_CPP11_VARIADIC_TEMPLATES_PRESENT -#endif //__TBB_CPP11_RVALUE_REF_PRESENT - - //! Insert range [first, last) - template - void insert( I first, I last ) { - for ( ; first != last; ++first ) - insert( *first ); - } - -#if __TBB_INITIALIZER_LISTS_PRESENT - //! Insert initializer list - void insert( std::initializer_list il ) { - insert( il.begin(), il.end() ); - } -#endif //__TBB_INITIALIZER_LISTS_PRESENT - - //! Erase item. - /** Return true if item was erased by particularly this call. */ - bool erase( const Key& key ); - - //! Erase item by const_accessor. - /** Return true if item was erased by particularly this call. */ - bool erase( const_accessor& item_accessor ) { - return exclude( item_accessor ); - } - - //! Erase item by accessor. - /** Return true if item was erased by particularly this call. */ - bool erase( accessor& item_accessor ) { - return exclude( item_accessor ); - } - -protected: - //! Insert or find item and optionally acquire a lock on the item. - bool lookup(bool op_insert, const Key &key, const T *t, const_accessor *result, bool write, node* (*allocate_node)(node_allocator_type& , const Key &, const T * ), node *tmp_n = 0 ) ; - - struct accessor_not_used { void release(){}}; - friend const_accessor* accessor_location( accessor_not_used const& ){ return NULL;} - friend const_accessor* accessor_location( const_accessor & a ) { return &a;} - - friend bool is_write_access_needed( accessor const& ) { return true;} - friend bool is_write_access_needed( const_accessor const& ) { return false;} - friend bool is_write_access_needed( accessor_not_used const& ) { return false;} - -#if __TBB_CPP11_RVALUE_REF_PRESENT - template - bool generic_move_insert( Accessor && result, value_type && value ) { - result.release(); - return lookup(/*insert*/true, value.first, &value.second, accessor_location(result), is_write_access_needed(result), &allocate_node_move_construct ); - } - -#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT - template - bool generic_emplace( Accessor && result, Args &&... args ) { - result.release(); - node * node_ptr = allocate_node_emplace_construct(my_allocator, std::forward(args)...); - return lookup(/*insert*/true, node_ptr->item.first, NULL, accessor_location(result), is_write_access_needed(result), &do_not_allocate_node, node_ptr ); - } -#endif //__TBB_CPP11_VARIADIC_TEMPLATES_PRESENT -#endif //__TBB_CPP11_RVALUE_REF_PRESENT - - //! delete item by accessor - bool exclude( const_accessor &item_accessor ); - - //! Returns an iterator for an item defined by the key, or for the next item after it (if upper==true) - template - std::pair internal_equal_range( const Key& key, I end ) const; - - //! Copy "source" to *this, where *this must start out empty. - void internal_copy( const concurrent_hash_map& source ); - - template - void internal_copy( I first, I last ); - - //! Fast find when no concurrent erasure is used. For internal use inside TBB only! - /** Return pointer to item with given key, or NULL if no such item exists. - Must not be called concurrently with erasure operations. */ - const_pointer internal_fast_find( const Key& key ) const { - hashcode_t h = my_hash_compare.hash( key ); - hashcode_t m = (hashcode_t) itt_load_word_with_acquire( my_mask ); - node *n; - restart: - __TBB_ASSERT((m&(m+1))==0, "data structure is invalid"); - bucket *b = get_bucket( h & m ); - // TODO: actually, notification is unnecessary here, just hiding double-check - if( itt_load_word_with_acquire(b->node_list) == internal::rehash_req ) - { - bucket::scoped_t lock; - if( lock.try_acquire( b->mutex, /*write=*/true ) ) { - if( b->node_list == internal::rehash_req) - const_cast(this)->rehash_bucket( b, h & m ); //recursive rehashing - } - else lock.acquire( b->mutex, /*write=*/false ); - __TBB_ASSERT(b->node_list!=internal::rehash_req,NULL); - } - n = search_bucket( key, b ); - if( n ) - return &n->item; - else if( check_mask_race( h, m ) ) - goto restart; - return 0; - } -}; - -template -bool concurrent_hash_map::lookup( bool op_insert, const Key &key, const T *t, const_accessor *result, bool write, node* (*allocate_node)(node_allocator_type& , const Key&, const T*), node *tmp_n ) { - __TBB_ASSERT( !result || !result->my_node, NULL ); - bool return_value; - hashcode_t const h = my_hash_compare.hash( key ); - hashcode_t m = (hashcode_t) itt_load_word_with_acquire( my_mask ); - segment_index_t grow_segment = 0; - node *n; - restart: - {//lock scope - __TBB_ASSERT((m&(m+1))==0, "data structure is invalid"); - return_value = false; - // get bucket - bucket_accessor b( this, h & m ); - - // find a node - n = search_bucket( key, b() ); - if( op_insert ) { - // [opt] insert a key - if( !n ) { - if( !tmp_n ) { - tmp_n = allocate_node(my_allocator, key, t); - } - if( !b.is_writer() && !b.upgrade_to_writer() ) { // TODO: improved insertion - // Rerun search_list, in case another thread inserted the item during the upgrade. - n = search_bucket( key, b() ); - if( is_valid(n) ) { // unfortunately, it did - b.downgrade_to_reader(); - goto exists; - } - } - if( check_mask_race(h, m) ) - goto restart; // b.release() is done in ~b(). - // insert and set flag to grow the container - grow_segment = insert_new_node( b(), n = tmp_n, m ); - tmp_n = 0; - return_value = true; - } - } else { // find or count - if( !n ) { - if( check_mask_race( h, m ) ) - goto restart; // b.release() is done in ~b(). TODO: replace by continue - return false; - } - return_value = true; - } - exists: - if( !result ) goto check_growth; - // TODO: the following seems as generic/regular operation - // acquire the item - if( !result->try_acquire( n->mutex, write ) ) { - for( tbb::internal::atomic_backoff backoff(true);; ) { - if( result->try_acquire( n->mutex, write ) ) break; - if( !backoff.bounded_pause() ) { - // the wait takes really long, restart the operation - b.release(); - __TBB_ASSERT( !op_insert || !return_value, "Can't acquire new item in locked bucket?" ); - __TBB_Yield(); - m = (hashcode_t) itt_load_word_with_acquire( my_mask ); - goto restart; - } - } - } - }//lock scope - result->my_node = n; - result->my_hash = h; -check_growth: - // [opt] grow the container - if( grow_segment ) { -#if __TBB_STATISTICS - my_info_resizes++; // concurrent ones -#endif - enable_segment( grow_segment ); - } - if( tmp_n ) // if op_insert only - delete_node( tmp_n ); - return return_value; -} - -template -template -std::pair concurrent_hash_map::internal_equal_range( const Key& key, I end_ ) const { - hashcode_t h = my_hash_compare.hash( key ); - hashcode_t m = my_mask; - __TBB_ASSERT((m&(m+1))==0, "data structure is invalid"); - h &= m; - bucket *b = get_bucket( h ); - while( b->node_list == internal::rehash_req ) { - m = ( 1u<<__TBB_Log2( h ) ) - 1; // get parent mask from the topmost bit - b = get_bucket( h &= m ); - } - node *n = search_bucket( key, b ); - if( !n ) - return std::make_pair(end_, end_); - iterator lower(*this, h, b, n), upper(lower); - return std::make_pair(lower, ++upper); -} - -template -bool concurrent_hash_map::exclude( const_accessor &item_accessor ) { - __TBB_ASSERT( item_accessor.my_node, NULL ); - node_base *const n = item_accessor.my_node; - hashcode_t const h = item_accessor.my_hash; - hashcode_t m = (hashcode_t) itt_load_word_with_acquire( my_mask ); - do { - // get bucket - bucket_accessor b( this, h & m, /*writer=*/true ); - node_base **p = &b()->node_list; - while( *p && *p != n ) - p = &(*p)->next; - if( !*p ) { // someone else was first - if( check_mask_race( h, m ) ) - continue; - item_accessor.release(); - return false; - } - __TBB_ASSERT( *p == n, NULL ); - *p = n->next; // remove from container - my_size--; - break; - } while(true); - if( !item_accessor.is_writer() ) // need to get exclusive lock - item_accessor.upgrade_to_writer(); // return value means nothing here - item_accessor.release(); - delete_node( n ); // Only one thread can delete it - return true; -} - -template -bool concurrent_hash_map::erase( const Key &key ) { - node_base *n; - hashcode_t const h = my_hash_compare.hash( key ); - hashcode_t m = (hashcode_t) itt_load_word_with_acquire( my_mask ); -restart: - {//lock scope - // get bucket - bucket_accessor b( this, h & m ); - search: - node_base **p = &b()->node_list; - n = *p; - while( is_valid(n) && !my_hash_compare.equal(key, static_cast(n)->item.first ) ) { - p = &n->next; - n = *p; - } - if( !n ) { // not found, but mask could be changed - if( check_mask_race( h, m ) ) - goto restart; - return false; - } - else if( !b.is_writer() && !b.upgrade_to_writer() ) { - if( check_mask_race( h, m ) ) // contended upgrade, check mask - goto restart; - goto search; - } - *p = n->next; - my_size--; - } - { - typename node::scoped_t item_locker( n->mutex, /*write=*/true ); - } - // note: there should be no threads pretending to acquire this mutex again, do not try to upgrade const_accessor! - delete_node( n ); // Only one thread can delete it due to write lock on the bucket - return true; -} - -template -void concurrent_hash_map::swap(concurrent_hash_map &table) { - //TODO: respect C++11 allocator_traits::propogate_on_constainer_swap - using std::swap; - swap(this->my_allocator, table.my_allocator); - swap(this->my_hash_compare, table.my_hash_compare); - internal_swap(table); -} - -template -void concurrent_hash_map::rehash(size_type sz) { - reserve( sz ); // TODO: add reduction of number of buckets as well - hashcode_t mask = my_mask; - hashcode_t b = (mask+1)>>1; // size or first index of the last segment - __TBB_ASSERT((b&(b-1))==0, NULL); // zero or power of 2 - bucket *bp = get_bucket( b ); // only the last segment should be scanned for rehashing - for(; b <= mask; b++, bp++ ) { - node_base *n = bp->node_list; - __TBB_ASSERT( is_valid(n) || n == internal::empty_rehashed || n == internal::rehash_req, "Broken internal structure" ); - __TBB_ASSERT( *reinterpret_cast(&bp->mutex) == 0, "concurrent or unexpectedly terminated operation during rehash() execution" ); - if( n == internal::rehash_req ) { // rehash bucket, conditional because rehashing of a previous bucket may affect this one - hashcode_t h = b; bucket *b_old = bp; - do { - __TBB_ASSERT( h > 1, "The lowermost buckets can't be rehashed" ); - hashcode_t m = ( 1u<<__TBB_Log2( h ) ) - 1; // get parent mask from the topmost bit - b_old = get_bucket( h &= m ); - } while( b_old->node_list == internal::rehash_req ); - // now h - is index of the root rehashed bucket b_old - mark_rehashed_levels( h ); // mark all non-rehashed children recursively across all segments - for( node_base **p = &b_old->node_list, *q = *p; is_valid(q); q = *p ) { - hashcode_t c = my_hash_compare.hash( static_cast(q)->item.first ); - if( (c & mask) != h ) { // should be rehashed - *p = q->next; // exclude from b_old - bucket *b_new = get_bucket( c & mask ); - __TBB_ASSERT( b_new->node_list != internal::rehash_req, "hash() function changed for key in table or internal error" ); - add_to_bucket( b_new, q ); - } else p = &q->next; // iterate to next item - } - } - } -#if TBB_USE_PERFORMANCE_WARNINGS - int current_size = int(my_size), buckets = int(mask)+1, empty_buckets = 0, overpopulated_buckets = 0; // usage statistics - static bool reported = false; -#endif -#if TBB_USE_ASSERT || TBB_USE_PERFORMANCE_WARNINGS - for( b = 0; b <= mask; b++ ) {// only last segment should be scanned for rehashing - if( b & (b-2) ) ++bp; // not the beginning of a segment - else bp = get_bucket( b ); - node_base *n = bp->node_list; - __TBB_ASSERT( *reinterpret_cast(&bp->mutex) == 0, "concurrent or unexpectedly terminated operation during rehash() execution" ); - __TBB_ASSERT( is_valid(n) || n == internal::empty_rehashed, "Broken internal structure" ); -#if TBB_USE_PERFORMANCE_WARNINGS - if( n == internal::empty_rehashed ) empty_buckets++; - else if( n->next ) overpopulated_buckets++; -#endif -#if TBB_USE_ASSERT - for( ; is_valid(n); n = n->next ) { - hashcode_t h = my_hash_compare.hash( static_cast(n)->item.first ) & mask; - __TBB_ASSERT( h == b, "hash() function changed for key in table or internal error" ); - } -#endif - } -#endif // TBB_USE_ASSERT || TBB_USE_PERFORMANCE_WARNINGS -#if TBB_USE_PERFORMANCE_WARNINGS - if( buckets > current_size) empty_buckets -= buckets - current_size; - else overpopulated_buckets -= current_size - buckets; // TODO: load_factor? - if( !reported && buckets >= 512 && ( 2*empty_buckets > current_size || 2*overpopulated_buckets > current_size ) ) { - tbb::internal::runtime_warning( - "Performance is not optimal because the hash function produces bad randomness in lower bits in %s.\nSize: %d Empties: %d Overlaps: %d", -#if __TBB_USE_OPTIONAL_RTTI - typeid(*this).name(), -#else - "concurrent_hash_map", -#endif - current_size, empty_buckets, overpopulated_buckets ); - reported = true; - } -#endif -} - -template -void concurrent_hash_map::clear() { - hashcode_t m = my_mask; - __TBB_ASSERT((m&(m+1))==0, "data structure is invalid"); -#if TBB_USE_ASSERT || TBB_USE_PERFORMANCE_WARNINGS || __TBB_STATISTICS -#if TBB_USE_PERFORMANCE_WARNINGS || __TBB_STATISTICS - int current_size = int(my_size), buckets = int(m)+1, empty_buckets = 0, overpopulated_buckets = 0; // usage statistics - static bool reported = false; -#endif - bucket *bp = 0; - // check consistency - for( segment_index_t b = 0; b <= m; b++ ) { - if( b & (b-2) ) ++bp; // not the beginning of a segment - else bp = get_bucket( b ); - node_base *n = bp->node_list; - __TBB_ASSERT( is_valid(n) || n == internal::empty_rehashed || n == internal::rehash_req, "Broken internal structure" ); - __TBB_ASSERT( *reinterpret_cast(&bp->mutex) == 0, "concurrent or unexpectedly terminated operation during clear() execution" ); -#if TBB_USE_PERFORMANCE_WARNINGS || __TBB_STATISTICS - if( n == internal::empty_rehashed ) empty_buckets++; - else if( n == internal::rehash_req ) buckets--; - else if( n->next ) overpopulated_buckets++; -#endif -#if __TBB_EXTRA_DEBUG - for(; is_valid(n); n = n->next ) { - hashcode_t h = my_hash_compare.hash( static_cast(n)->item.first ); - h &= m; - __TBB_ASSERT( h == b || get_bucket(h)->node_list == internal::rehash_req, "hash() function changed for key in table or internal error" ); - } -#endif - } -#if TBB_USE_PERFORMANCE_WARNINGS || __TBB_STATISTICS -#if __TBB_STATISTICS - printf( "items=%d buckets: capacity=%d rehashed=%d empty=%d overpopulated=%d" - " concurrent: resizes=%u rehashes=%u restarts=%u\n", - current_size, int(m+1), buckets, empty_buckets, overpopulated_buckets, - unsigned(my_info_resizes), unsigned(my_info_rehashes), unsigned(my_info_restarts) ); - my_info_resizes = 0; // concurrent ones - my_info_restarts = 0; // race collisions - my_info_rehashes = 0; // invocations of rehash_bucket -#endif - if( buckets > current_size) empty_buckets -= buckets - current_size; - else overpopulated_buckets -= current_size - buckets; // TODO: load_factor? - if( !reported && buckets >= 512 && ( 2*empty_buckets > current_size || 2*overpopulated_buckets > current_size ) ) { - tbb::internal::runtime_warning( - "Performance is not optimal because the hash function produces bad randomness in lower bits in %s.\nSize: %d Empties: %d Overlaps: %d", -#if __TBB_USE_OPTIONAL_RTTI - typeid(*this).name(), -#else - "concurrent_hash_map", -#endif - current_size, empty_buckets, overpopulated_buckets ); - reported = true; - } -#endif -#endif//TBB_USE_ASSERT || TBB_USE_PERFORMANCE_WARNINGS || __TBB_STATISTICS - my_size = 0; - segment_index_t s = segment_index_of( m ); - __TBB_ASSERT( s+1 == pointers_per_table || !my_table[s+1], "wrong mask or concurrent grow" ); - cache_aligned_allocator alloc; - do { - __TBB_ASSERT( is_valid( my_table[s] ), "wrong mask or concurrent grow" ); - segment_ptr_t buckets_ptr = my_table[s]; - size_type sz = segment_size( s ? s : 1 ); - for( segment_index_t i = 0; i < sz; i++ ) - for( node_base *n = buckets_ptr[i].node_list; is_valid(n); n = buckets_ptr[i].node_list ) { - buckets_ptr[i].node_list = n->next; - delete_node( n ); - } - if( s >= first_block) // the first segment or the next - alloc.deallocate( buckets_ptr, sz ); - else if( s == embedded_block && embedded_block != first_block ) - alloc.deallocate( buckets_ptr, segment_size(first_block)-embedded_buckets ); - if( s >= embedded_block ) my_table[s] = 0; - } while(s-- > 0); - my_mask = embedded_buckets - 1; -} - -template -void concurrent_hash_map::internal_copy( const concurrent_hash_map& source ) { - reserve( source.my_size ); // TODO: load_factor? - hashcode_t mask = source.my_mask; - if( my_mask == mask ) { // optimized version - bucket *dst = 0, *src = 0; - bool rehash_required = false; - for( hashcode_t k = 0; k <= mask; k++ ) { - if( k & (k-2) ) ++dst,src++; // not the beginning of a segment - else { dst = get_bucket( k ); src = source.get_bucket( k ); } - __TBB_ASSERT( dst->node_list != internal::rehash_req, "Invalid bucket in destination table"); - node *n = static_cast( src->node_list ); - if( n == internal::rehash_req ) { // source is not rehashed, items are in previous buckets - rehash_required = true; - dst->node_list = internal::rehash_req; - } else for(; n; n = static_cast( n->next ) ) { - add_to_bucket( dst, new( my_allocator ) node(n->item.first, n->item.second) ); - ++my_size; // TODO: replace by non-atomic op - } - } - if( rehash_required ) rehash(); - } else internal_copy( source.begin(), source.end() ); -} - -template -template -void concurrent_hash_map::internal_copy(I first, I last) { - hashcode_t m = my_mask; - for(; first != last; ++first) { - hashcode_t h = my_hash_compare.hash( (*first).first ); - bucket *b = get_bucket( h & m ); - __TBB_ASSERT( b->node_list != internal::rehash_req, "Invalid bucket in destination table"); - node *n = new( my_allocator ) node(*first); - add_to_bucket( b, n ); - ++my_size; // TODO: replace by non-atomic op - } -} - -} // namespace interface5 - -using interface5::concurrent_hash_map; - - -template -inline bool operator==(const concurrent_hash_map &a, const concurrent_hash_map &b) { - if(a.size() != b.size()) return false; - typename concurrent_hash_map::const_iterator i(a.begin()), i_end(a.end()); - typename concurrent_hash_map::const_iterator j, j_end(b.end()); - for(; i != i_end; ++i) { - j = b.equal_range(i->first).first; - if( j == j_end || !(i->second == j->second) ) return false; - } - return true; -} - -template -inline bool operator!=(const concurrent_hash_map &a, const concurrent_hash_map &b) -{ return !(a == b); } - -template -inline void swap(concurrent_hash_map &a, concurrent_hash_map &b) -{ a.swap( b ); } - -#if _MSC_VER && !defined(__INTEL_COMPILER) - #pragma warning( pop ) -#endif // warning 4127 is back - -} // namespace tbb - -#endif /* __TBB_concurrent_hash_map_H */ diff --git a/inst/include/tbb/concurrent_lru_cache.h b/inst/include/tbb/concurrent_lru_cache.h deleted file mode 100644 index 8aacf241d..000000000 --- a/inst/include/tbb/concurrent_lru_cache.h +++ /dev/null @@ -1,235 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_concurrent_lru_cache_H -#define __TBB_concurrent_lru_cache_H - -#if ! TBB_PREVIEW_CONCURRENT_LRU_CACHE - #error Set TBB_PREVIEW_CONCURRENT_LRU_CACHE to include concurrent_lru_cache.h -#endif - -#include -#include - -#include "tbb_stddef.h" -#include "atomic.h" -#include "internal/_aggregator_impl.h" - -namespace tbb{ -namespace interface6 { - - -template -class concurrent_lru_cache : internal::no_assign{ -private: - typedef concurrent_lru_cache self_type; - typedef value_functor_type value_function_type; - typedef std::size_t ref_counter_type; - struct map_value_type; - typedef std::map map_storage_type; - typedef std::list lru_list_type; - struct map_value_type { - value_type my_value; - ref_counter_type my_ref_counter; - typename lru_list_type::iterator my_lru_list_iterator; - bool my_is_ready; - - map_value_type (value_type const& a_value, ref_counter_type a_ref_counter, typename lru_list_type::iterator a_lru_list_iterator, bool a_is_ready) - : my_value(a_value), my_ref_counter(a_ref_counter), my_lru_list_iterator (a_lru_list_iterator), my_is_ready(a_is_ready) - {} - }; - - class handle_object; - - struct aggregator_operation; - typedef aggregator_operation aggregated_operation_type; - typedef tbb::internal::aggregating_functor aggregator_function_type; - friend class tbb::internal::aggregating_functor; - typedef tbb::internal::aggregator aggregator_type; - -private: - value_function_type my_value_function; - std::size_t const my_number_of_lru_history_items; - map_storage_type my_map_storage; - lru_list_type my_lru_list; - aggregator_type my_aggregator; - -public: - typedef handle_object handle; - -public: - concurrent_lru_cache(value_function_type f, std::size_t number_of_lru_history_items) - : my_value_function(f),my_number_of_lru_history_items(number_of_lru_history_items) - { - my_aggregator.initialize_handler(aggregator_function_type(this)); - } - - handle_object operator[](key_type k){ - retrieve_aggregator_operation op(k); - my_aggregator.execute(&op); - if (op.is_new_value_needed()){ - op.result().second.my_value = my_value_function(k); - __TBB_store_with_release(op.result().second.my_is_ready, true); - }else{ - tbb::internal::spin_wait_while_eq(op.result().second.my_is_ready,false); - } - return handle_object(*this,op.result()); - } -private: - void signal_end_of_usage(typename map_storage_type::reference value_ref){ - signal_end_of_usage_aggregator_operation op(value_ref); - my_aggregator.execute(&op); - } - -private: - struct handle_move_t:no_assign{ - concurrent_lru_cache & my_cache_ref; - typename map_storage_type::reference my_map_record_ref; - handle_move_t(concurrent_lru_cache & cache_ref, typename map_storage_type::reference value_ref):my_cache_ref(cache_ref),my_map_record_ref(value_ref) {}; - }; - class handle_object { - concurrent_lru_cache * my_cache_pointer; - typename map_storage_type::reference my_map_record_ref; - public: - handle_object(concurrent_lru_cache & cache_ref, typename map_storage_type::reference value_ref):my_cache_pointer(&cache_ref), my_map_record_ref(value_ref) {} - handle_object(handle_move_t m):my_cache_pointer(&m.my_cache_ref), my_map_record_ref(m.my_map_record_ref){} - operator handle_move_t(){ return move(*this);} - value_type& value(){ - __TBB_ASSERT(my_cache_pointer,"get value from moved from object?"); - return my_map_record_ref.second.my_value; - } - ~handle_object(){ - if (my_cache_pointer){ - my_cache_pointer->signal_end_of_usage(my_map_record_ref); - } - } - private: - friend handle_move_t move(handle_object& h){ - return handle_object::move(h); - } - static handle_move_t move(handle_object& h){ - __TBB_ASSERT(h.my_cache_pointer,"move from the same object twice ?"); - concurrent_lru_cache * cache_pointer = h.my_cache_pointer; - h.my_cache_pointer = NULL; - return handle_move_t(*cache_pointer,h.my_map_record_ref); - } - private: - void operator=(handle_object&); -#if __SUNPRO_CC - // Presumably due to a compiler error, private copy constructor - // breaks expressions like handle h = cache[key]; - public: -#endif - handle_object(handle_object &); - }; -private: - //TODO: looks like aggregator_operation is a perfect match for statically typed variant type - struct aggregator_operation : tbb::internal::aggregated_operation{ - enum e_op_type {op_retive, op_signal_end_of_usage}; - //TODO: try to use pointer to function apply_visitor here - //TODO: try virtual functions and measure the difference - e_op_type my_operation_type; - aggregator_operation(e_op_type operation_type): my_operation_type(operation_type) {} - void cast_and_handle(self_type& container ){ - if (my_operation_type==op_retive){ - static_cast(this)->handle(container); - }else{ - static_cast(this)->handle(container); - } - } - }; - struct retrieve_aggregator_operation : aggregator_operation, private internal::no_assign { - key_type my_key; - typename map_storage_type::pointer my_result_map_record_pointer; - bool my_is_new_value_needed; - retrieve_aggregator_operation(key_type key):aggregator_operation(aggregator_operation::op_retive),my_key(key),my_is_new_value_needed(false){} - void handle(self_type& container ){ - my_result_map_record_pointer = & container.retrieve_serial(my_key,my_is_new_value_needed); - } - typename map_storage_type::reference result(){ return * my_result_map_record_pointer; } - bool is_new_value_needed(){return my_is_new_value_needed;} - }; - struct signal_end_of_usage_aggregator_operation : aggregator_operation, private internal::no_assign { - typename map_storage_type::reference my_map_record_ref; - signal_end_of_usage_aggregator_operation(typename map_storage_type::reference map_record_ref):aggregator_operation(aggregator_operation::op_signal_end_of_usage),my_map_record_ref(map_record_ref){} - void handle(self_type& container ){ - container.signal_end_of_usage_serial(my_map_record_ref); - } - }; - -private: - void handle_operations(aggregator_operation* op_list){ - while(op_list){ - op_list->cast_and_handle(*this); - aggregator_operation* tmp = op_list; - op_list=op_list->next; - tbb::internal::itt_store_word_with_release(tmp->status, uintptr_t(1)); - } - } - -private: - typename map_storage_type::reference retrieve_serial(key_type k, bool& is_new_value_needed){ - typename map_storage_type::iterator it = my_map_storage.find(k); - if (it == my_map_storage.end()){ - it = my_map_storage.insert(it,std::make_pair(k,map_value_type(value_type(),0,my_lru_list.end(),false))); - is_new_value_needed = true; - }else { - typename lru_list_type::iterator list_it = it->second.my_lru_list_iterator; - if (list_it!=my_lru_list.end()) { - __TBB_ASSERT(!it->second.my_ref_counter,"item to be evicted should not have a live references"); - //item is going to be used. Therefore it is not a subject for eviction - //so - remove it from LRU history. - my_lru_list.erase(list_it); - it->second.my_lru_list_iterator= my_lru_list.end(); - } - } - ++(it->second.my_ref_counter); - return *it; - } - - void signal_end_of_usage_serial(typename map_storage_type::reference map_record_ref){ - typename map_storage_type::iterator it = my_map_storage.find(map_record_ref.first); - __TBB_ASSERT(it!=my_map_storage.end(),"cache should not return past-end iterators to outer world"); - __TBB_ASSERT(&(*it) == &map_record_ref,"dangling reference has been returned to outside world? data race ?"); - __TBB_ASSERT( my_lru_list.end()== std::find(my_lru_list.begin(),my_lru_list.end(),it), - "object in use should not be in list of unused objects "); - if (! --(it->second.my_ref_counter)){ - //it was the last reference so put it to the LRU history - if (my_lru_list.size()>=my_number_of_lru_history_items){ - //evict items in order to get a space - size_t number_of_elements_to_evict = 1 + my_lru_list.size() - my_number_of_lru_history_items; - for (size_t i=0; isecond.my_ref_counter,"item to be evicted should not have a live references"); - my_lru_list.pop_back(); - my_map_storage.erase(it_to_evict); - } - } - my_lru_list.push_front(it); - it->second.my_lru_list_iterator = my_lru_list.begin(); - } - } -}; -} // namespace interface6 - -using interface6::concurrent_lru_cache; - -} // namespace tbb -#endif //__TBB_concurrent_lru_cache_H diff --git a/inst/include/tbb/concurrent_priority_queue.h b/inst/include/tbb/concurrent_priority_queue.h deleted file mode 100644 index 89297d851..000000000 --- a/inst/include/tbb/concurrent_priority_queue.h +++ /dev/null @@ -1,490 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_concurrent_priority_queue_H -#define __TBB_concurrent_priority_queue_H - -#include "atomic.h" -#include "cache_aligned_allocator.h" -#include "tbb_exception.h" -#include "tbb_stddef.h" -#include "tbb_profiling.h" -#include "internal/_aggregator_impl.h" -#include -#include -#include -#include __TBB_STD_SWAP_HEADER - -#if __TBB_INITIALIZER_LISTS_PRESENT - #include -#endif - -#if __TBB_CPP11_IS_COPY_CONSTRUCTIBLE_PRESENT - #include -#endif - -namespace tbb { -namespace interface5 { -namespace internal { -#if __TBB_CPP11_IS_COPY_CONSTRUCTIBLE_PRESENT - template::value> - struct use_element_copy_constructor { - typedef tbb::internal::true_type type; - }; - template - struct use_element_copy_constructor { - typedef tbb::internal::false_type type; - }; -#else - template - struct use_element_copy_constructor { - typedef tbb::internal::true_type type; - }; -#endif -} // namespace internal - -using namespace tbb::internal; - -//! Concurrent priority queue -template , typename A=cache_aligned_allocator > -class concurrent_priority_queue { - public: - //! Element type in the queue. - typedef T value_type; - - //! Reference type - typedef T& reference; - - //! Const reference type - typedef const T& const_reference; - - //! Integral type for representing size of the queue. - typedef size_t size_type; - - //! Difference type for iterator - typedef ptrdiff_t difference_type; - - //! Allocator type - typedef A allocator_type; - - //! Constructs a new concurrent_priority_queue with default capacity - explicit concurrent_priority_queue(const allocator_type& a = allocator_type()) : mark(0), my_size(0), data(a) - { - my_aggregator.initialize_handler(my_functor_t(this)); - } - - //! Constructs a new concurrent_priority_queue with init_sz capacity - explicit concurrent_priority_queue(size_type init_capacity, const allocator_type& a = allocator_type()) : - mark(0), my_size(0), data(a) - { - data.reserve(init_capacity); - my_aggregator.initialize_handler(my_functor_t(this)); - } - - //! [begin,end) constructor - template - concurrent_priority_queue(InputIterator begin, InputIterator end, const allocator_type& a = allocator_type()) : - mark(0), data(begin, end, a) - { - my_aggregator.initialize_handler(my_functor_t(this)); - heapify(); - my_size = data.size(); - } - -#if __TBB_INITIALIZER_LISTS_PRESENT - //! Constructor from std::initializer_list - concurrent_priority_queue(std::initializer_list init_list, const allocator_type &a = allocator_type()) : - mark(0),data(init_list.begin(), init_list.end(), a) - { - my_aggregator.initialize_handler(my_functor_t(this)); - heapify(); - my_size = data.size(); - } -#endif //# __TBB_INITIALIZER_LISTS_PRESENT - - //! Copy constructor - /** This operation is unsafe if there are pending concurrent operations on the src queue. */ - explicit concurrent_priority_queue(const concurrent_priority_queue& src) : mark(src.mark), - my_size(src.my_size), data(src.data.begin(), src.data.end(), src.data.get_allocator()) - { - my_aggregator.initialize_handler(my_functor_t(this)); - heapify(); - } - - //! Copy constructor with specific allocator - /** This operation is unsafe if there are pending concurrent operations on the src queue. */ - concurrent_priority_queue(const concurrent_priority_queue& src, const allocator_type& a) : mark(src.mark), - my_size(src.my_size), data(src.data.begin(), src.data.end(), a) - { - my_aggregator.initialize_handler(my_functor_t(this)); - heapify(); - } - - //! Assignment operator - /** This operation is unsafe if there are pending concurrent operations on the src queue. */ - concurrent_priority_queue& operator=(const concurrent_priority_queue& src) { - if (this != &src) { - vector_t(src.data.begin(), src.data.end(), src.data.get_allocator()).swap(data); - mark = src.mark; - my_size = src.my_size; - } - return *this; - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - //! Move constructor - /** This operation is unsafe if there are pending concurrent operations on the src queue. */ - concurrent_priority_queue(concurrent_priority_queue&& src) : mark(src.mark), - my_size(src.my_size), data(std::move(src.data)) - { - my_aggregator.initialize_handler(my_functor_t(this)); - } - - //! Move constructor with specific allocator - /** This operation is unsafe if there are pending concurrent operations on the src queue. */ - concurrent_priority_queue(concurrent_priority_queue&& src, const allocator_type& a) : mark(src.mark), - my_size(src.my_size), -#if __TBB_ALLOCATOR_TRAITS_PRESENT - data(std::move(src.data), a) -#else - // Some early version of C++11 STL vector does not have a constructor of vector(vector&& , allocator). - // It seems that the reason is absence of support of allocator_traits (stateful allocators). - data(a) -#endif //__TBB_ALLOCATOR_TRAITS_PRESENT - { - my_aggregator.initialize_handler(my_functor_t(this)); -#if !__TBB_ALLOCATOR_TRAITS_PRESENT - if (a != src.data.get_allocator()){ - data.reserve(src.data.size()); - data.assign(std::make_move_iterator(src.data.begin()), std::make_move_iterator(src.data.end())); - }else{ - data = std::move(src.data); - } -#endif //!__TBB_ALLOCATOR_TRAITS_PRESENT - } - - //! Move assignment operator - /** This operation is unsafe if there are pending concurrent operations on the src queue. */ - concurrent_priority_queue& operator=( concurrent_priority_queue&& src) { - if (this != &src) { - mark = src.mark; - my_size = src.my_size; -#if !__TBB_ALLOCATOR_TRAITS_PRESENT - if (data.get_allocator() != src.data.get_allocator()){ - vector_t(std::make_move_iterator(src.data.begin()), std::make_move_iterator(src.data.end()), data.get_allocator()).swap(data); - }else -#endif //!__TBB_ALLOCATOR_TRAITS_PRESENT - { - data = std::move(src.data); - } - } - return *this; - } -#endif //__TBB_CPP11_RVALUE_REF_PRESENT - - //! Assign the queue from [begin,end) range, not thread-safe - template - void assign(InputIterator begin, InputIterator end) { - vector_t(begin, end, data.get_allocator()).swap(data); - mark = 0; - my_size = data.size(); - heapify(); - } - -#if __TBB_INITIALIZER_LISTS_PRESENT - //! Assign the queue from std::initializer_list, not thread-safe - void assign(std::initializer_list il) { this->assign(il.begin(), il.end()); } - - //! Assign from std::initializer_list, not thread-safe - concurrent_priority_queue& operator=(std::initializer_list il) { - this->assign(il.begin(), il.end()); - return *this; - } -#endif //# __TBB_INITIALIZER_LISTS_PRESENT - - //! Returns true if empty, false otherwise - /** Returned value may not reflect results of pending operations. - This operation reads shared data and will trigger a race condition. */ - bool empty() const { return size()==0; } - - //! Returns the current number of elements contained in the queue - /** Returned value may not reflect results of pending operations. - This operation reads shared data and will trigger a race condition. */ - size_type size() const { return __TBB_load_with_acquire(my_size); } - - //! Pushes elem onto the queue, increasing capacity of queue if necessary - /** This operation can be safely used concurrently with other push, try_pop or emplace operations. */ - void push(const_reference elem) { -#if __TBB_CPP11_IS_COPY_CONSTRUCTIBLE_PRESENT - __TBB_STATIC_ASSERT( std::is_copy_constructible::value, "The type is not copy constructible. Copying push operation is impossible." ); -#endif - cpq_operation op_data(elem, PUSH_OP); - my_aggregator.execute(&op_data); - if (op_data.status == FAILED) // exception thrown - throw_exception(eid_bad_alloc); - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - //! Pushes elem onto the queue, increasing capacity of queue if necessary - /** This operation can be safely used concurrently with other push, try_pop or emplace operations. */ - void push(value_type &&elem) { - cpq_operation op_data(elem, PUSH_RVALUE_OP); - my_aggregator.execute(&op_data); - if (op_data.status == FAILED) // exception thrown - throw_exception(eid_bad_alloc); - } - -#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT - //! Constructs a new element using args as the arguments for its construction and pushes it onto the queue */ - /** This operation can be safely used concurrently with other push, try_pop or emplace operations. */ - template - void emplace(Args&&... args) { - push(value_type(std::forward(args)...)); - } -#endif /* __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT */ -#endif /* __TBB_CPP11_RVALUE_REF_PRESENT */ - - //! Gets a reference to and removes highest priority element - /** If a highest priority element was found, sets elem and returns true, - otherwise returns false. - This operation can be safely used concurrently with other push, try_pop or emplace operations. */ - bool try_pop(reference elem) { - cpq_operation op_data(POP_OP); - op_data.elem = &elem; - my_aggregator.execute(&op_data); - return op_data.status==SUCCEEDED; - } - - //! Clear the queue; not thread-safe - /** This operation is unsafe if there are pending concurrent operations on the queue. - Resets size, effectively emptying queue; does not free space. - May not clear elements added in pending operations. */ - void clear() { - data.clear(); - mark = 0; - my_size = 0; - } - - //! Swap this queue with another; not thread-safe - /** This operation is unsafe if there are pending concurrent operations on the queue. */ - void swap(concurrent_priority_queue& q) { - using std::swap; - data.swap(q.data); - swap(mark, q.mark); - swap(my_size, q.my_size); - } - - //! Return allocator object - allocator_type get_allocator() const { return data.get_allocator(); } - - private: - enum operation_type {INVALID_OP, PUSH_OP, POP_OP, PUSH_RVALUE_OP}; - enum operation_status { WAIT=0, SUCCEEDED, FAILED }; - - class cpq_operation : public aggregated_operation { - public: - operation_type type; - union { - value_type *elem; - size_type sz; - }; - cpq_operation(const_reference e, operation_type t) : - type(t), elem(const_cast(&e)) {} - cpq_operation(operation_type t) : type(t) {} - }; - - class my_functor_t { - concurrent_priority_queue *cpq; - public: - my_functor_t() {} - my_functor_t(concurrent_priority_queue *cpq_) : cpq(cpq_) {} - void operator()(cpq_operation* op_list) { - cpq->handle_operations(op_list); - } - }; - - typedef tbb::internal::aggregator< my_functor_t, cpq_operation > aggregator_t; - aggregator_t my_aggregator; - //! Padding added to avoid false sharing - char padding1[NFS_MaxLineSize - sizeof(aggregator_t)]; - //! The point at which unsorted elements begin - size_type mark; - __TBB_atomic size_type my_size; - Compare compare; - //! Padding added to avoid false sharing - char padding2[NFS_MaxLineSize - (2*sizeof(size_type)) - sizeof(Compare)]; - //! Storage for the heap of elements in queue, plus unheapified elements - /** data has the following structure: - - binary unheapified - heap elements - ____|_______|____ - | | | - v v v - [_|...|_|_|...|_| |...| ] - 0 ^ ^ ^ - | | |__capacity - | |__my_size - |__mark - - Thus, data stores the binary heap starting at position 0 through - mark-1 (it may be empty). Then there are 0 or more elements - that have not yet been inserted into the heap, in positions - mark through my_size-1. */ - typedef std::vector vector_t; - vector_t data; - - void handle_operations(cpq_operation *op_list) { - cpq_operation *tmp, *pop_list=NULL; - - __TBB_ASSERT(mark == data.size(), NULL); - - // First pass processes all constant (amortized; reallocation may happen) time pushes and pops. - while (op_list) { - // ITT note: &(op_list->status) tag is used to cover accesses to op_list - // node. This thread is going to handle the operation, and so will acquire it - // and perform the associated operation w/o triggering a race condition; the - // thread that created the operation is waiting on the status field, so when - // this thread is done with the operation, it will perform a - // store_with_release to give control back to the waiting thread in - // aggregator::insert_operation. - call_itt_notify(acquired, &(op_list->status)); - __TBB_ASSERT(op_list->type != INVALID_OP, NULL); - tmp = op_list; - op_list = itt_hide_load_word(op_list->next); - if (tmp->type == POP_OP) { - if (mark < data.size() && - compare(data[0], data[data.size()-1])) { - // there are newly pushed elems and the last one - // is higher than top - *(tmp->elem) = tbb::internal::move(data[data.size()-1]); - __TBB_store_with_release(my_size, my_size-1); - itt_store_word_with_release(tmp->status, uintptr_t(SUCCEEDED)); - data.pop_back(); - __TBB_ASSERT(mark<=data.size(), NULL); - } - else { // no convenient item to pop; postpone - itt_hide_store_word(tmp->next, pop_list); - pop_list = tmp; - } - } else { // PUSH_OP or PUSH_RVALUE_OP - __TBB_ASSERT(tmp->type == PUSH_OP || tmp->type == PUSH_RVALUE_OP, "Unknown operation" ); - __TBB_TRY{ - if (tmp->type == PUSH_OP) { - push_back_helper(*(tmp->elem), typename internal::use_element_copy_constructor::type()); - } else { - data.push_back(tbb::internal::move(*(tmp->elem))); - } - __TBB_store_with_release(my_size, my_size + 1); - itt_store_word_with_release(tmp->status, uintptr_t(SUCCEEDED)); - } __TBB_CATCH(...) { - itt_store_word_with_release(tmp->status, uintptr_t(FAILED)); - } - } - } - - // second pass processes pop operations - while (pop_list) { - tmp = pop_list; - pop_list = itt_hide_load_word(pop_list->next); - __TBB_ASSERT(tmp->type == POP_OP, NULL); - if (data.empty()) { - itt_store_word_with_release(tmp->status, uintptr_t(FAILED)); - } - else { - __TBB_ASSERT(mark<=data.size(), NULL); - if (mark < data.size() && - compare(data[0], data[data.size()-1])) { - // there are newly pushed elems and the last one is - // higher than top - *(tmp->elem) = tbb::internal::move(data[data.size()-1]); - __TBB_store_with_release(my_size, my_size-1); - itt_store_word_with_release(tmp->status, uintptr_t(SUCCEEDED)); - data.pop_back(); - } - else { // extract top and push last element down heap - *(tmp->elem) = tbb::internal::move(data[0]); - __TBB_store_with_release(my_size, my_size-1); - itt_store_word_with_release(tmp->status, uintptr_t(SUCCEEDED)); - reheap(); - } - } - } - - // heapify any leftover pushed elements before doing the next - // batch of operations - if (mark0) mark = 1; - for (; mark>1; - if (!compare(data[parent], to_place)) break; - data[cur_pos] = tbb::internal::move(data[parent]); - cur_pos = parent; - } while( cur_pos ); - data[cur_pos] = tbb::internal::move(to_place); - } - } - - //! Re-heapify after an extraction - /** Re-heapify by pushing last element down the heap from the root. */ - void reheap() { - size_type cur_pos=0, child=1; - - while (child < mark) { - size_type target = child; - if (child+1 < mark && compare(data[child], data[child+1])) - ++target; - // target now has the higher priority child - if (compare(data[target], data[data.size()-1])) break; - data[cur_pos] = tbb::internal::move(data[target]); - cur_pos = target; - child = (cur_pos<<1)+1; - } - if (cur_pos != data.size()-1) - data[cur_pos] = tbb::internal::move(data[data.size()-1]); - data.pop_back(); - if (mark > data.size()) mark = data.size(); - } - - void push_back_helper(const T& t, tbb::internal::true_type) { - data.push_back(t); - } - - void push_back_helper(const T&, tbb::internal::false_type) { - __TBB_ASSERT( false, "The type is not copy constructible. Copying push operation is impossible." ); - } -}; - -} // namespace interface5 - -using interface5::concurrent_priority_queue; - -} // namespace tbb - -#endif /* __TBB_concurrent_priority_queue_H */ diff --git a/inst/include/tbb/concurrent_queue.h b/inst/include/tbb/concurrent_queue.h deleted file mode 100644 index 81db58a33..000000000 --- a/inst/include/tbb/concurrent_queue.h +++ /dev/null @@ -1,458 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_concurrent_queue_H -#define __TBB_concurrent_queue_H - -#include "internal/_concurrent_queue_impl.h" - -namespace tbb { - -namespace strict_ppl { - -//! A high-performance thread-safe non-blocking concurrent queue. -/** Multiple threads may each push and pop concurrently. - Assignment construction is not allowed. - @ingroup containers */ -template > -class concurrent_queue: public internal::concurrent_queue_base_v3 { - template friend class internal::concurrent_queue_iterator; - - //! Allocator type - typedef typename A::template rebind::other page_allocator_type; - page_allocator_type my_allocator; - - //! Allocates a block of size n (bytes) - virtual void *allocate_block( size_t n ) __TBB_override { - void *b = reinterpret_cast(my_allocator.allocate( n )); - if( !b ) - internal::throw_exception(internal::eid_bad_alloc); - return b; - } - - //! Deallocates block created by allocate_block. - virtual void deallocate_block( void *b, size_t n ) __TBB_override { - my_allocator.deallocate( reinterpret_cast(b), n ); - } - - static void copy_construct_item(T* location, const void* src){ - new (location) T(*static_cast(src)); - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - static void move_construct_item(T* location, const void* src) { - new (location) T( std::move(*static_cast(const_cast(src))) ); - } -#endif /* __TBB_CPP11_RVALUE_REF_PRESENT */ -public: - //! Element type in the queue. - typedef T value_type; - - //! Reference type - typedef T& reference; - - //! Const reference type - typedef const T& const_reference; - - //! Integral type for representing size of the queue. - typedef size_t size_type; - - //! Difference type for iterator - typedef ptrdiff_t difference_type; - - //! Allocator type - typedef A allocator_type; - - //! Construct empty queue - explicit concurrent_queue(const allocator_type& a = allocator_type()) : - my_allocator( a ) - { - } - - //! [begin,end) constructor - template - concurrent_queue( InputIterator begin, InputIterator end, const allocator_type& a = allocator_type()) : - my_allocator( a ) - { - for( ; begin != end; ++begin ) - this->push(*begin); - } - - //! Copy constructor - concurrent_queue( const concurrent_queue& src, const allocator_type& a = allocator_type()) : - internal::concurrent_queue_base_v3(), my_allocator( a ) - { - this->assign( src, copy_construct_item ); - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - //! Move constructors - concurrent_queue( concurrent_queue&& src ) : - internal::concurrent_queue_base_v3(), my_allocator( std::move(src.my_allocator) ) - { - this->internal_swap( src ); - } - - concurrent_queue( concurrent_queue&& src, const allocator_type& a ) : - internal::concurrent_queue_base_v3(), my_allocator( a ) - { - // checking that memory allocated by one instance of allocator can be deallocated - // with another - if( my_allocator == src.my_allocator) { - this->internal_swap( src ); - } else { - // allocators are different => performing per-element move - this->assign( src, move_construct_item ); - src.clear(); - } - } -#endif /* __TBB_CPP11_RVALUE_REF_PRESENT */ - - //! Destroy queue - ~concurrent_queue(); - - //! Enqueue an item at tail of queue. - void push( const T& source ) { - this->internal_push( &source, copy_construct_item ); - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - void push( T&& source ) { - this->internal_push( &source, move_construct_item ); - } - -#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT - template - void emplace( Arguments&&... args ) { - push( T(std::forward( args )...) ); - } -#endif //__TBB_CPP11_VARIADIC_TEMPLATES_PRESENT -#endif /* __TBB_CPP11_RVALUE_REF_PRESENT */ - - //! Attempt to dequeue an item from head of queue. - /** Does not wait for item to become available. - Returns true if successful; false otherwise. */ - bool try_pop( T& result ) { - return this->internal_try_pop( &result ); - } - - //! Return the number of items in the queue; thread unsafe - size_type unsafe_size() const {return this->internal_size();} - - //! Equivalent to size()==0. - bool empty() const {return this->internal_empty();} - - //! Clear the queue. not thread-safe. - void clear() ; - - //! Return allocator object - allocator_type get_allocator() const { return this->my_allocator; } - - typedef internal::concurrent_queue_iterator iterator; - typedef internal::concurrent_queue_iterator const_iterator; - - //------------------------------------------------------------------------ - // The iterators are intended only for debugging. They are slow and not thread safe. - //------------------------------------------------------------------------ - iterator unsafe_begin() {return iterator(*this);} - iterator unsafe_end() {return iterator();} - const_iterator unsafe_begin() const {return const_iterator(*this);} - const_iterator unsafe_end() const {return const_iterator();} -} ; - -template -concurrent_queue::~concurrent_queue() { - clear(); - this->internal_finish_clear(); -} - -template -void concurrent_queue::clear() { - T value; - while( !empty() ) try_pop(value); -} - -} // namespace strict_ppl - -//! A high-performance thread-safe blocking concurrent bounded queue. -/** This is the pre-PPL TBB concurrent queue which supports boundedness and blocking semantics. - Note that method names agree with the PPL-style concurrent queue. - Multiple threads may each push and pop concurrently. - Assignment construction is not allowed. - @ingroup containers */ -template > -class concurrent_bounded_queue: public internal::concurrent_queue_base_v8 { - template friend class internal::concurrent_queue_iterator; - - //! Allocator type - typedef typename A::template rebind::other page_allocator_type; - page_allocator_type my_allocator; - - typedef typename concurrent_queue_base_v3::padded_page padded_page; - typedef typename concurrent_queue_base_v3::copy_specifics copy_specifics; - - //! Class used to ensure exception-safety of method "pop" - class destroyer: internal::no_copy { - T& my_value; - public: - destroyer( T& value ) : my_value(value) {} - ~destroyer() {my_value.~T();} - }; - - T& get_ref( page& p, size_t index ) { - __TBB_ASSERT( index(static_cast(&p))->last)[index]; - } - - virtual void copy_item( page& dst, size_t index, const void* src ) __TBB_override { - new( &get_ref(dst,index) ) T(*static_cast(src)); - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - virtual void move_item( page& dst, size_t index, const void* src ) __TBB_override { - new( &get_ref(dst,index) ) T( std::move(*static_cast(const_cast(src))) ); - } -#else - virtual void move_item( page&, size_t, const void* ) __TBB_override { - __TBB_ASSERT( false, "Unreachable code" ); - } -#endif - - virtual void copy_page_item( page& dst, size_t dindex, const page& src, size_t sindex ) __TBB_override { - new( &get_ref(dst,dindex) ) T( get_ref( const_cast(src), sindex ) ); - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - virtual void move_page_item( page& dst, size_t dindex, const page& src, size_t sindex ) __TBB_override { - new( &get_ref(dst,dindex) ) T( std::move(get_ref( const_cast(src), sindex )) ); - } -#else - virtual void move_page_item( page&, size_t, const page&, size_t ) __TBB_override { - __TBB_ASSERT( false, "Unreachable code" ); - } -#endif - - virtual void assign_and_destroy_item( void* dst, page& src, size_t index ) __TBB_override { - T& from = get_ref(src,index); - destroyer d(from); - *static_cast(dst) = tbb::internal::move( from ); - } - - virtual page *allocate_page() __TBB_override { - size_t n = sizeof(padded_page) + (items_per_page-1)*sizeof(T); - page *p = reinterpret_cast(my_allocator.allocate( n )); - if( !p ) - internal::throw_exception(internal::eid_bad_alloc); - return p; - } - - virtual void deallocate_page( page *p ) __TBB_override { - size_t n = sizeof(padded_page) + (items_per_page-1)*sizeof(T); - my_allocator.deallocate( reinterpret_cast(p), n ); - } - -public: - //! Element type in the queue. - typedef T value_type; - - //! Allocator type - typedef A allocator_type; - - //! Reference type - typedef T& reference; - - //! Const reference type - typedef const T& const_reference; - - //! Integral type for representing size of the queue. - /** Note that the size_type is a signed integral type. - This is because the size can be negative if there are pending pops without corresponding pushes. */ - typedef std::ptrdiff_t size_type; - - //! Difference type for iterator - typedef std::ptrdiff_t difference_type; - - //! Construct empty queue - explicit concurrent_bounded_queue(const allocator_type& a = allocator_type()) : - concurrent_queue_base_v8( sizeof(T) ), my_allocator( a ) - { - } - - //! Copy constructor - concurrent_bounded_queue( const concurrent_bounded_queue& src, const allocator_type& a = allocator_type()) - : concurrent_queue_base_v8( sizeof(T) ), my_allocator( a ) - { - assign( src ); - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - //! Move constructors - concurrent_bounded_queue( concurrent_bounded_queue&& src ) - : concurrent_queue_base_v8( sizeof(T) ), my_allocator( std::move(src.my_allocator) ) - { - internal_swap( src ); - } - - concurrent_bounded_queue( concurrent_bounded_queue&& src, const allocator_type& a ) - : concurrent_queue_base_v8( sizeof(T) ), my_allocator( a ) - { - // checking that memory allocated by one instance of allocator can be deallocated - // with another - if( my_allocator == src.my_allocator) { - this->internal_swap( src ); - } else { - // allocators are different => performing per-element move - this->move_content( src ); - src.clear(); - } - } -#endif /* __TBB_CPP11_RVALUE_REF_PRESENT */ - - //! [begin,end) constructor - template - concurrent_bounded_queue( InputIterator begin, InputIterator end, - const allocator_type& a = allocator_type()) - : concurrent_queue_base_v8( sizeof(T) ), my_allocator( a ) - { - for( ; begin != end; ++begin ) - internal_push_if_not_full(&*begin); - } - - //! Destroy queue - ~concurrent_bounded_queue(); - - //! Enqueue an item at tail of queue. - void push( const T& source ) { - internal_push( &source ); - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - //! Move an item at tail of queue. - void push( T&& source ) { - internal_push_move( &source ); - } - -#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT - template - void emplace( Arguments&&... args ) { - push( T(std::forward( args )...) ); - } -#endif /* __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT */ -#endif /* __TBB_CPP11_RVALUE_REF_PRESENT */ - - //! Dequeue item from head of queue. - /** Block until an item becomes available, and then dequeue it. */ - void pop( T& destination ) { - internal_pop( &destination ); - } - -#if TBB_USE_EXCEPTIONS - //! Abort all pending queue operations - void abort() { - internal_abort(); - } -#endif - - //! Enqueue an item at tail of queue if queue is not already full. - /** Does not wait for queue to become not full. - Returns true if item is pushed; false if queue was already full. */ - bool try_push( const T& source ) { - return internal_push_if_not_full( &source ); - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - //! Move an item at tail of queue if queue is not already full. - /** Does not wait for queue to become not full. - Returns true if item is pushed; false if queue was already full. */ - bool try_push( T&& source ) { - return internal_push_move_if_not_full( &source ); - } -#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT - template - bool try_emplace( Arguments&&... args ) { - return try_push( T(std::forward( args )...) ); - } -#endif /* __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT */ -#endif /* __TBB_CPP11_RVALUE_REF_PRESENT */ - - //! Attempt to dequeue an item from head of queue. - /** Does not wait for item to become available. - Returns true if successful; false otherwise. */ - bool try_pop( T& destination ) { - return internal_pop_if_present( &destination ); - } - - //! Return number of pushes minus number of pops. - /** Note that the result can be negative if there are pops waiting for the - corresponding pushes. The result can also exceed capacity() if there - are push operations in flight. */ - size_type size() const {return internal_size();} - - //! Equivalent to size()<=0. - bool empty() const {return internal_empty();} - - //! Maximum number of allowed elements - size_type capacity() const { - return my_capacity; - } - - //! Set the capacity - /** Setting the capacity to 0 causes subsequent try_push operations to always fail, - and subsequent push operations to block forever. */ - void set_capacity( size_type new_capacity ) { - internal_set_capacity( new_capacity, sizeof(T) ); - } - - //! return allocator object - allocator_type get_allocator() const { return this->my_allocator; } - - //! clear the queue. not thread-safe. - void clear() ; - - typedef internal::concurrent_queue_iterator iterator; - typedef internal::concurrent_queue_iterator const_iterator; - - //------------------------------------------------------------------------ - // The iterators are intended only for debugging. They are slow and not thread safe. - //------------------------------------------------------------------------ - iterator unsafe_begin() {return iterator(*this);} - iterator unsafe_end() {return iterator();} - const_iterator unsafe_begin() const {return const_iterator(*this);} - const_iterator unsafe_end() const {return const_iterator();} - -}; - -template -concurrent_bounded_queue::~concurrent_bounded_queue() { - clear(); - internal_finish_clear(); -} - -template -void concurrent_bounded_queue::clear() { - T value; - while( try_pop(value) ) /*noop*/; -} - -using strict_ppl::concurrent_queue; - -} // namespace tbb - -#endif /* __TBB_concurrent_queue_H */ diff --git a/inst/include/tbb/concurrent_unordered_map.h b/inst/include/tbb/concurrent_unordered_map.h deleted file mode 100644 index c959a7ec7..000000000 --- a/inst/include/tbb/concurrent_unordered_map.h +++ /dev/null @@ -1,299 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -/* Container implementations in this header are based on PPL implementations - provided by Microsoft. */ - -#ifndef __TBB_concurrent_unordered_map_H -#define __TBB_concurrent_unordered_map_H - -#include "internal/_concurrent_unordered_impl.h" - -namespace tbb -{ - -namespace interface5 { - -// Template class for hash map traits -template -class concurrent_unordered_map_traits -{ -protected: - typedef std::pair value_type; - typedef Key key_type; - typedef Hash_compare hash_compare; - typedef typename Allocator::template rebind::other allocator_type; - enum { allow_multimapping = Allow_multimapping }; - - concurrent_unordered_map_traits() : my_hash_compare() {} - concurrent_unordered_map_traits(const hash_compare& hc) : my_hash_compare(hc) {} - - template - static const Key& get_key(const std::pair& value) { - return (value.first); - } - - hash_compare my_hash_compare; // the comparator predicate for keys -}; - -template , typename Key_equality = std::equal_to, - typename Allocator = tbb::tbb_allocator > > -class concurrent_unordered_map : - public internal::concurrent_unordered_base< concurrent_unordered_map_traits, Allocator, false> > -{ - // Base type definitions - typedef internal::hash_compare hash_compare; - typedef concurrent_unordered_map_traits traits_type; - typedef internal::concurrent_unordered_base< traits_type > base_type; -#if __TBB_EXTRA_DEBUG -public: -#endif - using traits_type::allow_multimapping; -public: - using base_type::end; - using base_type::find; - using base_type::insert; - - // Type definitions - typedef Key key_type; - typedef typename base_type::value_type value_type; - typedef T mapped_type; - typedef Hasher hasher; - typedef Key_equality key_equal; - typedef hash_compare key_compare; - - typedef typename base_type::allocator_type allocator_type; - typedef typename base_type::pointer pointer; - typedef typename base_type::const_pointer const_pointer; - typedef typename base_type::reference reference; - typedef typename base_type::const_reference const_reference; - - typedef typename base_type::size_type size_type; - typedef typename base_type::difference_type difference_type; - - typedef typename base_type::iterator iterator; - typedef typename base_type::const_iterator const_iterator; - typedef typename base_type::iterator local_iterator; - typedef typename base_type::const_iterator const_local_iterator; - - // Construction/destruction/copying - explicit concurrent_unordered_map(size_type n_of_buckets = base_type::initial_bucket_number, - const hasher& _Hasher = hasher(), const key_equal& _Key_equality = key_equal(), - const allocator_type& a = allocator_type()) - : base_type(n_of_buckets, key_compare(_Hasher, _Key_equality), a) - {} - - explicit concurrent_unordered_map(const Allocator& a) : base_type(base_type::initial_bucket_number, key_compare(), a) - {} - - template - concurrent_unordered_map(Iterator first, Iterator last, size_type n_of_buckets = base_type::initial_bucket_number, - const hasher& _Hasher = hasher(), const key_equal& _Key_equality = key_equal(), - const allocator_type& a = allocator_type()) - : base_type(n_of_buckets, key_compare(_Hasher, _Key_equality), a) - { - insert(first, last); - } - -#if __TBB_INITIALIZER_LISTS_PRESENT - //! Constructor from initializer_list - concurrent_unordered_map(std::initializer_list il, size_type n_of_buckets = base_type::initial_bucket_number, - const hasher& _Hasher = hasher(), const key_equal& _Key_equality = key_equal(), - const allocator_type& a = allocator_type()) - : base_type(n_of_buckets, key_compare(_Hasher, _Key_equality), a) - { - this->insert(il.begin(),il.end()); - } -#endif //# __TBB_INITIALIZER_LISTS_PRESENT - -#if __TBB_CPP11_RVALUE_REF_PRESENT -#if !__TBB_IMPLICIT_MOVE_PRESENT - concurrent_unordered_map(const concurrent_unordered_map& table) - : base_type(table) - {} - - concurrent_unordered_map& operator=(const concurrent_unordered_map& table) - { - return static_cast(base_type::operator=(table)); - } - - concurrent_unordered_map(concurrent_unordered_map&& table) - : base_type(std::move(table)) - {} - - concurrent_unordered_map& operator=(concurrent_unordered_map&& table) - { - return static_cast(base_type::operator=(std::move(table))); - } -#endif //!__TBB_IMPLICIT_MOVE_PRESENT - - concurrent_unordered_map(concurrent_unordered_map&& table, const Allocator& a) : base_type(std::move(table), a) - {} -#endif //__TBB_CPP11_RVALUE_REF_PRESENT - - concurrent_unordered_map(const concurrent_unordered_map& table, const Allocator& a) - : base_type(table, a) - {} - - // Observers - mapped_type& operator[](const key_type& key) - { - iterator where = find(key); - - if (where == end()) - { - where = insert(std::pair(key, mapped_type())).first; - } - - return ((*where).second); - } - - mapped_type& at(const key_type& key) - { - iterator where = find(key); - - if (where == end()) - { - tbb::internal::throw_exception(tbb::internal::eid_invalid_key); - } - - return ((*where).second); - } - - const mapped_type& at(const key_type& key) const - { - const_iterator where = find(key); - - if (where == end()) - { - tbb::internal::throw_exception(tbb::internal::eid_invalid_key); - } - - return ((*where).second); - } -}; - -template < typename Key, typename T, typename Hasher = tbb::tbb_hash, typename Key_equality = std::equal_to, - typename Allocator = tbb::tbb_allocator > > -class concurrent_unordered_multimap : - public internal::concurrent_unordered_base< concurrent_unordered_map_traits< Key, T, - internal::hash_compare, Allocator, true> > -{ - // Base type definitions - typedef internal::hash_compare hash_compare; - typedef concurrent_unordered_map_traits traits_type; - typedef internal::concurrent_unordered_base base_type; -#if __TBB_EXTRA_DEBUG -public: -#endif - using traits_type::allow_multimapping; -public: - using base_type::insert; - - // Type definitions - typedef Key key_type; - typedef typename base_type::value_type value_type; - typedef T mapped_type; - typedef Hasher hasher; - typedef Key_equality key_equal; - typedef hash_compare key_compare; - - typedef typename base_type::allocator_type allocator_type; - typedef typename base_type::pointer pointer; - typedef typename base_type::const_pointer const_pointer; - typedef typename base_type::reference reference; - typedef typename base_type::const_reference const_reference; - - typedef typename base_type::size_type size_type; - typedef typename base_type::difference_type difference_type; - - typedef typename base_type::iterator iterator; - typedef typename base_type::const_iterator const_iterator; - typedef typename base_type::iterator local_iterator; - typedef typename base_type::const_iterator const_local_iterator; - - // Construction/destruction/copying - explicit concurrent_unordered_multimap(size_type n_of_buckets = base_type::initial_bucket_number, - const hasher& _Hasher = hasher(), const key_equal& _Key_equality = key_equal(), - const allocator_type& a = allocator_type()) - : base_type(n_of_buckets, key_compare(_Hasher, _Key_equality), a) - {} - - explicit concurrent_unordered_multimap(const Allocator& a) : base_type(base_type::initial_bucket_number, key_compare(), a) - {} - - template - concurrent_unordered_multimap(Iterator first, Iterator last, size_type n_of_buckets = base_type::initial_bucket_number, - const hasher& _Hasher = hasher(), const key_equal& _Key_equality = key_equal(), - const allocator_type& a = allocator_type()) - : base_type(n_of_buckets,key_compare(_Hasher,_Key_equality), a) - { - insert(first, last); - } - -#if __TBB_INITIALIZER_LISTS_PRESENT - //! Constructor from initializer_list - concurrent_unordered_multimap(std::initializer_list il, size_type n_of_buckets = base_type::initial_bucket_number, - const hasher& _Hasher = hasher(), const key_equal& _Key_equality = key_equal(), - const allocator_type& a = allocator_type()) - : base_type(n_of_buckets, key_compare(_Hasher, _Key_equality), a) - { - this->insert(il.begin(),il.end()); - } -#endif //# __TBB_INITIALIZER_LISTS_PRESENT - -#if __TBB_CPP11_RVALUE_REF_PRESENT -#if !__TBB_IMPLICIT_MOVE_PRESENT - concurrent_unordered_multimap(const concurrent_unordered_multimap& table) - : base_type(table) - {} - - concurrent_unordered_multimap& operator=(const concurrent_unordered_multimap& table) - { - return static_cast(base_type::operator=(table)); - } - - concurrent_unordered_multimap(concurrent_unordered_multimap&& table) - : base_type(std::move(table)) - {} - - concurrent_unordered_multimap& operator=(concurrent_unordered_multimap&& table) - { - return static_cast(base_type::operator=(std::move(table))); - } -#endif //!__TBB_IMPLICIT_MOVE_PRESENT - - concurrent_unordered_multimap(concurrent_unordered_multimap&& table, const Allocator& a) : base_type(std::move(table), a) - {} -#endif //__TBB_CPP11_RVALUE_REF_PRESENT - - concurrent_unordered_multimap(const concurrent_unordered_multimap& table, const Allocator& a) - : base_type(table, a) - {} -}; -} // namespace interface5 - -using interface5::concurrent_unordered_map; -using interface5::concurrent_unordered_multimap; - -} // namespace tbb - -#endif// __TBB_concurrent_unordered_map_H diff --git a/inst/include/tbb/concurrent_unordered_set.h b/inst/include/tbb/concurrent_unordered_set.h deleted file mode 100644 index a26fee1a7..000000000 --- a/inst/include/tbb/concurrent_unordered_set.h +++ /dev/null @@ -1,256 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -/* Container implementations in this header are based on PPL implementations - provided by Microsoft. */ - -#ifndef __TBB_concurrent_unordered_set_H -#define __TBB_concurrent_unordered_set_H - -#include "internal/_concurrent_unordered_impl.h" - -namespace tbb -{ - -namespace interface5 { - -// Template class for hash set traits -template -class concurrent_unordered_set_traits -{ -protected: - typedef Key value_type; - typedef Key key_type; - typedef Hash_compare hash_compare; - typedef typename Allocator::template rebind::other allocator_type; - enum { allow_multimapping = Allow_multimapping }; - - concurrent_unordered_set_traits() : my_hash_compare() {} - concurrent_unordered_set_traits(const hash_compare& hc) : my_hash_compare(hc) {} - - static const Key& get_key(const value_type& value) { - return value; - } - - hash_compare my_hash_compare; // the comparator predicate for keys -}; - -template , typename Key_equality = std::equal_to, typename Allocator = tbb::tbb_allocator > -class concurrent_unordered_set : public internal::concurrent_unordered_base< concurrent_unordered_set_traits, Allocator, false> > -{ - // Base type definitions - typedef internal::hash_compare hash_compare; - typedef concurrent_unordered_set_traits traits_type; - typedef internal::concurrent_unordered_base< traits_type > base_type; -#if __TBB_EXTRA_DEBUG -public: -#endif - using traits_type::allow_multimapping; -public: - using base_type::insert; - - // Type definitions - typedef Key key_type; - typedef typename base_type::value_type value_type; - typedef Key mapped_type; - typedef Hasher hasher; - typedef Key_equality key_equal; - typedef hash_compare key_compare; - - typedef typename base_type::allocator_type allocator_type; - typedef typename base_type::pointer pointer; - typedef typename base_type::const_pointer const_pointer; - typedef typename base_type::reference reference; - typedef typename base_type::const_reference const_reference; - - typedef typename base_type::size_type size_type; - typedef typename base_type::difference_type difference_type; - - typedef typename base_type::iterator iterator; - typedef typename base_type::const_iterator const_iterator; - typedef typename base_type::iterator local_iterator; - typedef typename base_type::const_iterator const_local_iterator; - - // Construction/destruction/copying - explicit concurrent_unordered_set(size_type n_of_buckets = base_type::initial_bucket_number, const hasher& a_hasher = hasher(), - const key_equal& a_keyeq = key_equal(), const allocator_type& a = allocator_type()) - : base_type(n_of_buckets, key_compare(a_hasher, a_keyeq), a) - {} - - explicit concurrent_unordered_set(const Allocator& a) : base_type(base_type::initial_bucket_number, key_compare(), a) - {} - - template - concurrent_unordered_set(Iterator first, Iterator last, size_type n_of_buckets = base_type::initial_bucket_number, const hasher& a_hasher = hasher(), - const key_equal& a_keyeq = key_equal(), const allocator_type& a = allocator_type()) - : base_type(n_of_buckets, key_compare(a_hasher, a_keyeq), a) - { - insert(first, last); - } - -#if __TBB_INITIALIZER_LISTS_PRESENT - //! Constructor from initializer_list - concurrent_unordered_set(std::initializer_list il, size_type n_of_buckets = base_type::initial_bucket_number, const hasher& a_hasher = hasher(), - const key_equal& a_keyeq = key_equal(), const allocator_type& a = allocator_type()) - : base_type(n_of_buckets, key_compare(a_hasher, a_keyeq), a) - { - this->insert(il.begin(),il.end()); - } -#endif //# __TBB_INITIALIZER_LISTS_PRESENT - -#if __TBB_CPP11_RVALUE_REF_PRESENT -#if !__TBB_IMPLICIT_MOVE_PRESENT - concurrent_unordered_set(const concurrent_unordered_set& table) - : base_type(table) - {} - - concurrent_unordered_set& operator=(const concurrent_unordered_set& table) - { - return static_cast(base_type::operator=(table)); - } - - concurrent_unordered_set(concurrent_unordered_set&& table) - : base_type(std::move(table)) - {} - - concurrent_unordered_set& operator=(concurrent_unordered_set&& table) - { - return static_cast(base_type::operator=(std::move(table))); - } -#endif //!__TBB_IMPLICIT_MOVE_PRESENT - - concurrent_unordered_set(concurrent_unordered_set&& table, const Allocator& a) - : base_type(std::move(table), a) - {} -#endif //__TBB_CPP11_RVALUE_REF_PRESENT - - concurrent_unordered_set(const concurrent_unordered_set& table, const Allocator& a) - : base_type(table, a) - {} - -}; - -template , typename Key_equality = std::equal_to, - typename Allocator = tbb::tbb_allocator > -class concurrent_unordered_multiset : - public internal::concurrent_unordered_base< concurrent_unordered_set_traits, Allocator, true> > -{ - // Base type definitions - typedef internal::hash_compare hash_compare; - typedef concurrent_unordered_set_traits traits_type; - typedef internal::concurrent_unordered_base< traits_type > base_type; -#if __TBB_EXTRA_DEBUG -public: -#endif - using traits_type::allow_multimapping; -public: - using base_type::insert; - - // Type definitions - typedef Key key_type; - typedef typename base_type::value_type value_type; - typedef Key mapped_type; - typedef Hasher hasher; - typedef Key_equality key_equal; - typedef hash_compare key_compare; - - typedef typename base_type::allocator_type allocator_type; - typedef typename base_type::pointer pointer; - typedef typename base_type::const_pointer const_pointer; - typedef typename base_type::reference reference; - typedef typename base_type::const_reference const_reference; - - typedef typename base_type::size_type size_type; - typedef typename base_type::difference_type difference_type; - - typedef typename base_type::iterator iterator; - typedef typename base_type::const_iterator const_iterator; - typedef typename base_type::iterator local_iterator; - typedef typename base_type::const_iterator const_local_iterator; - - // Construction/destruction/copying - explicit concurrent_unordered_multiset(size_type n_of_buckets = base_type::initial_bucket_number, - const hasher& _Hasher = hasher(), const key_equal& _Key_equality = key_equal(), - const allocator_type& a = allocator_type()) - : base_type(n_of_buckets, key_compare(_Hasher, _Key_equality), a) - {} - - explicit concurrent_unordered_multiset(const Allocator& a) : base_type(base_type::initial_bucket_number, key_compare(), a) - {} - - template - concurrent_unordered_multiset(Iterator first, Iterator last, size_type n_of_buckets = base_type::initial_bucket_number, - const hasher& _Hasher = hasher(), const key_equal& _Key_equality = key_equal(), - const allocator_type& a = allocator_type()) - : base_type(n_of_buckets, key_compare(_Hasher, _Key_equality), a) - { - insert(first, last); - } - -#if __TBB_INITIALIZER_LISTS_PRESENT - //! Constructor from initializer_list - concurrent_unordered_multiset(std::initializer_list il, size_type n_of_buckets = base_type::initial_bucket_number, const hasher& a_hasher = hasher(), - const key_equal& a_keyeq = key_equal(), const allocator_type& a = allocator_type()) - : base_type(n_of_buckets, key_compare(a_hasher, a_keyeq), a) - { - this->insert(il.begin(),il.end()); - } -#endif //# __TBB_INITIALIZER_LISTS_PRESENT - -#if __TBB_CPP11_RVALUE_REF_PRESENT -#if !__TBB_IMPLICIT_MOVE_PRESENT - concurrent_unordered_multiset(const concurrent_unordered_multiset& table) - : base_type(table) - {} - - concurrent_unordered_multiset& operator=(const concurrent_unordered_multiset& table) - { - return static_cast(base_type::operator=(table)); - } - - concurrent_unordered_multiset(concurrent_unordered_multiset&& table) - : base_type(std::move(table)) - {} - - concurrent_unordered_multiset& operator=(concurrent_unordered_multiset&& table) - { - return static_cast(base_type::operator=(std::move(table))); - } -#endif //!__TBB_IMPLICIT_MOVE_PRESENT - - concurrent_unordered_multiset(concurrent_unordered_multiset&& table, const Allocator& a) - : base_type(std::move(table), a) - { - } -#endif //__TBB_CPP11_RVALUE_REF_PRESENT - - concurrent_unordered_multiset(const concurrent_unordered_multiset& table, const Allocator& a) - : base_type(table, a) - {} -}; -} // namespace interface5 - -using interface5::concurrent_unordered_set; -using interface5::concurrent_unordered_multiset; - -} // namespace tbb - -#endif// __TBB_concurrent_unordered_set_H diff --git a/inst/include/tbb/concurrent_vector.h b/inst/include/tbb/concurrent_vector.h deleted file mode 100644 index bd93615e6..000000000 --- a/inst/include/tbb/concurrent_vector.h +++ /dev/null @@ -1,1374 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_concurrent_vector_H -#define __TBB_concurrent_vector_H - -#include "tbb_stddef.h" -#include "tbb_exception.h" -#include "atomic.h" -#include "cache_aligned_allocator.h" -#include "blocked_range.h" -#include "tbb_machine.h" -#include "tbb_profiling.h" -#include -#include // for memset() -#include __TBB_STD_SWAP_HEADER - -#if !TBB_USE_EXCEPTIONS && _MSC_VER - // Suppress "C++ exception handler used, but unwind semantics are not enabled" warning in STL headers - #pragma warning (push) - #pragma warning (disable: 4530) -#endif - -#include -#include - -#if !TBB_USE_EXCEPTIONS && _MSC_VER - #pragma warning (pop) -#endif - -#if _MSC_VER==1500 && !__INTEL_COMPILER - // VS2008/VC9 seems to have an issue; limits pull in math.h - #pragma warning( push ) - #pragma warning( disable: 4985 ) -#endif -#include /* std::numeric_limits */ -#if _MSC_VER==1500 && !__INTEL_COMPILER - #pragma warning( pop ) -#endif - -#if __TBB_INITIALIZER_LISTS_PRESENT - #include -#endif - -#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) - // Workaround for overzealous compiler warnings in /Wp64 mode - #pragma warning (push) -#if defined(_Wp64) - #pragma warning (disable: 4267) -#endif - #pragma warning (disable: 4127) //warning C4127: conditional expression is constant -#endif - -namespace tbb { - -template > -class concurrent_vector; - -//! @cond INTERNAL -namespace internal { - - template - class vector_iterator; - - //! Bad allocation marker - static void *const vector_allocation_error_flag = reinterpret_cast(size_t(63)); - - //! Exception helper function - template - void handle_unconstructed_elements(T* array, size_t n_of_elements){ - std::memset( (void*) array, 0, n_of_elements * sizeof( T ) ); - } - - //! Base class of concurrent vector implementation. - /** @ingroup containers */ - class concurrent_vector_base_v3 { - protected: - - // Basic types declarations - typedef size_t segment_index_t; - typedef size_t size_type; - - // Using enumerations due to Mac linking problems of static const variables - enum { - // Size constants - default_initial_segments = 1, // 2 initial items - //! Number of slots for segment pointers inside the class - pointers_per_short_table = 3, // to fit into 8 words of entire structure - pointers_per_long_table = sizeof(segment_index_t) * 8 // one segment per bit - }; - public: - struct segment_not_used {}; - struct segment_allocated {}; - struct segment_allocation_failed {}; - protected: - class segment_t; - class segment_value_t { - void* array; - private: - //TODO: More elegant way to grant access to selected functions _only_? - friend class segment_t; - explicit segment_value_t(void* an_array):array(an_array) {} - public: - friend bool operator==(segment_value_t const& lhs, segment_not_used ) { return lhs.array == 0;} - friend bool operator==(segment_value_t const& lhs, segment_allocated) { return lhs.array > internal::vector_allocation_error_flag;} - friend bool operator==(segment_value_t const& lhs, segment_allocation_failed) { return lhs.array == internal::vector_allocation_error_flag;} - template - friend bool operator!=(segment_value_t const& lhs, argument_type arg) { return ! (lhs == arg);} - - template - T* pointer() const { return static_cast(const_cast(array)); } - }; - - friend void enforce_segment_allocated(segment_value_t const& s, internal::exception_id exception = eid_bad_last_alloc){ - if(s != segment_allocated()){ - internal::throw_exception(exception); - } - } - - // Segment pointer. - class segment_t { - atomic array; - public: - segment_t(){ store(segment_not_used());} - //Copy ctor and assignment operator are defined to ease using of stl algorithms. - //These algorithms usually not a synchronization point, so, semantic is - //intentionally relaxed here. - segment_t(segment_t const& rhs ){ array.store(rhs.array.load());} - - void swap(segment_t & rhs ){ - tbb::internal::swap(array, rhs.array); - } - - segment_t& operator=(segment_t const& rhs ){ - array.store(rhs.array.load()); - return *this; - } - - template - segment_value_t load() const { return segment_value_t(array.load());} - - template - void store(segment_not_used) { - array.store(0); - } - - template - void store(segment_allocation_failed) { - __TBB_ASSERT(load() != segment_allocated(),"transition from \"allocated\" to \"allocation failed\" state looks non-logical"); - array.store(internal::vector_allocation_error_flag); - } - - template - void store(void* allocated_segment_pointer) __TBB_NOEXCEPT(true) { - __TBB_ASSERT(segment_value_t(allocated_segment_pointer) == segment_allocated(), - "other overloads of store should be used for marking segment as not_used or allocation_failed" ); - array.store(allocated_segment_pointer); - } - -#if TBB_USE_ASSERT - ~segment_t() { - __TBB_ASSERT(load() != segment_allocated(), "should have been freed by clear" ); - } -#endif /* TBB_USE_ASSERT */ - }; - friend void swap(segment_t & , segment_t & ) __TBB_NOEXCEPT(true); - - // Data fields - - //! allocator function pointer - void* (*vector_allocator_ptr)(concurrent_vector_base_v3 &, size_t); - - //! count of segments in the first block - atomic my_first_block; - - //! Requested size of vector - atomic my_early_size; - - //! Pointer to the segments table - atomic my_segment; - - //! embedded storage of segment pointers - segment_t my_storage[pointers_per_short_table]; - - // Methods - - concurrent_vector_base_v3() { - //Here the semantic is intentionally relaxed. - //The reason this is next: - //Object that is in middle of construction (i.e. its constructor is not yet finished) - //cannot be used concurrently until the construction is finished. - //Thus to flag other threads that construction is finished, some synchronization with - //acquire-release semantic should be done by the (external) code that uses the vector. - //So, no need to do the synchronization inside the vector. - - my_early_size.store(0); - my_first_block.store(0); // here is not default_initial_segments - my_segment.store(my_storage); - } - - __TBB_EXPORTED_METHOD ~concurrent_vector_base_v3(); - - //these helpers methods use the fact that segments are allocated so - //that every segment size is a (increasing) power of 2. - //with one exception 0 segment has size of 2 as well segment 1; - //e.g. size of segment with index of 3 is 2^3=8; - static segment_index_t segment_index_of( size_type index ) { - return segment_index_t( __TBB_Log2( index|1 ) ); - } - - static segment_index_t segment_base( segment_index_t k ) { - return (segment_index_t(1)< - friend class vector_iterator; - - }; - - inline void swap(concurrent_vector_base_v3::segment_t & lhs, concurrent_vector_base_v3::segment_t & rhs) __TBB_NOEXCEPT(true) { - lhs.swap(rhs); - } - - typedef concurrent_vector_base_v3 concurrent_vector_base; - - //! Meets requirements of a forward iterator for STL and a Value for a blocked_range.*/ - /** Value is either the T or const T type of the container. - @ingroup containers */ - template - class vector_iterator - { - //! concurrent_vector over which we are iterating. - Container* my_vector; - - //! Index into the vector - size_t my_index; - - //! Caches my_vector->internal_subscript(my_index) - /** NULL if cached value is not available */ - mutable Value* my_item; - - template - friend vector_iterator operator+( ptrdiff_t offset, const vector_iterator& v ); - - template - friend bool operator==( const vector_iterator& i, const vector_iterator& j ); - - template - friend bool operator<( const vector_iterator& i, const vector_iterator& j ); - - template - friend ptrdiff_t operator-( const vector_iterator& i, const vector_iterator& j ); - - template - friend class internal::vector_iterator; - -#if !__TBB_TEMPLATE_FRIENDS_BROKEN - template - friend class tbb::concurrent_vector; -#else -public: -#endif - - vector_iterator( const Container& vector, size_t index, void *ptr = 0 ) : - my_vector(const_cast(&vector)), - my_index(index), - my_item(static_cast(ptr)) - {} - - public: - //! Default constructor - vector_iterator() : my_vector(NULL), my_index(~size_t(0)), my_item(NULL) {} - - vector_iterator( const vector_iterator& other ) : - my_vector(other.my_vector), - my_index(other.my_index), - my_item(other.my_item) - {} - - vector_iterator operator+( ptrdiff_t offset ) const { - return vector_iterator( *my_vector, my_index+offset ); - } - vector_iterator &operator+=( ptrdiff_t offset ) { - my_index+=offset; - my_item = NULL; - return *this; - } - vector_iterator operator-( ptrdiff_t offset ) const { - return vector_iterator( *my_vector, my_index-offset ); - } - vector_iterator &operator-=( ptrdiff_t offset ) { - my_index-=offset; - my_item = NULL; - return *this; - } - Value& operator*() const { - Value* item = my_item; - if( !item ) { - item = my_item = &my_vector->internal_subscript(my_index); - } - __TBB_ASSERT( item==&my_vector->internal_subscript(my_index), "corrupt cache" ); - return *item; - } - Value& operator[]( ptrdiff_t k ) const { - return my_vector->internal_subscript(my_index+k); - } - Value* operator->() const {return &operator*();} - - //! Pre increment - vector_iterator& operator++() { - size_t element_index = ++my_index; - if( my_item ) { - //TODO: consider using of knowledge about "first_block optimization" here as well? - if( concurrent_vector_base::is_first_element_in_segment(element_index)) { - //if the iterator crosses a segment boundary, the pointer become invalid - //as possibly next segment is in another memory location - my_item= NULL; - } else { - ++my_item; - } - } - return *this; - } - - //! Pre decrement - vector_iterator& operator--() { - __TBB_ASSERT( my_index>0, "operator--() applied to iterator already at beginning of concurrent_vector" ); - size_t element_index = my_index--; - if( my_item ) { - if(concurrent_vector_base::is_first_element_in_segment(element_index)) { - //if the iterator crosses a segment boundary, the pointer become invalid - //as possibly next segment is in another memory location - my_item= NULL; - } else { - --my_item; - } - } - return *this; - } - - //! Post increment - vector_iterator operator++(int) { - vector_iterator result = *this; - operator++(); - return result; - } - - //! Post decrement - vector_iterator operator--(int) { - vector_iterator result = *this; - operator--(); - return result; - } - - // STL support - - typedef ptrdiff_t difference_type; - typedef Value value_type; - typedef Value* pointer; - typedef Value& reference; - typedef std::random_access_iterator_tag iterator_category; - }; - - template - vector_iterator operator+( ptrdiff_t offset, const vector_iterator& v ) { - return vector_iterator( *v.my_vector, v.my_index+offset ); - } - - template - bool operator==( const vector_iterator& i, const vector_iterator& j ) { - return i.my_index==j.my_index && i.my_vector == j.my_vector; - } - - template - bool operator!=( const vector_iterator& i, const vector_iterator& j ) { - return !(i==j); - } - - template - bool operator<( const vector_iterator& i, const vector_iterator& j ) { - return i.my_index - bool operator>( const vector_iterator& i, const vector_iterator& j ) { - return j - bool operator>=( const vector_iterator& i, const vector_iterator& j ) { - return !(i - bool operator<=( const vector_iterator& i, const vector_iterator& j ) { - return !(j - ptrdiff_t operator-( const vector_iterator& i, const vector_iterator& j ) { - return ptrdiff_t(i.my_index)-ptrdiff_t(j.my_index); - } - - template - class allocator_base { - public: - typedef typename A::template - rebind::other allocator_type; - allocator_type my_allocator; - - allocator_base(const allocator_type &a = allocator_type() ) : my_allocator(a) {} - - }; - -} // namespace internal -//! @endcond - -//! Concurrent vector container -/** concurrent_vector is a container having the following main properties: - - It provides random indexed access to its elements. The index of the first element is 0. - - It ensures safe concurrent growing its size (different threads can safely append new elements). - - Adding new elements does not invalidate existing iterators and does not change indices of existing items. - -@par Compatibility - The class meets all Container Requirements and Reversible Container Requirements from - C++ Standard (See ISO/IEC 14882:2003(E), clause 23.1). But it doesn't meet - Sequence Requirements due to absence of insert() and erase() methods. - -@par Exception Safety - Methods working with memory allocation and/or new elements construction can throw an - exception if allocator fails to allocate memory or element's default constructor throws one. - Concurrent vector's element of type T must conform to the following requirements: - - Throwing an exception is forbidden for destructor of T. - - Default constructor of T must not throw an exception OR its non-virtual destructor must safely work when its object memory is zero-initialized. - . - Otherwise, the program's behavior is undefined. -@par - If an exception happens inside growth or assignment operation, an instance of the vector becomes invalid unless it is stated otherwise in the method documentation. - Invalid state means: - - There are no guarantees that all items were initialized by a constructor. The rest of items is zero-filled, including item where exception happens. - - An invalid vector instance cannot be repaired; it is unable to grow anymore. - - Size and capacity reported by the vector are incorrect, and calculated as if the failed operation were successful. - - Attempt to access not allocated elements using operator[] or iterators results in access violation or segmentation fault exception, and in case of using at() method a C++ exception is thrown. - . - If a concurrent grow operation successfully completes, all the elements it has added to the vector will remain valid and accessible even if one of subsequent grow operations fails. - -@par Fragmentation - Unlike an STL vector, a concurrent_vector does not move existing elements if it needs - to allocate more memory. The container is divided into a series of contiguous arrays of - elements. The first reservation, growth, or assignment operation determines the size of - the first array. Using small number of elements as initial size incurs fragmentation that - may increase element access time. Internal layout can be optimized by method compact() that - merges several smaller arrays into one solid. - -@par Changes since TBB 2.1 - - Fixed guarantees of concurrent_vector::size() and grow_to_at_least() methods to assure elements are allocated. - - Methods end()/rbegin()/back() are partly thread-safe since they use size() to get the end of vector - - Added resize() methods (not thread-safe) - - Added cbegin/cend/crbegin/crend methods - - Changed return type of methods grow* and push_back to iterator - -@par Changes since TBB 2.0 - - Implemented exception-safety guarantees - - Added template argument for allocator - - Added allocator argument in constructors - - Faster index calculation - - First growth call specifies a number of segments to be merged in the first allocation. - - Fixed memory blow up for swarm of vector's instances of small size - - Added grow_by(size_type n, const_reference t) growth using copying constructor to init new items. - - Added STL-like constructors. - - Added operators ==, < and derivatives - - Added at() method, approved for using after an exception was thrown inside the vector - - Added get_allocator() method. - - Added assign() methods - - Added compact() method to defragment first segments - - Added swap() method - - range() defaults on grainsize = 1 supporting auto grainsize algorithms. - - @ingroup containers */ -template -class concurrent_vector: protected internal::allocator_base, - private internal::concurrent_vector_base { -private: - template - class generic_range_type: public blocked_range { - public: - typedef T value_type; - typedef T& reference; - typedef const T& const_reference; - typedef I iterator; - typedef ptrdiff_t difference_type; - generic_range_type( I begin_, I end_, size_t grainsize_ = 1) : blocked_range(begin_,end_,grainsize_) {} - template - generic_range_type( const generic_range_type& r) : blocked_range(r.begin(),r.end(),r.grainsize()) {} - generic_range_type( generic_range_type& r, split ) : blocked_range(r,split()) {} - }; - - template - friend class internal::vector_iterator; - -public: - //------------------------------------------------------------------------ - // STL compatible types - //------------------------------------------------------------------------ - typedef internal::concurrent_vector_base_v3::size_type size_type; - typedef typename internal::allocator_base::allocator_type allocator_type; - - typedef T value_type; - typedef ptrdiff_t difference_type; - typedef T& reference; - typedef const T& const_reference; - typedef T *pointer; - typedef const T *const_pointer; - - typedef internal::vector_iterator iterator; - typedef internal::vector_iterator const_iterator; - -#if !defined(_MSC_VER) || _CPPLIB_VER>=300 - // Assume ISO standard definition of std::reverse_iterator - typedef std::reverse_iterator reverse_iterator; - typedef std::reverse_iterator const_reverse_iterator; -#else - // Use non-standard std::reverse_iterator - typedef std::reverse_iterator reverse_iterator; - typedef std::reverse_iterator const_reverse_iterator; -#endif /* defined(_MSC_VER) && (_MSC_VER<1300) */ - - //------------------------------------------------------------------------ - // Parallel algorithm support - //------------------------------------------------------------------------ - typedef generic_range_type range_type; - typedef generic_range_type const_range_type; - - //------------------------------------------------------------------------ - // STL compatible constructors & destructors - //------------------------------------------------------------------------ - - //! Construct empty vector. - explicit concurrent_vector(const allocator_type &a = allocator_type()) - : internal::allocator_base(a), internal::concurrent_vector_base() - { - vector_allocator_ptr = &internal_allocator; - } - - //Constructors are not required to have synchronization - //(for more details see comment in the concurrent_vector_base constructor). -#if __TBB_INITIALIZER_LISTS_PRESENT - //! Constructor from initializer_list - concurrent_vector(std::initializer_list init_list, const allocator_type &a = allocator_type()) - : internal::allocator_base(a), internal::concurrent_vector_base() - { - vector_allocator_ptr = &internal_allocator; - __TBB_TRY { - internal_assign_iterators(init_list.begin(), init_list.end()); - } __TBB_CATCH(...) { - segment_t *table = my_segment.load();; - internal_free_segments( table, internal_clear(&destroy_array), my_first_block.load()); - __TBB_RETHROW(); - } - - } -#endif //# __TBB_INITIALIZER_LISTS_PRESENT - - //! Copying constructor - concurrent_vector( const concurrent_vector& vector, const allocator_type& a = allocator_type() ) - : internal::allocator_base(a), internal::concurrent_vector_base() - { - vector_allocator_ptr = &internal_allocator; - __TBB_TRY { - internal_copy(vector, sizeof(T), ©_array); - } __TBB_CATCH(...) { - segment_t *table = my_segment.load(); - internal_free_segments( table, internal_clear(&destroy_array), my_first_block.load()); - __TBB_RETHROW(); - } - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - //! Move constructor - //TODO add __TBB_NOEXCEPT(true) and static_assert(std::has_nothrow_move_constructor::value) - concurrent_vector( concurrent_vector&& source) - : internal::allocator_base(std::move(source)), internal::concurrent_vector_base() - { - vector_allocator_ptr = &internal_allocator; - concurrent_vector_base_v3::internal_swap(source); - } - - concurrent_vector( concurrent_vector&& source, const allocator_type& a) - : internal::allocator_base(a), internal::concurrent_vector_base() - { - vector_allocator_ptr = &internal_allocator; - //C++ standard requires instances of an allocator being compared for equality, - //which means that memory allocated by one instance is possible to deallocate with the other one. - if (a == source.my_allocator) { - concurrent_vector_base_v3::internal_swap(source); - } else { - __TBB_TRY { - internal_copy(source, sizeof(T), &move_array); - } __TBB_CATCH(...) { - segment_t *table = my_segment.load(); - internal_free_segments( table, internal_clear(&destroy_array), my_first_block.load()); - __TBB_RETHROW(); - } - } - } - -#endif - - //! Copying constructor for vector with different allocator type - template - concurrent_vector( const concurrent_vector& vector, const allocator_type& a = allocator_type() ) - : internal::allocator_base(a), internal::concurrent_vector_base() - { - vector_allocator_ptr = &internal_allocator; - __TBB_TRY { - internal_copy(vector.internal_vector_base(), sizeof(T), ©_array); - } __TBB_CATCH(...) { - segment_t *table = my_segment.load(); - internal_free_segments( table, internal_clear(&destroy_array), my_first_block.load() ); - __TBB_RETHROW(); - } - } - - //! Construction with initial size specified by argument n - explicit concurrent_vector(size_type n) - { - vector_allocator_ptr = &internal_allocator; - __TBB_TRY { - internal_resize( n, sizeof(T), max_size(), NULL, &destroy_array, &initialize_array ); - } __TBB_CATCH(...) { - segment_t *table = my_segment.load(); - internal_free_segments( table, internal_clear(&destroy_array), my_first_block.load() ); - __TBB_RETHROW(); - } - } - - //! Construction with initial size specified by argument n, initialization by copying of t, and given allocator instance - concurrent_vector(size_type n, const_reference t, const allocator_type& a = allocator_type()) - : internal::allocator_base(a) - { - vector_allocator_ptr = &internal_allocator; - __TBB_TRY { - internal_resize( n, sizeof(T), max_size(), static_cast(&t), &destroy_array, &initialize_array_by ); - } __TBB_CATCH(...) { - segment_t *table = my_segment.load(); - internal_free_segments( table, internal_clear(&destroy_array), my_first_block.load() ); - __TBB_RETHROW(); - } - } - - //! Construction with copying iteration range and given allocator instance - template - concurrent_vector(I first, I last, const allocator_type &a = allocator_type()) - : internal::allocator_base(a) - { - vector_allocator_ptr = &internal_allocator; - __TBB_TRY { - internal_assign_range(first, last, static_cast::is_integer> *>(0) ); - } __TBB_CATCH(...) { - segment_t *table = my_segment.load(); - internal_free_segments( table, internal_clear(&destroy_array), my_first_block.load() ); - __TBB_RETHROW(); - } - } - - //! Assignment - concurrent_vector& operator=( const concurrent_vector& vector ) { - if( this != &vector ) - internal_assign(vector, sizeof(T), &destroy_array, &assign_array, ©_array); - return *this; - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - //TODO: add __TBB_NOEXCEPT() - //! Move assignment - concurrent_vector& operator=( concurrent_vector&& other ) { - __TBB_ASSERT(this != &other, "Move assignment to itself is prohibited "); - typedef typename tbb::internal::allocator_traits::propagate_on_container_move_assignment pocma_t; - if(pocma_t::value || this->my_allocator == other.my_allocator) { - concurrent_vector trash (std::move(*this)); - internal_swap(other); - if (pocma_t::value) { - this->my_allocator = std::move(other.my_allocator); - } - } else { - internal_assign(other, sizeof(T), &destroy_array, &move_assign_array, &move_array); - } - return *this; - } -#endif - //TODO: add an template assignment operator? (i.e. with different element type) - - //! Assignment for vector with different allocator type - template - concurrent_vector& operator=( const concurrent_vector& vector ) { - if( static_cast( this ) != static_cast( &vector ) ) - internal_assign(vector.internal_vector_base(), - sizeof(T), &destroy_array, &assign_array, ©_array); - return *this; - } - -#if __TBB_INITIALIZER_LISTS_PRESENT - //! Assignment for initializer_list - concurrent_vector& operator=( std::initializer_list init_list ) { - internal_clear(&destroy_array); - internal_assign_iterators(init_list.begin(), init_list.end()); - return *this; - } -#endif //#if __TBB_INITIALIZER_LISTS_PRESENT - - //------------------------------------------------------------------------ - // Concurrent operations - //------------------------------------------------------------------------ - //! Grow by "delta" elements. - /** Returns iterator pointing to the first new element. */ - iterator grow_by( size_type delta ) { - return iterator(*this, delta ? internal_grow_by( delta, sizeof(T), &initialize_array, NULL ) : my_early_size.load()); - } - - //! Grow by "delta" elements using copying constructor. - /** Returns iterator pointing to the first new element. */ - iterator grow_by( size_type delta, const_reference t ) { - return iterator(*this, delta ? internal_grow_by( delta, sizeof(T), &initialize_array_by, static_cast(&t) ) : my_early_size.load()); - } - - /** Returns iterator pointing to the first new element. */ - template - iterator grow_by( I first, I last ) { - typename std::iterator_traits::difference_type delta = std::distance(first, last); - __TBB_ASSERT( delta >= 0, NULL); - - return iterator(*this, delta ? internal_grow_by(delta, sizeof(T), ©_range, static_cast(&first)) : my_early_size.load()); - } - -#if __TBB_INITIALIZER_LISTS_PRESENT - /** Returns iterator pointing to the first new element. */ - iterator grow_by( std::initializer_list init_list ) { - return grow_by( init_list.begin(), init_list.end() ); - } -#endif //#if __TBB_INITIALIZER_LISTS_PRESENT - - //! Append minimal sequence of elements such that size()>=n. - /** The new elements are default constructed. Blocks until all elements in range [0..n) are allocated. - May return while other elements are being constructed by other threads. - Returns iterator that points to beginning of appended sequence. - If no elements were appended, returns iterator pointing to nth element. */ - iterator grow_to_at_least( size_type n ) { - size_type m=0; - if( n ) { - m = internal_grow_to_at_least_with_result( n, sizeof(T), &initialize_array, NULL ); - if( m>n ) m=n; - } - return iterator(*this, m); - }; - - /** Analogous to grow_to_at_least( size_type n ) with exception that the new - elements are initialized by copying of t instead of default construction. */ - iterator grow_to_at_least( size_type n, const_reference t ) { - size_type m=0; - if( n ) { - m = internal_grow_to_at_least_with_result( n, sizeof(T), &initialize_array_by, &t); - if( m>n ) m=n; - } - return iterator(*this, m); - }; - - //! Push item - /** Returns iterator pointing to the new element. */ - iterator push_back( const_reference item ) - { - push_back_helper prolog(*this); - new(prolog.internal_push_back_result()) T(item); - return prolog.return_iterator_and_dismiss(); - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - //! Push item, move-aware - /** Returns iterator pointing to the new element. */ - iterator push_back( T&& item ) - { - push_back_helper prolog(*this); - new(prolog.internal_push_back_result()) T(std::move(item)); - return prolog.return_iterator_and_dismiss(); - } -#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT - //! Push item, create item "in place" with provided arguments - /** Returns iterator pointing to the new element. */ - template - iterator emplace_back( Args&&... args ) - { - push_back_helper prolog(*this); - new(prolog.internal_push_back_result()) T(std::forward(args)...); - return prolog.return_iterator_and_dismiss(); - } -#endif //__TBB_CPP11_VARIADIC_TEMPLATES_PRESENT -#endif //__TBB_CPP11_RVALUE_REF_PRESENT - //! Get reference to element at given index. - /** This method is thread-safe for concurrent reads, and also while growing the vector, - as long as the calling thread has checked that index < size(). */ - reference operator[]( size_type index ) { - return internal_subscript(index); - } - - //! Get const reference to element at given index. - const_reference operator[]( size_type index ) const { - return internal_subscript(index); - } - - //! Get reference to element at given index. Throws exceptions on errors. - reference at( size_type index ) { - return internal_subscript_with_exceptions(index); - } - - //! Get const reference to element at given index. Throws exceptions on errors. - const_reference at( size_type index ) const { - return internal_subscript_with_exceptions(index); - } - - //! Get range for iterating with parallel algorithms - range_type range( size_t grainsize = 1 ) { - return range_type( begin(), end(), grainsize ); - } - - //! Get const range for iterating with parallel algorithms - const_range_type range( size_t grainsize = 1 ) const { - return const_range_type( begin(), end(), grainsize ); - } - - //------------------------------------------------------------------------ - // Capacity - //------------------------------------------------------------------------ - //! Return size of vector. It may include elements under construction - size_type size() const { - size_type sz = my_early_size, cp = internal_capacity(); - return cp < sz ? cp : sz; - } - - //! Return false if vector is not empty or has elements under construction at least. - bool empty() const {return !my_early_size;} - - //! Maximum size to which array can grow without allocating more memory. Concurrent allocations are not included in the value. - size_type capacity() const {return internal_capacity();} - - //! Allocate enough space to grow to size n without having to allocate more memory later. - /** Like most of the methods provided for STL compatibility, this method is *not* thread safe. - The capacity afterwards may be bigger than the requested reservation. */ - void reserve( size_type n ) { - if( n ) - internal_reserve(n, sizeof(T), max_size()); - } - - //! Resize the vector. Not thread-safe. - void resize( size_type n ) { - internal_resize( n, sizeof(T), max_size(), NULL, &destroy_array, &initialize_array ); - } - - //! Resize the vector, copy t for new elements. Not thread-safe. - void resize( size_type n, const_reference t ) { - internal_resize( n, sizeof(T), max_size(), static_cast(&t), &destroy_array, &initialize_array_by ); - } - - //! Optimize memory usage and fragmentation. - void shrink_to_fit(); - - //! Upper bound on argument to reserve. - size_type max_size() const {return (~size_type(0))/sizeof(T);} - - //------------------------------------------------------------------------ - // STL support - //------------------------------------------------------------------------ - - //! start iterator - iterator begin() {return iterator(*this,0);} - //! end iterator - iterator end() {return iterator(*this,size());} - //! start const iterator - const_iterator begin() const {return const_iterator(*this,0);} - //! end const iterator - const_iterator end() const {return const_iterator(*this,size());} - //! start const iterator - const_iterator cbegin() const {return const_iterator(*this,0);} - //! end const iterator - const_iterator cend() const {return const_iterator(*this,size());} - //! reverse start iterator - reverse_iterator rbegin() {return reverse_iterator(end());} - //! reverse end iterator - reverse_iterator rend() {return reverse_iterator(begin());} - //! reverse start const iterator - const_reverse_iterator rbegin() const {return const_reverse_iterator(end());} - //! reverse end const iterator - const_reverse_iterator rend() const {return const_reverse_iterator(begin());} - //! reverse start const iterator - const_reverse_iterator crbegin() const {return const_reverse_iterator(end());} - //! reverse end const iterator - const_reverse_iterator crend() const {return const_reverse_iterator(begin());} - //! the first item - reference front() { - __TBB_ASSERT( size()>0, NULL); - const segment_value_t& segment_value = my_segment[0].template load(); - return (segment_value.template pointer())[0]; - } - //! the first item const - const_reference front() const { - __TBB_ASSERT( size()>0, NULL); - const segment_value_t& segment_value = my_segment[0].template load(); - return (segment_value.template pointer())[0]; - } - //! the last item - reference back() { - __TBB_ASSERT( size()>0, NULL); - return internal_subscript( size()-1 ); - } - //! the last item const - const_reference back() const { - __TBB_ASSERT( size()>0, NULL); - return internal_subscript( size()-1 ); - } - //! return allocator object - allocator_type get_allocator() const { return this->my_allocator; } - - //! assign n items by copying t item - void assign(size_type n, const_reference t) { - clear(); - internal_resize( n, sizeof(T), max_size(), static_cast(&t), &destroy_array, &initialize_array_by ); - } - - //! assign range [first, last) - template - void assign(I first, I last) { - clear(); internal_assign_range( first, last, static_cast::is_integer> *>(0) ); - } - -#if __TBB_INITIALIZER_LISTS_PRESENT - //! assigns an initializer list - void assign(std::initializer_list init_list) { - clear(); internal_assign_iterators( init_list.begin(), init_list.end()); - } -#endif //# __TBB_INITIALIZER_LISTS_PRESENT - - //! swap two instances - void swap(concurrent_vector &vector) { - using std::swap; - if( this != &vector ) { - concurrent_vector_base_v3::internal_swap(static_cast(vector)); - swap(this->my_allocator, vector.my_allocator); - } - } - - //! Clear container while keeping memory allocated. - /** To free up the memory, use in conjunction with method compact(). Not thread safe **/ - void clear() { - internal_clear(&destroy_array); - } - - //! Clear and destroy vector. - ~concurrent_vector() { - segment_t *table = my_segment.load(); - internal_free_segments( table, internal_clear(&destroy_array), my_first_block.load() ); - // base class destructor call should be then - } - - const internal::concurrent_vector_base_v3 &internal_vector_base() const { return *this; } -private: - //! Allocate k items - static void *internal_allocator(internal::concurrent_vector_base_v3 &vb, size_t k) { - return static_cast&>(vb).my_allocator.allocate(k); - } - //! Free k segments from table - void internal_free_segments(segment_t table[], segment_index_t k, segment_index_t first_block); - - //! Get reference to element at given index. - T& internal_subscript( size_type index ) const; - - //! Get reference to element at given index with errors checks - T& internal_subscript_with_exceptions( size_type index ) const; - - //! assign n items by copying t - void internal_assign_n(size_type n, const_pointer p) { - internal_resize( n, sizeof(T), max_size(), static_cast(p), &destroy_array, p? &initialize_array_by : &initialize_array ); - } - - //! helper class - template class is_integer_tag; - - //! assign integer items by copying when arguments are treated as iterators. See C++ Standard 2003 23.1.1p9 - template - void internal_assign_range(I first, I last, is_integer_tag *) { - internal_assign_n(static_cast(first), &static_cast(last)); - } - //! inline proxy assign by iterators - template - void internal_assign_range(I first, I last, is_integer_tag *) { - internal_assign_iterators(first, last); - } - //! assign by iterators - template - void internal_assign_iterators(I first, I last); - - //these functions are marked __TBB_EXPORTED_FUNC as they are called from within the library - - //! Construct n instances of T, starting at "begin". - static void __TBB_EXPORTED_FUNC initialize_array( void* begin, const void*, size_type n ); - - //! Copy-construct n instances of T, starting at "begin". - static void __TBB_EXPORTED_FUNC initialize_array_by( void* begin, const void* src, size_type n ); - - //! Copy-construct n instances of T by copying single element pointed to by src, starting at "dst". - static void __TBB_EXPORTED_FUNC copy_array( void* dst, const void* src, size_type n ); - -#if __TBB_MOVE_IF_NOEXCEPT_PRESENT - //! Either opy or move-construct n instances of T, starting at "dst" by copying according element of src array. - static void __TBB_EXPORTED_FUNC move_array_if_noexcept( void* dst, const void* src, size_type n ); -#endif //__TBB_MOVE_IF_NO_EXCEPT_PRESENT - -#if __TBB_CPP11_RVALUE_REF_PRESENT - //! Move-construct n instances of T, starting at "dst" by copying according element of src array. - static void __TBB_EXPORTED_FUNC move_array( void* dst, const void* src, size_type n ); - - //! Move-assign (using operator=) n instances of T, starting at "dst" by assigning according element of src array. - static void __TBB_EXPORTED_FUNC move_assign_array( void* dst, const void* src, size_type n ); -#endif - //! Copy-construct n instances of T, starting at "dst" by iterator range of [p_type_erased_iterator, p_type_erased_iterator+n). - template - static void __TBB_EXPORTED_FUNC copy_range( void* dst, const void* p_type_erased_iterator, size_type n ); - - //! Assign (using operator=) n instances of T, starting at "dst" by assigning according element of src array. - static void __TBB_EXPORTED_FUNC assign_array( void* dst, const void* src, size_type n ); - - //! Destroy n instances of T, starting at "begin". - static void __TBB_EXPORTED_FUNC destroy_array( void* begin, size_type n ); - - //! Exception-aware helper class for filling a segment by exception-danger operators of user class - class internal_loop_guide : internal::no_copy { - public: - const pointer array; - const size_type n; - size_type i; - - static const T* as_const_pointer(const void *ptr) { return static_cast(ptr); } - static T* as_pointer(const void *src) { return static_cast(const_cast(src)); } - - internal_loop_guide(size_type ntrials, void *ptr) - : array(as_pointer(ptr)), n(ntrials), i(0) {} - void init() { for(; i < n; ++i) new( &array[i] ) T(); } - void init(const void *src) { for(; i < n; ++i) new( &array[i] ) T(*as_const_pointer(src)); } - void copy(const void *src) { for(; i < n; ++i) new( &array[i] ) T(as_const_pointer(src)[i]); } - void assign(const void *src) { for(; i < n; ++i) array[i] = as_const_pointer(src)[i]; } -#if __TBB_CPP11_RVALUE_REF_PRESENT - void move_assign(const void *src) { for(; i < n; ++i) array[i] = std::move(as_pointer(src)[i]); } - void move_construct(const void *src) { for(; i < n; ++i) new( &array[i] ) T( std::move(as_pointer(src)[i]) ); } -#endif -#if __TBB_MOVE_IF_NOEXCEPT_PRESENT - void move_construct_if_noexcept(const void *src) { for(; i < n; ++i) new( &array[i] ) T( std::move_if_noexcept(as_pointer(src)[i]) ); } -#endif //__TBB_MOVE_IF_NOEXCEPT_PRESENT - - //TODO: rename to construct_range - template void iterate(I &src) { for(; i < n; ++i, ++src) new( &array[i] ) T( *src ); } - ~internal_loop_guide() { - if(i < n) {// if an exception was raised, fill the rest of items with zeros - internal::handle_unconstructed_elements(array+i, n-i); - } - } - }; - - struct push_back_helper : internal::no_copy{ - struct element_construction_guard : internal::no_copy{ - pointer element; - - element_construction_guard(pointer an_element) : element (an_element){} - void dismiss(){ element = NULL; } - ~element_construction_guard(){ - if (element){ - internal::handle_unconstructed_elements(element, 1); - } - } - }; - - concurrent_vector & v; - size_type k; - element_construction_guard g; - - push_back_helper(concurrent_vector & vector) : - v(vector), - g (static_cast(v.internal_push_back(sizeof(T),k))) - {} - - pointer internal_push_back_result(){ return g.element;} - iterator return_iterator_and_dismiss(){ - pointer ptr = g.element; - g.dismiss(); - return iterator(v, k, ptr); - } - }; -}; - -#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) -#pragma warning (push) -#pragma warning (disable: 4701) // potentially uninitialized local variable "old" -#endif -template -void concurrent_vector::shrink_to_fit() { - internal_segments_table old; - __TBB_TRY { - internal_array_op2 copy_or_move_array = -#if __TBB_MOVE_IF_NOEXCEPT_PRESENT - &move_array_if_noexcept -#else - ©_array -#endif - ; - if( internal_compact( sizeof(T), &old, &destroy_array, copy_or_move_array ) ) - internal_free_segments( old.table, pointers_per_long_table, old.first_block ); // free joined and unnecessary segments - } __TBB_CATCH(...) { - if( old.first_block ) // free segment allocated for compacting. Only for support of exceptions in ctor of user T[ype] - internal_free_segments( old.table, 1, old.first_block ); - __TBB_RETHROW(); - } -} -#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) -#pragma warning (pop) -#endif // warning 4701 is back - -template -void concurrent_vector::internal_free_segments(segment_t table[], segment_index_t k, segment_index_t first_block) { - // Free the arrays - while( k > first_block ) { - --k; - segment_value_t segment_value = table[k].load(); - table[k].store(segment_not_used()); - if( segment_value == segment_allocated() ) // check for correct segment pointer - this->my_allocator.deallocate( (segment_value.pointer()), segment_size(k) ); - } - segment_value_t segment_value = table[0].load(); - if( segment_value == segment_allocated() ) { - __TBB_ASSERT( first_block > 0, NULL ); - while(k > 0) table[--k].store(segment_not_used()); - this->my_allocator.deallocate( (segment_value.pointer()), segment_size(first_block) ); - } -} - -template -T& concurrent_vector::internal_subscript( size_type index ) const { - //TODO: unify both versions of internal_subscript - __TBB_ASSERT( index < my_early_size, "index out of bounds" ); - size_type j = index; - segment_index_t k = segment_base_index_of( j ); - __TBB_ASSERT( my_segment.load() != my_storage || k < pointers_per_short_table, "index is being allocated" ); - //no need in load with acquire (load) since thread works in own space or gets - //the information about added elements via some form of external synchronization - //TODO: why not make a load of my_segment relaxed as well ? - //TODO: add an assertion that my_segment[k] is properly aligned to please ITT - segment_value_t segment_value = my_segment[k].template load(); - __TBB_ASSERT( segment_value != segment_allocation_failed(), "the instance is broken by bad allocation. Use at() instead" ); - __TBB_ASSERT( segment_value != segment_not_used(), "index is being allocated" ); - return (( segment_value.pointer()))[j]; -} - -template -T& concurrent_vector::internal_subscript_with_exceptions( size_type index ) const { - if( index >= my_early_size ) - internal::throw_exception(internal::eid_out_of_range); // throw std::out_of_range - size_type j = index; - segment_index_t k = segment_base_index_of( j ); - //TODO: refactor this condition into separate helper function, e.g. fits_into_small_table - if( my_segment.load() == my_storage && k >= pointers_per_short_table ) - internal::throw_exception(internal::eid_segment_range_error); // throw std::range_error - // no need in load with acquire (load) since thread works in own space or gets - //the information about added elements via some form of external synchronization - //TODO: why not make a load of my_segment relaxed as well ? - //TODO: add an assertion that my_segment[k] is properly aligned to please ITT - segment_value_t segment_value = my_segment[k].template load(); - enforce_segment_allocated(segment_value, internal::eid_index_range_error); - return (segment_value.pointer())[j]; -} - -template template -void concurrent_vector::internal_assign_iterators(I first, I last) { - __TBB_ASSERT(my_early_size == 0, NULL); - size_type n = std::distance(first, last); - if( !n ) return; - internal_reserve(n, sizeof(T), max_size()); - my_early_size = n; - segment_index_t k = 0; - //TODO: unify segment iteration code with concurrent_base_v3::helper - size_type sz = segment_size( my_first_block ); - while( sz < n ) { - internal_loop_guide loop(sz, my_segment[k].template load().template pointer()); - loop.iterate(first); - n -= sz; - if( !k ) k = my_first_block; - else { ++k; sz <<= 1; } - } - internal_loop_guide loop(n, my_segment[k].template load().template pointer()); - loop.iterate(first); -} - -template -void concurrent_vector::initialize_array( void* begin, const void *, size_type n ) { - internal_loop_guide loop(n, begin); loop.init(); -} - -template -void concurrent_vector::initialize_array_by( void* begin, const void *src, size_type n ) { - internal_loop_guide loop(n, begin); loop.init(src); -} - -template -void concurrent_vector::copy_array( void* dst, const void* src, size_type n ) { - internal_loop_guide loop(n, dst); loop.copy(src); -} - -#if __TBB_CPP11_RVALUE_REF_PRESENT -template -void concurrent_vector::move_array( void* dst, const void* src, size_type n ) { - internal_loop_guide loop(n, dst); loop.move_construct(src); -} -template -void concurrent_vector::move_assign_array( void* dst, const void* src, size_type n ) { - internal_loop_guide loop(n, dst); loop.move_assign(src); -} -#endif - -#if __TBB_MOVE_IF_NOEXCEPT_PRESENT -template -void concurrent_vector::move_array_if_noexcept( void* dst, const void* src, size_type n ) { - internal_loop_guide loop(n, dst); loop.move_construct_if_noexcept(src); -} -#endif //__TBB_MOVE_IF_NOEXCEPT_PRESENT - -template -template -void concurrent_vector::copy_range( void* dst, const void* p_type_erased_iterator, size_type n ){ - I & iterator ((*const_cast(static_cast(p_type_erased_iterator)))); - internal_loop_guide loop(n, dst); loop.iterate(iterator); -} - -template -void concurrent_vector::assign_array( void* dst, const void* src, size_type n ) { - internal_loop_guide loop(n, dst); loop.assign(src); -} - -#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) - // Workaround for overzealous compiler warning - #pragma warning (push) - #pragma warning (disable: 4189) -#endif -template -void concurrent_vector::destroy_array( void* begin, size_type n ) { - T* array = static_cast(begin); - for( size_type j=n; j>0; --j ) - array[j-1].~T(); // destructors are supposed to not throw any exceptions -} -#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) - #pragma warning (pop) -#endif // warning 4189 is back - -// concurrent_vector's template functions -template -inline bool operator==(const concurrent_vector &a, const concurrent_vector &b) { - //TODO: call size() only once per vector (in operator==) - // Simply: return a.size() == b.size() && std::equal(a.begin(), a.end(), b.begin()); - if(a.size() != b.size()) return false; - typename concurrent_vector::const_iterator i(a.begin()); - typename concurrent_vector::const_iterator j(b.begin()); - for(; i != a.end(); ++i, ++j) - if( !(*i == *j) ) return false; - return true; -} - -template -inline bool operator!=(const concurrent_vector &a, const concurrent_vector &b) -{ return !(a == b); } - -template -inline bool operator<(const concurrent_vector &a, const concurrent_vector &b) -{ return (std::lexicographical_compare(a.begin(), a.end(), b.begin(), b.end())); } - -template -inline bool operator>(const concurrent_vector &a, const concurrent_vector &b) -{ return b < a; } - -template -inline bool operator<=(const concurrent_vector &a, const concurrent_vector &b) -{ return !(b < a); } - -template -inline bool operator>=(const concurrent_vector &a, const concurrent_vector &b) -{ return !(a < b); } - -template -inline void swap(concurrent_vector &a, concurrent_vector &b) -{ a.swap( b ); } - -} // namespace tbb - -#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) - #pragma warning (pop) -#endif // warning 4267,4127 are back - -#endif /* __TBB_concurrent_vector_H */ diff --git a/inst/include/tbb/critical_section.h b/inst/include/tbb/critical_section.h deleted file mode 100644 index 324b3e138..000000000 --- a/inst/include/tbb/critical_section.h +++ /dev/null @@ -1,133 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef _TBB_CRITICAL_SECTION_H_ -#define _TBB_CRITICAL_SECTION_H_ - -#if _WIN32||_WIN64 -#include "machine/windows_api.h" -#else -#include -#include -#endif // _WIN32||WIN64 - -#include "tbb_stddef.h" -#include "tbb_thread.h" -#include "tbb_exception.h" - -#include "tbb_profiling.h" - -namespace tbb { - - namespace internal { -class critical_section_v4 : internal::no_copy { -#if _WIN32||_WIN64 - CRITICAL_SECTION my_impl; -#else - pthread_mutex_t my_impl; -#endif - tbb_thread::id my_tid; -public: - - void __TBB_EXPORTED_METHOD internal_construct(); - - critical_section_v4() { -#if _WIN32||_WIN64 - InitializeCriticalSectionEx( &my_impl, 4000, 0 ); -#else - pthread_mutex_init(&my_impl, NULL); -#endif - internal_construct(); - } - - ~critical_section_v4() { - __TBB_ASSERT(my_tid == tbb_thread::id(), "Destroying a still-held critical section"); -#if _WIN32||_WIN64 - DeleteCriticalSection(&my_impl); -#else - pthread_mutex_destroy(&my_impl); -#endif - } - - class scoped_lock : internal::no_copy { - private: - critical_section_v4 &my_crit; - public: - scoped_lock( critical_section_v4& lock_me) :my_crit(lock_me) { - my_crit.lock(); - } - - ~scoped_lock() { - my_crit.unlock(); - } - }; - - void lock() { - tbb_thread::id local_tid = this_tbb_thread::get_id(); - if(local_tid == my_tid) throw_exception( eid_improper_lock ); -#if _WIN32||_WIN64 - EnterCriticalSection( &my_impl ); -#else - int rval = pthread_mutex_lock(&my_impl); - __TBB_ASSERT_EX(!rval, "critical_section::lock: pthread_mutex_lock failed"); -#endif - __TBB_ASSERT(my_tid == tbb_thread::id(), NULL); - my_tid = local_tid; - } - - bool try_lock() { - bool gotlock; - tbb_thread::id local_tid = this_tbb_thread::get_id(); - if(local_tid == my_tid) return false; -#if _WIN32||_WIN64 - gotlock = TryEnterCriticalSection( &my_impl ) != 0; -#else - int rval = pthread_mutex_trylock(&my_impl); - // valid returns are 0 (locked) and [EBUSY] - __TBB_ASSERT(rval == 0 || rval == EBUSY, "critical_section::trylock: pthread_mutex_trylock failed"); - gotlock = rval == 0; -#endif - if(gotlock) { - my_tid = local_tid; - } - return gotlock; - } - - void unlock() { - __TBB_ASSERT(this_tbb_thread::get_id() == my_tid, "thread unlocking critical_section is not thread that locked it"); - my_tid = tbb_thread::id(); -#if _WIN32||_WIN64 - LeaveCriticalSection( &my_impl ); -#else - int rval = pthread_mutex_unlock(&my_impl); - __TBB_ASSERT_EX(!rval, "critical_section::unlock: pthread_mutex_unlock failed"); -#endif - } - - static const bool is_rw_mutex = false; - static const bool is_recursive_mutex = false; - static const bool is_fair_mutex = true; -}; // critical_section_v4 -} // namespace internal -typedef internal::critical_section_v4 critical_section; - -__TBB_DEFINE_PROFILING_SET_NAME(critical_section) -} // namespace tbb -#endif // _TBB_CRITICAL_SECTION_H_ diff --git a/inst/include/tbb/enumerable_thread_specific.h b/inst/include/tbb/enumerable_thread_specific.h deleted file mode 100644 index f31f533b9..000000000 --- a/inst/include/tbb/enumerable_thread_specific.h +++ /dev/null @@ -1,1137 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_enumerable_thread_specific_H -#define __TBB_enumerable_thread_specific_H - -#include "atomic.h" -#include "concurrent_vector.h" -#include "tbb_thread.h" -#include "tbb_allocator.h" -#include "cache_aligned_allocator.h" -#include "aligned_space.h" -#include "internal/_template_helpers.h" -#include "internal/_tbb_hash_compare_impl.h" -#include "tbb_profiling.h" -#include // for memcpy - -#if _WIN32||_WIN64 -#include "machine/windows_api.h" -#else -#include -#endif - -#define __TBB_ETS_USE_CPP11 \ - (__TBB_CPP11_RVALUE_REF_PRESENT && __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT \ - && __TBB_CPP11_DECLTYPE_PRESENT && __TBB_CPP11_LAMBDAS_PRESENT) - -namespace tbb { - -//! enum for selecting between single key and key-per-instance versions -enum ets_key_usage_type { ets_key_per_instance, ets_no_key }; - -namespace interface6 { - - // Forward declaration to use in internal classes - template - class enumerable_thread_specific; - - //! @cond - namespace internal { - - using namespace tbb::internal; - - template - class ets_base: tbb::internal::no_copy { - protected: - typedef tbb_thread::id key_type; -#if __TBB_PROTECTED_NESTED_CLASS_BROKEN - public: -#endif - struct slot; - - struct array { - array* next; - size_t lg_size; - slot& at( size_t k ) { - return ((slot*)(void*)(this+1))[k]; - } - size_t size() const {return size_t(1)<>(8*sizeof(size_t)-lg_size); - } - }; - struct slot { - key_type key; - void* ptr; - bool empty() const {return key == key_type();} - bool match( key_type k ) const {return key == k;} - bool claim( key_type k ) { - // TODO: maybe claim ptr, because key_type is not guaranteed to fit into word size - return atomic_compare_and_swap(key, k, key_type()) == key_type(); - } - }; -#if __TBB_PROTECTED_NESTED_CLASS_BROKEN - protected: -#endif - - //! Root of linked list of arrays of decreasing size. - /** NULL if and only if my_count==0. - Each array in the list is half the size of its predecessor. */ - atomic my_root; - atomic my_count; - virtual void* create_local() = 0; - virtual void* create_array(size_t _size) = 0; // _size in bytes - virtual void free_array(void* ptr, size_t _size) = 0; // _size in bytes - array* allocate( size_t lg_size ) { - size_t n = size_t(1)<(create_array( sizeof(array)+n*sizeof(slot) )); - a->lg_size = lg_size; - std::memset( a+1, 0, n*sizeof(slot) ); - return a; - } - void free(array* a) { - size_t n = size_t(1)<<(a->lg_size); - free_array( (void *)a, size_t(sizeof(array)+n*sizeof(slot)) ); - } - - ets_base() {my_root=NULL; my_count=0;} - virtual ~ets_base(); // g++ complains if this is not virtual - void* table_lookup( bool& exists ); - void table_clear(); - // The following functions are not used in concurrent context, - // so we don't need synchronization and ITT annotations there. - void table_elementwise_copy( const ets_base& other, - void*(*add_element)(ets_base&, void*) ) { - __TBB_ASSERT(!my_root,NULL); - __TBB_ASSERT(!my_count,NULL); - if( !other.my_root ) return; - array* root = my_root = allocate(other.my_root->lg_size); - root->next = NULL; - my_count = other.my_count; - size_t mask = root->mask(); - for( array* r=other.my_root; r; r=r->next ) { - for( size_t i=0; isize(); ++i ) { - slot& s1 = r->at(i); - if( !s1.empty() ) { - for( size_t j = root->start(tbb::tbb_hash()(s1.key)); ; j=(j+1)&mask ) { - slot& s2 = root->at(j); - if( s2.empty() ) { - s2.ptr = add_element(*this, s1.ptr); - s2.key = s1.key; - break; - } - else if( s2.match(s1.key) ) - break; - } - } - } - } - } - void table_swap( ets_base& other ) { - __TBB_ASSERT(this!=&other, "Don't swap an instance with itself"); - tbb::internal::swap(my_root, other.my_root); - tbb::internal::swap(my_count, other.my_count); - } - }; - - template - ets_base::~ets_base() { - __TBB_ASSERT(!my_root, NULL); - } - - template - void ets_base::table_clear() { - while( array* r = my_root ) { - my_root = r->next; - free(r); - } - my_count = 0; - } - - template - void* ets_base::table_lookup( bool& exists ) { - const key_type k = tbb::this_tbb_thread::get_id(); - - __TBB_ASSERT(k != key_type(),NULL); - void* found; - size_t h = tbb::tbb_hash()(k); - for( array* r=my_root; r; r=r->next ) { - call_itt_notify(acquired,r); - size_t mask=r->mask(); - for(size_t i = r->start(h); ;i=(i+1)&mask) { - slot& s = r->at(i); - if( s.empty() ) break; - if( s.match(k) ) { - if( r==my_root ) { - // Success at top level - exists = true; - return s.ptr; - } else { - // Success at some other level. Need to insert at top level. - exists = true; - found = s.ptr; - goto insert; - } - } - } - } - // Key does not yet exist. The density of slots in the table does not exceed 0.5, - // for if this will occur a new table is allocated with double the current table - // size, which is swapped in as the new root table. So an empty slot is guaranteed. - exists = false; - found = create_local(); - { - size_t c = ++my_count; - array* r = my_root; - call_itt_notify(acquired,r); - if( !r || c>r->size()/2 ) { - size_t s = r ? r->lg_size : 2; - while( c>size_t(1)<<(s-1) ) ++s; - array* a = allocate(s); - for(;;) { - a->next = r; - call_itt_notify(releasing,a); - array* new_r = my_root.compare_and_swap(a,r); - if( new_r==r ) break; - call_itt_notify(acquired, new_r); - if( new_r->lg_size>=s ) { - // Another thread inserted an equal or bigger array, so our array is superfluous. - free(a); - break; - } - r = new_r; - } - } - } - insert: - // Whether a slot has been found in an older table, or if it has been inserted at this level, - // it has already been accounted for in the total. Guaranteed to be room for it, and it is - // not present, so search for empty slot and use it. - array* ir = my_root; - call_itt_notify(acquired, ir); - size_t mask = ir->mask(); - for(size_t i = ir->start(h);;i=(i+1)&mask) { - slot& s = ir->at(i); - if( s.empty() ) { - if( s.claim(k) ) { - s.ptr = found; - return found; - } - } - } - } - - //! Specialization that exploits native TLS - template <> - class ets_base: protected ets_base { - typedef ets_base super; -#if _WIN32||_WIN64 -#if __TBB_WIN8UI_SUPPORT - typedef DWORD tls_key_t; - void create_key() { my_key = FlsAlloc(NULL); } - void destroy_key() { FlsFree(my_key); } - void set_tls(void * value) { FlsSetValue(my_key, (LPVOID)value); } - void* get_tls() { return (void *)FlsGetValue(my_key); } -#else - typedef DWORD tls_key_t; - void create_key() { my_key = TlsAlloc(); } - void destroy_key() { TlsFree(my_key); } - void set_tls(void * value) { TlsSetValue(my_key, (LPVOID)value); } - void* get_tls() { return (void *)TlsGetValue(my_key); } -#endif -#else - typedef pthread_key_t tls_key_t; - void create_key() { pthread_key_create(&my_key, NULL); } - void destroy_key() { pthread_key_delete(my_key); } - void set_tls( void * value ) const { pthread_setspecific(my_key, value); } - void* get_tls() const { return pthread_getspecific(my_key); } -#endif - tls_key_t my_key; - virtual void* create_local() __TBB_override = 0; - virtual void* create_array(size_t _size) __TBB_override = 0; // _size in bytes - virtual void free_array(void* ptr, size_t _size) __TBB_override = 0; // size in bytes - protected: - ets_base() {create_key();} - ~ets_base() {destroy_key();} - void* table_lookup( bool& exists ) { - void* found = get_tls(); - if( found ) { - exists=true; - } else { - found = super::table_lookup(exists); - set_tls(found); - } - return found; - } - void table_clear() { - destroy_key(); - create_key(); - super::table_clear(); - } - void table_swap( ets_base& other ) { - using std::swap; - __TBB_ASSERT(this!=&other, "Don't swap an instance with itself"); - swap(my_key, other.my_key); - super::table_swap(other); - } - }; - - //! Random access iterator for traversing the thread local copies. - template< typename Container, typename Value > - class enumerable_thread_specific_iterator -#if defined(_WIN64) && defined(_MSC_VER) - // Ensure that Microsoft's internal template function _Val_type works correctly. - : public std::iterator -#endif /* defined(_WIN64) && defined(_MSC_VER) */ - { - //! current position in the concurrent_vector - - Container *my_container; - typename Container::size_type my_index; - mutable Value *my_value; - - template - friend enumerable_thread_specific_iterator - operator+( ptrdiff_t offset, const enumerable_thread_specific_iterator& v ); - - template - friend bool operator==( const enumerable_thread_specific_iterator& i, - const enumerable_thread_specific_iterator& j ); - - template - friend bool operator<( const enumerable_thread_specific_iterator& i, - const enumerable_thread_specific_iterator& j ); - - template - friend ptrdiff_t operator-( const enumerable_thread_specific_iterator& i, - const enumerable_thread_specific_iterator& j ); - - template - friend class enumerable_thread_specific_iterator; - - public: - - enumerable_thread_specific_iterator( const Container &container, typename Container::size_type index ) : - my_container(&const_cast(container)), my_index(index), my_value(NULL) {} - - //! Default constructor - enumerable_thread_specific_iterator() : my_container(NULL), my_index(0), my_value(NULL) {} - - template - enumerable_thread_specific_iterator( const enumerable_thread_specific_iterator& other ) : - my_container( other.my_container ), my_index( other.my_index), my_value( const_cast(other.my_value) ) {} - - enumerable_thread_specific_iterator operator+( ptrdiff_t offset ) const { - return enumerable_thread_specific_iterator(*my_container, my_index + offset); - } - - enumerable_thread_specific_iterator &operator+=( ptrdiff_t offset ) { - my_index += offset; - my_value = NULL; - return *this; - } - - enumerable_thread_specific_iterator operator-( ptrdiff_t offset ) const { - return enumerable_thread_specific_iterator( *my_container, my_index-offset ); - } - - enumerable_thread_specific_iterator &operator-=( ptrdiff_t offset ) { - my_index -= offset; - my_value = NULL; - return *this; - } - - Value& operator*() const { - Value* value = my_value; - if( !value ) { - value = my_value = (*my_container)[my_index].value(); - } - __TBB_ASSERT( value==(*my_container)[my_index].value(), "corrupt cache" ); - return *value; - } - - Value& operator[]( ptrdiff_t k ) const { - return (*my_container)[my_index + k].value; - } - - Value* operator->() const {return &operator*();} - - enumerable_thread_specific_iterator& operator++() { - ++my_index; - my_value = NULL; - return *this; - } - - enumerable_thread_specific_iterator& operator--() { - --my_index; - my_value = NULL; - return *this; - } - - //! Post increment - enumerable_thread_specific_iterator operator++(int) { - enumerable_thread_specific_iterator result = *this; - ++my_index; - my_value = NULL; - return result; - } - - //! Post decrement - enumerable_thread_specific_iterator operator--(int) { - enumerable_thread_specific_iterator result = *this; - --my_index; - my_value = NULL; - return result; - } - - // STL support - typedef ptrdiff_t difference_type; - typedef Value value_type; - typedef Value* pointer; - typedef Value& reference; - typedef std::random_access_iterator_tag iterator_category; - }; - - template - enumerable_thread_specific_iterator - operator+( ptrdiff_t offset, const enumerable_thread_specific_iterator& v ) { - return enumerable_thread_specific_iterator( v.my_container, v.my_index + offset ); - } - - template - bool operator==( const enumerable_thread_specific_iterator& i, - const enumerable_thread_specific_iterator& j ) { - return i.my_index==j.my_index && i.my_container == j.my_container; - } - - template - bool operator!=( const enumerable_thread_specific_iterator& i, - const enumerable_thread_specific_iterator& j ) { - return !(i==j); - } - - template - bool operator<( const enumerable_thread_specific_iterator& i, - const enumerable_thread_specific_iterator& j ) { - return i.my_index - bool operator>( const enumerable_thread_specific_iterator& i, - const enumerable_thread_specific_iterator& j ) { - return j - bool operator>=( const enumerable_thread_specific_iterator& i, - const enumerable_thread_specific_iterator& j ) { - return !(i - bool operator<=( const enumerable_thread_specific_iterator& i, - const enumerable_thread_specific_iterator& j ) { - return !(j - ptrdiff_t operator-( const enumerable_thread_specific_iterator& i, - const enumerable_thread_specific_iterator& j ) { - return i.my_index-j.my_index; - } - - template - class segmented_iterator -#if defined(_WIN64) && defined(_MSC_VER) - : public std::iterator -#endif - { - template - friend bool operator==(const segmented_iterator& i, const segmented_iterator& j); - - template - friend bool operator!=(const segmented_iterator& i, const segmented_iterator& j); - - template - friend class segmented_iterator; - - public: - - segmented_iterator() {my_segcont = NULL;} - - segmented_iterator( const SegmentedContainer& _segmented_container ) : - my_segcont(const_cast(&_segmented_container)), - outer_iter(my_segcont->end()) { } - - ~segmented_iterator() {} - - typedef typename SegmentedContainer::iterator outer_iterator; - typedef typename SegmentedContainer::value_type InnerContainer; - typedef typename InnerContainer::iterator inner_iterator; - - // STL support - typedef ptrdiff_t difference_type; - typedef Value value_type; - typedef typename SegmentedContainer::size_type size_type; - typedef Value* pointer; - typedef Value& reference; - typedef std::input_iterator_tag iterator_category; - - // Copy Constructor - template - segmented_iterator(const segmented_iterator& other) : - my_segcont(other.my_segcont), - outer_iter(other.outer_iter), - // can we assign a default-constructed iterator to inner if we're at the end? - inner_iter(other.inner_iter) - {} - - // assignment - template - segmented_iterator& operator=( const segmented_iterator& other) { - if(this != &other) { - my_segcont = other.my_segcont; - outer_iter = other.outer_iter; - if(outer_iter != my_segcont->end()) inner_iter = other.inner_iter; - } - return *this; - } - - // allow assignment of outer iterator to segmented iterator. Once it is - // assigned, move forward until a non-empty inner container is found or - // the end of the outer container is reached. - segmented_iterator& operator=(const outer_iterator& new_outer_iter) { - __TBB_ASSERT(my_segcont != NULL, NULL); - // check that this iterator points to something inside the segmented container - for(outer_iter = new_outer_iter ;outer_iter!=my_segcont->end(); ++outer_iter) { - if( !outer_iter->empty() ) { - inner_iter = outer_iter->begin(); - break; - } - } - return *this; - } - - // pre-increment - segmented_iterator& operator++() { - advance_me(); - return *this; - } - - // post-increment - segmented_iterator operator++(int) { - segmented_iterator tmp = *this; - operator++(); - return tmp; - } - - bool operator==(const outer_iterator& other_outer) const { - __TBB_ASSERT(my_segcont != NULL, NULL); - return (outer_iter == other_outer && - (outer_iter == my_segcont->end() || inner_iter == outer_iter->begin())); - } - - bool operator!=(const outer_iterator& other_outer) const { - return !operator==(other_outer); - - } - - // (i)* RHS - reference operator*() const { - __TBB_ASSERT(my_segcont != NULL, NULL); - __TBB_ASSERT(outer_iter != my_segcont->end(), "Dereferencing a pointer at end of container"); - __TBB_ASSERT(inner_iter != outer_iter->end(), NULL); // should never happen - return *inner_iter; - } - - // i-> - pointer operator->() const { return &operator*();} - - private: - SegmentedContainer* my_segcont; - outer_iterator outer_iter; - inner_iterator inner_iter; - - void advance_me() { - __TBB_ASSERT(my_segcont != NULL, NULL); - __TBB_ASSERT(outer_iter != my_segcont->end(), NULL); // not true if there are no inner containers - __TBB_ASSERT(inner_iter != outer_iter->end(), NULL); // not true if the inner containers are all empty. - ++inner_iter; - while(inner_iter == outer_iter->end() && ++outer_iter != my_segcont->end()) { - inner_iter = outer_iter->begin(); - } - } - }; // segmented_iterator - - template - bool operator==( const segmented_iterator& i, - const segmented_iterator& j ) { - if(i.my_segcont != j.my_segcont) return false; - if(i.my_segcont == NULL) return true; - if(i.outer_iter != j.outer_iter) return false; - if(i.outer_iter == i.my_segcont->end()) return true; - return i.inner_iter == j.inner_iter; - } - - // != - template - bool operator!=( const segmented_iterator& i, - const segmented_iterator& j ) { - return !(i==j); - } - - template - struct construct_by_default: tbb::internal::no_assign { - void construct(void*where) {new(where) T();} // C++ note: the () in T() ensure zero initialization. - construct_by_default( int ) {} - }; - - template - struct construct_by_exemplar: tbb::internal::no_assign { - const T exemplar; - void construct(void*where) {new(where) T(exemplar);} - construct_by_exemplar( const T& t ) : exemplar(t) {} -#if __TBB_ETS_USE_CPP11 - construct_by_exemplar( T&& t ) : exemplar(std::move(t)) {} -#endif - }; - - template - struct construct_by_finit: tbb::internal::no_assign { - Finit f; - void construct(void* where) {new(where) T(f());} - construct_by_finit( const Finit& f_ ) : f(f_) {} -#if __TBB_ETS_USE_CPP11 - construct_by_finit( Finit&& f_ ) : f(std::move(f_)) {} -#endif - }; - -#if __TBB_ETS_USE_CPP11 - template - struct construct_by_args: tbb::internal::no_assign { - internal::stored_pack pack; - void construct(void* where) { - internal::call( [where](const typename strip

::type&... args ){ - new(where) T(args...); - }, pack ); - } - construct_by_args( P&& ... args ) : pack(std::forward

(args)...) {} - }; -#endif - - // storage for initialization function pointer - // TODO: consider removing the template parameter T here and in callback_leaf - template - class callback_base { - public: - // Clone *this - virtual callback_base* clone() const = 0; - // Destruct and free *this - virtual void destroy() = 0; - // Need virtual destructor to satisfy GCC compiler warning - virtual ~callback_base() { } - // Construct T at where - virtual void construct(void* where) = 0; - }; - - template - class callback_leaf: public callback_base, Constructor { -#if __TBB_ETS_USE_CPP11 - template callback_leaf( P&& ... params ) : Constructor(std::forward

(params)...) {} -#else - template callback_leaf( const X& x ) : Constructor(x) {} -#endif - // TODO: make the construction/destruction consistent (use allocator.construct/destroy) - typedef typename tbb::tbb_allocator my_allocator_type; - - callback_base* clone() const __TBB_override { - return make(*this); - } - - void destroy() __TBB_override { - my_allocator_type().destroy(this); - my_allocator_type().deallocate(this,1); - } - - void construct(void* where) __TBB_override { - Constructor::construct(where); - } - public: -#if __TBB_ETS_USE_CPP11 - template - static callback_base* make( P&& ... params ) { - void* where = my_allocator_type().allocate(1); - return new(where) callback_leaf( std::forward

(params)... ); - } -#else - template - static callback_base* make( const X& x ) { - void* where = my_allocator_type().allocate(1); - return new(where) callback_leaf(x); - } -#endif - }; - - //! Template for recording construction of objects in table - /** All maintenance of the space will be done explicitly on push_back, - and all thread local copies must be destroyed before the concurrent - vector is deleted. - - The flag is_built is initialized to false. When the local is - successfully-constructed, set the flag to true or call value_committed(). - If the constructor throws, the flag will be false. - */ - template - struct ets_element { - tbb::aligned_space my_space; - bool is_built; - ets_element() { is_built = false; } // not currently-built - U* value() { return my_space.begin(); } - U* value_committed() { is_built = true; return my_space.begin(); } - ~ets_element() { - if(is_built) { - my_space.begin()->~U(); - is_built = false; - } - } - }; - - // A predicate that can be used for a compile-time compatibility check of ETS instances - // Ideally, it should have been declared inside the ETS class, but unfortunately - // in that case VS2013 does not enable the variadic constructor. - template struct is_compatible_ets { static const bool value = false; }; - template - struct is_compatible_ets< T, enumerable_thread_specific > { static const bool value = internal::is_same_type::value; }; - -#if __TBB_ETS_USE_CPP11 - // A predicate that checks whether, for a variable 'foo' of type T, foo() is a valid expression - template - class is_callable_no_args { - private: - typedef char yes[1]; - typedef char no [2]; - - template static yes& decide( decltype(declval()())* ); - template static no& decide(...); - public: - static const bool value = (sizeof(decide(NULL)) == sizeof(yes)); - }; -#endif - - } // namespace internal - //! @endcond - - //! The enumerable_thread_specific container - /** enumerable_thread_specific has the following properties: - - thread-local copies are lazily created, with default, exemplar or function initialization. - - thread-local copies do not move (during lifetime, and excepting clear()) so the address of a copy is invariant. - - the contained objects need not have operator=() defined if combine is not used. - - enumerable_thread_specific containers may be copy-constructed or assigned. - - thread-local copies can be managed by hash-table, or can be accessed via TLS storage for speed. - - outside of parallel contexts, the contents of all thread-local copies are accessible by iterator or using combine or combine_each methods - - @par Segmented iterator - When the thread-local objects are containers with input_iterators defined, a segmented iterator may - be used to iterate over all the elements of all thread-local copies. - - @par combine and combine_each - - Both methods are defined for enumerable_thread_specific. - - combine() requires the type T have operator=() defined. - - neither method modifies the contents of the object (though there is no guarantee that the applied methods do not modify the object.) - - Both are evaluated in serial context (the methods are assumed to be non-benign.) - - @ingroup containers */ - template , - ets_key_usage_type ETS_key_type=ets_no_key > - class enumerable_thread_specific: internal::ets_base { - - template friend class enumerable_thread_specific; - - typedef internal::padded< internal::ets_element > padded_element; - - //! A generic range, used to create range objects from the iterators - template - class generic_range_type: public blocked_range { - public: - typedef T value_type; - typedef T& reference; - typedef const T& const_reference; - typedef I iterator; - typedef ptrdiff_t difference_type; - generic_range_type( I begin_, I end_, size_t grainsize_ = 1) : blocked_range(begin_,end_,grainsize_) {} - template - generic_range_type( const generic_range_type& r) : blocked_range(r.begin(),r.end(),r.grainsize()) {} - generic_range_type( generic_range_type& r, split ) : blocked_range(r,split()) {} - }; - - typedef typename Allocator::template rebind< padded_element >::other padded_allocator_type; - typedef tbb::concurrent_vector< padded_element, padded_allocator_type > internal_collection_type; - - internal::callback_base *my_construct_callback; - - internal_collection_type my_locals; - - // TODO: consider unifying the callback mechanism for all create_local* methods below - // (likely non-compatible and requires interface version increase) - void* create_local() __TBB_override { - padded_element& lref = *my_locals.grow_by(1); - my_construct_callback->construct(lref.value()); - return lref.value_committed(); - } - - static void* create_local_by_copy( internal::ets_base& base, void* p ) { - enumerable_thread_specific& ets = static_cast(base); - padded_element& lref = *ets.my_locals.grow_by(1); - new(lref.value()) T(*static_cast(p)); - return lref.value_committed(); - } - -#if __TBB_ETS_USE_CPP11 - static void* create_local_by_move( internal::ets_base& base, void* p ) { - enumerable_thread_specific& ets = static_cast(base); - padded_element& lref = *ets.my_locals.grow_by(1); - new(lref.value()) T(std::move(*static_cast(p))); - return lref.value_committed(); - } -#endif - - typedef typename Allocator::template rebind< uintptr_t >::other array_allocator_type; - - // _size is in bytes - void* create_array(size_t _size) __TBB_override { - size_t nelements = (_size + sizeof(uintptr_t) -1) / sizeof(uintptr_t); - return array_allocator_type().allocate(nelements); - } - - void free_array( void* _ptr, size_t _size) __TBB_override { - size_t nelements = (_size + sizeof(uintptr_t) -1) / sizeof(uintptr_t); - array_allocator_type().deallocate( reinterpret_cast(_ptr),nelements); - } - - public: - - //! Basic types - typedef Allocator allocator_type; - typedef T value_type; - typedef T& reference; - typedef const T& const_reference; - typedef T* pointer; - typedef const T* const_pointer; - typedef typename internal_collection_type::size_type size_type; - typedef typename internal_collection_type::difference_type difference_type; - - // Iterator types - typedef typename internal::enumerable_thread_specific_iterator< internal_collection_type, value_type > iterator; - typedef typename internal::enumerable_thread_specific_iterator< internal_collection_type, const value_type > const_iterator; - - // Parallel range types - typedef generic_range_type< iterator > range_type; - typedef generic_range_type< const_iterator > const_range_type; - - //! Default constructor. Each local instance of T is default constructed. - enumerable_thread_specific() : my_construct_callback( - internal::callback_leaf >::make(/*dummy argument*/0) - ){} - - //! Constructor with initializer functor. Each local instance of T is constructed by T(finit()). - template ::type>::value>::type -#endif - > - explicit enumerable_thread_specific( Finit finit ) : my_construct_callback( - internal::callback_leaf >::make( tbb::internal::move(finit) ) - ){} - - //! Constructor with exemplar. Each local instance of T is copy-constructed from the exemplar. - explicit enumerable_thread_specific( const T& exemplar ) : my_construct_callback( - internal::callback_leaf >::make( exemplar ) - ){} - -#if __TBB_ETS_USE_CPP11 - explicit enumerable_thread_specific( T&& exemplar ) : my_construct_callback( - internal::callback_leaf >::make( std::move(exemplar) ) - ){} - - //! Variadic constructor with initializer arguments. Each local instance of T is constructed by T(args...) - template ::type>::value - && !internal::is_compatible_ets::type>::value - && !internal::is_same_type::type>::value - >::type> - enumerable_thread_specific( P1&& arg1, P&& ... args ) : my_construct_callback( - internal::callback_leaf >::make( std::forward(arg1), std::forward

(args)... ) - ){} -#endif - - //! Destructor - ~enumerable_thread_specific() { - if(my_construct_callback) my_construct_callback->destroy(); - // Deallocate the hash table before overridden free_array() becomes inaccessible - this->internal::ets_base::table_clear(); - } - - //! returns reference to local, discarding exists - reference local() { - bool exists; - return local(exists); - } - - //! Returns reference to calling thread's local copy, creating one if necessary - reference local(bool& exists) { - void* ptr = this->table_lookup(exists); - return *(T*)ptr; - } - - //! Get the number of local copies - size_type size() const { return my_locals.size(); } - - //! true if there have been no local copies created - bool empty() const { return my_locals.empty(); } - - //! begin iterator - iterator begin() { return iterator( my_locals, 0 ); } - //! end iterator - iterator end() { return iterator(my_locals, my_locals.size() ); } - - //! begin const iterator - const_iterator begin() const { return const_iterator(my_locals, 0); } - - //! end const iterator - const_iterator end() const { return const_iterator(my_locals, my_locals.size()); } - - //! Get range for parallel algorithms - range_type range( size_t grainsize=1 ) { return range_type( begin(), end(), grainsize ); } - - //! Get const range for parallel algorithms - const_range_type range( size_t grainsize=1 ) const { return const_range_type( begin(), end(), grainsize ); } - - //! Destroys local copies - void clear() { - my_locals.clear(); - this->table_clear(); - // callback is not destroyed - } - - private: - - template - void internal_copy(const enumerable_thread_specific& other) { -#if __TBB_ETS_USE_CPP11 && TBB_USE_ASSERT - // this tests is_compatible_ets - __TBB_STATIC_ASSERT( (internal::is_compatible_ets::type>::value), "is_compatible_ets fails" ); -#endif - // Initialize my_construct_callback first, so that it is valid even if rest of this routine throws an exception. - my_construct_callback = other.my_construct_callback->clone(); - __TBB_ASSERT(my_locals.size()==0,NULL); - my_locals.reserve(other.size()); - this->table_elementwise_copy( other, create_local_by_copy ); - } - - void internal_swap(enumerable_thread_specific& other) { - using std::swap; - __TBB_ASSERT( this!=&other, NULL ); - swap(my_construct_callback, other.my_construct_callback); - // concurrent_vector::swap() preserves storage space, - // so addresses to the vector kept in ETS hash table remain valid. - swap(my_locals, other.my_locals); - this->internal::ets_base::table_swap(other); - } - -#if __TBB_ETS_USE_CPP11 - template - void internal_move(enumerable_thread_specific&& other) { -#if TBB_USE_ASSERT - // this tests is_compatible_ets - __TBB_STATIC_ASSERT( (internal::is_compatible_ets::type>::value), "is_compatible_ets fails" ); -#endif - my_construct_callback = other.my_construct_callback; - other.my_construct_callback = NULL; - __TBB_ASSERT(my_locals.size()==0,NULL); - my_locals.reserve(other.size()); - this->table_elementwise_copy( other, create_local_by_move ); - } -#endif - - public: - - enumerable_thread_specific( const enumerable_thread_specific& other ) - : internal::ets_base() /* prevents GCC warnings with -Wextra */ - { - internal_copy(other); - } - - template - enumerable_thread_specific( const enumerable_thread_specific& other ) - { - internal_copy(other); - } - -#if __TBB_ETS_USE_CPP11 - enumerable_thread_specific( enumerable_thread_specific&& other ) : my_construct_callback() - { - internal_swap(other); - } - - template - enumerable_thread_specific( enumerable_thread_specific&& other ) : my_construct_callback() - { - internal_move(std::move(other)); - } -#endif - - enumerable_thread_specific& operator=( const enumerable_thread_specific& other ) - { - if( this != &other ) { - this->clear(); - my_construct_callback->destroy(); - internal_copy( other ); - } - return *this; - } - - template - enumerable_thread_specific& operator=( const enumerable_thread_specific& other ) - { - __TBB_ASSERT( static_cast(this)!=static_cast(&other), NULL ); // Objects of different types - this->clear(); - my_construct_callback->destroy(); - internal_copy(other); - return *this; - } - -#if __TBB_ETS_USE_CPP11 - enumerable_thread_specific& operator=( enumerable_thread_specific&& other ) - { - if( this != &other ) - internal_swap(other); - return *this; - } - - template - enumerable_thread_specific& operator=( enumerable_thread_specific&& other ) - { - __TBB_ASSERT( static_cast(this)!=static_cast(&other), NULL ); // Objects of different types - this->clear(); - my_construct_callback->destroy(); - internal_move(std::move(other)); - return *this; - } -#endif - - // combine_func_t has signature T(T,T) or T(const T&, const T&) - template - T combine(combine_func_t f_combine) { - if(begin() == end()) { - internal::ets_element location; - my_construct_callback->construct(location.value()); - return *location.value_committed(); - } - const_iterator ci = begin(); - T my_result = *ci; - while(++ci != end()) - my_result = f_combine( my_result, *ci ); - return my_result; - } - - // combine_func_t takes T by value or by [const] reference, and returns nothing - template - void combine_each(combine_func_t f_combine) { - for(iterator ci = begin(); ci != end(); ++ci) { - f_combine( *ci ); - } - } - - }; // enumerable_thread_specific - - template< typename Container > - class flattened2d { - - // This intermediate typedef is to address issues with VC7.1 compilers - typedef typename Container::value_type conval_type; - - public: - - //! Basic types - typedef typename conval_type::size_type size_type; - typedef typename conval_type::difference_type difference_type; - typedef typename conval_type::allocator_type allocator_type; - typedef typename conval_type::value_type value_type; - typedef typename conval_type::reference reference; - typedef typename conval_type::const_reference const_reference; - typedef typename conval_type::pointer pointer; - typedef typename conval_type::const_pointer const_pointer; - - typedef typename internal::segmented_iterator iterator; - typedef typename internal::segmented_iterator const_iterator; - - flattened2d( const Container &c, typename Container::const_iterator b, typename Container::const_iterator e ) : - my_container(const_cast(&c)), my_begin(b), my_end(e) { } - - explicit flattened2d( const Container &c ) : - my_container(const_cast(&c)), my_begin(c.begin()), my_end(c.end()) { } - - iterator begin() { return iterator(*my_container) = my_begin; } - iterator end() { return iterator(*my_container) = my_end; } - const_iterator begin() const { return const_iterator(*my_container) = my_begin; } - const_iterator end() const { return const_iterator(*my_container) = my_end; } - - size_type size() const { - size_type tot_size = 0; - for(typename Container::const_iterator i = my_begin; i != my_end; ++i) { - tot_size += i->size(); - } - return tot_size; - } - - private: - - Container *my_container; - typename Container::const_iterator my_begin; - typename Container::const_iterator my_end; - - }; - - template - flattened2d flatten2d(const Container &c, const typename Container::const_iterator b, const typename Container::const_iterator e) { - return flattened2d(c, b, e); - } - - template - flattened2d flatten2d(const Container &c) { - return flattened2d(c); - } - -} // interface6 - -namespace internal { -using interface6::internal::segmented_iterator; -} - -using interface6::enumerable_thread_specific; -using interface6::flattened2d; -using interface6::flatten2d; - -} // namespace tbb - -#endif diff --git a/inst/include/tbb/flow_graph.h b/inst/include/tbb/flow_graph.h deleted file mode 100644 index 7b03c0e2e..000000000 --- a/inst/include/tbb/flow_graph.h +++ /dev/null @@ -1,4075 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_flow_graph_H -#define __TBB_flow_graph_H - -#include "tbb_stddef.h" -#include "atomic.h" -#include "spin_mutex.h" -#include "null_mutex.h" -#include "spin_rw_mutex.h" -#include "null_rw_mutex.h" -#include "task.h" -#include "cache_aligned_allocator.h" -#include "tbb_exception.h" -#include "internal/_template_helpers.h" -#include "internal/_aggregator_impl.h" -#include "tbb_profiling.h" -#include "task_arena.h" -#include "flow_graph_abstractions.h" - -#if __TBB_PREVIEW_ASYNC_MSG -#include // std::vector in internal::async_storage -#include // std::shared_ptr in async_msg -#endif - -#if __TBB_PREVIEW_STREAMING_NODE -// For streaming_node -#include // std::array -#include // std::unordered_map -#include // std::decay, std::true_type, std::false_type -#endif // __TBB_PREVIEW_STREAMING_NODE - -#if TBB_DEPRECATED_FLOW_ENQUEUE -#define FLOW_SPAWN(a) tbb::task::enqueue((a)) -#else -#define FLOW_SPAWN(a) tbb::task::spawn((a)) -#endif - -// use the VC10 or gcc version of tuple if it is available. -#if __TBB_CPP11_TUPLE_PRESENT - #include -namespace tbb { - namespace flow { - using std::tuple; - using std::tuple_size; - using std::tuple_element; - using std::get; - } -} -#else - #include "compat/tuple" -#endif - -#include -#include - -/** @file - \brief The graph related classes and functions - - There are some applications that best express dependencies as messages - passed between nodes in a graph. These messages may contain data or - simply act as signals that a predecessors has completed. The graph - class and its associated node classes can be used to express such - applications. -*/ - -namespace tbb { -namespace flow { - -//! An enumeration the provides the two most common concurrency levels: unlimited and serial -enum concurrency { unlimited = 0, serial = 1 }; - -namespace internal { -static tbb::task * const SUCCESSFULLY_ENQUEUED = (task *)-1; -} - -namespace interface9 { - -using tbb::flow::internal::SUCCESSFULLY_ENQUEUED; - -namespace internal { - template class successor_cache; - template class broadcast_cache; - template class round_robin_cache; - template class predecessor_cache; - template class reservable_predecessor_cache; -} - -//A generic null type -struct null_type {}; - -//! An empty class used for messages that mean "I'm done" -class continue_msg {}; - -template< typename T > class sender; -template< typename T > class receiver; -class continue_receiver; - -template< typename T > class limiter_node; // needed for resetting decrementer -template< typename R, typename B > class run_and_put_task; - -// flags to modify the behavior of the graph reset(). Can be combined. -enum reset_flags { - rf_reset_protocol = 0, - rf_reset_bodies = 1<<0, // delete the current node body, reset to a copy of the initial node body. - rf_clear_edges = 1<<1 // delete edges -}; - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES -//* holder of edges both for caches and for those nodes which do not have predecessor caches. -// C == receiver< ... > or sender< ... >, depending. -namespace internal { -template -class edge_container { - -public: - typedef std::list > edge_list_type; - - void add_edge( C &s) { - built_edges.push_back( &s ); - } - - void delete_edge( C &s) { - for ( typename edge_list_type::iterator i = built_edges.begin(); i != built_edges.end(); ++i ) { - if ( *i == &s ) { - (void)built_edges.erase(i); - return; // only remove one predecessor per request - } - } - } - - void copy_edges( edge_list_type &v) { - v = built_edges; - } - - size_t edge_count() { - return (size_t)(built_edges.size()); - } - - void clear() { - built_edges.clear(); - } - - // methods remove the statement from all predecessors/successors liste in the edge - // container. - template< typename S > void sender_extract( S &s ); - template< typename R > void receiver_extract( R &r ); - -private: - edge_list_type built_edges; -}; // class edge_container -} // namespace internal -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ - -#if __TBB_PREVIEW_ASYNC_MSG - -#include "internal/_flow_graph_async_msg_impl.h" - -namespace internal { - -class untyped_receiver; - -class untyped_sender { - template< typename, typename > friend class internal::predecessor_cache; - template< typename, typename > friend class internal::reservable_predecessor_cache; -public: - //! The successor type for this node - typedef untyped_receiver successor_type; - - virtual ~untyped_sender() {} - - // NOTE: Following part of PUBLIC section is copy-paste from original sender class - - // TODO: Prevent untyped successor registration - - //! Add a new successor to this node - virtual bool register_successor( successor_type &r ) = 0; - - //! Removes a successor from this node - virtual bool remove_successor( successor_type &r ) = 0; - - //! Releases the reserved item - virtual bool try_release( ) { return false; } - - //! Consumes the reserved item - virtual bool try_consume( ) { return false; } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - //! interface to record edges for traversal & deletion - typedef internal::edge_container built_successors_type; - typedef built_successors_type::edge_list_type successor_list_type; - virtual built_successors_type &built_successors() = 0; - virtual void internal_add_built_successor( successor_type & ) = 0; - virtual void internal_delete_built_successor( successor_type & ) = 0; - virtual void copy_successors( successor_list_type &) = 0; - virtual size_t successor_count() = 0; -#endif -protected: - //! Request an item from the sender - template< typename X > - bool try_get( X &t ) { - return try_get_wrapper( internal::async_helpers::to_void_ptr(t), internal::async_helpers::is_async_type ); - } - - //! Reserves an item in the sender - template< typename X > - bool try_reserve( X &t ) { - return try_reserve_wrapper( internal::async_helpers::to_void_ptr(t), internal::async_helpers::is_async_type ); - } - - virtual bool try_get_wrapper( void* p, bool is_async ) = 0; - virtual bool try_reserve_wrapper( void* p, bool is_async ) = 0; -}; - -class untyped_receiver { - template< typename, typename > friend class run_and_put_task; - template< typename > friend class limiter_node; - - template< typename, typename > friend class internal::broadcast_cache; - template< typename, typename > friend class internal::round_robin_cache; - template< typename, typename > friend class internal::successor_cache; - -#if __TBB_PREVIEW_OPENCL_NODE - template< typename, typename > friend class proxy_dependency_receiver; -#endif /* __TBB_PREVIEW_OPENCL_NODE */ -public: - //! The predecessor type for this node - typedef untyped_sender predecessor_type; - - //! Destructor - virtual ~untyped_receiver() {} - - //! Put an item to the receiver - template - bool try_put(const X& t) { - task *res = try_put_task(t); - if (!res) return false; - if (res != SUCCESSFULLY_ENQUEUED) FLOW_SPAWN(*res); - return true; - } - - // NOTE: Following part of PUBLIC section is copy-paste from original receiver class - - // TODO: Prevent untyped predecessor registration - - //! Add a predecessor to the node - virtual bool register_predecessor( predecessor_type & ) { return false; } - - //! Remove a predecessor from the node - virtual bool remove_predecessor( predecessor_type & ) { return false; } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - typedef internal::edge_container built_predecessors_type; - typedef built_predecessors_type::edge_list_type predecessor_list_type; - virtual built_predecessors_type &built_predecessors() = 0; - virtual void internal_add_built_predecessor( predecessor_type & ) = 0; - virtual void internal_delete_built_predecessor( predecessor_type & ) = 0; - virtual void copy_predecessors( predecessor_list_type & ) = 0; - virtual size_t predecessor_count() = 0; -#endif -protected: - template - task *try_put_task(const X& t) { - return try_put_task_wrapper( internal::async_helpers::to_void_ptr(t), internal::async_helpers::is_async_type ); - } - - virtual task* try_put_task_wrapper( const void* p, bool is_async ) = 0; - - // NOTE: Following part of PROTECTED and PRIVATE sections is copy-paste from original receiver class - - //! put receiver back in initial state - virtual void reset_receiver(reset_flags f = rf_reset_protocol) = 0; - - virtual bool is_continue_receiver() { return false; } -}; - -} // namespace internal - -//! Pure virtual template class that defines a sender of messages of type T -template< typename T > -class sender : public internal::untyped_sender { -public: - //! The output type of this sender - typedef T output_type; - - typedef typename internal::async_helpers::filtered_type filtered_type; - - //! Request an item from the sender - virtual bool try_get( T & ) { return false; } - - //! Reserves an item in the sender - virtual bool try_reserve( T & ) { return false; } - -protected: - virtual bool try_get_wrapper( void* p, bool is_async ) __TBB_override { - // Both async OR both are NOT async - if ( internal::async_helpers::is_async_type == is_async ) { - return try_get( internal::async_helpers::from_void_ptr(p) ); - } - // Else: this (T) is async OR incoming 't' is async - __TBB_ASSERT(false, "async_msg interface does not support 'pull' protocol in try_get()"); - return false; - } - - virtual bool try_reserve_wrapper( void* p, bool is_async ) __TBB_override { - // Both async OR both are NOT async - if ( internal::async_helpers::is_async_type == is_async ) { - return try_reserve( internal::async_helpers::from_void_ptr(p) ); - } - // Else: this (T) is async OR incoming 't' is async - __TBB_ASSERT(false, "async_msg interface does not support 'pull' protocol in try_reserve()"); - return false; - } -}; // class sender - -//! Pure virtual template class that defines a receiver of messages of type T -template< typename T > -class receiver : public internal::untyped_receiver { - template< typename > friend class internal::async_storage; - template< typename, typename > friend struct internal::async_helpers; -public: - //! The input type of this receiver - typedef T input_type; - - typedef typename internal::async_helpers::filtered_type filtered_type; - - //! Put an item to the receiver - bool try_put( const typename internal::async_helpers::filtered_type& t ) { - return internal::untyped_receiver::try_put(t); - } - - bool try_put( const typename internal::async_helpers::async_type& t ) { - return internal::untyped_receiver::try_put(t); - } - -protected: - virtual task* try_put_task_wrapper( const void *p, bool is_async ) __TBB_override { - return internal::async_helpers::try_put_task_wrapper_impl(this, p, is_async); - } - - //! Put item to successor; return task to run the successor if possible. - virtual task *try_put_task(const T& t) = 0; - -}; // class receiver - -#else // __TBB_PREVIEW_ASYNC_MSG - -//! Pure virtual template class that defines a sender of messages of type T -template< typename T > -class sender { -public: - //! The output type of this sender - typedef T output_type; - - //! The successor type for this node - typedef receiver successor_type; - - virtual ~sender() {} - - // NOTE: Following part of PUBLIC section is partly copy-pasted in sender under #if __TBB_PREVIEW_ASYNC_MSG - - //! Add a new successor to this node - virtual bool register_successor( successor_type &r ) = 0; - - //! Removes a successor from this node - virtual bool remove_successor( successor_type &r ) = 0; - - //! Request an item from the sender - virtual bool try_get( T & ) { return false; } - - //! Reserves an item in the sender - virtual bool try_reserve( T & ) { return false; } - - //! Releases the reserved item - virtual bool try_release( ) { return false; } - - //! Consumes the reserved item - virtual bool try_consume( ) { return false; } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - //! interface to record edges for traversal & deletion - typedef typename internal::edge_container built_successors_type; - typedef typename built_successors_type::edge_list_type successor_list_type; - virtual built_successors_type &built_successors() = 0; - virtual void internal_add_built_successor( successor_type & ) = 0; - virtual void internal_delete_built_successor( successor_type & ) = 0; - virtual void copy_successors( successor_list_type &) = 0; - virtual size_t successor_count() = 0; -#endif -}; // class sender - -//! Pure virtual template class that defines a receiver of messages of type T -template< typename T > -class receiver { -public: - //! The input type of this receiver - typedef T input_type; - - //! The predecessor type for this node - typedef sender predecessor_type; - - //! Destructor - virtual ~receiver() {} - - //! Put an item to the receiver - bool try_put( const T& t ) { - task *res = try_put_task(t); - if (!res) return false; - if (res != SUCCESSFULLY_ENQUEUED) FLOW_SPAWN(*res); - return true; - } - - //! put item to successor; return task to run the successor if possible. -protected: - template< typename R, typename B > friend class run_and_put_task; - template< typename X, typename Y > friend class internal::broadcast_cache; - template< typename X, typename Y > friend class internal::round_robin_cache; - virtual task *try_put_task(const T& t) = 0; -public: - // NOTE: Following part of PUBLIC and PROTECTED sections is copy-pasted in receiver under #if __TBB_PREVIEW_ASYNC_MSG - - //! Add a predecessor to the node - virtual bool register_predecessor( predecessor_type & ) { return false; } - - //! Remove a predecessor from the node - virtual bool remove_predecessor( predecessor_type & ) { return false; } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - typedef typename internal::edge_container built_predecessors_type; - typedef typename built_predecessors_type::edge_list_type predecessor_list_type; - virtual built_predecessors_type &built_predecessors() = 0; - virtual void internal_add_built_predecessor( predecessor_type & ) = 0; - virtual void internal_delete_built_predecessor( predecessor_type & ) = 0; - virtual void copy_predecessors( predecessor_list_type & ) = 0; - virtual size_t predecessor_count() = 0; -#endif - -protected: - //! put receiver back in initial state - template friend class limiter_node; - virtual void reset_receiver(reset_flags f = rf_reset_protocol) = 0; - - template friend class internal::successor_cache; - virtual bool is_continue_receiver() { return false; } - -#if __TBB_PREVIEW_OPENCL_NODE - template< typename, typename > friend class proxy_dependency_receiver; -#endif /* __TBB_PREVIEW_OPENCL_NODE */ -}; // class receiver - -#endif // __TBB_PREVIEW_ASYNC_MSG - -// enqueue left task if necessary. Returns the non-enqueued task if there is one. -static inline tbb::task *combine_tasks( tbb::task * left, tbb::task * right) { - // if no RHS task, don't change left. - if(right == NULL) return left; - // right != NULL - if(left == NULL) return right; - if(left == SUCCESSFULLY_ENQUEUED) return right; - // left contains a task - if(right != SUCCESSFULLY_ENQUEUED) { - // both are valid tasks - FLOW_SPAWN(*left); - return right; - } - return left; -} - -//! Base class for receivers of completion messages -/** These receivers automatically reset, but cannot be explicitly waited on */ -class continue_receiver : public receiver< continue_msg > { -public: - - //! The input type - typedef continue_msg input_type; - - //! The predecessor type for this node - typedef receiver::predecessor_type predecessor_type; - - //! Constructor - explicit continue_receiver( int number_of_predecessors = 0 ) { - my_predecessor_count = my_initial_predecessor_count = number_of_predecessors; - my_current_count = 0; - } - - //! Copy constructor - continue_receiver( const continue_receiver& src ) : receiver() { - my_predecessor_count = my_initial_predecessor_count = src.my_initial_predecessor_count; - my_current_count = 0; - } - - //! Increments the trigger threshold - bool register_predecessor( predecessor_type & ) __TBB_override { - spin_mutex::scoped_lock l(my_mutex); - ++my_predecessor_count; - return true; - } - - //! Decrements the trigger threshold - /** Does not check to see if the removal of the predecessor now makes the current count - exceed the new threshold. So removing a predecessor while the graph is active can cause - unexpected results. */ - bool remove_predecessor( predecessor_type & ) __TBB_override { - spin_mutex::scoped_lock l(my_mutex); - --my_predecessor_count; - return true; - } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - typedef internal::edge_container built_predecessors_type; - typedef built_predecessors_type::edge_list_type predecessor_list_type; - built_predecessors_type &built_predecessors() __TBB_override { return my_built_predecessors; } - - void internal_add_built_predecessor( predecessor_type &s) __TBB_override { - spin_mutex::scoped_lock l(my_mutex); - my_built_predecessors.add_edge( s ); - } - - void internal_delete_built_predecessor( predecessor_type &s) __TBB_override { - spin_mutex::scoped_lock l(my_mutex); - my_built_predecessors.delete_edge(s); - } - - void copy_predecessors( predecessor_list_type &v) __TBB_override { - spin_mutex::scoped_lock l(my_mutex); - my_built_predecessors.copy_edges(v); - } - - size_t predecessor_count() __TBB_override { - spin_mutex::scoped_lock l(my_mutex); - return my_built_predecessors.edge_count(); - } - -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ - -protected: - template< typename R, typename B > friend class run_and_put_task; - template friend class internal::broadcast_cache; - template friend class internal::round_robin_cache; - // execute body is supposed to be too small to create a task for. - task *try_put_task( const input_type & ) __TBB_override { - { - spin_mutex::scoped_lock l(my_mutex); - if ( ++my_current_count < my_predecessor_count ) - return SUCCESSFULLY_ENQUEUED; - else - my_current_count = 0; - } - task * res = execute(); - return res? res : SUCCESSFULLY_ENQUEUED; - } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - // continue_receiver must contain its own built_predecessors because it does - // not have a node_cache. - built_predecessors_type my_built_predecessors; -#endif - spin_mutex my_mutex; - int my_predecessor_count; - int my_current_count; - int my_initial_predecessor_count; - // the friend declaration in the base class did not eliminate the "protected class" - // error in gcc 4.1.2 - template friend class limiter_node; - - void reset_receiver( reset_flags f ) __TBB_override { - my_current_count = 0; - if (f & rf_clear_edges) { -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - my_built_predecessors.clear(); -#endif - my_predecessor_count = my_initial_predecessor_count; - } - } - - //! Does whatever should happen when the threshold is reached - /** This should be very fast or else spawn a task. This is - called while the sender is blocked in the try_put(). */ - virtual task * execute() = 0; - template friend class internal::successor_cache; - bool is_continue_receiver() __TBB_override { return true; } - -}; // class continue_receiver -} // interface9 - -#if __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING - template - K key_from_message( const T &t ) { - return t.key(); - } -#endif /* __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING */ - - using interface9::sender; - using interface9::receiver; - using interface9::continue_receiver; -} // flow -} // tbb - -#include "internal/_flow_graph_trace_impl.h" -#include "internal/_tbb_hash_compare_impl.h" - -namespace tbb { -namespace flow { -namespace interface9 { - -#include "internal/_flow_graph_impl.h" -#include "internal/_flow_graph_types_impl.h" -using namespace internal::graph_policy_namespace; - -class graph; -class graph_node; - -template -class graph_iterator { - friend class graph; - friend class graph_node; -public: - typedef size_t size_type; - typedef GraphNodeType value_type; - typedef GraphNodeType* pointer; - typedef GraphNodeType& reference; - typedef const GraphNodeType& const_reference; - typedef std::forward_iterator_tag iterator_category; - - //! Default constructor - graph_iterator() : my_graph(NULL), current_node(NULL) {} - - //! Copy constructor - graph_iterator(const graph_iterator& other) : - my_graph(other.my_graph), current_node(other.current_node) - {} - - //! Assignment - graph_iterator& operator=(const graph_iterator& other) { - if (this != &other) { - my_graph = other.my_graph; - current_node = other.current_node; - } - return *this; - } - - //! Dereference - reference operator*() const; - - //! Dereference - pointer operator->() const; - - //! Equality - bool operator==(const graph_iterator& other) const { - return ((my_graph == other.my_graph) && (current_node == other.current_node)); - } - - //! Inequality - bool operator!=(const graph_iterator& other) const { return !(operator==(other)); } - - //! Pre-increment - graph_iterator& operator++() { - internal_forward(); - return *this; - } - - //! Post-increment - graph_iterator operator++(int) { - graph_iterator result = *this; - operator++(); - return result; - } - -private: - // the graph over which we are iterating - GraphContainerType *my_graph; - // pointer into my_graph's my_nodes list - pointer current_node; - - //! Private initializing constructor for begin() and end() iterators - graph_iterator(GraphContainerType *g, bool begin); - void internal_forward(); -}; // class graph_iterator - -//! The graph class -/** This class serves as a handle to the graph */ -class graph : tbb::internal::no_copy, public graph_proxy { - friend class graph_node; - - template< typename Body > - class run_task : public task { - public: - run_task( Body& body ) : my_body(body) {} - task *execute() __TBB_override { - my_body(); - return NULL; - } - private: - Body my_body; - }; - - template< typename Receiver, typename Body > - class run_and_put_task : public task { - public: - run_and_put_task( Receiver &r, Body& body ) : my_receiver(r), my_body(body) {} - task *execute() __TBB_override { - task *res = my_receiver.try_put_task( my_body() ); - if (res == SUCCESSFULLY_ENQUEUED) res = NULL; - return res; - } - private: - Receiver &my_receiver; - Body my_body; - }; - typedef std::list task_list_type; - - class wait_functor { - task* graph_root_task; - public: - wait_functor( task* t ) : graph_root_task(t) {} - void operator()() const { graph_root_task->wait_for_all(); } - }; - - void prepare_task_arena( bool reinit = false ) { - if (reinit) { - __TBB_ASSERT( my_task_arena, "task arena is NULL"); - my_task_arena->terminate(); - my_task_arena->initialize(tbb::task_arena::attach()); - } else { - __TBB_ASSERT(my_task_arena == NULL, "task arena is not NULL"); - my_task_arena = new tbb::task_arena(tbb::task_arena::attach()); - } - if (!my_task_arena->is_active()) // failed to attach - my_task_arena->initialize(); // create a new, default-initialized arena - __TBB_ASSERT(my_task_arena->is_active(), "task arena is not active"); - } - -public: - //! Constructs a graph with isolated task_group_context - graph() : my_nodes(NULL), my_nodes_last(NULL), my_task_arena(NULL) { - prepare_task_arena(); - own_context = true; - cancelled = false; - caught_exception = false; - my_context = new task_group_context(); - my_root_task = ( new ( task::allocate_root(*my_context) ) empty_task ); - my_root_task->set_ref_count(1); - tbb::internal::fgt_graph( this ); - my_is_active = true; - } - - //! Constructs a graph with use_this_context as context - explicit graph(task_group_context& use_this_context) : - my_context(&use_this_context), my_nodes(NULL), my_nodes_last(NULL), my_task_arena(NULL) { - prepare_task_arena(); - own_context = false; - my_root_task = ( new ( task::allocate_root(*my_context) ) empty_task ); - my_root_task->set_ref_count(1); - tbb::internal::fgt_graph( this ); - my_is_active = true; - } - - //! Destroys the graph. - /** Calls wait_for_all, then destroys the root task and context. */ - ~graph() { - wait_for_all(); - my_root_task->set_ref_count(0); - task::destroy( *my_root_task ); - if (own_context) delete my_context; - delete my_task_arena; - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) { - tbb::internal::fgt_graph_desc( this, name ); - } -#endif - - //! Used to register that an external entity may still interact with the graph. - /** The graph will not return from wait_for_all until a matching number of decrement_wait_count calls - is made. */ - void increment_wait_count() { - if (my_root_task) - my_root_task->increment_ref_count(); - } - - //! Deregisters an external entity that may have interacted with the graph. - /** The graph will not return from wait_for_all until all the number of decrement_wait_count calls - matches the number of increment_wait_count calls. */ - void decrement_wait_count() { - if (my_root_task) - my_root_task->decrement_ref_count(); - } - - void reserve_wait() __TBB_override { - increment_wait_count(); - } - - void release_wait() __TBB_override { - decrement_wait_count(); - } - - //! Spawns a task that runs a body and puts its output to a specific receiver - /** The task is spawned as a child of the graph. This is useful for running tasks - that need to block a wait_for_all() on the graph. For example a one-off source. */ - template< typename Receiver, typename Body > - void run( Receiver &r, Body body ) { - if(is_active()) { - FLOW_SPAWN( (* new ( task::allocate_additional_child_of( *root_task() ) ) - run_and_put_task< Receiver, Body >( r, body )) ); - } - } - - //! Spawns a task that runs a function object - /** The task is spawned as a child of the graph. This is useful for running tasks - that need to block a wait_for_all() on the graph. For example a one-off source. */ - template< typename Body > - void run( Body body ) { - if(is_active()) { - FLOW_SPAWN( * new ( task::allocate_additional_child_of( *root_task() ) ) run_task< Body >( body ) ); - } - } - - //! Wait until graph is idle and decrement_wait_count calls equals increment_wait_count calls. - /** The waiting thread will go off and steal work while it is block in the wait_for_all. */ - void wait_for_all() { - cancelled = false; - caught_exception = false; - if (my_root_task) { -#if TBB_USE_EXCEPTIONS - try { -#endif - my_task_arena->execute(wait_functor(my_root_task)); - cancelled = my_context->is_group_execution_cancelled(); -#if TBB_USE_EXCEPTIONS - } - catch(...) { - my_root_task->set_ref_count(1); - my_context->reset(); - caught_exception = true; - cancelled = true; - throw; - } -#endif - // TODO: the "if" condition below is just a work-around to support the concurrent wait - // mode. The cancellation and exception mechanisms are still broken in this mode. - // Consider using task group not to re-implement the same functionality. - if ( !(my_context->traits() & task_group_context::concurrent_wait) ) { - my_context->reset(); // consistent with behavior in catch() - my_root_task->set_ref_count(1); - } - } - } - - //! Returns the root task of the graph - task * root_task() { - return my_root_task; - } - - void set_active(bool a = true) { - my_is_active = a; - } - - bool is_active() { - return my_is_active; - } - - void add_task_to_reset_list(task *tp) { - my_reset_task_list.push_back(tp); - } - - // ITERATORS - template - friend class graph_iterator; - - // Graph iterator typedefs - typedef graph_iterator iterator; - typedef graph_iterator const_iterator; - - // Graph iterator constructors - //! start iterator - iterator begin() { return iterator(this, true); } - //! end iterator - iterator end() { return iterator(this, false); } - //! start const iterator - const_iterator begin() const { return const_iterator(this, true); } - //! end const iterator - const_iterator end() const { return const_iterator(this, false); } - //! start const iterator - const_iterator cbegin() const { return const_iterator(this, true); } - //! end const iterator - const_iterator cend() const { return const_iterator(this, false); } - - //! return status of graph execution - bool is_cancelled() { return cancelled; } - bool exception_thrown() { return caught_exception; } - - // thread-unsafe state reset. - void reset(reset_flags f = rf_reset_protocol); - -private: - task *my_root_task; - task_group_context *my_context; - bool own_context; - bool cancelled; - bool caught_exception; - bool my_is_active; - task_list_type my_reset_task_list; - - graph_node *my_nodes, *my_nodes_last; - - spin_mutex nodelist_mutex; - void register_node(graph_node *n); - void remove_node(graph_node *n); - - template < typename Input, typename Output, typename Policy, typename Allocator > - friend class async_node; - task_arena* my_task_arena; -}; // class graph - -template -graph_iterator::graph_iterator(C *g, bool begin) : my_graph(g), current_node(NULL) -{ - if (begin) current_node = my_graph->my_nodes; - //else it is an end iterator by default -} - -template -typename graph_iterator::reference graph_iterator::operator*() const { - __TBB_ASSERT(current_node, "graph_iterator at end"); - return *operator->(); -} - -template -typename graph_iterator::pointer graph_iterator::operator->() const { - return current_node; -} - -template -void graph_iterator::internal_forward() { - if (current_node) current_node = current_node->next; -} - -//! The base of all graph nodes. -class graph_node : tbb::internal::no_copy { - friend class graph; - template - friend class graph_iterator; -protected: - graph& my_graph; - graph_node *next, *prev; -public: - explicit graph_node(graph& g) : my_graph(g) { - my_graph.register_node(this); - } - virtual ~graph_node() { - my_graph.remove_node(this); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - virtual void set_name( const char *name ) = 0; -#endif - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - virtual void extract( ) = 0; -#endif - -protected: - // performs the reset on an individual node. - virtual void reset_node(reset_flags f=rf_reset_protocol) = 0; -}; // class graph_node - -inline void graph::register_node(graph_node *n) { - n->next = NULL; - { - spin_mutex::scoped_lock lock(nodelist_mutex); - n->prev = my_nodes_last; - if (my_nodes_last) my_nodes_last->next = n; - my_nodes_last = n; - if (!my_nodes) my_nodes = n; - } -} - -inline void graph::remove_node(graph_node *n) { - { - spin_mutex::scoped_lock lock(nodelist_mutex); - __TBB_ASSERT(my_nodes && my_nodes_last, "graph::remove_node: Error: no registered nodes"); - if (n->prev) n->prev->next = n->next; - if (n->next) n->next->prev = n->prev; - if (my_nodes_last == n) my_nodes_last = n->prev; - if (my_nodes == n) my_nodes = n->next; - } - n->prev = n->next = NULL; -} - -inline void graph::reset( reset_flags f ) { - // reset context - set_active(false); - if(my_context) my_context->reset(); - cancelled = false; - caught_exception = false; - // reset all the nodes comprising the graph - for(iterator ii = begin(); ii != end(); ++ii) { - graph_node *my_p = &(*ii); - my_p->reset_node(f); - } - // Reattach the arena. Might be useful to run the graph in a particular task_arena - // while not limiting graph lifetime to a single task_arena::execute() call. - prepare_task_arena( /*reinit=*/true ); - set_active(true); - // now spawn the tasks necessary to start the graph - for(task_list_type::iterator rti = my_reset_task_list.begin(); rti != my_reset_task_list.end(); ++rti) { - FLOW_SPAWN(*(*rti)); - } - my_reset_task_list.clear(); -} - -#include "internal/_flow_graph_node_impl.h" - -//! An executable node that acts as a source, i.e. it has no predecessors -template < typename Output > -class source_node : public graph_node, public sender< Output > { -public: - //! The type of the output message, which is complete - typedef Output output_type; - - //! The type of successors of this node - typedef typename sender::successor_type successor_type; - - //Source node has no input type - typedef null_type input_type; - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - typedef typename sender::built_successors_type built_successors_type; - typedef typename sender::successor_list_type successor_list_type; -#endif - - //! Constructor for a node with a successor - template< typename Body > - source_node( graph &g, Body body, bool is_active = true ) - : graph_node(g), my_active(is_active), init_my_active(is_active), - my_body( new internal::source_body_leaf< output_type, Body>(body) ), - my_init_body( new internal::source_body_leaf< output_type, Body>(body) ), - my_reserved(false), my_has_cached_item(false) - { - my_successors.set_owner(this); - tbb::internal::fgt_node_with_body( tbb::internal::FLOW_SOURCE_NODE, &this->my_graph, - static_cast *>(this), this->my_body ); - } - - //! Copy constructor - source_node( const source_node& src ) : - graph_node(src.my_graph), sender(), - my_active(src.init_my_active), - init_my_active(src.init_my_active), my_body( src.my_init_body->clone() ), my_init_body(src.my_init_body->clone() ), - my_reserved(false), my_has_cached_item(false) - { - my_successors.set_owner(this); - tbb::internal::fgt_node_with_body( tbb::internal::FLOW_SOURCE_NODE, &this->my_graph, - static_cast *>(this), this->my_body ); - } - - //! The destructor - ~source_node() { delete my_body; delete my_init_body; } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif - - //! Add a new successor to this node - bool register_successor( successor_type &r ) __TBB_override { - spin_mutex::scoped_lock lock(my_mutex); - my_successors.register_successor(r); - if ( my_active ) - spawn_put(); - return true; - } - - //! Removes a successor from this node - bool remove_successor( successor_type &r ) __TBB_override { - spin_mutex::scoped_lock lock(my_mutex); - my_successors.remove_successor(r); - return true; - } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - - built_successors_type &built_successors() __TBB_override { return my_successors.built_successors(); } - - void internal_add_built_successor( successor_type &r) __TBB_override { - spin_mutex::scoped_lock lock(my_mutex); - my_successors.internal_add_built_successor(r); - } - - void internal_delete_built_successor( successor_type &r) __TBB_override { - spin_mutex::scoped_lock lock(my_mutex); - my_successors.internal_delete_built_successor(r); - } - - size_t successor_count() __TBB_override { - spin_mutex::scoped_lock lock(my_mutex); - return my_successors.successor_count(); - } - - void copy_successors(successor_list_type &v) __TBB_override { - spin_mutex::scoped_lock l(my_mutex); - my_successors.copy_successors(v); - } -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ - - //! Request an item from the node - bool try_get( output_type &v ) __TBB_override { - spin_mutex::scoped_lock lock(my_mutex); - if ( my_reserved ) - return false; - - if ( my_has_cached_item ) { - v = my_cached_item; - my_has_cached_item = false; - return true; - } - // we've been asked to provide an item, but we have none. enqueue a task to - // provide one. - spawn_put(); - return false; - } - - //! Reserves an item. - bool try_reserve( output_type &v ) __TBB_override { - spin_mutex::scoped_lock lock(my_mutex); - if ( my_reserved ) { - return false; - } - - if ( my_has_cached_item ) { - v = my_cached_item; - my_reserved = true; - return true; - } else { - return false; - } - } - - //! Release a reserved item. - /** true = item has been released and so remains in sender, dest must request or reserve future items */ - bool try_release( ) __TBB_override { - spin_mutex::scoped_lock lock(my_mutex); - __TBB_ASSERT( my_reserved && my_has_cached_item, "releasing non-existent reservation" ); - my_reserved = false; - if(!my_successors.empty()) - spawn_put(); - return true; - } - - //! Consumes a reserved item - bool try_consume( ) __TBB_override { - spin_mutex::scoped_lock lock(my_mutex); - __TBB_ASSERT( my_reserved && my_has_cached_item, "consuming non-existent reservation" ); - my_reserved = false; - my_has_cached_item = false; - if ( !my_successors.empty() ) { - spawn_put(); - } - return true; - } - - //! Activates a node that was created in the inactive state - void activate() { - spin_mutex::scoped_lock lock(my_mutex); - my_active = true; - if ( !my_successors.empty() ) - spawn_put(); - } - - template - Body copy_function_object() { - internal::source_body &body_ref = *this->my_body; - return dynamic_cast< internal::source_body_leaf & >(body_ref).get_body(); - } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - void extract( ) __TBB_override { - my_successors.built_successors().sender_extract(*this); // removes "my_owner" == this from each successor - my_active = init_my_active; - my_reserved = false; - if(my_has_cached_item) my_has_cached_item = false; - } -#endif - -protected: - - //! resets the source_node to its initial state - void reset_node( reset_flags f) __TBB_override { - my_active = init_my_active; - my_reserved =false; - if(my_has_cached_item) { - my_has_cached_item = false; - } - if(f & rf_clear_edges) my_successors.clear(); - if(f & rf_reset_bodies) { - internal::source_body *tmp = my_init_body->clone(); - delete my_body; - my_body = tmp; - } - if(my_active) - this->my_graph.add_task_to_reset_list(create_put_task()); - } - -private: - spin_mutex my_mutex; - bool my_active; - bool init_my_active; - internal::source_body *my_body; - internal::source_body *my_init_body; - internal::broadcast_cache< output_type > my_successors; - bool my_reserved; - bool my_has_cached_item; - output_type my_cached_item; - - // used by apply_body_bypass, can invoke body of node. - bool try_reserve_apply_body(output_type &v) { - spin_mutex::scoped_lock lock(my_mutex); - if ( my_reserved ) { - return false; - } - if ( !my_has_cached_item ) { - tbb::internal::fgt_begin_body( my_body ); - bool r = (*my_body)(my_cached_item); - tbb::internal::fgt_end_body( my_body ); - if (r) { - my_has_cached_item = true; - } - } - if ( my_has_cached_item ) { - v = my_cached_item; - my_reserved = true; - return true; - } else { - return false; - } - } - - // when resetting, and if the source_node was created with my_active == true, then - // when we reset the node we must store a task to run the node, and spawn it only - // after the reset is complete and is_active() is again true. This is why we don't - // test for is_active() here. - task* create_put_task() { - return ( new ( task::allocate_additional_child_of( *(this->my_graph.root_task()) ) ) - internal:: source_task_bypass < source_node< output_type > >( *this ) ); - } - - //! Spawns a task that applies the body - void spawn_put( ) { - if(this->my_graph.is_active()) { - FLOW_SPAWN( *create_put_task()); - } - } - - friend class internal::source_task_bypass< source_node< output_type > >; - //! Applies the body. Returning SUCCESSFULLY_ENQUEUED okay; forward_task_bypass will handle it. - task * apply_body_bypass( ) { - output_type v; - if ( !try_reserve_apply_body(v) ) - return NULL; - - task *last_task = my_successors.try_put_task(v); - if ( last_task ) - try_consume(); - else - try_release(); - return last_task; - } -}; // class source_node - -template -struct allocate_buffer { - static const bool value = false; -}; - -template<> -struct allocate_buffer { - static const bool value = true; -}; - -//! Implements a function node that supports Input -> Output -template < typename Input, typename Output = continue_msg, typename Policy = queueing, typename Allocator=cache_aligned_allocator > -class function_node : public graph_node, public internal::function_input, public internal::function_output { -public: - typedef Input input_type; - typedef Output output_type; - typedef internal::function_input fInput_type; - typedef internal::function_input_queue input_queue_type; - typedef internal::function_output fOutput_type; - typedef typename fInput_type::predecessor_type predecessor_type; - typedef typename fOutput_type::successor_type successor_type; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - typedef typename fInput_type::predecessor_list_type predecessor_list_type; - typedef typename fOutput_type::successor_list_type successor_list_type; -#endif - using fInput_type::my_predecessors; - - //! Constructor - // input_queue_type is allocated here, but destroyed in the function_input_base. - // TODO: pass the graph_buffer_policy to the function_input_base so it can all - // be done in one place. This would be an interface-breaking change. - template< typename Body > - function_node( graph &g, size_t concurrency, Body body ) : - graph_node(g), fInput_type(g, concurrency, body, allocate_buffer::value ? - new input_queue_type( ) : NULL ) { - tbb::internal::fgt_node_with_body( tbb::internal::FLOW_FUNCTION_NODE, &this->my_graph, - static_cast *>(this), static_cast *>(this), this->my_body ); - } - - //! Copy constructor - function_node( const function_node& src ) : - graph_node(src.my_graph), - fInput_type(src, allocate_buffer::value ? new input_queue_type : NULL), - fOutput_type() { - tbb::internal::fgt_node_with_body( tbb::internal::FLOW_FUNCTION_NODE, &this->my_graph, - static_cast *>(this), static_cast *>(this), this->my_body ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - void extract( ) __TBB_override { - my_predecessors.built_predecessors().receiver_extract(*this); - successors().built_successors().sender_extract(*this); - } -#endif - -protected: - template< typename R, typename B > friend class run_and_put_task; - template friend class internal::broadcast_cache; - template friend class internal::round_robin_cache; - using fInput_type::try_put_task; - - internal::broadcast_cache &successors () __TBB_override { return fOutput_type::my_successors; } - - void reset_node(reset_flags f) __TBB_override { - fInput_type::reset_function_input(f); - // TODO: use clear() instead. - if(f & rf_clear_edges) { - successors().clear(); - my_predecessors.clear(); - } - __TBB_ASSERT(!(f & rf_clear_edges) || successors().empty(), "function_node successors not empty"); - __TBB_ASSERT(this->my_predecessors.empty(), "function_node predecessors not empty"); - } - -}; // class function_node - -//! implements a function node that supports Input -> (set of outputs) -// Output is a tuple of output types. -template < typename Input, typename Output, typename Policy = queueing, typename Allocator=cache_aligned_allocator > -class multifunction_node : - public graph_node, - public internal::multifunction_input - < - Input, - typename internal::wrap_tuple_elements< - tbb::flow::tuple_size::value, // #elements in tuple - internal::multifunction_output, // wrap this around each element - Output // the tuple providing the types - >::type, - Allocator - > { -protected: - static const int N = tbb::flow::tuple_size::value; -public: - typedef Input input_type; - typedef null_type output_type; - typedef typename internal::wrap_tuple_elements::type output_ports_type; - typedef internal::multifunction_input fInput_type; - typedef internal::function_input_queue input_queue_type; -private: - typedef typename internal::multifunction_input base_type; - using fInput_type::my_predecessors; -public: - template - multifunction_node( graph &g, size_t concurrency, Body body ) : - graph_node(g), base_type(g,concurrency, body, allocate_buffer::value ? new input_queue_type : NULL) { - tbb::internal::fgt_multioutput_node_with_body( tbb::internal::FLOW_MULTIFUNCTION_NODE, - &this->my_graph, static_cast *>(this), - this->output_ports(), this->my_body ); - } - - multifunction_node( const multifunction_node &other) : - graph_node(other.my_graph), base_type(other, allocate_buffer::value ? new input_queue_type : NULL) { - tbb::internal::fgt_multioutput_node_with_body( tbb::internal::FLOW_MULTIFUNCTION_NODE, - &this->my_graph, static_cast *>(this), - this->output_ports(), this->my_body ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_multioutput_node_desc( this, name ); - } -#endif - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - void extract( ) __TBB_override { - my_predecessors.built_predecessors().receiver_extract(*this); - base_type::extract(); - } -#endif - // all the guts are in multifunction_input... -protected: - void reset_node(reset_flags f) __TBB_override { base_type::reset(f); } -}; // multifunction_node - -//! split_node: accepts a tuple as input, forwards each element of the tuple to its -// successors. The node has unlimited concurrency, so it does not reject inputs. -template > -class split_node : public graph_node, public receiver { - static const int N = tbb::flow::tuple_size::value; - typedef receiver base_type; -public: - typedef TupleType input_type; - typedef Allocator allocator_type; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - typedef typename base_type::predecessor_type predecessor_type; - typedef typename base_type::predecessor_list_type predecessor_list_type; - typedef internal::predecessor_cache predecessor_cache_type; - typedef typename predecessor_cache_type::built_predecessors_type built_predecessors_type; -#endif - - typedef typename internal::wrap_tuple_elements< - N, // #elements in tuple - internal::multifunction_output, // wrap this around each element - TupleType // the tuple providing the types - >::type output_ports_type; - - explicit split_node(graph &g) : graph_node(g) - { - tbb::internal::fgt_multioutput_node(tbb::internal::FLOW_SPLIT_NODE, &this->my_graph, - static_cast *>(this), this->output_ports()); - } - split_node( const split_node & other) : graph_node(other.my_graph), base_type(other) - { - tbb::internal::fgt_multioutput_node(tbb::internal::FLOW_SPLIT_NODE, &this->my_graph, - static_cast *>(this), this->output_ports()); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_multioutput_node_desc( this, name ); - } -#endif - - output_ports_type &output_ports() { return my_output_ports; } - -protected: - task *try_put_task(const TupleType& t) __TBB_override { - // Sending split messages in parallel is not justified, as overheads would prevail - internal::emit_element::emit_this(t, output_ports()); - - //we do not have successors here.So we just tell the task is successful. - return SUCCESSFULLY_ENQUEUED; - } - void reset_node(reset_flags f) __TBB_override { - if (f & rf_clear_edges) - internal::clear_element::clear_this(my_output_ports); - - __TBB_ASSERT(!(f & rf_clear_edges) || internal::clear_element::this_empty(my_output_ports), "split_node reset failed"); - } - void reset_receiver(reset_flags /*f*/) __TBB_override {} - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES -private: //! split_node doesn't use this "predecessors" functionality; so, we have "dummies" here; - void extract() __TBB_override {} - - //! Adds to list of predecessors added by make_edge - void internal_add_built_predecessor(predecessor_type&) __TBB_override {} - - //! removes from to list of predecessors (used by remove_edge) - void internal_delete_built_predecessor(predecessor_type&) __TBB_override {} - - size_t predecessor_count() __TBB_override { return 0; } - - void copy_predecessors(predecessor_list_type&) __TBB_override {} - - built_predecessors_type &built_predecessors() __TBB_override { return my_predessors; } - - //! dummy member - built_predecessors_type my_predessors; -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ - -private: - output_ports_type my_output_ports; -}; - -//! Implements an executable node that supports continue_msg -> Output -template -class continue_node : public graph_node, public internal::continue_input, public internal::function_output { -public: - typedef continue_msg input_type; - typedef Output output_type; - typedef internal::continue_input fInput_type; - typedef internal::function_output fOutput_type; - typedef typename fInput_type::predecessor_type predecessor_type; - typedef typename fOutput_type::successor_type successor_type; - - //! Constructor for executable node with continue_msg -> Output - template - continue_node( graph &g, Body body ) : - graph_node(g), internal::continue_input( g, body ) { - tbb::internal::fgt_node_with_body( tbb::internal::FLOW_CONTINUE_NODE, &this->my_graph, - static_cast *>(this), - static_cast *>(this), this->my_body ); - } - - - //! Constructor for executable node with continue_msg -> Output - template - continue_node( graph &g, int number_of_predecessors, Body body ) : - graph_node(g), internal::continue_input( g, number_of_predecessors, body ) { - tbb::internal::fgt_node_with_body( tbb::internal::FLOW_CONTINUE_NODE, &this->my_graph, - static_cast *>(this), - static_cast *>(this), this->my_body ); - } - - //! Copy constructor - continue_node( const continue_node& src ) : - graph_node(src.my_graph), internal::continue_input(src), - internal::function_output() { - tbb::internal::fgt_node_with_body( tbb::internal::FLOW_CONTINUE_NODE, &this->my_graph, - static_cast *>(this), - static_cast *>(this), this->my_body ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - void extract() __TBB_override { - fInput_type::my_built_predecessors.receiver_extract(*this); - successors().built_successors().sender_extract(*this); - } -#endif - -protected: - template< typename R, typename B > friend class run_and_put_task; - template friend class internal::broadcast_cache; - template friend class internal::round_robin_cache; - using fInput_type::try_put_task; - internal::broadcast_cache &successors () __TBB_override { return fOutput_type::my_successors; } - - void reset_node(reset_flags f) __TBB_override { - fInput_type::reset_receiver(f); - if(f & rf_clear_edges)successors().clear(); - __TBB_ASSERT(!(f & rf_clear_edges) || successors().empty(), "continue_node not reset"); - } -}; // continue_node - -template< typename T > -class overwrite_node : public graph_node, public receiver, public sender { -public: - typedef T input_type; - typedef T output_type; - typedef typename receiver::predecessor_type predecessor_type; - typedef typename sender::successor_type successor_type; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - typedef typename receiver::built_predecessors_type built_predecessors_type; - typedef typename sender::built_successors_type built_successors_type; - typedef typename receiver::predecessor_list_type predecessor_list_type; - typedef typename sender::successor_list_type successor_list_type; -#endif - - explicit overwrite_node(graph &g) : graph_node(g), my_buffer_is_valid(false) { - my_successors.set_owner( this ); - tbb::internal::fgt_node( tbb::internal::FLOW_OVERWRITE_NODE, &this->my_graph, - static_cast *>(this), static_cast *>(this) ); - } - - // Copy constructor; doesn't take anything from src; default won't work - overwrite_node( const overwrite_node& src ) : - graph_node(src.my_graph), receiver(), sender(), my_buffer_is_valid(false) - { - my_successors.set_owner( this ); - tbb::internal::fgt_node( tbb::internal::FLOW_OVERWRITE_NODE, &this->my_graph, - static_cast *>(this), static_cast *>(this) ); - } - - ~overwrite_node() {} - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif - - bool register_successor( successor_type &s ) __TBB_override { - spin_mutex::scoped_lock l( my_mutex ); - if (my_buffer_is_valid && this->my_graph.is_active()) { - // We have a valid value that must be forwarded immediately. - if ( s.try_put( my_buffer ) || !s.register_predecessor( *this ) ) { - // We add the successor: it accepted our put or it rejected it but won't let us become a predecessor - my_successors.register_successor( s ); - } else { - // We don't add the successor: it rejected our put and we became its predecessor instead - return false; - } - } else { - // No valid value yet, just add as successor - my_successors.register_successor( s ); - } - return true; - } - - bool remove_successor( successor_type &s ) __TBB_override { - spin_mutex::scoped_lock l( my_mutex ); - my_successors.remove_successor(s); - return true; - } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - built_predecessors_type &built_predecessors() __TBB_override { return my_built_predecessors; } - built_successors_type &built_successors() __TBB_override { return my_successors.built_successors(); } - - void internal_add_built_successor( successor_type &s) __TBB_override { - spin_mutex::scoped_lock l( my_mutex ); - my_successors.internal_add_built_successor(s); - } - - void internal_delete_built_successor( successor_type &s) __TBB_override { - spin_mutex::scoped_lock l( my_mutex ); - my_successors.internal_delete_built_successor(s); - } - - size_t successor_count() __TBB_override { - spin_mutex::scoped_lock l( my_mutex ); - return my_successors.successor_count(); - } - - void copy_successors(successor_list_type &v) __TBB_override { - spin_mutex::scoped_lock l( my_mutex ); - my_successors.copy_successors(v); - } - - void internal_add_built_predecessor( predecessor_type &p) __TBB_override { - spin_mutex::scoped_lock l( my_mutex ); - my_built_predecessors.add_edge(p); - } - - void internal_delete_built_predecessor( predecessor_type &p) __TBB_override { - spin_mutex::scoped_lock l( my_mutex ); - my_built_predecessors.delete_edge(p); - } - - size_t predecessor_count() __TBB_override { - spin_mutex::scoped_lock l( my_mutex ); - return my_built_predecessors.edge_count(); - } - - void copy_predecessors(predecessor_list_type &v) __TBB_override { - spin_mutex::scoped_lock l( my_mutex ); - my_built_predecessors.copy_edges(v); - } - - void extract() __TBB_override { - my_buffer_is_valid = false; - built_successors().sender_extract(*this); - built_predecessors().receiver_extract(*this); - } - -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ - - bool try_get( input_type &v ) __TBB_override { - spin_mutex::scoped_lock l( my_mutex ); - if ( my_buffer_is_valid ) { - v = my_buffer; - return true; - } - return false; - } - - bool is_valid() { - spin_mutex::scoped_lock l( my_mutex ); - return my_buffer_is_valid; - } - - void clear() { - spin_mutex::scoped_lock l( my_mutex ); - my_buffer_is_valid = false; - } - -protected: - template< typename R, typename B > friend class run_and_put_task; - template friend class internal::broadcast_cache; - template friend class internal::round_robin_cache; - task * try_put_task( const input_type &v ) __TBB_override { - spin_mutex::scoped_lock l( my_mutex ); - return try_put_task_impl(v); - } - - task * try_put_task_impl(const input_type &v) { - my_buffer = v; - my_buffer_is_valid = true; - task * rtask = my_successors.try_put_task(v); - if (!rtask) rtask = SUCCESSFULLY_ENQUEUED; - return rtask; - } - - spin_mutex my_mutex; - internal::broadcast_cache< input_type, null_rw_mutex > my_successors; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - internal::edge_container my_built_predecessors; -#endif - input_type my_buffer; - bool my_buffer_is_valid; - void reset_receiver(reset_flags /*f*/) __TBB_override {} - - void reset_node( reset_flags f) __TBB_override { - my_buffer_is_valid = false; - if (f&rf_clear_edges) { - my_successors.clear(); - } - } -}; // overwrite_node - -template< typename T > -class write_once_node : public overwrite_node { -public: - typedef T input_type; - typedef T output_type; - typedef typename receiver::predecessor_type predecessor_type; - typedef typename sender::successor_type successor_type; - - //! Constructor - explicit write_once_node(graph& g) : overwrite_node(g) { - tbb::internal::fgt_node( tbb::internal::FLOW_WRITE_ONCE_NODE, &(this->my_graph), - static_cast *>(this), - static_cast *>(this) ); - } - - //! Copy constructor: call base class copy constructor - write_once_node( const write_once_node& src ) : overwrite_node(src) { - tbb::internal::fgt_node( tbb::internal::FLOW_WRITE_ONCE_NODE, &(this->my_graph), - static_cast *>(this), - static_cast *>(this) ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif - -protected: - template< typename R, typename B > friend class run_and_put_task; - template friend class internal::broadcast_cache; - template friend class internal::round_robin_cache; - task *try_put_task( const T &v ) __TBB_override { - spin_mutex::scoped_lock l( this->my_mutex ); - return this->my_buffer_is_valid ? NULL : this->try_put_task_impl(v); - } -}; - -//! Forwards messages of type T to all successors -template -class broadcast_node : public graph_node, public receiver, public sender { -public: - typedef T input_type; - typedef T output_type; - typedef typename receiver::predecessor_type predecessor_type; - typedef typename sender::successor_type successor_type; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - typedef typename receiver::predecessor_list_type predecessor_list_type; - typedef typename sender::successor_list_type successor_list_type; -#endif -private: - internal::broadcast_cache my_successors; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - internal::edge_container my_built_predecessors; - spin_mutex pred_mutex; // serialize accesses on edge_container -#endif -public: - - explicit broadcast_node(graph& g) : graph_node(g) { - my_successors.set_owner( this ); - tbb::internal::fgt_node( tbb::internal::FLOW_BROADCAST_NODE, &this->my_graph, - static_cast *>(this), static_cast *>(this) ); - } - - // Copy constructor - broadcast_node( const broadcast_node& src ) : - graph_node(src.my_graph), receiver(), sender() - { - my_successors.set_owner( this ); - tbb::internal::fgt_node( tbb::internal::FLOW_BROADCAST_NODE, &this->my_graph, - static_cast *>(this), static_cast *>(this) ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif - - //! Adds a successor - bool register_successor( successor_type &r ) __TBB_override { - my_successors.register_successor( r ); - return true; - } - - //! Removes s as a successor - bool remove_successor( successor_type &r ) __TBB_override { - my_successors.remove_successor( r ); - return true; - } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - typedef typename sender::built_successors_type built_successors_type; - - built_successors_type &built_successors() __TBB_override { return my_successors.built_successors(); } - - void internal_add_built_successor(successor_type &r) __TBB_override { - my_successors.internal_add_built_successor(r); - } - - void internal_delete_built_successor(successor_type &r) __TBB_override { - my_successors.internal_delete_built_successor(r); - } - - size_t successor_count() __TBB_override { - return my_successors.successor_count(); - } - - void copy_successors(successor_list_type &v) __TBB_override { - my_successors.copy_successors(v); - } - - typedef typename receiver::built_predecessors_type built_predecessors_type; - - built_predecessors_type &built_predecessors() __TBB_override { return my_built_predecessors; } - - void internal_add_built_predecessor( predecessor_type &p) __TBB_override { - spin_mutex::scoped_lock l(pred_mutex); - my_built_predecessors.add_edge(p); - } - - void internal_delete_built_predecessor( predecessor_type &p) __TBB_override { - spin_mutex::scoped_lock l(pred_mutex); - my_built_predecessors.delete_edge(p); - } - - size_t predecessor_count() __TBB_override { - spin_mutex::scoped_lock l(pred_mutex); - return my_built_predecessors.edge_count(); - } - - void copy_predecessors(predecessor_list_type &v) __TBB_override { - spin_mutex::scoped_lock l(pred_mutex); - my_built_predecessors.copy_edges(v); - } - - void extract() __TBB_override { - my_built_predecessors.receiver_extract(*this); - my_successors.built_successors().sender_extract(*this); - } -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ - -protected: - template< typename R, typename B > friend class run_and_put_task; - template friend class internal::broadcast_cache; - template friend class internal::round_robin_cache; - //! build a task to run the successor if possible. Default is old behavior. - task *try_put_task(const T& t) __TBB_override { - task *new_task = my_successors.try_put_task(t); - if (!new_task) new_task = SUCCESSFULLY_ENQUEUED; - return new_task; - } - - void reset_receiver(reset_flags /*f*/) __TBB_override {} - - void reset_node(reset_flags f) __TBB_override { - if (f&rf_clear_edges) { - my_successors.clear(); -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - my_built_predecessors.clear(); -#endif - } - __TBB_ASSERT(!(f & rf_clear_edges) || my_successors.empty(), "Error resetting broadcast_node"); - } -}; // broadcast_node - -//! Forwards messages in arbitrary order -template > -class buffer_node : public graph_node, public internal::reservable_item_buffer, public receiver, public sender { -public: - typedef T input_type; - typedef T output_type; - typedef typename receiver::predecessor_type predecessor_type; - typedef typename sender::successor_type successor_type; - typedef buffer_node class_type; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - typedef typename receiver::predecessor_list_type predecessor_list_type; - typedef typename sender::successor_list_type successor_list_type; -#endif -protected: - typedef size_t size_type; - internal::round_robin_cache< T, null_rw_mutex > my_successors; - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - internal::edge_container my_built_predecessors; -#endif - - friend class internal::forward_task_bypass< buffer_node< T, A > >; - - enum op_type {reg_succ, rem_succ, req_item, res_item, rel_res, con_res, put_item, try_fwd_task -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - , add_blt_succ, del_blt_succ, - add_blt_pred, del_blt_pred, - blt_succ_cnt, blt_pred_cnt, - blt_succ_cpy, blt_pred_cpy // create vector copies of preds and succs -#endif - }; - - // implements the aggregator_operation concept - class buffer_operation : public internal::aggregated_operation< buffer_operation > { - public: - char type; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - task * ltask; - union { - input_type *elem; - successor_type *r; - predecessor_type *p; - size_t cnt_val; - successor_list_type *svec; - predecessor_list_type *pvec; - }; -#else - T *elem; - task * ltask; - successor_type *r; -#endif - buffer_operation(const T& e, op_type t) : type(char(t)) - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - , ltask(NULL), elem(const_cast(&e)) -#else - , elem(const_cast(&e)) , ltask(NULL) -#endif - {} - buffer_operation(op_type t) : type(char(t)), ltask(NULL) {} - }; - - bool forwarder_busy; - typedef internal::aggregating_functor handler_type; - friend class internal::aggregating_functor; - internal::aggregator< handler_type, buffer_operation> my_aggregator; - - virtual void handle_operations(buffer_operation *op_list) { - handle_operations_impl(op_list, this); - } - - template - void handle_operations_impl(buffer_operation *op_list, derived_type* derived) { - __TBB_ASSERT(static_cast(derived) == this, "'this' is not a base class for derived"); - - buffer_operation *tmp = NULL; - bool try_forwarding = false; - while (op_list) { - tmp = op_list; - op_list = op_list->next; - switch (tmp->type) { - case reg_succ: internal_reg_succ(tmp); try_forwarding = true; break; - case rem_succ: internal_rem_succ(tmp); break; - case req_item: internal_pop(tmp); break; - case res_item: internal_reserve(tmp); break; - case rel_res: internal_release(tmp); try_forwarding = true; break; - case con_res: internal_consume(tmp); try_forwarding = true; break; - case put_item: try_forwarding = internal_push(tmp); break; - case try_fwd_task: internal_forward_task(tmp); break; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - // edge recording - case add_blt_succ: internal_add_built_succ(tmp); break; - case del_blt_succ: internal_del_built_succ(tmp); break; - case add_blt_pred: internal_add_built_pred(tmp); break; - case del_blt_pred: internal_del_built_pred(tmp); break; - case blt_succ_cnt: internal_succ_cnt(tmp); break; - case blt_pred_cnt: internal_pred_cnt(tmp); break; - case blt_succ_cpy: internal_copy_succs(tmp); break; - case blt_pred_cpy: internal_copy_preds(tmp); break; -#endif - } - } - - derived->order(); - - if (try_forwarding && !forwarder_busy) { - if(this->my_graph.is_active()) { - forwarder_busy = true; - task *new_task = new(task::allocate_additional_child_of(*(this->my_graph.root_task()))) internal:: - forward_task_bypass - < buffer_node >(*this); - // tmp should point to the last item handled by the aggregator. This is the operation - // the handling thread enqueued. So modifying that record will be okay. - tbb::task *z = tmp->ltask; - tmp->ltask = combine_tasks(z, new_task); // in case the op generated a task - } - } - } // handle_operations - - inline task *grab_forwarding_task( buffer_operation &op_data) { - return op_data.ltask; - } - - inline bool enqueue_forwarding_task(buffer_operation &op_data) { - task *ft = grab_forwarding_task(op_data); - if(ft) { - FLOW_SPAWN(*ft); - return true; - } - return false; - } - - //! This is executed by an enqueued task, the "forwarder" - virtual task *forward_task() { - buffer_operation op_data(try_fwd_task); - task *last_task = NULL; - do { - op_data.status = internal::WAIT; - op_data.ltask = NULL; - my_aggregator.execute(&op_data); - tbb::task *xtask = op_data.ltask; - last_task = combine_tasks(last_task, xtask); - } while (op_data.status ==internal::SUCCEEDED); - return last_task; - } - - //! Register successor - virtual void internal_reg_succ(buffer_operation *op) { - my_successors.register_successor(*(op->r)); - __TBB_store_with_release(op->status, internal::SUCCEEDED); - } - - //! Remove successor - virtual void internal_rem_succ(buffer_operation *op) { - my_successors.remove_successor(*(op->r)); - __TBB_store_with_release(op->status, internal::SUCCEEDED); - } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - typedef typename sender::built_successors_type built_successors_type; - - built_successors_type &built_successors() __TBB_override { return my_successors.built_successors(); } - - virtual void internal_add_built_succ(buffer_operation *op) { - my_successors.internal_add_built_successor(*(op->r)); - __TBB_store_with_release(op->status, internal::SUCCEEDED); - } - - virtual void internal_del_built_succ(buffer_operation *op) { - my_successors.internal_delete_built_successor(*(op->r)); - __TBB_store_with_release(op->status, internal::SUCCEEDED); - } - - typedef typename receiver::built_predecessors_type built_predecessors_type; - - built_predecessors_type &built_predecessors() __TBB_override { return my_built_predecessors; } - - virtual void internal_add_built_pred(buffer_operation *op) { - my_built_predecessors.add_edge(*(op->p)); - __TBB_store_with_release(op->status, internal::SUCCEEDED); - } - - virtual void internal_del_built_pred(buffer_operation *op) { - my_built_predecessors.delete_edge(*(op->p)); - __TBB_store_with_release(op->status, internal::SUCCEEDED); - } - - virtual void internal_succ_cnt(buffer_operation *op) { - op->cnt_val = my_successors.successor_count(); - __TBB_store_with_release(op->status, internal::SUCCEEDED); - } - - virtual void internal_pred_cnt(buffer_operation *op) { - op->cnt_val = my_built_predecessors.edge_count(); - __TBB_store_with_release(op->status, internal::SUCCEEDED); - } - - virtual void internal_copy_succs(buffer_operation *op) { - my_successors.copy_successors(*(op->svec)); - __TBB_store_with_release(op->status, internal::SUCCEEDED); - } - - virtual void internal_copy_preds(buffer_operation *op) { - my_built_predecessors.copy_edges(*(op->pvec)); - __TBB_store_with_release(op->status, internal::SUCCEEDED); - } - -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ - -private: - void order() {} - - bool is_item_valid() { - return this->my_item_valid(this->my_tail - 1); - } - - void try_put_and_add_task(task*& last_task) { - task *new_task = my_successors.try_put_task(this->back()); - if (new_task) { - last_task = combine_tasks(last_task, new_task); - this->destroy_back(); - } - } - -protected: - //! Tries to forward valid items to successors - virtual void internal_forward_task(buffer_operation *op) { - internal_forward_task_impl(op, this); - } - - template - void internal_forward_task_impl(buffer_operation *op, derived_type* derived) { - __TBB_ASSERT(static_cast(derived) == this, "'this' is not a base class for derived"); - - if (this->my_reserved || !derived->is_item_valid()) { - __TBB_store_with_release(op->status, internal::FAILED); - this->forwarder_busy = false; - return; - } - // Try forwarding, giving each successor a chance - task * last_task = NULL; - size_type counter = my_successors.size(); - for (; counter > 0 && derived->is_item_valid(); --counter) - derived->try_put_and_add_task(last_task); - - op->ltask = last_task; // return task - if (last_task && !counter) { - __TBB_store_with_release(op->status, internal::SUCCEEDED); - } - else { - __TBB_store_with_release(op->status, internal::FAILED); - forwarder_busy = false; - } - } - - virtual bool internal_push(buffer_operation *op) { - this->push_back(*(op->elem)); - __TBB_store_with_release(op->status, internal::SUCCEEDED); - return true; - } - - virtual void internal_pop(buffer_operation *op) { - if(this->pop_back(*(op->elem))) { - __TBB_store_with_release(op->status, internal::SUCCEEDED); - } - else { - __TBB_store_with_release(op->status, internal::FAILED); - } - } - - virtual void internal_reserve(buffer_operation *op) { - if(this->reserve_front(*(op->elem))) { - __TBB_store_with_release(op->status, internal::SUCCEEDED); - } - else { - __TBB_store_with_release(op->status, internal::FAILED); - } - } - - virtual void internal_consume(buffer_operation *op) { - this->consume_front(); - __TBB_store_with_release(op->status, internal::SUCCEEDED); - } - - virtual void internal_release(buffer_operation *op) { - this->release_front(); - __TBB_store_with_release(op->status, internal::SUCCEEDED); - } - -public: - //! Constructor - explicit buffer_node( graph &g ) : graph_node(g), internal::reservable_item_buffer(), - forwarder_busy(false) { - my_successors.set_owner(this); - my_aggregator.initialize_handler(handler_type(this)); - tbb::internal::fgt_node( tbb::internal::FLOW_BUFFER_NODE, &this->my_graph, - static_cast *>(this), static_cast *>(this) ); - } - - //! Copy constructor - buffer_node( const buffer_node& src ) : graph_node(src.my_graph), - internal::reservable_item_buffer(), receiver(), sender() { - forwarder_busy = false; - my_successors.set_owner(this); - my_aggregator.initialize_handler(handler_type(this)); - tbb::internal::fgt_node( tbb::internal::FLOW_BUFFER_NODE, &this->my_graph, - static_cast *>(this), static_cast *>(this) ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif - - // - // message sender implementation - // - - //! Adds a new successor. - /** Adds successor r to the list of successors; may forward tasks. */ - bool register_successor( successor_type &r ) __TBB_override { - buffer_operation op_data(reg_succ); - op_data.r = &r; - my_aggregator.execute(&op_data); - (void)enqueue_forwarding_task(op_data); - return true; - } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - void internal_add_built_successor( successor_type &r) __TBB_override { - buffer_operation op_data(add_blt_succ); - op_data.r = &r; - my_aggregator.execute(&op_data); - } - - void internal_delete_built_successor( successor_type &r) __TBB_override { - buffer_operation op_data(del_blt_succ); - op_data.r = &r; - my_aggregator.execute(&op_data); - } - - void internal_add_built_predecessor( predecessor_type &p) __TBB_override { - buffer_operation op_data(add_blt_pred); - op_data.p = &p; - my_aggregator.execute(&op_data); - } - - void internal_delete_built_predecessor( predecessor_type &p) __TBB_override { - buffer_operation op_data(del_blt_pred); - op_data.p = &p; - my_aggregator.execute(&op_data); - } - - size_t predecessor_count() __TBB_override { - buffer_operation op_data(blt_pred_cnt); - my_aggregator.execute(&op_data); - return op_data.cnt_val; - } - - size_t successor_count() __TBB_override { - buffer_operation op_data(blt_succ_cnt); - my_aggregator.execute(&op_data); - return op_data.cnt_val; - } - - void copy_predecessors( predecessor_list_type &v ) __TBB_override { - buffer_operation op_data(blt_pred_cpy); - op_data.pvec = &v; - my_aggregator.execute(&op_data); - } - - void copy_successors( successor_list_type &v ) __TBB_override { - buffer_operation op_data(blt_succ_cpy); - op_data.svec = &v; - my_aggregator.execute(&op_data); - } - -#endif - - //! Removes a successor. - /** Removes successor r from the list of successors. - It also calls r.remove_predecessor(*this) to remove this node as a predecessor. */ - bool remove_successor( successor_type &r ) __TBB_override { - r.remove_predecessor(*this); - buffer_operation op_data(rem_succ); - op_data.r = &r; - my_aggregator.execute(&op_data); - // even though this operation does not cause a forward, if we are the handler, and - // a forward is scheduled, we may be the first to reach this point after the aggregator, - // and so should check for the task. - (void)enqueue_forwarding_task(op_data); - return true; - } - - //! Request an item from the buffer_node - /** true = v contains the returned item
- false = no item has been returned */ - bool try_get( T &v ) __TBB_override { - buffer_operation op_data(req_item); - op_data.elem = &v; - my_aggregator.execute(&op_data); - (void)enqueue_forwarding_task(op_data); - return (op_data.status==internal::SUCCEEDED); - } - - //! Reserves an item. - /** false = no item can be reserved
- true = an item is reserved */ - bool try_reserve( T &v ) __TBB_override { - buffer_operation op_data(res_item); - op_data.elem = &v; - my_aggregator.execute(&op_data); - (void)enqueue_forwarding_task(op_data); - return (op_data.status==internal::SUCCEEDED); - } - - //! Release a reserved item. - /** true = item has been released and so remains in sender */ - bool try_release() __TBB_override { - buffer_operation op_data(rel_res); - my_aggregator.execute(&op_data); - (void)enqueue_forwarding_task(op_data); - return true; - } - - //! Consumes a reserved item. - /** true = item is removed from sender and reservation removed */ - bool try_consume() __TBB_override { - buffer_operation op_data(con_res); - my_aggregator.execute(&op_data); - (void)enqueue_forwarding_task(op_data); - return true; - } - -protected: - - template< typename R, typename B > friend class run_and_put_task; - template friend class internal::broadcast_cache; - template friend class internal::round_robin_cache; - //! receive an item, return a task *if possible - task *try_put_task(const T &t) __TBB_override { - buffer_operation op_data(t, put_item); - my_aggregator.execute(&op_data); - task *ft = grab_forwarding_task(op_data); - // sequencer_nodes can return failure (if an item has been previously inserted) - // We have to spawn the returned task if our own operation fails. - - if(ft && op_data.status ==internal::FAILED) { - // we haven't succeeded queueing the item, but for some reason the - // call returned a task (if another request resulted in a successful - // forward this could happen.) Queue the task and reset the pointer. - FLOW_SPAWN(*ft); ft = NULL; - } - else if(!ft && op_data.status ==internal::SUCCEEDED) { - ft = SUCCESSFULLY_ENQUEUED; - } - return ft; - } - - void reset_receiver(reset_flags /*f*/) __TBB_override { } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES -public: - void extract() __TBB_override { - my_built_predecessors.receiver_extract(*this); - my_successors.built_successors().sender_extract(*this); - } -#endif - -protected: - void reset_node( reset_flags f) __TBB_override { - internal::reservable_item_buffer::reset(); - // TODO: just clear structures - if (f&rf_clear_edges) { - my_successors.clear(); -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - my_built_predecessors.clear(); -#endif - } - forwarder_busy = false; - } -}; // buffer_node - -//! Forwards messages in FIFO order -template > -class queue_node : public buffer_node { -protected: - typedef buffer_node base_type; - typedef typename base_type::size_type size_type; - typedef typename base_type::buffer_operation queue_operation; - typedef queue_node class_type; - -private: - template friend class buffer_node; - - bool is_item_valid() { - return this->my_item_valid(this->my_head); - } - - void try_put_and_add_task(task*& last_task) { - task *new_task = this->my_successors.try_put_task(this->front()); - if (new_task) { - last_task = combine_tasks(last_task, new_task); - this->destroy_front(); - } - } - -protected: - void internal_forward_task(queue_operation *op) __TBB_override { - this->internal_forward_task_impl(op, this); - } - - void internal_pop(queue_operation *op) __TBB_override { - if ( this->my_reserved || !this->my_item_valid(this->my_head)){ - __TBB_store_with_release(op->status, internal::FAILED); - } - else { - this->pop_front(*(op->elem)); - __TBB_store_with_release(op->status, internal::SUCCEEDED); - } - } - void internal_reserve(queue_operation *op) __TBB_override { - if (this->my_reserved || !this->my_item_valid(this->my_head)) { - __TBB_store_with_release(op->status, internal::FAILED); - } - else { - this->reserve_front(*(op->elem)); - __TBB_store_with_release(op->status, internal::SUCCEEDED); - } - } - void internal_consume(queue_operation *op) __TBB_override { - this->consume_front(); - __TBB_store_with_release(op->status, internal::SUCCEEDED); - } - -public: - typedef T input_type; - typedef T output_type; - typedef typename receiver::predecessor_type predecessor_type; - typedef typename sender::successor_type successor_type; - - //! Constructor - explicit queue_node( graph &g ) : base_type(g) { - tbb::internal::fgt_node( tbb::internal::FLOW_QUEUE_NODE, &(this->my_graph), - static_cast *>(this), - static_cast *>(this) ); - } - - //! Copy constructor - queue_node( const queue_node& src) : base_type(src) { - tbb::internal::fgt_node( tbb::internal::FLOW_QUEUE_NODE, &(this->my_graph), - static_cast *>(this), - static_cast *>(this) ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif - -protected: - void reset_node( reset_flags f) __TBB_override { - base_type::reset_node(f); - } -}; // queue_node - -//! Forwards messages in sequence order -template< typename T, typename A=cache_aligned_allocator > -class sequencer_node : public queue_node { - internal::function_body< T, size_t > *my_sequencer; - // my_sequencer should be a benign function and must be callable - // from a parallel context. Does this mean it needn't be reset? -public: - typedef T input_type; - typedef T output_type; - typedef typename receiver::predecessor_type predecessor_type; - typedef typename sender::successor_type successor_type; - - //! Constructor - template< typename Sequencer > - sequencer_node( graph &g, const Sequencer& s ) : queue_node(g), - my_sequencer(new internal::function_body_leaf< T, size_t, Sequencer>(s) ) { - tbb::internal::fgt_node( tbb::internal::FLOW_SEQUENCER_NODE, &(this->my_graph), - static_cast *>(this), - static_cast *>(this) ); - } - - //! Copy constructor - sequencer_node( const sequencer_node& src ) : queue_node(src), - my_sequencer( src.my_sequencer->clone() ) { - tbb::internal::fgt_node( tbb::internal::FLOW_SEQUENCER_NODE, &(this->my_graph), - static_cast *>(this), - static_cast *>(this) ); - } - - //! Destructor - ~sequencer_node() { delete my_sequencer; } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif - -protected: - typedef typename buffer_node::size_type size_type; - typedef typename buffer_node::buffer_operation sequencer_operation; - -private: - bool internal_push(sequencer_operation *op) __TBB_override { - size_type tag = (*my_sequencer)(*(op->elem)); -#if !TBB_DEPRECATED_SEQUENCER_DUPLICATES - if (tag < this->my_head) { - // have already emitted a message with this tag - __TBB_store_with_release(op->status, internal::FAILED); - return false; - } -#endif - // cannot modify this->my_tail now; the buffer would be inconsistent. - size_t new_tail = (tag+1 > this->my_tail) ? tag+1 : this->my_tail; - - if (this->size(new_tail) > this->capacity()) { - this->grow_my_array(this->size(new_tail)); - } - this->my_tail = new_tail; - - const internal::op_stat res = this->place_item(tag, *(op->elem)) ? internal::SUCCEEDED : internal::FAILED; - __TBB_store_with_release(op->status, res); - return res ==internal::SUCCEEDED; - } -}; // sequencer_node - -//! Forwards messages in priority order -template< typename T, typename Compare = std::less, typename A=cache_aligned_allocator > -class priority_queue_node : public buffer_node { -public: - typedef T input_type; - typedef T output_type; - typedef buffer_node base_type; - typedef priority_queue_node class_type; - typedef typename receiver::predecessor_type predecessor_type; - typedef typename sender::successor_type successor_type; - - //! Constructor - explicit priority_queue_node( graph &g ) : buffer_node(g), mark(0) { - tbb::internal::fgt_node( tbb::internal::FLOW_PRIORITY_QUEUE_NODE, &(this->my_graph), - static_cast *>(this), - static_cast *>(this) ); - } - - //! Copy constructor - priority_queue_node( const priority_queue_node &src ) : buffer_node(src), mark(0) { - tbb::internal::fgt_node( tbb::internal::FLOW_PRIORITY_QUEUE_NODE, &(this->my_graph), - static_cast *>(this), - static_cast *>(this) ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif - -protected: - - void reset_node( reset_flags f) __TBB_override { - mark = 0; - base_type::reset_node(f); - } - - typedef typename buffer_node::size_type size_type; - typedef typename buffer_node::item_type item_type; - typedef typename buffer_node::buffer_operation prio_operation; - - //! Tries to forward valid items to successors - void internal_forward_task(prio_operation *op) __TBB_override { - this->internal_forward_task_impl(op, this); - } - - void handle_operations(prio_operation *op_list) __TBB_override { - this->handle_operations_impl(op_list, this); - } - - bool internal_push(prio_operation *op) __TBB_override { - prio_push(*(op->elem)); - __TBB_store_with_release(op->status, internal::SUCCEEDED); - return true; - } - - void internal_pop(prio_operation *op) __TBB_override { - // if empty or already reserved, don't pop - if ( this->my_reserved == true || this->my_tail == 0 ) { - __TBB_store_with_release(op->status, internal::FAILED); - return; - } - - *(op->elem) = prio(); - __TBB_store_with_release(op->status, internal::SUCCEEDED); - prio_pop(); - - } - - // pops the highest-priority item, saves copy - void internal_reserve(prio_operation *op) __TBB_override { - if (this->my_reserved == true || this->my_tail == 0) { - __TBB_store_with_release(op->status, internal::FAILED); - return; - } - this->my_reserved = true; - *(op->elem) = prio(); - reserved_item = *(op->elem); - __TBB_store_with_release(op->status, internal::SUCCEEDED); - prio_pop(); - } - - void internal_consume(prio_operation *op) __TBB_override { - __TBB_store_with_release(op->status, internal::SUCCEEDED); - this->my_reserved = false; - reserved_item = input_type(); - } - - void internal_release(prio_operation *op) __TBB_override { - __TBB_store_with_release(op->status, internal::SUCCEEDED); - prio_push(reserved_item); - this->my_reserved = false; - reserved_item = input_type(); - } - -private: - template friend class buffer_node; - - void order() { - if (mark < this->my_tail) heapify(); - __TBB_ASSERT(mark == this->my_tail, "mark unequal after heapify"); - } - - bool is_item_valid() { - return this->my_tail > 0; - } - - void try_put_and_add_task(task*& last_task) { - task * new_task = this->my_successors.try_put_task(this->prio()); - if (new_task) { - last_task = combine_tasks(last_task, new_task); - prio_pop(); - } - } - -private: - Compare compare; - size_type mark; - - input_type reserved_item; - - // in case a reheap has not been done after a push, check if the mark item is higher than the 0'th item - bool prio_use_tail() { - __TBB_ASSERT(mark <= this->my_tail, "mark outside bounds before test"); - return mark < this->my_tail && compare(this->get_my_item(0), this->get_my_item(this->my_tail - 1)); - } - - // prio_push: checks that the item will fit, expand array if necessary, put at end - void prio_push(const T &src) { - if ( this->my_tail >= this->my_array_size ) - this->grow_my_array( this->my_tail + 1 ); - (void) this->place_item(this->my_tail, src); - ++(this->my_tail); - __TBB_ASSERT(mark < this->my_tail, "mark outside bounds after push"); - } - - // prio_pop: deletes highest priority item from the array, and if it is item - // 0, move last item to 0 and reheap. If end of array, just destroy and decrement tail - // and mark. Assumes the array has already been tested for emptiness; no failure. - void prio_pop() { - if (prio_use_tail()) { - // there are newly pushed elements; last one higher than top - // copy the data - this->destroy_item(this->my_tail-1); - --(this->my_tail); - __TBB_ASSERT(mark <= this->my_tail, "mark outside bounds after pop"); - return; - } - this->destroy_item(0); - if(this->my_tail > 1) { - // push the last element down heap - __TBB_ASSERT(this->my_item_valid(this->my_tail - 1), NULL); - this->move_item(0,this->my_tail - 1); - } - --(this->my_tail); - if(mark > this->my_tail) --mark; - if (this->my_tail > 1) // don't reheap for heap of size 1 - reheap(); - __TBB_ASSERT(mark <= this->my_tail, "mark outside bounds after pop"); - } - - const T& prio() { - return this->get_my_item(prio_use_tail() ? this->my_tail-1 : 0); - } - - // turn array into heap - void heapify() { - if(this->my_tail == 0) { - mark = 0; - return; - } - if (!mark) mark = 1; - for (; markmy_tail; ++mark) { // for each unheaped element - size_type cur_pos = mark; - input_type to_place; - this->fetch_item(mark,to_place); - do { // push to_place up the heap - size_type parent = (cur_pos-1)>>1; - if (!compare(this->get_my_item(parent), to_place)) - break; - this->move_item(cur_pos, parent); - cur_pos = parent; - } while( cur_pos ); - (void) this->place_item(cur_pos, to_place); - } - } - - // otherwise heapified array with new root element; rearrange to heap - void reheap() { - size_type cur_pos=0, child=1; - while (child < mark) { - size_type target = child; - if (child+1get_my_item(child), - this->get_my_item(child+1))) - ++target; - // target now has the higher priority child - if (compare(this->get_my_item(target), - this->get_my_item(cur_pos))) - break; - // swap - this->swap_items(cur_pos, target); - cur_pos = target; - child = (cur_pos<<1)+1; - } - } -}; // priority_queue_node - -//! Forwards messages only if the threshold has not been reached -/** This node forwards items until its threshold is reached. - It contains no buffering. If the downstream node rejects, the - message is dropped. */ -template< typename T > -class limiter_node : public graph_node, public receiver< T >, public sender< T > { -public: - typedef T input_type; - typedef T output_type; - typedef typename receiver::predecessor_type predecessor_type; - typedef typename sender::successor_type successor_type; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - typedef typename receiver::built_predecessors_type built_predecessors_type; - typedef typename sender::built_successors_type built_successors_type; - typedef typename receiver::predecessor_list_type predecessor_list_type; - typedef typename sender::successor_list_type successor_list_type; -#endif - //TODO: There is a lack of predefined types for its controlling "decrementer" port. It should be fixed later. - -private: - size_t my_threshold; - size_t my_count; //number of successful puts - size_t my_tries; //number of active put attempts - internal::reservable_predecessor_cache< T, spin_mutex > my_predecessors; - spin_mutex my_mutex; - internal::broadcast_cache< T > my_successors; - int init_decrement_predecessors; - - friend class internal::forward_task_bypass< limiter_node >; - - // Let decrementer call decrement_counter() - friend class internal::decrementer< limiter_node >; - - bool check_conditions() { // always called under lock - return ( my_count + my_tries < my_threshold && !my_predecessors.empty() && !my_successors.empty() ); - } - - // only returns a valid task pointer or NULL, never SUCCESSFULLY_ENQUEUED - task *forward_task() { - input_type v; - task *rval = NULL; - bool reserved = false; - { - spin_mutex::scoped_lock lock(my_mutex); - if ( check_conditions() ) - ++my_tries; - else - return NULL; - } - - //SUCCESS - // if we can reserve and can put, we consume the reservation - // we increment the count and decrement the tries - if ( (my_predecessors.try_reserve(v)) == true ){ - reserved=true; - if ( (rval = my_successors.try_put_task(v)) != NULL ){ - { - spin_mutex::scoped_lock lock(my_mutex); - ++my_count; - --my_tries; - my_predecessors.try_consume(); - if ( check_conditions() ) { - if ( this->my_graph.is_active() ) { - task *rtask = new ( task::allocate_additional_child_of( *(this->my_graph.root_task()) ) ) - internal::forward_task_bypass< limiter_node >( *this ); - FLOW_SPAWN (*rtask); - } - } - } - return rval; - } - } - //FAILURE - //if we can't reserve, we decrement the tries - //if we can reserve but can't put, we decrement the tries and release the reservation - { - spin_mutex::scoped_lock lock(my_mutex); - --my_tries; - if (reserved) my_predecessors.try_release(); - if ( check_conditions() ) { - if ( this->my_graph.is_active() ) { - task *rtask = new ( task::allocate_additional_child_of( *(this->my_graph.root_task()) ) ) - internal::forward_task_bypass< limiter_node >( *this ); - __TBB_ASSERT(!rval, "Have two tasks to handle"); - return rtask; - } - } - return rval; - } - } - - void forward() { - __TBB_ASSERT(false, "Should never be called"); - return; - } - - task * decrement_counter() { - { - spin_mutex::scoped_lock lock(my_mutex); - if(my_count) --my_count; - } - return forward_task(); - } - -public: - //! The internal receiver< continue_msg > that decrements the count - internal::decrementer< limiter_node > decrement; - - //! Constructor - limiter_node(graph &g, size_t threshold, int num_decrement_predecessors=0) : - graph_node(g), my_threshold(threshold), my_count(0), my_tries(0), - init_decrement_predecessors(num_decrement_predecessors), - decrement(num_decrement_predecessors) - { - my_predecessors.set_owner(this); - my_successors.set_owner(this); - decrement.set_owner(this); - tbb::internal::fgt_node( tbb::internal::FLOW_LIMITER_NODE, &this->my_graph, - static_cast *>(this), static_cast *>(&decrement), - static_cast *>(this) ); - } - - //! Copy constructor - limiter_node( const limiter_node& src ) : - graph_node(src.my_graph), receiver(), sender(), - my_threshold(src.my_threshold), my_count(0), my_tries(0), - init_decrement_predecessors(src.init_decrement_predecessors), - decrement(src.init_decrement_predecessors) - { - my_predecessors.set_owner(this); - my_successors.set_owner(this); - decrement.set_owner(this); - tbb::internal::fgt_node( tbb::internal::FLOW_LIMITER_NODE, &this->my_graph, - static_cast *>(this), static_cast *>(&decrement), - static_cast *>(this) ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif - - //! Replace the current successor with this new successor - bool register_successor( successor_type &r ) __TBB_override { - spin_mutex::scoped_lock lock(my_mutex); - bool was_empty = my_successors.empty(); - my_successors.register_successor(r); - //spawn a forward task if this is the only successor - if ( was_empty && !my_predecessors.empty() && my_count + my_tries < my_threshold ) { - if ( this->my_graph.is_active() ) { - FLOW_SPAWN( (* new ( task::allocate_additional_child_of( *(this->my_graph.root_task()) ) ) - internal::forward_task_bypass < limiter_node >( *this ) ) ); - } - } - return true; - } - - //! Removes a successor from this node - /** r.remove_predecessor(*this) is also called. */ - bool remove_successor( successor_type &r ) __TBB_override { - r.remove_predecessor(*this); - my_successors.remove_successor(r); - return true; - } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - built_successors_type &built_successors() __TBB_override { return my_successors.built_successors(); } - built_predecessors_type &built_predecessors() __TBB_override { return my_predecessors.built_predecessors(); } - - void internal_add_built_successor(successor_type &src) __TBB_override { - my_successors.internal_add_built_successor(src); - } - - void internal_delete_built_successor(successor_type &src) __TBB_override { - my_successors.internal_delete_built_successor(src); - } - - size_t successor_count() __TBB_override { return my_successors.successor_count(); } - - void copy_successors(successor_list_type &v) __TBB_override { - my_successors.copy_successors(v); - } - - void internal_add_built_predecessor(predecessor_type &src) __TBB_override { - my_predecessors.internal_add_built_predecessor(src); - } - - void internal_delete_built_predecessor(predecessor_type &src) __TBB_override { - my_predecessors.internal_delete_built_predecessor(src); - } - - size_t predecessor_count() __TBB_override { return my_predecessors.predecessor_count(); } - - void copy_predecessors(predecessor_list_type &v) __TBB_override { - my_predecessors.copy_predecessors(v); - } - - void extract() __TBB_override { - my_count = 0; - my_successors.built_successors().sender_extract(*this); - my_predecessors.built_predecessors().receiver_extract(*this); - decrement.built_predecessors().receiver_extract(decrement); - } -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ - - //! Adds src to the list of cached predecessors. - bool register_predecessor( predecessor_type &src ) __TBB_override { - spin_mutex::scoped_lock lock(my_mutex); - my_predecessors.add( src ); - if ( my_count + my_tries < my_threshold && !my_successors.empty() && this->my_graph.is_active() ) { - FLOW_SPAWN( (* new ( task::allocate_additional_child_of( *(this->my_graph.root_task()) ) ) - internal::forward_task_bypass < limiter_node >( *this ) ) ); - } - return true; - } - - //! Removes src from the list of cached predecessors. - bool remove_predecessor( predecessor_type &src ) __TBB_override { - my_predecessors.remove( src ); - return true; - } - -protected: - - template< typename R, typename B > friend class run_and_put_task; - template friend class internal::broadcast_cache; - template friend class internal::round_robin_cache; - //! Puts an item to this receiver - task *try_put_task( const T &t ) __TBB_override { - { - spin_mutex::scoped_lock lock(my_mutex); - if ( my_count + my_tries >= my_threshold ) - return NULL; - else - ++my_tries; - } - - task * rtask = my_successors.try_put_task(t); - - if ( !rtask ) { // try_put_task failed. - spin_mutex::scoped_lock lock(my_mutex); - --my_tries; - if ( check_conditions() && this->my_graph.is_active() ) { - rtask = new ( task::allocate_additional_child_of( *(this->my_graph.root_task()) ) ) - internal::forward_task_bypass< limiter_node >( *this ); - } - } - else { - spin_mutex::scoped_lock lock(my_mutex); - ++my_count; - --my_tries; - } - return rtask; - } - - void reset_receiver(reset_flags /*f*/) __TBB_override { - __TBB_ASSERT(false,NULL); // should never be called - } - - void reset_node( reset_flags f) __TBB_override { - my_count = 0; - if(f & rf_clear_edges) { - my_predecessors.clear(); - my_successors.clear(); - } - else - { - my_predecessors.reset( ); - } - decrement.reset_receiver(f); - } -}; // limiter_node - -#include "internal/_flow_graph_join_impl.h" - -using internal::reserving_port; -using internal::queueing_port; -using internal::key_matching_port; -using internal::input_port; -using internal::tag_value; - -template class join_node; - -template -class join_node: public internal::unfolded_join_node::value, reserving_port, OutputTuple, reserving> { -private: - static const int N = tbb::flow::tuple_size::value; - typedef typename internal::unfolded_join_node unfolded_type; -public: - typedef OutputTuple output_type; - typedef typename unfolded_type::input_ports_type input_ports_type; - explicit join_node(graph &g) : unfolded_type(g) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_JOIN_NODE_RESERVING, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - join_node(const join_node &other) : unfolded_type(other) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_JOIN_NODE_RESERVING, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif - -}; - -template -class join_node: public internal::unfolded_join_node::value, queueing_port, OutputTuple, queueing> { -private: - static const int N = tbb::flow::tuple_size::value; - typedef typename internal::unfolded_join_node unfolded_type; -public: - typedef OutputTuple output_type; - typedef typename unfolded_type::input_ports_type input_ports_type; - explicit join_node(graph &g) : unfolded_type(g) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_JOIN_NODE_QUEUEING, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - join_node(const join_node &other) : unfolded_type(other) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_JOIN_NODE_QUEUEING, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif - -}; - -// template for key_matching join_node -// tag_matching join_node is a specialization of key_matching, and is source-compatible. -template -class join_node > : public internal::unfolded_join_node::value, - key_matching_port, OutputTuple, key_matching > { -private: - static const int N = tbb::flow::tuple_size::value; - typedef typename internal::unfolded_join_node > unfolded_type; -public: - typedef OutputTuple output_type; - typedef typename unfolded_type::input_ports_type input_ports_type; - -#if __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING - join_node(graph &g) : unfolded_type(g) {} -#endif /* __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING */ - - template - join_node(graph &g, __TBB_B0 b0, __TBB_B1 b1) : unfolded_type(g, b0, b1) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_JOIN_NODE_TAG_MATCHING, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - template - join_node(graph &g, __TBB_B0 b0, __TBB_B1 b1, __TBB_B2 b2) : unfolded_type(g, b0, b1, b2) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_JOIN_NODE_TAG_MATCHING, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - template - join_node(graph &g, __TBB_B0 b0, __TBB_B1 b1, __TBB_B2 b2, __TBB_B3 b3) : unfolded_type(g, b0, b1, b2, b3) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_JOIN_NODE_TAG_MATCHING, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - template - join_node(graph &g, __TBB_B0 b0, __TBB_B1 b1, __TBB_B2 b2, __TBB_B3 b3, __TBB_B4 b4) : - unfolded_type(g, b0, b1, b2, b3, b4) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_JOIN_NODE_TAG_MATCHING, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } -#if __TBB_VARIADIC_MAX >= 6 - template - join_node(graph &g, __TBB_B0 b0, __TBB_B1 b1, __TBB_B2 b2, __TBB_B3 b3, __TBB_B4 b4, __TBB_B5 b5) : - unfolded_type(g, b0, b1, b2, b3, b4, b5) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_JOIN_NODE_TAG_MATCHING, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } -#endif -#if __TBB_VARIADIC_MAX >= 7 - template - join_node(graph &g, __TBB_B0 b0, __TBB_B1 b1, __TBB_B2 b2, __TBB_B3 b3, __TBB_B4 b4, __TBB_B5 b5, __TBB_B6 b6) : - unfolded_type(g, b0, b1, b2, b3, b4, b5, b6) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_JOIN_NODE_TAG_MATCHING, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } -#endif -#if __TBB_VARIADIC_MAX >= 8 - template - join_node(graph &g, __TBB_B0 b0, __TBB_B1 b1, __TBB_B2 b2, __TBB_B3 b3, __TBB_B4 b4, __TBB_B5 b5, __TBB_B6 b6, - __TBB_B7 b7) : unfolded_type(g, b0, b1, b2, b3, b4, b5, b6, b7) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_JOIN_NODE_TAG_MATCHING, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } -#endif -#if __TBB_VARIADIC_MAX >= 9 - template - join_node(graph &g, __TBB_B0 b0, __TBB_B1 b1, __TBB_B2 b2, __TBB_B3 b3, __TBB_B4 b4, __TBB_B5 b5, __TBB_B6 b6, - __TBB_B7 b7, __TBB_B8 b8) : unfolded_type(g, b0, b1, b2, b3, b4, b5, b6, b7, b8) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_JOIN_NODE_TAG_MATCHING, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } -#endif -#if __TBB_VARIADIC_MAX >= 10 - template - join_node(graph &g, __TBB_B0 b0, __TBB_B1 b1, __TBB_B2 b2, __TBB_B3 b3, __TBB_B4 b4, __TBB_B5 b5, __TBB_B6 b6, - __TBB_B7 b7, __TBB_B8 b8, __TBB_B9 b9) : unfolded_type(g, b0, b1, b2, b3, b4, b5, b6, b7, b8, b9) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_JOIN_NODE_TAG_MATCHING, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } -#endif - join_node(const join_node &other) : unfolded_type(other) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_JOIN_NODE_TAG_MATCHING, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif - -}; - -// indexer node -#include "internal/_flow_graph_indexer_impl.h" - -// TODO: Implement interface with variadic template or tuple -template class indexer_node; - -//indexer node specializations -template -class indexer_node : public internal::unfolded_indexer_node > { -private: - static const int N = 1; -public: - typedef tuple InputTuple; - typedef typename internal::tagged_msg output_type; - typedef typename internal::unfolded_indexer_node unfolded_type; - indexer_node(graph& g) : unfolded_type(g) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_INDEXER_NODE, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - // Copy constructor - indexer_node( const indexer_node& other ) : unfolded_type(other) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_INDEXER_NODE, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) { - tbb::internal::fgt_node_desc( this, name ); - } -#endif -}; - -template -class indexer_node : public internal::unfolded_indexer_node > { -private: - static const int N = 2; -public: - typedef tuple InputTuple; - typedef typename internal::tagged_msg output_type; - typedef typename internal::unfolded_indexer_node unfolded_type; - indexer_node(graph& g) : unfolded_type(g) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_INDEXER_NODE, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - // Copy constructor - indexer_node( const indexer_node& other ) : unfolded_type(other) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_INDEXER_NODE, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) { - tbb::internal::fgt_node_desc( this, name ); - } -#endif -}; - -template -class indexer_node : public internal::unfolded_indexer_node > { -private: - static const int N = 3; -public: - typedef tuple InputTuple; - typedef typename internal::tagged_msg output_type; - typedef typename internal::unfolded_indexer_node unfolded_type; - indexer_node(graph& g) : unfolded_type(g) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_INDEXER_NODE, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - // Copy constructor - indexer_node( const indexer_node& other ) : unfolded_type(other) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_INDEXER_NODE, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) { - tbb::internal::fgt_node_desc( this, name ); - } -#endif -}; - -template -class indexer_node : public internal::unfolded_indexer_node > { -private: - static const int N = 4; -public: - typedef tuple InputTuple; - typedef typename internal::tagged_msg output_type; - typedef typename internal::unfolded_indexer_node unfolded_type; - indexer_node(graph& g) : unfolded_type(g) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_INDEXER_NODE, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - // Copy constructor - indexer_node( const indexer_node& other ) : unfolded_type(other) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_INDEXER_NODE, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif -}; - -template -class indexer_node : public internal::unfolded_indexer_node > { -private: - static const int N = 5; -public: - typedef tuple InputTuple; - typedef typename internal::tagged_msg output_type; - typedef typename internal::unfolded_indexer_node unfolded_type; - indexer_node(graph& g) : unfolded_type(g) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_INDEXER_NODE, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - // Copy constructor - indexer_node( const indexer_node& other ) : unfolded_type(other) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_INDEXER_NODE, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif -}; - -#if __TBB_VARIADIC_MAX >= 6 -template -class indexer_node : public internal::unfolded_indexer_node > { -private: - static const int N = 6; -public: - typedef tuple InputTuple; - typedef typename internal::tagged_msg output_type; - typedef typename internal::unfolded_indexer_node unfolded_type; - indexer_node(graph& g) : unfolded_type(g) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_INDEXER_NODE, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - // Copy constructor - indexer_node( const indexer_node& other ) : unfolded_type(other) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_INDEXER_NODE, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif -}; -#endif //variadic max 6 - -#if __TBB_VARIADIC_MAX >= 7 -template -class indexer_node : public internal::unfolded_indexer_node > { -private: - static const int N = 7; -public: - typedef tuple InputTuple; - typedef typename internal::tagged_msg output_type; - typedef typename internal::unfolded_indexer_node unfolded_type; - indexer_node(graph& g) : unfolded_type(g) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_INDEXER_NODE, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - // Copy constructor - indexer_node( const indexer_node& other ) : unfolded_type(other) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_INDEXER_NODE, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif -}; -#endif //variadic max 7 - -#if __TBB_VARIADIC_MAX >= 8 -template -class indexer_node : public internal::unfolded_indexer_node > { -private: - static const int N = 8; -public: - typedef tuple InputTuple; - typedef typename internal::tagged_msg output_type; - typedef typename internal::unfolded_indexer_node unfolded_type; - indexer_node(graph& g) : unfolded_type(g) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_INDEXER_NODE, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - // Copy constructor - indexer_node( const indexer_node& other ) : unfolded_type(other) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_INDEXER_NODE, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif -}; -#endif //variadic max 8 - -#if __TBB_VARIADIC_MAX >= 9 -template -class indexer_node : public internal::unfolded_indexer_node > { -private: - static const int N = 9; -public: - typedef tuple InputTuple; - typedef typename internal::tagged_msg output_type; - typedef typename internal::unfolded_indexer_node unfolded_type; - indexer_node(graph& g) : unfolded_type(g) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_INDEXER_NODE, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - // Copy constructor - indexer_node( const indexer_node& other ) : unfolded_type(other) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_INDEXER_NODE, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif -}; -#endif //variadic max 9 - -#if __TBB_VARIADIC_MAX >= 10 -template -class indexer_node/*default*/ : public internal::unfolded_indexer_node > { -private: - static const int N = 10; -public: - typedef tuple InputTuple; - typedef typename internal::tagged_msg output_type; - typedef typename internal::unfolded_indexer_node unfolded_type; - indexer_node(graph& g) : unfolded_type(g) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_INDEXER_NODE, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - // Copy constructor - indexer_node( const indexer_node& other ) : unfolded_type(other) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_INDEXER_NODE, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif -}; -#endif //variadic max 10 - -#if __TBB_PREVIEW_ASYNC_MSG -inline void internal_make_edge( internal::untyped_sender &p, internal::untyped_receiver &s ) { -#else -template< typename T > -inline void internal_make_edge( sender &p, receiver &s ) { -#endif -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - s.internal_add_built_predecessor(p); - p.internal_add_built_successor(s); -#endif - p.register_successor( s ); - tbb::internal::fgt_make_edge( &p, &s ); -} - -//! Makes an edge between a single predecessor and a single successor -template< typename T > -inline void make_edge( sender &p, receiver &s ) { - internal_make_edge( p, s ); -} - -#if __TBB_PREVIEW_ASYNC_MSG -template< typename TS, typename TR, - typename = typename tbb::internal::enable_if::value - || tbb::internal::is_same_type::value>::type> -inline void make_edge( TS &p, TR &s ) { - internal_make_edge( p, s ); -} - -template< typename T > -inline void make_edge( sender &p, receiver &s ) { - internal_make_edge( p, s ); -} - -template< typename T > -inline void make_edge( sender &p, receiver &s ) { - internal_make_edge( p, s ); -} - -#endif // __TBB_PREVIEW_ASYNC_MSG - -#if __TBB_FLOW_GRAPH_CPP11_FEATURES -//Makes an edge from port 0 of a multi-output predecessor to port 0 of a multi-input successor. -template< typename T, typename V, - typename = typename T::output_ports_type, typename = typename V::input_ports_type > -inline void make_edge( T& output, V& input) { - make_edge(get<0>(output.output_ports()), get<0>(input.input_ports())); -} - -//Makes an edge from port 0 of a multi-output predecessor to a receiver. -template< typename T, typename R, - typename = typename T::output_ports_type > -inline void make_edge( T& output, receiver& input) { - make_edge(get<0>(output.output_ports()), input); -} - -//Makes an edge from a sender to port 0 of a multi-input successor. -template< typename S, typename V, - typename = typename V::input_ports_type > -inline void make_edge( sender& output, V& input) { - make_edge(output, get<0>(input.input_ports())); -} -#endif - -#if __TBB_PREVIEW_ASYNC_MSG -inline void internal_remove_edge( internal::untyped_sender &p, internal::untyped_receiver &s ) { -#else -template< typename T > -inline void internal_remove_edge( sender &p, receiver &s ) { -#endif - p.remove_successor( s ); -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - // TODO: should we try to remove p from the predecessor list of s, in case the edge is reversed? - p.internal_delete_built_successor(s); - s.internal_delete_built_predecessor(p); -#endif - tbb::internal::fgt_remove_edge( &p, &s ); -} - -//! Removes an edge between a single predecessor and a single successor -template< typename T > -inline void remove_edge( sender &p, receiver &s ) { - internal_remove_edge( p, s ); -} - -#if __TBB_PREVIEW_ASYNC_MSG -template< typename TS, typename TR, - typename = typename tbb::internal::enable_if::value - || tbb::internal::is_same_type::value>::type> -inline void remove_edge( TS &p, TR &s ) { - internal_remove_edge( p, s ); -} - -template< typename T > -inline void remove_edge( sender &p, receiver &s ) { - internal_remove_edge( p, s ); -} - -template< typename T > -inline void remove_edge( sender &p, receiver &s ) { - internal_remove_edge( p, s ); -} -#endif // __TBB_PREVIEW_ASYNC_MSG - -#if __TBB_FLOW_GRAPH_CPP11_FEATURES -//Removes an edge between port 0 of a multi-output predecessor and port 0 of a multi-input successor. -template< typename T, typename V, - typename = typename T::output_ports_type, typename = typename V::input_ports_type > -inline void remove_edge( T& output, V& input) { - remove_edge(get<0>(output.output_ports()), get<0>(input.input_ports())); -} - -//Removes an edge between port 0 of a multi-output predecessor and a receiver. -template< typename T, typename R, - typename = typename T::output_ports_type > -inline void remove_edge( T& output, receiver& input) { - remove_edge(get<0>(output.output_ports()), input); -} -//Removes an edge between a sender and port 0 of a multi-input successor. -template< typename S, typename V, - typename = typename V::input_ports_type > -inline void remove_edge( sender& output, V& input) { - remove_edge(output, get<0>(input.input_ports())); -} -#endif - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES -template -template< typename S > -void internal::edge_container::sender_extract( S &s ) { - edge_list_type e = built_edges; - for ( typename edge_list_type::iterator i = e.begin(); i != e.end(); ++i ) { - remove_edge(s, **i); - } -} - -template -template< typename R > -void internal::edge_container::receiver_extract( R &r ) { - edge_list_type e = built_edges; - for ( typename edge_list_type::iterator i = e.begin(); i != e.end(); ++i ) { - remove_edge(**i, r); - } -} -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ - -//! Returns a copy of the body from a function or continue node -template< typename Body, typename Node > -Body copy_body( Node &n ) { - return n.template copy_function_object(); -} - -#if __TBB_FLOW_GRAPH_CPP11_FEATURES - -//composite_node -template< typename InputTuple, typename OutputTuple > class composite_node; - -template< typename... InputTypes, typename... OutputTypes> -class composite_node , tbb::flow::tuple > : public graph_node{ - -public: - typedef tbb::flow::tuple< receiver&... > input_ports_type; - typedef tbb::flow::tuple< sender&... > output_ports_type; - -private: -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - const char *my_type_name; -#endif - input_ports_type *my_input_ports; - output_ports_type *my_output_ports; - - static const size_t NUM_INPUTS = sizeof...(InputTypes); - static const size_t NUM_OUTPUTS = sizeof...(OutputTypes); - -protected: - void reset_node(reset_flags) __TBB_override {} - -public: -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - composite_node( graph &g, const char *type_name = "composite_node") : graph_node(g), my_type_name(type_name), my_input_ports(NULL), my_output_ports(NULL) { - tbb::internal::itt_make_task_group( tbb::internal::ITT_DOMAIN_FLOW, this, tbb::internal::FLOW_NODE, &g, tbb::internal::FLOW_GRAPH, tbb::internal::FLOW_COMPOSITE_NODE ); - tbb::internal::fgt_multiinput_multioutput_node_desc( this, my_type_name ); - } -#else - composite_node( graph &g) : graph_node(g), my_input_ports(NULL), my_output_ports(NULL) {} -#endif - - template - void set_external_ports(T1&& input_ports_tuple, T2&& output_ports_tuple) { - __TBB_STATIC_ASSERT(NUM_INPUTS == tbb::flow::tuple_size::value, "number of arguments does not match number of input ports"); - __TBB_STATIC_ASSERT(NUM_OUTPUTS == tbb::flow::tuple_size::value, "number of arguments does not match number of output ports"); - my_input_ports = new input_ports_type(std::forward(input_ports_tuple)); - my_output_ports = new output_ports_type(std::forward(output_ports_tuple)); - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - tbb::internal::fgt_internal_input_helper::register_port( this, input_ports_tuple); - tbb::internal::fgt_internal_output_helper::register_port( this, output_ports_tuple); -#endif - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - template< typename... NodeTypes > - void add_visible_nodes(const NodeTypes&... n) { internal::add_nodes_impl(this, true, n...); } - - template< typename... NodeTypes > - void add_nodes(const NodeTypes&... n) { internal::add_nodes_impl(this, false, n...); } -#else - template void add_nodes(Nodes&...) { } - template void add_visible_nodes(Nodes&...) { } -#endif - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_multiinput_multioutput_node_desc( this, name ); - } -#endif - - input_ports_type input_ports() { - __TBB_ASSERT(my_input_ports, "input ports not set, call set_external_ports to set input ports"); - return *my_input_ports; - } - - output_ports_type output_ports() { - __TBB_ASSERT(my_output_ports, "output ports not set, call set_external_ports to set output ports"); - return *my_output_ports; - } - - virtual ~composite_node() { - if(my_input_ports) delete my_input_ports; - if(my_output_ports) delete my_output_ports; - } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - void extract() __TBB_override { - __TBB_ASSERT(false, "Current composite_node implementation does not support extract"); - } -#endif -}; // class composite_node - -//composite_node with only input ports -template< typename... InputTypes> -class composite_node , tbb::flow::tuple<> > : public graph_node { -public: - typedef tbb::flow::tuple< receiver&... > input_ports_type; - -private: -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - const char *my_type_name; -#endif - input_ports_type *my_input_ports; - static const size_t NUM_INPUTS = sizeof...(InputTypes); - -protected: - void reset_node(reset_flags) __TBB_override {} - -public: -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - composite_node( graph &g, const char *type_name = "composite_node") : graph_node(g), my_type_name(type_name), my_input_ports(NULL) { - tbb::internal::itt_make_task_group( tbb::internal::ITT_DOMAIN_FLOW, this, tbb::internal::FLOW_NODE, &g, tbb::internal::FLOW_GRAPH, tbb::internal::FLOW_COMPOSITE_NODE ); - tbb::internal::fgt_multiinput_multioutput_node_desc( this, my_type_name ); - } -#else - composite_node( graph &g) : graph_node(g), my_input_ports(NULL) {} -#endif - - template - void set_external_ports(T&& input_ports_tuple) { - __TBB_STATIC_ASSERT(NUM_INPUTS == tbb::flow::tuple_size::value, "number of arguments does not match number of input ports"); - - my_input_ports = new input_ports_type(std::forward(input_ports_tuple)); - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - tbb::internal::fgt_internal_input_helper::register_port( this, std::forward(input_ports_tuple)); -#endif - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - template< typename... NodeTypes > - void add_visible_nodes(const NodeTypes&... n) { internal::add_nodes_impl(this, true, n...); } - - template< typename... NodeTypes > - void add_nodes( const NodeTypes&... n) { internal::add_nodes_impl(this, false, n...); } -#else - template void add_nodes(Nodes&...) {} - template void add_visible_nodes(Nodes&...) {} -#endif - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_multiinput_multioutput_node_desc( this, name ); - } -#endif - - input_ports_type input_ports() { - __TBB_ASSERT(my_input_ports, "input ports not set, call set_external_ports to set input ports"); - return *my_input_ports; - } - - virtual ~composite_node() { - if(my_input_ports) delete my_input_ports; - } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - void extract() __TBB_override { - __TBB_ASSERT(false, "Current composite_node implementation does not support extract"); - } -#endif - -}; // class composite_node - -//composite_nodes with only output_ports -template -class composite_node , tbb::flow::tuple > : public graph_node { -public: - typedef tbb::flow::tuple< sender&... > output_ports_type; - -private: -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - const char *my_type_name; -#endif - output_ports_type *my_output_ports; - static const size_t NUM_OUTPUTS = sizeof...(OutputTypes); - -protected: - void reset_node(reset_flags) __TBB_override {} - -public: -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - composite_node( graph &g, const char *type_name = "composite_node") : graph_node(g), my_type_name(type_name), my_output_ports(NULL) { - tbb::internal::itt_make_task_group( tbb::internal::ITT_DOMAIN_FLOW, this, tbb::internal::FLOW_NODE, &g, tbb::internal::FLOW_GRAPH, tbb::internal::FLOW_COMPOSITE_NODE ); - tbb::internal::fgt_multiinput_multioutput_node_desc( this, my_type_name ); - } -#else - composite_node( graph &g) : graph_node(g), my_output_ports(NULL) {} -#endif - - template - void set_external_ports(T&& output_ports_tuple) { - __TBB_STATIC_ASSERT(NUM_OUTPUTS == tbb::flow::tuple_size::value, "number of arguments does not match number of output ports"); - - my_output_ports = new output_ports_type(std::forward(output_ports_tuple)); - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - tbb::internal::fgt_internal_output_helper::register_port( this, std::forward(output_ports_tuple)); -#endif - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - template - void add_visible_nodes(const NodeTypes&... n) { internal::add_nodes_impl(this, true, n...); } - - template - void add_nodes(const NodeTypes&... n) { internal::add_nodes_impl(this, false, n...); } -#else - template void add_nodes(Nodes&...) {} - template void add_visible_nodes(Nodes&...) {} -#endif - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_multiinput_multioutput_node_desc( this, name ); - } -#endif - - output_ports_type output_ports() { - __TBB_ASSERT(my_output_ports, "output ports not set, call set_external_ports to set output ports"); - return *my_output_ports; - } - - virtual ~composite_node() { - if(my_output_ports) delete my_output_ports; - } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - void extract() __TBB_override { - __TBB_ASSERT(false, "Current composite_node implementation does not support extract"); - } -#endif - -}; // class composite_node - -#endif // __TBB_FLOW_GRAPH_CPP11_FEATURES - -namespace internal { - -template -class async_body_base: tbb::internal::no_assign { -public: - typedef Gateway gateway_type; - - async_body_base(gateway_type *gateway): my_gateway(gateway) { } - void set_gateway(gateway_type *gateway) { - my_gateway = gateway; - } - -protected: - gateway_type *my_gateway; -}; - -template -class async_body: public async_body_base { -public: - typedef async_body_base base_type; - typedef Gateway gateway_type; - - async_body(const Body &body, gateway_type *gateway) - : base_type(gateway), my_body(body) { } - - void operator()( const Input &v, Ports & ) { - my_body(v, *this->my_gateway); - } - - Body get_body() { return my_body; } - -private: - Body my_body; -}; - -} - -//! Implements async node -template < typename Input, typename Output, typename Policy = queueing, typename Allocator=cache_aligned_allocator > -class async_node : public multifunction_node< Input, tuple< Output >, Policy, Allocator >, public sender< Output > { - typedef multifunction_node< Input, tuple< Output >, Policy, Allocator > base_type; - typedef typename internal::multifunction_input mfn_input_type; - -public: - typedef Input input_type; - typedef Output output_type; - typedef receiver receiver_type; - typedef typename receiver_type::predecessor_type predecessor_type; - typedef typename sender::successor_type successor_type; - typedef receiver_gateway gateway_type; - typedef internal::async_body_base async_body_base_type; - -private: - struct try_put_functor { - typedef internal::multifunction_output output_port_type; - output_port_type *port; - const Output *value; - bool result; - try_put_functor(output_port_type &p, const Output &v) : port(&p), value(&v), result(false) { } - void operator()() { - result = port->try_put(*value); - } - }; - - class receiver_gateway_impl: public receiver_gateway { - public: - receiver_gateway_impl(async_node* node): my_node(node) {} - void reserve_wait() __TBB_override { - my_node->my_graph.reserve_wait(); - tbb::internal::fgt_async_reserve(static_cast(my_node), &my_node->my_graph); - } - - void release_wait() __TBB_override { - my_node->my_graph.release_wait(); - tbb::internal::fgt_async_commit(static_cast(my_node), &my_node->my_graph); - } - - //! Implements gateway_type::try_put for an external activity to submit a message to FG - bool try_put(const Output &i) __TBB_override { - return my_node->try_put_impl(i); - } - - private: - async_node* my_node; - } my_gateway; - - //The substitute of 'this' for member construction, to prevent compiler warnings - async_node* self() { return this; } - - //! Implements gateway_type::try_put for an external activity to submit a message to FG - bool try_put_impl(const Output &i) { - internal::multifunction_output &port_0 = internal::output_port<0>(*this); - tbb::internal::fgt_async_try_put_begin(this, &port_0); - __TBB_ASSERT(this->my_graph.my_task_arena && this->my_graph.my_task_arena->is_active(), NULL); - try_put_functor tpf(port_0, i); - this->my_graph.my_task_arena->execute(tpf); - tbb::internal::fgt_async_try_put_end(this, &port_0); - return tpf.result; - } - -public: - template - async_node( graph &g, size_t concurrency, Body body ) : - base_type( g, concurrency, internal::async_body(body, &my_gateway) ), my_gateway(self()) { - tbb::internal::fgt_multioutput_node<1>( tbb::internal::FLOW_ASYNC_NODE, - &this->my_graph, - static_cast *>(this), - this->output_ports() ); - } - - async_node( const async_node &other ) : base_type(other), sender(), my_gateway(self()) { - static_cast(this->my_body->get_body_ptr())->set_gateway(&my_gateway); - static_cast(this->my_init_body->get_body_ptr())->set_gateway(&my_gateway); - - tbb::internal::fgt_multioutput_node<1>( tbb::internal::FLOW_ASYNC_NODE, &this->my_graph, static_cast *>(this), this->output_ports() ); - } - - gateway_type& gateway() { - return my_gateway; - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif - - // Define sender< Output > - - //! Add a new successor to this node - bool register_successor( successor_type &r ) __TBB_override { - return internal::output_port<0>(*this).register_successor(r); - } - - //! Removes a successor from this node - bool remove_successor( successor_type &r ) __TBB_override { - return internal::output_port<0>(*this).remove_successor(r); - } - - template - Body copy_function_object() { - typedef internal::multifunction_body mfn_body_type; - typedef internal::async_body async_body_type; - mfn_body_type &body_ref = *this->my_body; - async_body_type ab = *static_cast(dynamic_cast< internal::multifunction_body_leaf & >(body_ref).get_body_ptr()); - return ab.get_body(); - } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - //! interface to record edges for traversal & deletion - typedef typename internal::edge_container built_successors_type; - typedef typename built_successors_type::edge_list_type successor_list_type; - built_successors_type &built_successors() __TBB_override { - return internal::output_port<0>(*this).built_successors(); - } - - void internal_add_built_successor( successor_type &r ) __TBB_override { - internal::output_port<0>(*this).internal_add_built_successor(r); - } - - void internal_delete_built_successor( successor_type &r ) __TBB_override { - internal::output_port<0>(*this).internal_delete_built_successor(r); - } - - void copy_successors( successor_list_type &l ) __TBB_override { - internal::output_port<0>(*this).copy_successors(l); - } - - size_t successor_count() __TBB_override { - return internal::output_port<0>(*this).successor_count(); - } -#endif - -protected: - - void reset_node( reset_flags f) __TBB_override { - base_type::reset_node(f); - } -}; - -#if __TBB_PREVIEW_STREAMING_NODE -#include "internal/_flow_graph_streaming_node.h" -#endif // __TBB_PREVIEW_STREAMING_NODE - -} // interface9 - - using interface9::reset_flags; - using interface9::rf_reset_protocol; - using interface9::rf_reset_bodies; - using interface9::rf_clear_edges; - - using interface9::graph; - using interface9::graph_node; - using interface9::continue_msg; - - using interface9::source_node; - using interface9::function_node; - using interface9::multifunction_node; - using interface9::split_node; - using interface9::internal::output_port; - using interface9::indexer_node; - using interface9::internal::tagged_msg; - using interface9::internal::cast_to; - using interface9::internal::is_a; - using interface9::continue_node; - using interface9::overwrite_node; - using interface9::write_once_node; - using interface9::broadcast_node; - using interface9::buffer_node; - using interface9::queue_node; - using interface9::sequencer_node; - using interface9::priority_queue_node; - using interface9::limiter_node; - using namespace interface9::internal::graph_policy_namespace; - using interface9::join_node; - using interface9::input_port; - using interface9::copy_body; - using interface9::make_edge; - using interface9::remove_edge; - using interface9::internal::tag_value; -#if __TBB_FLOW_GRAPH_CPP11_FEATURES - using interface9::composite_node; -#endif - using interface9::async_node; -#if __TBB_PREVIEW_ASYNC_MSG - using interface9::async_msg; -#endif -#if __TBB_PREVIEW_STREAMING_NODE - using interface9::port_ref; - using interface9::streaming_node; -#endif // __TBB_PREVIEW_STREAMING_NODE - -} // flow -} // tbb - -#undef __TBB_PFG_RESET_ARG -#undef __TBB_COMMA - -#endif // __TBB_flow_graph_H diff --git a/inst/include/tbb/flow_graph_opencl_node.h b/inst/include/tbb/flow_graph_opencl_node.h deleted file mode 100644 index 89f4da7b1..000000000 --- a/inst/include/tbb/flow_graph_opencl_node.h +++ /dev/null @@ -1,1537 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_flow_graph_opencl_node_H -#define __TBB_flow_graph_opencl_node_H - -#include "tbb/tbb_config.h" -#if __TBB_PREVIEW_OPENCL_NODE - -#include "flow_graph.h" - -#include -#include -#include -#include -#include -#include -#include - -#ifdef __APPLE__ -#include -#else -#include -#endif - -namespace tbb { -namespace flow { - -namespace interface9 { - -class opencl_foundation; -class opencl_device_list; - -template -class opencl_buffer_impl; - -template -class opencl_program; - -class default_opencl_factory; - -class opencl_graph : public graph { -public: - //! Constructs a graph with isolated task_group_context - opencl_graph() : my_opencl_foundation( NULL ) {} - //! Constructs a graph with an user context - explicit opencl_graph( task_group_context& context ) : graph( context ), my_opencl_foundation( NULL ) {} - //! Destroys a graph - ~opencl_graph(); - //! Available devices - const opencl_device_list& available_devices(); - default_opencl_factory& opencl_factory(); -protected: - opencl_foundation *my_opencl_foundation; - opencl_foundation &get_opencl_foundation(); - - template - friend class opencl_buffer; - template - friend class opencl_image2d; - template - friend class opencl_node; - template - friend class opencl_factory; -}; - -template -class opencl_factory; - -template -class dependency_msg; - - -inline void enforce_cl_retcode( cl_int err, std::string msg ) { - if ( err != CL_SUCCESS ) { - std::cerr << msg << "; error code: " << err << std::endl; - throw msg; - } -} - -template -T event_info( cl_event e, cl_event_info i ) { - T res; - enforce_cl_retcode( clGetEventInfo( e, i, sizeof( res ), &res, NULL ), "Failed to get OpenCL event information" ); - return res; -} - -template -T device_info( cl_device_id d, cl_device_info i ) { - T res; - enforce_cl_retcode( clGetDeviceInfo( d, i, sizeof( res ), &res, NULL ), "Failed to get OpenCL device information" ); - return res; -} -template <> -std::string device_info( cl_device_id d, cl_device_info i ) { - size_t required; - enforce_cl_retcode( clGetDeviceInfo( d, i, 0, NULL, &required ), "Failed to get OpenCL device information" ); - - char *buff = (char*)alloca( required ); - enforce_cl_retcode( clGetDeviceInfo( d, i, required, buff, NULL ), "Failed to get OpenCL device information" ); - - return buff; -} -template -T platform_info( cl_platform_id p, cl_platform_info i ) { - T res; - enforce_cl_retcode( clGetPlatformInfo( p, i, sizeof( res ), &res, NULL ), "Failed to get OpenCL platform information" ); - return res; -} -template <> -std::string platform_info( cl_platform_id p, cl_platform_info i ) { - size_t required; - enforce_cl_retcode( clGetPlatformInfo( p, i, 0, NULL, &required ), "Failed to get OpenCL platform information" ); - - char *buff = (char*)alloca( required ); - enforce_cl_retcode( clGetPlatformInfo( p, i, required, buff, NULL ), "Failed to get OpenCL platform information" ); - - return buff; -} - - -class opencl_device { -public: - typedef size_t device_id_type; - enum : device_id_type { - unknown = device_id_type( -2 ), - host = device_id_type( -1 ) - }; - - opencl_device() : my_device_id( unknown ) {} - - opencl_device( cl_device_id cl_d_id, device_id_type device_id ) : my_device_id( device_id ), my_cl_device_id( cl_d_id ) {} - - std::string platform_profile() const { - return platform_info( platform(), CL_PLATFORM_PROFILE ); - } - std::string platform_version() const { - return platform_info( platform(), CL_PLATFORM_VERSION ); - } - std::string platform_name() const { - return platform_info( platform(), CL_PLATFORM_NAME ); - } - std::string platform_vendor() const { - return platform_info( platform(), CL_PLATFORM_VENDOR ); - } - std::string platform_extensions() const { - return platform_info( platform(), CL_PLATFORM_EXTENSIONS ); - } - - template - void info( cl_device_info i, T &t ) const { - t = device_info( my_cl_device_id, i ); - } - std::string version() const { - // The version string format: OpenCL - return device_info( my_cl_device_id, CL_DEVICE_VERSION ); - } - int major_version() const { - int major; - std::sscanf( version().c_str(), "OpenCL %d", &major ); - return major; - } - int minor_version() const { - int major, minor; - std::sscanf( version().c_str(), "OpenCL %d.%d", &major, &minor ); - return minor; - } - bool out_of_order_exec_mode_on_host_present() const { -#if CL_VERSION_2_0 - if ( major_version() >= 2 ) - return (device_info( my_cl_device_id, CL_DEVICE_QUEUE_ON_HOST_PROPERTIES ) & CL_QUEUE_OUT_OF_ORDER_EXEC_MODE_ENABLE) != 0; - else -#endif /* CL_VERSION_2_0 */ - return (device_info( my_cl_device_id, CL_DEVICE_QUEUE_PROPERTIES ) & CL_QUEUE_OUT_OF_ORDER_EXEC_MODE_ENABLE) != 0; - } - bool out_of_order_exec_mode_on_device_present() const { -#if CL_VERSION_2_0 - if ( major_version() >= 2 ) - return (device_info( my_cl_device_id, CL_DEVICE_QUEUE_ON_DEVICE_PROPERTIES ) & CL_QUEUE_OUT_OF_ORDER_EXEC_MODE_ENABLE) != 0; - else -#endif /* CL_VERSION_2_0 */ - return false; - } - std::array max_work_item_sizes() const { - return device_info>( my_cl_device_id, CL_DEVICE_MAX_WORK_ITEM_SIZES ); - } - size_t max_work_group_size() const { - return device_info( my_cl_device_id, CL_DEVICE_MAX_WORK_GROUP_SIZE ); - } - bool built_in_kernel_available( const std::string& k ) const { - const std::string semi = ";"; - // Added semicolumns to force an exact match (to avoid a partial match, e.g. "add" is partly matched with "madd"). - return (semi + built_in_kernels() + semi).find( semi + k + semi ) != std::string::npos; - } - std::string built_in_kernels() const { - return device_info( my_cl_device_id, CL_DEVICE_BUILT_IN_KERNELS ); - } - std::string name() const { - return device_info( my_cl_device_id, CL_DEVICE_NAME ); - } - cl_bool available() const { - return device_info( my_cl_device_id, CL_DEVICE_AVAILABLE ); - } - cl_bool compiler_available() const { - return device_info( my_cl_device_id, CL_DEVICE_COMPILER_AVAILABLE ); - } - cl_bool linker_available() const { - return device_info( my_cl_device_id, CL_DEVICE_LINKER_AVAILABLE ); - } - bool extension_available( const std::string &ext ) const { - const std::string space = " "; - // Added space to force an exact match (to avoid a partial match, e.g. "ext" is partly matched with "ext2"). - return (space + extensions() + space).find( space + ext + space ) != std::string::npos; - } - std::string extensions() const { - return device_info( my_cl_device_id, CL_DEVICE_EXTENSIONS ); - } - - cl_device_type type() const { - return device_info( my_cl_device_id, CL_DEVICE_TYPE ); - } - - std::string vendor() const { - return device_info( my_cl_device_id, CL_DEVICE_VENDOR ); - } - - cl_uint address_bits() const { - return device_info( my_cl_device_id, CL_DEVICE_ADDRESS_BITS ); - } - - cl_device_id device_id() const { - return my_cl_device_id; - } - - cl_command_queue command_queue() const { - return my_cl_command_queue; - } - - void set_command_queue( cl_command_queue cmd_queue ) { - my_cl_command_queue = cmd_queue; - } - -private: - opencl_device( cl_device_id d_id ) : my_device_id( unknown ), my_cl_device_id( d_id ) {} - - cl_platform_id platform() const { - return device_info( my_cl_device_id, CL_DEVICE_PLATFORM ); - } - - device_id_type my_device_id; - cl_device_id my_cl_device_id; - cl_command_queue my_cl_command_queue; - - friend bool operator==(opencl_device d1, opencl_device d2) { return d1.my_cl_device_id == d2.my_cl_device_id; } - - template - friend class opencl_factory; - template - friend class opencl_memory; - template - friend class opencl_program; - friend class opencl_foundation; - -#if TBB_USE_ASSERT - template - friend class opencl_buffer; -#endif -}; - -class opencl_device_list { - typedef std::vector container_type; -public: - typedef container_type::iterator iterator; - typedef container_type::const_iterator const_iterator; - typedef container_type::size_type size_type; - - opencl_device_list() {} - opencl_device_list( std::initializer_list il ) : my_container( il ) {} - - void add( opencl_device d ) { my_container.push_back( d ); } - size_type size() const { return my_container.size(); } - bool empty() const { return my_container.empty(); } - iterator begin() { return my_container.begin(); } - iterator end() { return my_container.end(); } - const_iterator begin() const { return my_container.begin(); } - const_iterator end() const { return my_container.end(); } - const_iterator cbegin() const { return my_container.cbegin(); } - const_iterator cend() const { return my_container.cend(); } -private: - container_type my_container; -}; - -class callback_base : tbb::internal::no_copy { -public: - virtual void call() = 0; - virtual ~callback_base() {} -}; - -template -class callback : public callback_base { - graph &my_graph; - Callback my_callback; - T my_data; -public: - callback( graph &g, Callback c, const T& t ) : my_graph( g ), my_callback( c ), my_data( t ) { - // Extend the graph lifetime until the callback completion. - my_graph.increment_wait_count(); - } - ~callback() { - // Release the reference to the graph. - my_graph.decrement_wait_count(); - } - void call() __TBB_override { - my_callback( my_data ); - } -}; - -template -class dependency_msg : public async_msg { -public: - typedef T value_type; - - dependency_msg() : my_callback_flag_ptr( std::make_shared< tbb::atomic>() ) { - my_callback_flag_ptr->store(false); - } - - explicit dependency_msg( const T& data ) : my_data(data), my_callback_flag_ptr( std::make_shared>() ) { - my_callback_flag_ptr->store(false); - } - - dependency_msg( opencl_graph &g, const T& data ) : my_data(data), my_graph(&g), my_callback_flag_ptr( std::make_shared>() ) { - my_callback_flag_ptr->store(false); - } - - dependency_msg( const T& data, cl_event event ) : my_data(data), my_event(event), my_is_event(true), my_callback_flag_ptr( std::make_shared>() ) { - my_callback_flag_ptr->store(false); - enforce_cl_retcode( clRetainEvent( my_event ), "Failed to retain an event" ); - } - - T& data( bool wait = true ) { - if ( my_is_event && wait ) { - enforce_cl_retcode( clWaitForEvents( 1, &my_event ), "Failed to wait for an event" ); - enforce_cl_retcode( clReleaseEvent( my_event ), "Failed to release an event" ); - my_is_event = false; - } - return my_data; - } - - const T& data( bool wait = true ) const { - if ( my_is_event && wait ) { - enforce_cl_retcode( clWaitForEvents( 1, &my_event ), "Failed to wait for an event" ); - enforce_cl_retcode( clReleaseEvent( my_event ), "Failed to release an event" ); - my_is_event = false; - } - return my_data; - } - - dependency_msg( const dependency_msg &dmsg ) : async_msg(dmsg), - my_data(dmsg.my_data), my_event(dmsg.my_event), my_is_event( dmsg.my_is_event ), my_graph( dmsg.my_graph ), - my_callback_flag_ptr(dmsg.my_callback_flag_ptr) - { - if ( my_is_event ) - enforce_cl_retcode( clRetainEvent( my_event ), "Failed to retain an event" ); - } - - dependency_msg( dependency_msg &&dmsg ) : async_msg(std::move(dmsg)), - my_data(std::move(dmsg.my_data)), my_event(dmsg.my_event), my_is_event(dmsg.my_is_event), my_graph(dmsg.my_graph), - my_callback_flag_ptr( std::move(dmsg.my_callback_flag_ptr) ) - { - dmsg.my_is_event = false; - } - - dependency_msg& operator=(const dependency_msg &dmsg) { - async_msg::operator =(dmsg); - - // Release original event - if ( my_is_event ) - enforce_cl_retcode( clReleaseEvent( my_event ), "Failed to retain an event" ); - - my_data = dmsg.my_data; - my_event = dmsg.my_event; - my_is_event = dmsg.my_is_event; - my_graph = dmsg.my_graph; - - // Retain copied event - if ( my_is_event ) - enforce_cl_retcode( clRetainEvent( my_event ), "Failed to retain an event" ); - - my_callback_flag_ptr = dmsg.my_callback_flag_ptr; - return *this; - } - - ~dependency_msg() { - if ( my_is_event ) - enforce_cl_retcode( clReleaseEvent( my_event ), "Failed to release an event" ); - } - - cl_event const * get_event() const { return my_is_event ? &my_event : NULL; } - void set_event( cl_event e ) const { - if ( my_is_event ) { - cl_command_queue cq = event_info( my_event, CL_EVENT_COMMAND_QUEUE ); - if ( cq != event_info( e, CL_EVENT_COMMAND_QUEUE ) ) - enforce_cl_retcode( clFlush( cq ), "Failed to flush an OpenCL command queue" ); - enforce_cl_retcode( clReleaseEvent( my_event ), "Failed to release an event" ); - } - my_is_event = true; - my_event = e; - clRetainEvent( my_event ); - } - - void set_graph( graph &g ) { - my_graph = &g; - } - - void clear_event() const { - if ( my_is_event ) { - enforce_cl_retcode( clFlush( event_info( my_event, CL_EVENT_COMMAND_QUEUE ) ), "Failed to flush an OpenCL command queue" ); - enforce_cl_retcode( clReleaseEvent( my_event ), "Failed to release an event" ); - } - my_is_event = false; - } - - template - void register_callback( Callback c ) const { - __TBB_ASSERT( my_is_event, "The OpenCL event is not set" ); - __TBB_ASSERT( my_graph, "The graph is not set" ); - enforce_cl_retcode( clSetEventCallback( my_event, CL_COMPLETE, register_callback_func, new callback( *my_graph, c, my_data ) ), "Failed to set an OpenCL callback" ); - } - - operator T&() { return data(); } - operator const T&() const { return data(); } - -protected: - // Overridden in this derived class to inform that - // async calculation chain is over - void finalize() const __TBB_override { - receive_if_memory_object(*this); - if (! my_callback_flag_ptr->fetch_and_store(true)) { - dependency_msg a(*this); - if (my_is_event) { - register_callback([a](const T& t) mutable { - a.set(t); - }); - } - else { - a.set(my_data); - } - } - clear_event(); - } - -private: - static void CL_CALLBACK register_callback_func( cl_event, cl_int event_command_exec_status, void *data ) { - tbb::internal::suppress_unused_warning( event_command_exec_status ); - __TBB_ASSERT( event_command_exec_status == CL_COMPLETE, NULL ); - __TBB_ASSERT( data, NULL ); - callback_base *c = static_cast(data); - c->call(); - delete c; - } - - T my_data; - mutable cl_event my_event; - mutable bool my_is_event = false; - graph *my_graph = NULL; - - std::shared_ptr< tbb::atomic > my_callback_flag_ptr; -}; - -template -K key_from_message( const dependency_msg &dmsg ) { - using tbb::flow::key_from_message; - const T &t = dmsg.data( false ); - __TBB_STATIC_ASSERT( true, "" ); - return key_from_message( t ); -} - -template -class opencl_memory { -public: - opencl_memory() {} - opencl_memory( Factory &f ) : my_host_ptr( NULL ), my_factory( &f ), my_sending_event_present( false ) { - my_curr_device_id = my_factory->devices().begin()->my_device_id; - } - - ~opencl_memory() { - if ( my_sending_event_present ) enforce_cl_retcode( clReleaseEvent( my_sending_event ), "Failed to release an event for the OpenCL buffer" ); - enforce_cl_retcode( clReleaseMemObject( my_cl_mem ), "Failed to release an memory object" ); - } - - cl_mem get_cl_mem() const { - return my_cl_mem; - } - - void* get_host_ptr() { - if ( !my_host_ptr ) { - dependency_msg d = receive( NULL ); - d.data(); - __TBB_ASSERT( d.data() == my_host_ptr, NULL ); - } - return my_host_ptr; - } - - Factory *factory() const { return my_factory; } - - dependency_msg send( opencl_device d, const cl_event *e ); - dependency_msg receive( const cl_event *e ); - virtual void map_memory( opencl_device, dependency_msg & ) = 0; -protected: - cl_mem my_cl_mem; - tbb::atomic my_curr_device_id; - void* my_host_ptr; - Factory *my_factory; - - tbb::spin_mutex my_sending_lock; - bool my_sending_event_present; - cl_event my_sending_event; -}; - -template -class opencl_buffer_impl : public opencl_memory { - size_t my_size; -public: - opencl_buffer_impl( size_t size, Factory& f ) : opencl_memory( f ), my_size( size ) { - cl_int err; - this->my_cl_mem = clCreateBuffer( this->my_factory->context(), CL_MEM_ALLOC_HOST_PTR, size, NULL, &err ); - enforce_cl_retcode( err, "Failed to create an OpenCL buffer" ); - } - - // The constructor for subbuffers. - opencl_buffer_impl( cl_mem m, size_t index, size_t size, Factory& f ) : opencl_memory( f ), my_size( size ) { - cl_int err; - cl_buffer_region region = { index, size }; - this->my_cl_mem = clCreateSubBuffer( m, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err ); - enforce_cl_retcode( err, "Failed to create an OpenCL subbuffer" ); - } - - size_t size() const { - return my_size; - } - - void map_memory( opencl_device device, dependency_msg &dmsg ) __TBB_override { - this->my_factory->enque_map_buffer( device, *this, dmsg ); - } - -#if TBB_USE_ASSERT - template - friend class opencl_buffer; -#endif -}; - -enum access_type { - read_write, - write_only, - read_only -}; - -template -class opencl_subbuffer; - -template -class opencl_buffer { -public: - typedef cl_mem native_object_type; - typedef opencl_buffer memory_object_type; - typedef Factory opencl_factory_type; - - template using iterator = T*; - - template - iterator access() const { - T* ptr = (T*)my_impl->get_host_ptr(); - __TBB_ASSERT( ptr, NULL ); - return iterator( ptr ); - } - - T* data() const { return &access()[0]; } - - template - iterator begin() const { return access(); } - - template - iterator end() const { return access()+my_impl->size()/sizeof(T); } - - size_t size() const { return my_impl->size()/sizeof(T); } - - T& operator[] ( ptrdiff_t k ) { return begin()[k]; } - - opencl_buffer() {} - opencl_buffer( opencl_graph &g, size_t size ); - opencl_buffer( Factory &f, size_t size ) : my_impl( std::make_shared( size*sizeof(T), f ) ) {} - - cl_mem native_object() const { - return my_impl->get_cl_mem(); - } - - const opencl_buffer& memory_object() const { - return *this; - } - - void send( opencl_device device, dependency_msg &dependency ) const { - __TBB_ASSERT( dependency.data( /*wait = */false ) == *this, NULL ); - dependency_msg d = my_impl->send( device, dependency.get_event() ); - const cl_event *e = d.get_event(); - if ( e ) dependency.set_event( *e ); - else dependency.clear_event(); - } - void receive( const dependency_msg &dependency ) const { - __TBB_ASSERT( dependency.data( /*wait = */false ) == *this, NULL ); - dependency_msg d = my_impl->receive( dependency.get_event() ); - const cl_event *e = d.get_event(); - if ( e ) dependency.set_event( *e ); - else dependency.clear_event(); - } - - opencl_subbuffer subbuffer( size_t index, size_t size ) const; -private: - // The constructor for subbuffers. - opencl_buffer( Factory &f, cl_mem m, size_t index, size_t size ) : my_impl( std::make_shared( m, index*sizeof(T), size*sizeof(T), f ) ) {} - - typedef opencl_buffer_impl impl_type; - - std::shared_ptr my_impl; - - friend bool operator==(const opencl_buffer &lhs, const opencl_buffer &rhs) { - return lhs.my_impl == rhs.my_impl; - } - - template - friend class opencl_factory; - template - friend class opencl_subbuffer; -}; - -template -class opencl_subbuffer : public opencl_buffer { - opencl_buffer my_owner; -public: - opencl_subbuffer() {} - opencl_subbuffer( const opencl_buffer &owner, size_t index, size_t size ) : - opencl_buffer( *owner.my_impl->factory(), owner.native_object(), index, size ), my_owner( owner ) {} -}; - -template -opencl_subbuffer opencl_buffer::subbuffer( size_t index, size_t size ) const { - return opencl_subbuffer( *this, index, size ); -} - - -#define is_typedef(type) \ - template \ - struct is_##type { \ - template \ - static std::true_type check( typename C::type* ); \ - template \ - static std::false_type check( ... ); \ - \ - static const bool value = decltype(check(0))::value; \ - } - -is_typedef( native_object_type ); -is_typedef( memory_object_type ); - -template -typename std::enable_if::value, typename T::native_object_type>::type get_native_object( const T &t ) { - return t.native_object(); -} - -template -typename std::enable_if::value, T>::type get_native_object( T t ) { - return t; -} - -// send_if_memory_object checks if the T type has memory_object_type and call the send method for the object. -template -typename std::enable_if::value>::type send_if_memory_object( opencl_device device, dependency_msg &dmsg ) { - const T &t = dmsg.data( false ); - typedef typename T::memory_object_type mem_obj_t; - mem_obj_t mem_obj = t.memory_object(); - dependency_msg d( mem_obj ); - if ( dmsg.get_event() ) d.set_event( *dmsg.get_event() ); - mem_obj.send( device, d ); - if ( d.get_event() ) dmsg.set_event( *d.get_event() ); -} - -template -typename std::enable_if::value>::type send_if_memory_object( opencl_device device, T &t ) { - typedef typename T::memory_object_type mem_obj_t; - mem_obj_t mem_obj = t.memory_object(); - dependency_msg dmsg( mem_obj ); - mem_obj.send( device, dmsg ); -} - -template -typename std::enable_if::value>::type send_if_memory_object( opencl_device, T& ) {}; - -// receive_if_memory_object checks if the T type has memory_object_type and call the receive method for the object. -template -typename std::enable_if::value>::type receive_if_memory_object( const dependency_msg &dmsg ) { - const T &t = dmsg.data( false ); - typedef typename T::memory_object_type mem_obj_t; - mem_obj_t mem_obj = t.memory_object(); - dependency_msg d( mem_obj ); - if ( dmsg.get_event() ) d.set_event( *dmsg.get_event() ); - mem_obj.receive( d ); - if ( d.get_event() ) dmsg.set_event( *d.get_event() ); -} - -template -typename std::enable_if::value>::type receive_if_memory_object( const T& ) {} - -class opencl_range { -public: - typedef size_t range_index_type; - typedef std::array nd_range_type; - - template , typename L = std::initializer_list, - typename = typename std::enable_if::type, opencl_range>::value>::type> - opencl_range(G&& global_work = std::initializer_list({ 0 }), L&& local_work = std::initializer_list({ 0, 0, 0 })) { - auto g_it = global_work.begin(); - auto l_it = local_work.begin(); - my_global_work_size = { size_t(-1), size_t(-1), size_t(-1) }; - // my_local_work_size is still uninitialized - for (int s = 0; s < 3 && g_it != global_work.end(); ++g_it, ++l_it, ++s) { - __TBB_ASSERT(l_it != local_work.end(), "global_work & local_work must have same size"); - my_global_work_size[s] = *g_it; - my_local_work_size[s] = *l_it; - } - } - - const nd_range_type& global_range() const { return my_global_work_size; } - const nd_range_type& local_range() const { return my_local_work_size; } - -private: - nd_range_type my_global_work_size; - nd_range_type my_local_work_size; -}; - -template -class opencl_factory { -public: - template using async_msg_type = dependency_msg>; - typedef opencl_device device_type; - - class kernel : tbb::internal::no_assign { - public: - kernel( const kernel& k ) : my_factory( k.my_factory ) { - // Clone my_cl_kernel via opencl_program - size_t ret_size = 0; - - std::vector kernel_name; - for ( size_t curr_size = 32;; curr_size <<= 1 ) { - kernel_name.resize( curr_size <<= 1 ); - enforce_cl_retcode( clGetKernelInfo( k.my_cl_kernel, CL_KERNEL_FUNCTION_NAME, curr_size, kernel_name.data(), &ret_size ), "Failed to get kernel info" ); - if ( ret_size < curr_size ) break; - } - - cl_program program; - enforce_cl_retcode( clGetKernelInfo( k.my_cl_kernel, CL_KERNEL_PROGRAM, sizeof(program), &program, &ret_size ), "Failed to get kernel info" ); - __TBB_ASSERT( ret_size == sizeof(program), NULL ); - - my_cl_kernel = opencl_program< factory_type >( my_factory, program ).get_cl_kernel( kernel_name.data() ); - } - - ~kernel() { - enforce_cl_retcode( clReleaseKernel( my_cl_kernel ), "Failed to release a kernel" ); - } - - private: - typedef opencl_factory factory_type; - - kernel( const cl_kernel& k, factory_type& f ) : my_cl_kernel( k ), my_factory( f ) {} - - // Data - cl_kernel my_cl_kernel; - factory_type& my_factory; - - template - friend class opencl_factory; - - template - friend class opencl_program; - }; - - typedef kernel kernel_type; - - // 'range_type' enables kernel_executor with range support - // it affects expectations for enqueue_kernel(.....) interface method - typedef opencl_range range_type; - - opencl_factory( opencl_graph &g ) : my_graph( g ) {} - ~opencl_factory() { - if ( my_devices.size() ) { - for ( auto d = my_devices.begin(); d != my_devices.end(); ++d ) { - enforce_cl_retcode( clReleaseCommandQueue( (*d).my_cl_command_queue ), "Failed to release a command queue" ); - } - enforce_cl_retcode( clReleaseContext( my_cl_context ), "Failed to release a context" ); - } - } - - bool init( const opencl_device_list &device_list ) { - tbb::spin_mutex::scoped_lock lock( my_devices_mutex ); - if ( !my_devices.size() ) { - my_devices = device_list; - return true; - } - return false; - } - - -private: - template - void enque_map_buffer( opencl_device device, opencl_buffer_impl &buffer, dependency_msg& dmsg ) { - cl_event const* e1 = dmsg.get_event(); - cl_event e2; - cl_int err; - void *ptr = clEnqueueMapBuffer( device.my_cl_command_queue, buffer.get_cl_mem(), false, CL_MAP_READ | CL_MAP_WRITE, 0, buffer.size(), - e1 == NULL ? 0 : 1, e1, &e2, &err ); - enforce_cl_retcode( err, "Failed to map a buffer" ); - dmsg.data( false ) = ptr; - dmsg.set_event( e2 ); - enforce_cl_retcode( clReleaseEvent( e2 ), "Failed to release an event" ); - } - - - template - void enque_unmap_buffer( opencl_device device, opencl_memory &memory, dependency_msg& dmsg ) { - cl_event const* e1 = dmsg.get_event(); - cl_event e2; - enforce_cl_retcode( - clEnqueueUnmapMemObject( device.my_cl_command_queue, memory.get_cl_mem(), memory.get_host_ptr(), e1 == NULL ? 0 : 1, e1, &e2 ), - "Failed to unmap a buffer" ); - dmsg.set_event( e2 ); - enforce_cl_retcode( clReleaseEvent( e2 ), "Failed to release an event" ); - } - - // --------- Kernel argument & event list helpers --------- // - template - void process_one_arg( const kernel_type& kernel, std::array&, int&, int& place, const T& t ) { - auto p = get_native_object(t); - enforce_cl_retcode( clSetKernelArg(kernel.my_cl_kernel, place++, sizeof(p), &p), "Failed to set a kernel argument" ); - } - - template - void process_one_arg( const kernel_type& kernel, std::array& events, int& num_events, int& place, const dependency_msg& msg ) { - __TBB_ASSERT((static_cast::size_type>(num_events) < events.size()), NULL); - - const cl_event * const e = msg.get_event(); - if (e != NULL) { - events[num_events++] = *e; - } - - process_one_arg( kernel, events, num_events, place, msg.data(false) ); - } - - template - void process_arg_list( const kernel_type& kernel, std::array& events, int& num_events, int& place, const T& t, const Rest&... args ) { - process_one_arg( kernel, events, num_events, place, t ); - process_arg_list( kernel, events, num_events, place, args... ); - } - - template - void process_arg_list( const kernel_type&, std::array&, int&, int& ) {} - // ------------------------------------------- // - template - void update_one_arg( cl_event, T& ) {} - - template - void update_one_arg( cl_event e, dependency_msg& msg ) { - msg.set_event( e ); - msg.set_graph( my_graph ); - } - - template - void update_arg_list( cl_event e, T& t, Rest&... args ) { - update_one_arg( e, t ); - update_arg_list( e, args... ); - } - - void update_arg_list( cl_event ) {} - // ------------------------------------------- // -public: - template - void send_kernel( opencl_device device, const kernel_type& kernel, const range_type& work_size, Args&... args ) { - std::array events; - int num_events = 0; - int place = 0; - process_arg_list( kernel, events, num_events, place, args... ); - - const cl_event e = send_kernel_impl( device, kernel.my_cl_kernel, work_size, num_events, events.data() ); - - update_arg_list(e, args...); - - // Release our own reference to cl_event - enforce_cl_retcode( clReleaseEvent(e), "Failed to release an event" ); - } - - // ------------------------------------------- // - template - void send_data(opencl_device device, T& t, Rest&... args) { - send_if_memory_object( device, t ); - send_data( device, args... ); - } - - void send_data(opencl_device) {} - // ------------------------------------------- // - -private: - cl_event send_kernel_impl( opencl_device device, const cl_kernel& kernel, - const range_type& work_size, cl_uint num_events, cl_event* event_list ) { - const typename range_type::nd_range_type g_offset = { { 0, 0, 0 } }; - const typename range_type::nd_range_type& g_size = work_size.global_range(); - const typename range_type::nd_range_type& l_size = work_size.local_range(); - cl_uint s; - for ( s = 1; s < 3 && g_size[s] != size_t(-1); ++s) {} - cl_event event; - enforce_cl_retcode( - clEnqueueNDRangeKernel( device.my_cl_command_queue, kernel, s, - g_offset.data(), g_size.data(), l_size[0] ? l_size.data() : NULL, num_events, num_events ? event_list : NULL, &event ), - "Failed to enqueue a kernel" ); - return event; - } - - // ------------------------------------------- // - template - bool get_event_from_one_arg( cl_event&, const T& ) { - return false; - } - - template - bool get_event_from_one_arg( cl_event& e, const dependency_msg& msg) { - cl_event const *e_ptr = msg.get_event(); - - if ( e_ptr != NULL ) { - e = *e_ptr; - return true; - } - - return false; - } - - template - bool get_event_from_args( cl_event& e, const T& t, const Rest&... args ) { - if ( get_event_from_one_arg( e, t ) ) { - return true; - } - - return get_event_from_args( e, args... ); - } - - bool get_event_from_args( cl_event& ) { - return false; - } - // ------------------------------------------- // - - struct finalize_fn : tbb::internal::no_assign { - virtual ~finalize_fn() {} - virtual void operator() () {} - }; - - template - struct finalize_fn_leaf : public finalize_fn { - Fn my_fn; - finalize_fn_leaf(Fn fn) : my_fn(fn) {} - void operator() () __TBB_override { my_fn(); } - }; - - static void CL_CALLBACK finalize_callback(cl_event, cl_int event_command_exec_status, void *data) { - tbb::internal::suppress_unused_warning(event_command_exec_status); - __TBB_ASSERT(event_command_exec_status == CL_COMPLETE, NULL); - - finalize_fn * const fn_ptr = static_cast(data); - __TBB_ASSERT(fn_ptr != NULL, "Invalid finalize function pointer"); - (*fn_ptr)(); - - // Function pointer was created by 'new' & this callback must be called once only - delete fn_ptr; - } -public: - template - void finalize( opencl_device device, FinalizeFn fn, Args&... args ) { - cl_event e; - - if ( get_event_from_args( e, args... ) ) { - enforce_cl_retcode( clSetEventCallback( e, CL_COMPLETE, finalize_callback, - new finalize_fn_leaf(fn) ), "Failed to set a callback" ); - } - - enforce_cl_retcode( clFlush( device.my_cl_command_queue ), "Failed to flush an OpenCL command queue" ); - } - - const opencl_device_list& devices() { - std::call_once( my_once_flag, &opencl_factory::init_once, this ); - return my_devices; - } - -private: - bool is_same_context( opencl_device::device_id_type d1, opencl_device::device_id_type d2 ) { - __TBB_ASSERT( d1 != opencl_device::unknown && d2 != opencl_device::unknown, NULL ); - // Currently, factory supports only one context so if the both devices are not host it means the are in the same context. - if ( d1 != opencl_device::host && d2 != opencl_device::host ) - return true; - return d1 == d2; - } -private: - opencl_factory( const opencl_factory& ); - opencl_factory& operator=(const opencl_factory&); - - cl_context context() { - std::call_once( my_once_flag, &opencl_factory::init_once, this ); - return my_cl_context; - } - - void init_once(); - - std::once_flag my_once_flag; - opencl_device_list my_devices; - cl_context my_cl_context; - opencl_graph &my_graph; - - tbb::spin_mutex my_devices_mutex; - - template - friend class opencl_program; - template - friend class opencl_buffer_impl; - template - friend class opencl_memory; -}; - -template -dependency_msg opencl_memory::receive( const cl_event *e ) { - dependency_msg d = e ? dependency_msg( my_host_ptr, *e ) : dependency_msg( my_host_ptr ); - // Concurrent receives are prohibited so we do not worry about synchronization. - if ( my_curr_device_id.load() != opencl_device::host ) { - map_memory( *my_factory->devices().begin(), d ); - my_curr_device_id.store( opencl_device::host ); - my_host_ptr = d.data( false ); - } - // Release the sending event - if ( my_sending_event_present ) { - enforce_cl_retcode( clReleaseEvent( my_sending_event ), "Failed to release an event" ); - my_sending_event_present = false; - } - return d; -} - -template -dependency_msg opencl_memory::send( opencl_device device, const cl_event *e ) { - opencl_device::device_id_type device_id = device.my_device_id; - if ( !my_factory->is_same_context( my_curr_device_id.load(), device_id ) ) { - { - tbb::spin_mutex::scoped_lock lock( my_sending_lock ); - if ( !my_factory->is_same_context( my_curr_device_id.load(), device_id ) ) { - __TBB_ASSERT( my_host_ptr, "The buffer has not been mapped" ); - dependency_msg d( my_host_ptr ); - my_factory->enque_unmap_buffer( device, *this, d ); - my_sending_event = *d.get_event(); - my_sending_event_present = true; - enforce_cl_retcode( clRetainEvent( my_sending_event ), "Failed to retain an event" ); - my_host_ptr = NULL; - my_curr_device_id.store(device_id); - } - } - __TBB_ASSERT( my_sending_event_present, NULL ); - } - - // !e means that buffer has come from the host - if ( !e && my_sending_event_present ) e = &my_sending_event; - - __TBB_ASSERT( !my_host_ptr, "The buffer has not been unmapped" ); - return e ? dependency_msg( NULL, *e ) : dependency_msg( NULL ); -} - -struct default_opencl_factory_device_filter { - opencl_device_list operator()( const opencl_device_list &devices ) { - opencl_device_list dl; - dl.add( *devices.begin() ); - return dl; - } -}; - -class default_opencl_factory : public opencl_factory < default_opencl_factory_device_filter > { -public: - template using async_msg_type = dependency_msg; - - default_opencl_factory( opencl_graph &g ) : opencl_factory( g ) {} -private: - default_opencl_factory( const default_opencl_factory& ); - default_opencl_factory& operator=(const default_opencl_factory&); -}; - -class opencl_foundation : tbb::internal::no_assign { - struct default_device_selector_type { - opencl_device operator()( default_opencl_factory& f ) { - __TBB_ASSERT( ! f.devices().empty(), "No available devices" ); - return *( f.devices().begin() ); - } - }; -public: - opencl_foundation(opencl_graph &g) : my_default_opencl_factory(g), my_default_device_selector() { - cl_uint num_platforms; - enforce_cl_retcode(clGetPlatformIDs(0, NULL, &num_platforms), "clGetPlatformIDs failed"); - - std::vector platforms(num_platforms); - enforce_cl_retcode(clGetPlatformIDs(num_platforms, platforms.data(), NULL), "clGetPlatformIDs failed"); - - cl_uint num_devices; - std::vector::iterator platforms_it = platforms.begin(); - cl_uint num_all_devices = 0; - while (platforms_it != platforms.end()) { - cl_int err = clGetDeviceIDs(*platforms_it, CL_DEVICE_TYPE_ALL, 0, NULL, &num_devices); - if (err == CL_DEVICE_NOT_FOUND) { - platforms_it = platforms.erase(platforms_it); - } else { - enforce_cl_retcode(err, "clGetDeviceIDs failed"); - num_all_devices += num_devices; - ++platforms_it; - } - } - - std::vector devices(num_all_devices); - std::vector::iterator devices_it = devices.begin(); - for (auto p = platforms.begin(); p != platforms.end(); ++p) { - enforce_cl_retcode(clGetDeviceIDs((*p), CL_DEVICE_TYPE_ALL, (cl_uint)std::distance(devices_it, devices.end()), &*devices_it, &num_devices), "clGetDeviceIDs failed"); - devices_it += num_devices; - } - - for (auto d = devices.begin(); d != devices.end(); ++d) { - my_devices.add(opencl_device((*d))); - } - } - - default_opencl_factory &get_default_opencl_factory() { - return my_default_opencl_factory; - } - - const opencl_device_list &get_all_devices() { - return my_devices; - } - - default_device_selector_type get_default_device_selector() { return my_default_device_selector; } - -private: - default_opencl_factory my_default_opencl_factory; - opencl_device_list my_devices; - - const default_device_selector_type my_default_device_selector; -}; - -opencl_foundation &opencl_graph::get_opencl_foundation() { - opencl_foundation* INITIALIZATION = (opencl_foundation*)1; - if ( my_opencl_foundation <= INITIALIZATION ) { - if ( tbb::internal::as_atomic( my_opencl_foundation ).compare_and_swap( INITIALIZATION, NULL ) == 0 ) { - my_opencl_foundation = new opencl_foundation( *this ); - } - else { - tbb::internal::spin_wait_while_eq( my_opencl_foundation, INITIALIZATION ); - } - } - - __TBB_ASSERT( my_opencl_foundation > INITIALIZATION, "opencl_foundation is not initialized"); - return *my_opencl_foundation; -} - -opencl_graph::~opencl_graph() { - if ( my_opencl_foundation ) - delete my_opencl_foundation; -} - -template -void opencl_factory::init_once() { - { - tbb::spin_mutex::scoped_lock lock( my_devices_mutex ); - if ( !my_devices.size() ) - my_devices = DeviceFilter()(my_graph.get_opencl_foundation().get_all_devices()); - } - - enforce_cl_retcode( my_devices.size() ? CL_SUCCESS : CL_INVALID_DEVICE, "No devices in the device list" ); - cl_platform_id platform_id = my_devices.begin()->platform(); - for ( opencl_device_list::iterator it = ++my_devices.begin(); it != my_devices.end(); ++it ) - enforce_cl_retcode( it->platform() == platform_id ? CL_SUCCESS : CL_INVALID_PLATFORM, "All devices should be in the same platform" ); - - std::vector cl_device_ids; - for (auto d = my_devices.begin(); d != my_devices.end(); ++d) { - cl_device_ids.push_back((*d).my_cl_device_id); - } - - cl_context_properties context_properties[3] = { CL_CONTEXT_PLATFORM, (cl_context_properties)platform_id, (cl_context_properties)NULL }; - cl_int err; - cl_context ctx = clCreateContext( context_properties, - (cl_uint)cl_device_ids.size(), - cl_device_ids.data(), - NULL, NULL, &err ); - enforce_cl_retcode( err, "Failed to create context" ); - my_cl_context = ctx; - - size_t device_counter = 0; - for ( auto d = my_devices.begin(); d != my_devices.end(); d++ ) { - (*d).my_device_id = device_counter++; - cl_int err2; - cl_command_queue cq; -#if CL_VERSION_2_0 - if ( (*d).major_version() >= 2 ) { - if ( (*d).out_of_order_exec_mode_on_host_present() ) { - cl_queue_properties props[] = { CL_QUEUE_PROPERTIES, CL_QUEUE_OUT_OF_ORDER_EXEC_MODE_ENABLE, 0 }; - cq = clCreateCommandQueueWithProperties( ctx, (*d).my_cl_device_id, props, &err2 ); - } else { - cl_queue_properties props[] = { 0 }; - cq = clCreateCommandQueueWithProperties( ctx, (*d).my_cl_device_id, props, &err2 ); - } - } else -#endif - { - cl_command_queue_properties props = (*d).out_of_order_exec_mode_on_host_present() ? CL_QUEUE_OUT_OF_ORDER_EXEC_MODE_ENABLE : 0; - // Suppress "declared deprecated" warning for the next line. -#if __TBB_GCC_WARNING_SUPPRESSION_PRESENT -#pragma GCC diagnostic push -// #pragma GCC diagnostic ignored "-Wdeprecated-declarations" -#endif -#if _MSC_VER || __INTEL_COMPILER -#pragma warning( push ) -#if __INTEL_COMPILER -#pragma warning (disable: 1478) -#else -#pragma warning (disable: 4996) -#endif -#endif - cq = clCreateCommandQueue( ctx, (*d).my_cl_device_id, props, &err2 ); -#if _MSC_VER || __INTEL_COMPILER -#pragma warning( pop ) -#endif -#if __TBB_GCC_WARNING_SUPPRESSION_PRESENT -#pragma GCC diagnostic pop -#endif - } - enforce_cl_retcode( err2, "Failed to create command queue" ); - (*d).my_cl_command_queue = cq; - } -} - -const opencl_device_list &opencl_graph::available_devices() { - return get_opencl_foundation().get_all_devices(); -} - -default_opencl_factory &opencl_graph::opencl_factory() { - return get_opencl_foundation().get_default_opencl_factory(); -} - -template -opencl_buffer::opencl_buffer( opencl_graph &g, size_t size ) : my_impl( std::make_shared( size*sizeof(T), g.get_opencl_foundation().get_default_opencl_factory() ) ) {} - - -enum class opencl_program_type { - SOURCE, - PRECOMPILED, - SPIR -}; - -template -class opencl_program : tbb::internal::no_assign { -public: - typedef typename Factory::kernel_type kernel_type; - - opencl_program( Factory& factory, opencl_program_type type, const std::string& program_name ) : my_factory( factory ), my_type(type) , my_arg_str( program_name) {} - opencl_program( Factory& factory, const char* program_name ) : opencl_program( factory, std::string( program_name ) ) {} - opencl_program( Factory& factory, const std::string& program_name ) : opencl_program( factory, opencl_program_type::SOURCE, program_name ) {} - - opencl_program( opencl_graph& graph, opencl_program_type type, const std::string& program_name ) : opencl_program( graph.opencl_factory(), type, program_name ) {} - opencl_program( opencl_graph& graph, const char* program_name ) : opencl_program( graph.opencl_factory(), program_name ) {} - opencl_program( opencl_graph& graph, const std::string& program_name ) : opencl_program( graph.opencl_factory(), program_name ) {} - opencl_program( opencl_graph& graph, opencl_program_type type ) : opencl_program( graph.opencl_factory(), type ) {} - - opencl_program( const opencl_program &src ) : my_factory( src.my_factory ), my_type( src.type ), my_arg_str( src.my_arg_str ), my_cl_program( src.my_cl_program ) { - // Set my_do_once_flag to the called state. - std::call_once( my_do_once_flag, [](){} ); - } - - kernel_type get_kernel( const std::string& k ) const { - return kernel_type( get_cl_kernel(k), my_factory ); - } - -private: - opencl_program( Factory& factory, cl_program program ) : my_factory( factory ), my_cl_program( program ) { - // Set my_do_once_flag to the called state. - std::call_once( my_do_once_flag, [](){} ); - } - - cl_kernel get_cl_kernel( const std::string& k ) const { - std::call_once( my_do_once_flag, [this, &k](){ this->init( k ); } ); - cl_int err; - cl_kernel kernel = clCreateKernel( my_cl_program, k.c_str(), &err ); - enforce_cl_retcode( err, std::string( "Failed to create kernel: " ) + k ); - return kernel; - } - - class file_reader { - public: - file_reader( const std::string& filepath ) { - std::ifstream file_descriptor( filepath, std::ifstream::binary ); - if ( !file_descriptor.is_open() ) { - std::string str = std::string( "Could not open file: " ) + filepath; - std::cerr << str << std::endl; - throw str; - } - file_descriptor.seekg( 0, file_descriptor.end ); - size_t length = size_t( file_descriptor.tellg() ); - file_descriptor.seekg( 0, file_descriptor.beg ); - my_content.resize( length ); - char* begin = &*my_content.begin(); - file_descriptor.read( begin, length ); - file_descriptor.close(); - } - const char* content() { return &*my_content.cbegin(); } - size_t length() { return my_content.length(); } - private: - std::string my_content; - }; - - class opencl_program_builder { - public: - typedef void (CL_CALLBACK *cl_callback_type)(cl_program, void*); - opencl_program_builder( Factory& f, const std::string& name, cl_program program, - cl_uint num_devices, cl_device_id* device_list, - const char* options, cl_callback_type callback, - void* user_data ) { - cl_int err = clBuildProgram( program, num_devices, device_list, options, - callback, user_data ); - if( err == CL_SUCCESS ) - return; - std::string str = std::string( "Failed to build program: " ) + name; - if ( err == CL_BUILD_PROGRAM_FAILURE ) { - const opencl_device_list &devices = f.devices(); - for ( auto d = devices.begin(); d != devices.end(); ++d ) { - std::cerr << "Build log for device: " << (*d).name() << std::endl; - size_t log_size; - cl_int query_err = clGetProgramBuildInfo( - program, (*d).my_cl_device_id, CL_PROGRAM_BUILD_LOG, 0, NULL, - &log_size ); - enforce_cl_retcode( query_err, "Failed to get build log size" ); - if( log_size ) { - std::vector output; - output.resize( log_size ); - query_err = clGetProgramBuildInfo( - program, (*d).my_cl_device_id, CL_PROGRAM_BUILD_LOG, - output.size(), output.data(), NULL ); - enforce_cl_retcode( query_err, "Failed to get build output" ); - std::cerr << output.data() << std::endl; - } else { - std::cerr << "No build log available" << std::endl; - } - } - } - enforce_cl_retcode( err, str ); - } - }; - - class opencl_device_filter { - public: - template - opencl_device_filter( cl_uint& num_devices, cl_device_id* device_list, - Filter filter, const char* message ) { - for ( cl_uint i = 0; i < num_devices; ++i ) - if ( filter(device_list[i]) ) { - device_list[i--] = device_list[--num_devices]; - } - if ( !num_devices ) - enforce_cl_retcode( CL_DEVICE_NOT_AVAILABLE, message ); - } - }; - - void init( const std::string& ) const { - cl_uint num_devices; - enforce_cl_retcode( clGetContextInfo( my_factory.context(), CL_CONTEXT_NUM_DEVICES, sizeof( num_devices ), &num_devices, NULL ), - "Failed to get OpenCL context info" ); - if ( !num_devices ) - enforce_cl_retcode( CL_DEVICE_NOT_FOUND, "No supported devices found" ); - cl_device_id *device_list = (cl_device_id *)alloca( num_devices*sizeof( cl_device_id ) ); - enforce_cl_retcode( clGetContextInfo( my_factory.context(), CL_CONTEXT_DEVICES, num_devices*sizeof( cl_device_id ), device_list, NULL ), - "Failed to get OpenCL context info" ); - const char *options = NULL; - switch ( my_type ) { - case opencl_program_type::SOURCE: { - file_reader fr( my_arg_str ); - const char *s[] = { fr.content() }; - const size_t l[] = { fr.length() }; - cl_int err; - my_cl_program = clCreateProgramWithSource( my_factory.context(), 1, s, l, &err ); - enforce_cl_retcode( err, std::string( "Failed to create program: " ) + my_arg_str ); - opencl_device_filter( - num_devices, device_list, - []( const opencl_device& d ) -> bool { - return !d.compiler_available() || !d.linker_available(); - }, "No one device supports building program from sources" ); - opencl_program_builder( - my_factory, my_arg_str, my_cl_program, num_devices, device_list, - options, /*callback*/ NULL, /*user data*/NULL ); - break; - } - case opencl_program_type::SPIR: - options = "-x spir"; - case opencl_program_type::PRECOMPILED: { - file_reader fr( my_arg_str ); - std::vector s( - num_devices, reinterpret_cast(fr.content()) ); - std::vector l( num_devices, fr.length() ); - std::vector bin_statuses( num_devices, -1 ); - cl_int err; - my_cl_program = clCreateProgramWithBinary( my_factory.context(), num_devices, - device_list, l.data(), s.data(), - bin_statuses.data(), &err ); - if( err != CL_SUCCESS ) { - std::string statuses_str; - for (auto st = bin_statuses.begin(); st != bin_statuses.end(); ++st) { - statuses_str += std::to_string((*st)); - } - - enforce_cl_retcode( err, std::string( "Failed to create program, error " + std::to_string( err ) + " : " ) + my_arg_str + - std::string( ", binary_statuses = " ) + statuses_str ); - } - opencl_program_builder( - my_factory, my_arg_str, my_cl_program, num_devices, device_list, - options, /*callback*/ NULL, /*user data*/NULL ); - break; - } - default: - __TBB_ASSERT( false, "Unsupported program type" ); - } - } - - Factory& my_factory; - opencl_program_type my_type; - std::string my_arg_str; - mutable cl_program my_cl_program; - mutable std::once_flag my_do_once_flag; - - template - friend class opencl_factory; - - template - friend class opencl_factory::kernel; -}; - -template -class opencl_node; - -template -class opencl_node< tuple, JP, Factory > : public streaming_node< tuple, JP, Factory > { - typedef streaming_node < tuple, JP, Factory > base_type; -public: - typedef typename base_type::kernel_type kernel_type; - - opencl_node( opencl_graph &g, const kernel_type& kernel ) - : base_type( g, kernel, g.get_opencl_foundation().get_default_device_selector(), g.get_opencl_foundation().get_default_opencl_factory() ) - {} - - opencl_node( opencl_graph &g, const kernel_type& kernel, Factory &f ) - : base_type( g, kernel, g.get_opencl_foundation().get_default_device_selector(), f ) - {} - - template - opencl_node( opencl_graph &g, const kernel_type& kernel, DeviceSelector d, Factory &f) - : base_type( g, kernel, d, f) - {} -}; - -template -class opencl_node< tuple, JP > : public opencl_node < tuple, JP, default_opencl_factory > { - typedef opencl_node < tuple, JP, default_opencl_factory > base_type; -public: - typedef typename base_type::kernel_type kernel_type; - - opencl_node( opencl_graph &g, const kernel_type& kernel ) - : base_type( g, kernel, g.get_opencl_foundation().get_default_device_selector(), g.get_opencl_foundation().get_default_opencl_factory() ) - {} - - template - opencl_node( opencl_graph &g, const kernel_type& kernel, DeviceSelector d ) - : base_type( g, kernel, d, g.get_opencl_foundation().get_default_opencl_factory() ) - {} -}; - -template -class opencl_node< tuple > : public opencl_node < tuple, queueing, default_opencl_factory > { - typedef opencl_node < tuple, queueing, default_opencl_factory > base_type; -public: - typedef typename base_type::kernel_type kernel_type; - - opencl_node( opencl_graph &g, const kernel_type& kernel ) - : base_type( g, kernel, g.get_opencl_foundation().get_default_device_selector(), g.get_opencl_foundation().get_default_opencl_factory() ) - {} - - template - opencl_node( opencl_graph &g, const kernel_type& kernel, DeviceSelector d ) - : base_type( g, kernel, d, g.get_opencl_foundation().get_default_opencl_factory() ) - {} -}; - -} // namespace interface9 - -using interface9::opencl_graph; -using interface9::opencl_node; -using interface9::read_only; -using interface9::read_write; -using interface9::write_only; -using interface9::opencl_buffer; -using interface9::opencl_subbuffer; -using interface9::opencl_device; -using interface9::opencl_device_list; -using interface9::opencl_program; -using interface9::opencl_program_type; -using interface9::dependency_msg; -using interface9::opencl_factory; -using interface9::opencl_range; - -} // namespace flow -} // namespace tbb -#endif /* __TBB_PREVIEW_OPENCL_NODE */ - -#endif // __TBB_flow_graph_opencl_node_H diff --git a/inst/include/tbb/gfx_factory.h b/inst/include/tbb/gfx_factory.h deleted file mode 100644 index 76ca3559a..000000000 --- a/inst/include/tbb/gfx_factory.h +++ /dev/null @@ -1,359 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_flow_graph_gfx_factory_H -#define __TBB_flow_graph_gfx_factory_H - -#include "tbb/tbb_config.h" - -#if __TBB_PREVIEW_GFX_FACTORY - -#include -#include -#include -#include - -#include -#include -#include - -namespace tbb { - -namespace flow { - -namespace interface9 { - -template -class gfx_buffer; - -namespace gfx_offload { - - typedef GfxTaskId task_id_type; - - //----------------------------------------------------------------------- - // GFX errors checkers. - // For more debug output, set GFX_LOG_OFFLOAD=2 macro - //----------------------------------------------------------------------- - - // TODO: reconsider error handling approach. If exception is the right way - // then need to define and document a specific exception type. - inline void throw_gfx_exception() { - std::string msg = "GFX error occurred: " + std::to_string(_GFX_get_last_error()); - std::cerr << msg << std::endl; - throw msg; - } - - inline void check_enqueue_retcode(task_id_type err) { - if (err == 0) { - throw_gfx_exception(); - } - } - - inline void check_gfx_retcode(task_id_type err) { - if (err != GFX_SUCCESS) { - throw_gfx_exception(); - } - } - - //--------------------------------------------------------------------- - // GFX asynchronous offload and share API - //--------------------------------------------------------------------- - - // Sharing and unsharing data API - template - void share(DataType* p, SizeType n) { check_gfx_retcode(_GFX_share(p, sizeof(*p)*n)); } - template - void unshare(DataType* p) { check_gfx_retcode(_GFX_unshare(p)); } - - // Retrieving array pointer from shared gfx_buffer - // Other types remain the same - template - T* raw_data(gfx_buffer& buffer) { return buffer.data(); } - template - const T* raw_data(const gfx_buffer& buffer) { return buffer.data(); } - template - T& raw_data(T& data) { return data; } - template - const T& raw_data(const T& data) { return data; } - - // Kernel enqueuing on device with arguments - template - task_id_type run_kernel(F ptr, ArgType&... args) { - task_id_type id = _GFX_offload(ptr, raw_data(args)...); - - // Check if something during offload went wrong (ex: driver initialization failure) - gfx_offload::check_enqueue_retcode(id); - - return id; - } - - // Waiting for tasks completion - void wait_for_task(task_id_type id) { check_gfx_retcode(_GFX_wait(id)); } - -} // namespace gfx_offload - -template -class gfx_buffer { -public: - - typedef typename std::vector::iterator iterator; - typedef typename std::vector::const_iterator const_iterator; - - typedef std::size_t size_type; - - gfx_buffer() : my_vector_ptr(std::make_shared< std::vector >()) {} - gfx_buffer(size_type size) : my_vector_ptr(std::make_shared< std::vector >(size)) {} - - T* data() { return &(my_vector_ptr->front()); } - const T* data() const { return &(my_vector_ptr->front()); } - - size_type size() const { return my_vector_ptr->size(); } - - const_iterator cbegin() const { return my_vector_ptr->cbegin(); } - const_iterator cend() const { return my_vector_ptr->cend(); } - iterator begin() { return my_vector_ptr->begin(); } - iterator end() { return my_vector_ptr->end(); } - - T& operator[](size_type pos) { return (*my_vector_ptr)[pos]; } - const T& operator[](size_type pos) const { return (*my_vector_ptr)[pos]; } - -private: - std::shared_ptr< std::vector > my_vector_ptr; -}; - -template -class gfx_async_msg : public tbb::flow::async_msg { -public: - typedef gfx_offload::task_id_type kernel_id_type; - - gfx_async_msg() : my_task_id(0) {} - gfx_async_msg(const T& input_data) : my_data(input_data), my_task_id(0) {} - - T& data() { return my_data; } - const T& data() const { return my_data; } - - void set_task_id(kernel_id_type id) { my_task_id = id; } - kernel_id_type task_id() const { return my_task_id; } - -private: - T my_data; - kernel_id_type my_task_id; -}; - -class gfx_factory { -private: - - // Wrapper for GFX kernel which is just a function - class func_wrapper { - public: - - template - func_wrapper(F ptr) { my_ptr = reinterpret_cast(ptr); } - - template - void operator()(Args&&... args) {} - - operator void*() { return my_ptr; } - - private: - void* my_ptr; - }; - -public: - - // Device specific types - template using async_msg_type = gfx_async_msg; - - typedef func_wrapper kernel_type; - - // Empty device type that is needed for Factory Concept - // but is not used in gfx_factory - typedef struct {} device_type; - - typedef gfx_offload::task_id_type kernel_id_type; - - gfx_factory(tbb::flow::graph& g) : m_graph(g), current_task_id(0) {} - - // Upload data to the device - template - void send_data(device_type /*device*/, Args&... args) { - send_data_impl(args...); - } - - // Run kernel on the device - template - void send_kernel(device_type /*device*/, const kernel_type& kernel, Args&... args) { - // Get packed T data from async_msg and pass it to kernel - kernel_id_type id = gfx_offload::run_kernel(kernel, args.data()...); - - // Set id to async_msg - set_kernel_id(id, args...); - - // Extend the graph lifetime until the callback completion. - m_graph.reserve_wait(); - - // Mutex for future assignment - std::lock_guard lock(future_assignment_mutex); - - // Set callback that waits for kernel execution - callback_future = std::async(std::launch::async, &gfx_factory::callback, this, id, args...); - } - - // Finalization action after the kernel run - template - void finalize(device_type /*device*/, FinalizeFn fn, Args&... /*args*/) { - fn(); - } - - // Empty device selector. - // No way to choose a device with GFX API. - class dummy_device_selector { - public: - device_type operator()(gfx_factory& /*factory*/) { - return device_type(); - } - }; - -private: - - //--------------------------------------------------------------------- - // Callback for kernel result - //--------------------------------------------------------------------- - - template - void callback(kernel_id_type id, Args... args) { - // Waiting for specific tasks id to complete - { - std::lock_guard lock(task_wait_mutex); - if (current_task_id < id) { - gfx_offload::wait_for_task(id); - current_task_id = id; - } - } - - // Get result from device and set to async_msg (args) - receive_data(args...); - - // Data was sent to the graph, release the reference - m_graph.release_wait(); - } - - //--------------------------------------------------------------------- - // send_data() arguments processing - //--------------------------------------------------------------------- - - // GFX buffer shared data with device that will be executed on - template - void share_data(T) {} - - template - void share_data(gfx_buffer& buffer) { - gfx_offload::share(buffer.data(), buffer.size()); - } - - template - void send_arg(T) {} - - template - void send_arg(async_msg_type& msg) { - share_data(msg.data()); - } - - void send_data_impl() {} - - template - void send_data_impl(T& arg, Rest&... args) { - send_arg(arg); - send_data_impl(args...); - } - - //---------------------------------------------------------------------- - // send_kernel() arguments processing - //---------------------------------------------------------------------- - - template - void set_kernel_id_arg(kernel_id_type, T) {} - - template - void set_kernel_id_arg(kernel_id_type id, async_msg_type& msg) { - msg.set_task_id(id); - } - - void set_kernel_id(kernel_id_type) {} - - template - void set_kernel_id(kernel_id_type id, T& arg, Rest&... args) { - set_kernel_id_arg(id, arg); - set_kernel_id(id, args...); - } - - //----------------------------------------------------------------------- - // Arguments processing after kernel execution. - // Unsharing buffers and forwarding results to the graph - //----------------------------------------------------------------------- - - // After kernel execution the data should be unshared - template - void unshare_data(T) {} - - template - void unshare_data(gfx_buffer& buffer) { - gfx_offload::unshare(buffer.data()); - } - - template - void receive_arg(T) {} - - template - void receive_arg(async_msg_type& msg) { - unshare_data(msg.data()); - msg.set(msg.data()); - } - - void receive_data() {} - - template - void receive_data(T& arg, Rest&... args) { - receive_arg(arg); - receive_data(args...); - } - - //----------------------------------------------------------------------- - int current_task_id; - - std::future callback_future; - tbb::flow::graph& m_graph; - - std::mutex future_assignment_mutex; - std::mutex task_wait_mutex; -}; - -} // namespace interface9 - -using interface9::gfx_factory; -using interface9::gfx_buffer; - -} // namespace flow - -} // namespace tbb - -#endif // __TBB_PREVIEW_GFX_FACTORY - -#endif // __TBB_flow_graph_gfx_factory_H diff --git a/inst/include/tbb/global_control.h b/inst/include/tbb/global_control.h deleted file mode 100644 index fe7420203..000000000 --- a/inst/include/tbb/global_control.h +++ /dev/null @@ -1,86 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_global_control_H -#define __TBB_global_control_H - -#if !TBB_PREVIEW_GLOBAL_CONTROL && !__TBB_BUILD -#error Set TBB_PREVIEW_GLOBAL_CONTROL before including global_control.h -#endif - -#include "tbb_stddef.h" - -namespace tbb { -namespace interface9 { - -class global_control { -public: - enum parameter { - max_allowed_parallelism, - thread_stack_size, - parameter_max // insert new parameters above this point - }; - - global_control(parameter p, size_t value) : - my_value(value), my_next(NULL), my_param(p) { - __TBB_ASSERT(my_param < parameter_max, "Invalid parameter"); -#if __TBB_WIN8UI_SUPPORT - // For Windows Store* apps it's impossible to set stack size - if (p==thread_stack_size) - return; -#elif __TBB_x86_64 && (_WIN32 || _WIN64) - if (p==thread_stack_size) - __TBB_ASSERT_RELEASE((unsigned)value == value, "Stack size is limited to unsigned int range"); -#endif - if (my_param==max_allowed_parallelism) - __TBB_ASSERT_RELEASE(my_value>0, "max_allowed_parallelism cannot be 0."); - internal_create(); - } - - ~global_control() { - __TBB_ASSERT(my_param < parameter_max, "Invalid parameter. Probably the object was corrupted."); -#if __TBB_WIN8UI_SUPPORT - // For Windows Store* apps it's impossible to set stack size - if (my_param==thread_stack_size) - return; -#endif - internal_destroy(); - } - - static size_t active_value(parameter p) { - __TBB_ASSERT(p < parameter_max, "Invalid parameter"); - return active_value((int)p); - } -private: - size_t my_value; - global_control *my_next; - parameter my_param; - - void __TBB_EXPORTED_METHOD internal_create(); - void __TBB_EXPORTED_METHOD internal_destroy(); - static size_t __TBB_EXPORTED_FUNC active_value(int param); -}; -} // namespace interface9 - -using interface9::global_control; - -} // tbb - -#endif // __TBB_global_control_H diff --git a/inst/include/tbb/index.html b/inst/include/tbb/index.html deleted file mode 100644 index 2eddd450d..000000000 --- a/inst/include/tbb/index.html +++ /dev/null @@ -1,29 +0,0 @@ - - - -

Overview

-Include files for Intel® Threading Building Blocks classes and functions. - -
Click here to see all files in the directory. - -

Directories

-
-
compat -
Include files for source level compatibility with other frameworks. -
internal -
Include files with implementation details; not for direct use. -
machine -
Include files for low-level architecture specific functionality; not for direct use. -
- -
-Up to parent directory -

-Copyright © 2005-2017 Intel Corporation. All Rights Reserved. -

-Intel is a registered trademark or trademark of Intel Corporation -or its subsidiaries in the United States and other countries. -

-* Other names and brands may be claimed as the property of others. - - diff --git a/inst/include/tbb/internal/_aggregator_impl.h b/inst/include/tbb/internal/_aggregator_impl.h deleted file mode 100644 index 40bbd4916..000000000 --- a/inst/include/tbb/internal/_aggregator_impl.h +++ /dev/null @@ -1,184 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB__aggregator_impl_H -#define __TBB__aggregator_impl_H - -#include "../atomic.h" -#if !__TBBMALLOC_BUILD -#include "../tbb_profiling.h" -#endif - -namespace tbb { -namespace interface6 { -namespace internal { - -using namespace tbb::internal; - -//! aggregated_operation base class -template -class aggregated_operation { - public: - //! Zero value means "wait" status, all other values are "user" specified values and are defined into the scope of a class which uses "status". - uintptr_t status; - - Derived *next; - aggregated_operation() : status(0), next(NULL) {} -}; - -//! Aggregator base class -/** An aggregator for collecting operations coming from multiple sources and executing - them serially on a single thread. operation_type must be derived from - aggregated_operation. The parameter handler_type is a functor that will be passed the - list of operations and is expected to handle each operation appropriately, setting the - status of each operation to non-zero.*/ -template < typename operation_type > -class aggregator_generic { -public: - aggregator_generic() : handler_busy(false) { pending_operations = NULL; } - - //! Execute an operation - /** Places an operation into the waitlist (pending_operations), and either handles the list, - or waits for the operation to complete, or returns. - The long_life_time parameter specifies the life time of the given operation object. - Operations with long_life_time == true may be accessed after execution. - A "short" life time operation (long_life_time == false) can be destroyed - during execution, and so any access to it after it was put into the waitlist, - including status check, is invalid. As a consequence, waiting for completion - of such operation causes undefined behavior. - */ - template < typename handler_type > - void execute(operation_type *op, handler_type &handle_operations, bool long_life_time = true) { - operation_type *res; - // op->status should be read before inserting the operation into the - // aggregator waitlist since it can become invalid after executing a - // handler (if the operation has 'short' life time.) - const uintptr_t status = op->status; - - // ITT note: &(op->status) tag is used to cover accesses to this op node. This - // thread has created the operation, and now releases it so that the handler - // thread may handle the associated operation w/o triggering a race condition; - // thus this tag will be acquired just before the operation is handled in the - // handle_operations functor. - call_itt_notify(releasing, &(op->status)); - // insert the operation in the queue. - do { - // Tools may flag the following line as a race; it is a false positive: - // This is an atomic read; we don't provide itt_hide_load_word for atomics - op->next = res = pending_operations; // NOT A RACE - } while (pending_operations.compare_and_swap(op, res) != res); - if (!res) { // first in the list; handle the operations. - // ITT note: &pending_operations tag covers access to the handler_busy flag, - // which this waiting handler thread will try to set before entering - // handle_operations. - call_itt_notify(acquired, &pending_operations); - start_handle_operations(handle_operations); - // The operation with 'short' life time can already be destroyed. - if (long_life_time) - __TBB_ASSERT(op->status, NULL); - } - // not first; wait for op to be ready. - else if (!status) { // operation is blocking here. - __TBB_ASSERT(long_life_time, "Waiting for an operation object that might be destroyed during processing."); - call_itt_notify(prepare, &(op->status)); - spin_wait_while_eq(op->status, uintptr_t(0)); - itt_load_word_with_acquire(op->status); - } - } - - private: - //! An atomically updated list (aka mailbox) of pending operations - atomic pending_operations; - //! Controls thread access to handle_operations - uintptr_t handler_busy; - - //! Trigger the handling of operations when the handler is free - template < typename handler_type > - void start_handle_operations( handler_type &handle_operations ) { - operation_type *op_list; - - // ITT note: &handler_busy tag covers access to pending_operations as it is passed - // between active and waiting handlers. Below, the waiting handler waits until - // the active handler releases, and the waiting handler acquires &handler_busy as - // it becomes the active_handler. The release point is at the end of this - // function, when all operations in pending_operations have been handled by the - // owner of this aggregator. - call_itt_notify(prepare, &handler_busy); - // get the handler_busy: - // only one thread can possibly spin here at a time - spin_wait_until_eq(handler_busy, uintptr_t(0)); - call_itt_notify(acquired, &handler_busy); - // acquire fence not necessary here due to causality rule and surrounding atomics - __TBB_store_with_release(handler_busy, uintptr_t(1)); - - // ITT note: &pending_operations tag covers access to the handler_busy flag - // itself. Capturing the state of the pending_operations signifies that - // handler_busy has been set and a new active handler will now process that list's - // operations. - call_itt_notify(releasing, &pending_operations); - // grab pending_operations - op_list = pending_operations.fetch_and_store(NULL); - - // handle all the operations - handle_operations(op_list); - - // release the handler - itt_store_word_with_release(handler_busy, uintptr_t(0)); - } -}; - -template < typename handler_type, typename operation_type > -class aggregator : public aggregator_generic { - handler_type handle_operations; -public: - aggregator() {} - explicit aggregator(handler_type h) : handle_operations(h) {} - - void initialize_handler(handler_type h) { handle_operations = h; } - - void execute(operation_type *op) { - aggregator_generic::execute(op, handle_operations); - } -}; - -// the most-compatible friend declaration (vs, gcc, icc) is -// template friend class aggregating_functor; -template -class aggregating_functor { - aggregating_class *fi; -public: - aggregating_functor() {} - aggregating_functor(aggregating_class *fi_) : fi(fi_) {} - void operator()(operation_list* op_list) { fi->handle_operations(op_list); } -}; - -} // namespace internal -} // namespace interface6 - -namespace internal { - using interface6::internal::aggregated_operation; - using interface6::internal::aggregator_generic; - using interface6::internal::aggregator; - using interface6::internal::aggregating_functor; -} // namespace internal - -} // namespace tbb - -#endif // __TBB__aggregator_impl_H diff --git a/inst/include/tbb/internal/_concurrent_queue_impl.h b/inst/include/tbb/internal/_concurrent_queue_impl.h deleted file mode 100644 index a99a68f6e..000000000 --- a/inst/include/tbb/internal/_concurrent_queue_impl.h +++ /dev/null @@ -1,1086 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB__concurrent_queue_impl_H -#define __TBB__concurrent_queue_impl_H - -#ifndef __TBB_concurrent_queue_H -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#include "../tbb_stddef.h" -#include "../tbb_machine.h" -#include "../atomic.h" -#include "../spin_mutex.h" -#include "../cache_aligned_allocator.h" -#include "../tbb_exception.h" -#include "../tbb_profiling.h" -#include -#include __TBB_STD_SWAP_HEADER - -#if !TBB_USE_EXCEPTIONS && _MSC_VER - // Suppress "C++ exception handler used, but unwind semantics are not enabled" warning in STL headers - #pragma warning (push) - #pragma warning (disable: 4530) -#endif - -#include - -#if !TBB_USE_EXCEPTIONS && _MSC_VER - #pragma warning (pop) -#endif - -namespace tbb { - -#if !__TBB_TEMPLATE_FRIENDS_BROKEN - -// forward declaration -namespace strict_ppl { -template class concurrent_queue; -} - -template class concurrent_bounded_queue; - -#endif - -//! For internal use only. -namespace strict_ppl { - -//! @cond INTERNAL -namespace internal { - -using namespace tbb::internal; - -typedef size_t ticket; - -template class micro_queue ; -template class micro_queue_pop_finalizer ; -template class concurrent_queue_base_v3; -template struct concurrent_queue_rep; - -//! parts of concurrent_queue_rep that do not have references to micro_queue -/** - * For internal use only. - */ -struct concurrent_queue_rep_base : no_copy { - template friend class micro_queue; - template friend class concurrent_queue_base_v3; - -protected: - //! Approximately n_queue/golden ratio - static const size_t phi = 3; - -public: - // must be power of 2 - static const size_t n_queue = 8; - - //! Prefix on a page - struct page { - page* next; - uintptr_t mask; - }; - - atomic head_counter; - char pad1[NFS_MaxLineSize-sizeof(atomic)]; - atomic tail_counter; - char pad2[NFS_MaxLineSize-sizeof(atomic)]; - - //! Always a power of 2 - size_t items_per_page; - - //! Size of an item - size_t item_size; - - //! number of invalid entries in the queue - atomic n_invalid_entries; - - char pad3[NFS_MaxLineSize-sizeof(size_t)-sizeof(size_t)-sizeof(atomic)]; -} ; - -inline bool is_valid_page(const concurrent_queue_rep_base::page* p) { - return uintptr_t(p)>1; -} - -//! Abstract class to define interface for page allocation/deallocation -/** - * For internal use only. - */ -class concurrent_queue_page_allocator -{ - template friend class micro_queue ; - template friend class micro_queue_pop_finalizer ; -protected: - virtual ~concurrent_queue_page_allocator() {} -private: - virtual concurrent_queue_rep_base::page* allocate_page() = 0; - virtual void deallocate_page( concurrent_queue_rep_base::page* p ) = 0; -} ; - -#if _MSC_VER && !defined(__INTEL_COMPILER) -// unary minus operator applied to unsigned type, result still unsigned -#pragma warning( push ) -#pragma warning( disable: 4146 ) -#endif - -//! A queue using simple locking. -/** For efficiency, this class has no constructor. - The caller is expected to zero-initialize it. */ -template -class micro_queue : no_copy { -public: - typedef void (*item_constructor_t)(T* location, const void* src); -private: - typedef concurrent_queue_rep_base::page page; - - //! Class used to ensure exception-safety of method "pop" - class destroyer: no_copy { - T& my_value; - public: - destroyer( T& value ) : my_value(value) {} - ~destroyer() {my_value.~T();} - }; - - void copy_item( page& dst, size_t dindex, const void* src, item_constructor_t construct_item ) { - construct_item( &get_ref(dst, dindex), src ); - } - - void copy_item( page& dst, size_t dindex, const page& src, size_t sindex, - item_constructor_t construct_item ) - { - T& src_item = get_ref( const_cast(src), sindex ); - construct_item( &get_ref(dst, dindex), static_cast(&src_item) ); - } - - void assign_and_destroy_item( void* dst, page& src, size_t index ) { - T& from = get_ref(src,index); - destroyer d(from); - *static_cast(dst) = tbb::internal::move( from ); - } - - void spin_wait_until_my_turn( atomic& counter, ticket k, concurrent_queue_rep_base& rb ) const ; - -public: - friend class micro_queue_pop_finalizer; - - struct padded_page: page { - //! Not defined anywhere - exists to quiet warnings. - padded_page(); - //! Not defined anywhere - exists to quiet warnings. - void operator=( const padded_page& ); - //! Must be last field. - T last; - }; - - static T& get_ref( page& p, size_t index ) { - return (&static_cast(static_cast(&p))->last)[index]; - } - - atomic head_page; - atomic head_counter; - - atomic tail_page; - atomic tail_counter; - - spin_mutex page_mutex; - - void push( const void* item, ticket k, concurrent_queue_base_v3& base, - item_constructor_t construct_item ) ; - - bool pop( void* dst, ticket k, concurrent_queue_base_v3& base ) ; - - micro_queue& assign( const micro_queue& src, concurrent_queue_base_v3& base, - item_constructor_t construct_item ) ; - - page* make_copy( concurrent_queue_base_v3& base, const page* src_page, size_t begin_in_page, - size_t end_in_page, ticket& g_index, item_constructor_t construct_item ) ; - - void invalidate_page_and_rethrow( ticket k ) ; -}; - -template -void micro_queue::spin_wait_until_my_turn( atomic& counter, ticket k, concurrent_queue_rep_base& rb ) const { - for( atomic_backoff b(true);;b.pause() ) { - ticket c = counter; - if( c==k ) return; - else if( c&1 ) { - ++rb.n_invalid_entries; - throw_exception( eid_bad_last_alloc ); - } - } -} - -template -void micro_queue::push( const void* item, ticket k, concurrent_queue_base_v3& base, - item_constructor_t construct_item ) -{ - k &= -concurrent_queue_rep_base::n_queue; - page* p = NULL; - size_t index = modulo_power_of_two( k/concurrent_queue_rep_base::n_queue, base.my_rep->items_per_page); - if( !index ) { - __TBB_TRY { - concurrent_queue_page_allocator& pa = base; - p = pa.allocate_page(); - } __TBB_CATCH (...) { - ++base.my_rep->n_invalid_entries; - invalidate_page_and_rethrow( k ); - } - p->mask = 0; - p->next = NULL; - } - - if( tail_counter != k ) spin_wait_until_my_turn( tail_counter, k, *base.my_rep ); - call_itt_notify(acquired, &tail_counter); - - if( p ) { - spin_mutex::scoped_lock lock( page_mutex ); - page* q = tail_page; - if( is_valid_page(q) ) - q->next = p; - else - head_page = p; - tail_page = p; - } else { - p = tail_page; - } - - __TBB_TRY { - copy_item( *p, index, item, construct_item ); - // If no exception was thrown, mark item as present. - itt_hide_store_word(p->mask, p->mask | uintptr_t(1)<n_invalid_entries; - call_itt_notify(releasing, &tail_counter); - tail_counter += concurrent_queue_rep_base::n_queue; - __TBB_RETHROW(); - } -} - -template -bool micro_queue::pop( void* dst, ticket k, concurrent_queue_base_v3& base ) { - k &= -concurrent_queue_rep_base::n_queue; - if( head_counter!=k ) spin_wait_until_eq( head_counter, k ); - call_itt_notify(acquired, &head_counter); - if( tail_counter==k ) spin_wait_while_eq( tail_counter, k ); - call_itt_notify(acquired, &tail_counter); - page *p = head_page; - __TBB_ASSERT( p, NULL ); - size_t index = modulo_power_of_two( k/concurrent_queue_rep_base::n_queue, base.my_rep->items_per_page ); - bool success = false; - { - micro_queue_pop_finalizer finalizer( *this, base, k+concurrent_queue_rep_base::n_queue, index==base.my_rep->items_per_page-1 ? p : NULL ); - if( p->mask & uintptr_t(1)<n_invalid_entries; - } - } - return success; -} - -template -micro_queue& micro_queue::assign( const micro_queue& src, concurrent_queue_base_v3& base, - item_constructor_t construct_item ) -{ - head_counter = src.head_counter; - tail_counter = src.tail_counter; - - const page* srcp = src.head_page; - if( is_valid_page(srcp) ) { - ticket g_index = head_counter; - __TBB_TRY { - size_t n_items = (tail_counter-head_counter)/concurrent_queue_rep_base::n_queue; - size_t index = modulo_power_of_two( head_counter/concurrent_queue_rep_base::n_queue, base.my_rep->items_per_page ); - size_t end_in_first_page = (index+n_itemsitems_per_page)?(index+n_items):base.my_rep->items_per_page; - - head_page = make_copy( base, srcp, index, end_in_first_page, g_index, construct_item ); - page* cur_page = head_page; - - if( srcp != src.tail_page ) { - for( srcp = srcp->next; srcp!=src.tail_page; srcp=srcp->next ) { - cur_page->next = make_copy( base, srcp, 0, base.my_rep->items_per_page, g_index, construct_item ); - cur_page = cur_page->next; - } - - __TBB_ASSERT( srcp==src.tail_page, NULL ); - size_t last_index = modulo_power_of_two( tail_counter/concurrent_queue_rep_base::n_queue, base.my_rep->items_per_page ); - if( last_index==0 ) last_index = base.my_rep->items_per_page; - - cur_page->next = make_copy( base, srcp, 0, last_index, g_index, construct_item ); - cur_page = cur_page->next; - } - tail_page = cur_page; - } __TBB_CATCH (...) { - invalidate_page_and_rethrow( g_index ); - } - } else { - head_page = tail_page = NULL; - } - return *this; -} - -template -void micro_queue::invalidate_page_and_rethrow( ticket k ) { - // Append an invalid page at address 1 so that no more pushes are allowed. - page* invalid_page = (page*)uintptr_t(1); - { - spin_mutex::scoped_lock lock( page_mutex ); - itt_store_word_with_release(tail_counter, k+concurrent_queue_rep_base::n_queue+1); - page* q = tail_page; - if( is_valid_page(q) ) - q->next = invalid_page; - else - head_page = invalid_page; - tail_page = invalid_page; - } - __TBB_RETHROW(); -} - -template -concurrent_queue_rep_base::page* micro_queue::make_copy( concurrent_queue_base_v3& base, - const concurrent_queue_rep_base::page* src_page, size_t begin_in_page, size_t end_in_page, - ticket& g_index, item_constructor_t construct_item ) -{ - concurrent_queue_page_allocator& pa = base; - page* new_page = pa.allocate_page(); - new_page->next = NULL; - new_page->mask = src_page->mask; - for( ; begin_in_page!=end_in_page; ++begin_in_page, ++g_index ) - if( new_page->mask & uintptr_t(1)< -class micro_queue_pop_finalizer: no_copy { - typedef concurrent_queue_rep_base::page page; - ticket my_ticket; - micro_queue& my_queue; - page* my_page; - concurrent_queue_page_allocator& allocator; -public: - micro_queue_pop_finalizer( micro_queue& queue, concurrent_queue_base_v3& b, ticket k, page* p ) : - my_ticket(k), my_queue(queue), my_page(p), allocator(b) - {} - ~micro_queue_pop_finalizer() ; -}; - -template -micro_queue_pop_finalizer::~micro_queue_pop_finalizer() { - page* p = my_page; - if( is_valid_page(p) ) { - spin_mutex::scoped_lock lock( my_queue.page_mutex ); - page* q = p->next; - my_queue.head_page = q; - if( !is_valid_page(q) ) { - my_queue.tail_page = NULL; - } - } - itt_store_word_with_release(my_queue.head_counter, my_ticket); - if( is_valid_page(p) ) { - allocator.deallocate_page( p ); - } -} - -#if _MSC_VER && !defined(__INTEL_COMPILER) -#pragma warning( pop ) -#endif // warning 4146 is back - -template class concurrent_queue_iterator_rep ; -template class concurrent_queue_iterator_base_v3; - -//! representation of concurrent_queue_base -/** - * the class inherits from concurrent_queue_rep_base and defines an array of micro_queue's - */ -template -struct concurrent_queue_rep : public concurrent_queue_rep_base { - micro_queue array[n_queue]; - - //! Map ticket to an array index - static size_t index( ticket k ) { - return k*phi%n_queue; - } - - micro_queue& choose( ticket k ) { - // The formula here approximates LRU in a cache-oblivious way. - return array[index(k)]; - } -}; - -//! base class of concurrent_queue -/** - * The class implements the interface defined by concurrent_queue_page_allocator - * and has a pointer to an instance of concurrent_queue_rep. - */ -template -class concurrent_queue_base_v3: public concurrent_queue_page_allocator { -private: - //! Internal representation - concurrent_queue_rep* my_rep; - - friend struct concurrent_queue_rep; - friend class micro_queue; - friend class concurrent_queue_iterator_rep; - friend class concurrent_queue_iterator_base_v3; - -protected: - typedef typename concurrent_queue_rep::page page; - -private: - typedef typename micro_queue::padded_page padded_page; - typedef typename micro_queue::item_constructor_t item_constructor_t; - - virtual page *allocate_page() __TBB_override { - concurrent_queue_rep& r = *my_rep; - size_t n = sizeof(padded_page) + (r.items_per_page-1)*sizeof(T); - return reinterpret_cast(allocate_block ( n )); - } - - virtual void deallocate_page( concurrent_queue_rep_base::page *p ) __TBB_override { - concurrent_queue_rep& r = *my_rep; - size_t n = sizeof(padded_page) + (r.items_per_page-1)*sizeof(T); - deallocate_block( reinterpret_cast(p), n ); - } - - //! custom allocator - virtual void *allocate_block( size_t n ) = 0; - - //! custom de-allocator - virtual void deallocate_block( void *p, size_t n ) = 0; - -protected: - concurrent_queue_base_v3(); - - virtual ~concurrent_queue_base_v3() { -#if TBB_USE_ASSERT - size_t nq = my_rep->n_queue; - for( size_t i=0; iarray[i].tail_page==NULL, "pages were not freed properly" ); -#endif /* TBB_USE_ASSERT */ - cache_aligned_allocator >().deallocate(my_rep,1); - } - - //! Enqueue item at tail of queue - void internal_push( const void* src, item_constructor_t construct_item ) { - concurrent_queue_rep& r = *my_rep; - ticket k = r.tail_counter++; - r.choose(k).push( src, k, *this, construct_item ); - } - - //! Attempt to dequeue item from queue. - /** NULL if there was no item to dequeue. */ - bool internal_try_pop( void* dst ) ; - - //! Get size of queue; result may be invalid if queue is modified concurrently - size_t internal_size() const ; - - //! check if the queue is empty; thread safe - bool internal_empty() const ; - - //! free any remaining pages - /* note that the name may be misleading, but it remains so due to a historical accident. */ - void internal_finish_clear() ; - - //! Obsolete - void internal_throw_exception() const { - throw_exception( eid_bad_alloc ); - } - - //! copy or move internal representation - void assign( const concurrent_queue_base_v3& src, item_constructor_t construct_item ) ; - -#if __TBB_CPP11_RVALUE_REF_PRESENT - //! swap internal representation - void internal_swap( concurrent_queue_base_v3& src ) { - std::swap( my_rep, src.my_rep ); - } -#endif /* __TBB_CPP11_RVALUE_REF_PRESENT */ -}; - -template -concurrent_queue_base_v3::concurrent_queue_base_v3() { - const size_t item_size = sizeof(T); - my_rep = cache_aligned_allocator >().allocate(1); - __TBB_ASSERT( (size_t)my_rep % NFS_GetLineSize()==0, "alignment error" ); - __TBB_ASSERT( (size_t)&my_rep->head_counter % NFS_GetLineSize()==0, "alignment error" ); - __TBB_ASSERT( (size_t)&my_rep->tail_counter % NFS_GetLineSize()==0, "alignment error" ); - __TBB_ASSERT( (size_t)&my_rep->array % NFS_GetLineSize()==0, "alignment error" ); - memset(my_rep,0,sizeof(concurrent_queue_rep)); - my_rep->item_size = item_size; - my_rep->items_per_page = item_size<= 8 ? 32 : - item_size<= 16 ? 16 : - item_size<= 32 ? 8 : - item_size<= 64 ? 4 : - item_size<=128 ? 2 : - 1; -} - -template -bool concurrent_queue_base_v3::internal_try_pop( void* dst ) { - concurrent_queue_rep& r = *my_rep; - ticket k; - do { - k = r.head_counter; - for(;;) { - if( (ptrdiff_t)(r.tail_counter-k)<=0 ) { - // Queue is empty - return false; - } - // Queue had item with ticket k when we looked. Attempt to get that item. - ticket tk=k; -#if defined(_MSC_VER) && defined(_Wp64) - #pragma warning (push) - #pragma warning (disable: 4267) -#endif - k = r.head_counter.compare_and_swap( tk+1, tk ); -#if defined(_MSC_VER) && defined(_Wp64) - #pragma warning (pop) -#endif - if( k==tk ) - break; - // Another thread snatched the item, retry. - } - } while( !r.choose( k ).pop( dst, k, *this ) ); - return true; -} - -template -size_t concurrent_queue_base_v3::internal_size() const { - concurrent_queue_rep& r = *my_rep; - __TBB_ASSERT( sizeof(ptrdiff_t)<=sizeof(size_t), NULL ); - ticket hc = r.head_counter; - size_t nie = r.n_invalid_entries; - ticket tc = r.tail_counter; - __TBB_ASSERT( hc!=tc || !nie, NULL ); - ptrdiff_t sz = tc-hc-nie; - return sz<0 ? 0 : size_t(sz); -} - -template -bool concurrent_queue_base_v3::internal_empty() const { - concurrent_queue_rep& r = *my_rep; - ticket tc = r.tail_counter; - ticket hc = r.head_counter; - // if tc!=r.tail_counter, the queue was not empty at some point between the two reads. - return tc==r.tail_counter && tc==hc+r.n_invalid_entries ; -} - -template -void concurrent_queue_base_v3::internal_finish_clear() { - concurrent_queue_rep& r = *my_rep; - size_t nq = r.n_queue; - for( size_t i=0; i -void concurrent_queue_base_v3::assign( const concurrent_queue_base_v3& src, - item_constructor_t construct_item ) -{ - concurrent_queue_rep& r = *my_rep; - r.items_per_page = src.my_rep->items_per_page; - - // copy concurrent_queue_rep data - r.head_counter = src.my_rep->head_counter; - r.tail_counter = src.my_rep->tail_counter; - r.n_invalid_entries = src.my_rep->n_invalid_entries; - - // copy or move micro_queues - for( size_t i = 0; i < r.n_queue; ++i ) - r.array[i].assign( src.my_rep->array[i], *this, construct_item); - - __TBB_ASSERT( r.head_counter==src.my_rep->head_counter && r.tail_counter==src.my_rep->tail_counter, - "the source concurrent queue should not be concurrently modified." ); -} - -template class concurrent_queue_iterator; - -template -class concurrent_queue_iterator_rep: no_assign { - typedef typename micro_queue::padded_page padded_page; -public: - ticket head_counter; - const concurrent_queue_base_v3& my_queue; - typename concurrent_queue_base_v3::page* array[concurrent_queue_rep::n_queue]; - concurrent_queue_iterator_rep( const concurrent_queue_base_v3& queue ) : - head_counter(queue.my_rep->head_counter), - my_queue(queue) - { - for( size_t k=0; k::n_queue; ++k ) - array[k] = queue.my_rep->array[k].head_page; - } - - //! Set item to point to kth element. Return true if at end of queue or item is marked valid; false otherwise. - bool get_item( T*& item, size_t k ) ; -}; - -template -bool concurrent_queue_iterator_rep::get_item( T*& item, size_t k ) { - if( k==my_queue.my_rep->tail_counter ) { - item = NULL; - return true; - } else { - typename concurrent_queue_base_v3::page* p = array[concurrent_queue_rep::index(k)]; - __TBB_ASSERT(p,NULL); - size_t i = modulo_power_of_two( k/concurrent_queue_rep::n_queue, my_queue.my_rep->items_per_page ); - item = µ_queue::get_ref(*p,i); - return (p->mask & uintptr_t(1)< -class concurrent_queue_iterator_base_v3 : no_assign { - //! Represents concurrent_queue over which we are iterating. - /** NULL if one past last element in queue. */ - concurrent_queue_iterator_rep* my_rep; - - template - friend bool operator==( const concurrent_queue_iterator& i, const concurrent_queue_iterator& j ); - - template - friend bool operator!=( const concurrent_queue_iterator& i, const concurrent_queue_iterator& j ); -protected: - //! Pointer to current item - Value* my_item; - - //! Default constructor - concurrent_queue_iterator_base_v3() : my_rep(NULL), my_item(NULL) { -#if __TBB_GCC_OPTIMIZER_ORDERING_BROKEN - __TBB_compiler_fence(); -#endif - } - - //! Copy constructor - concurrent_queue_iterator_base_v3( const concurrent_queue_iterator_base_v3& i ) - : no_assign(), my_rep(NULL), my_item(NULL) { - assign(i); - } - - //! Construct iterator pointing to head of queue. - concurrent_queue_iterator_base_v3( const concurrent_queue_base_v3& queue ) ; - - //! Assignment - void assign( const concurrent_queue_iterator_base_v3& other ) ; - - //! Advance iterator one step towards tail of queue. - void advance() ; - - //! Destructor - ~concurrent_queue_iterator_base_v3() { - cache_aligned_allocator >().deallocate(my_rep, 1); - my_rep = NULL; - } -}; - -template -concurrent_queue_iterator_base_v3::concurrent_queue_iterator_base_v3( const concurrent_queue_base_v3& queue ) { - my_rep = cache_aligned_allocator >().allocate(1); - new( my_rep ) concurrent_queue_iterator_rep(queue); - size_t k = my_rep->head_counter; - if( !my_rep->get_item(my_item, k) ) advance(); -} - -template -void concurrent_queue_iterator_base_v3::assign( const concurrent_queue_iterator_base_v3& other ) { - if( my_rep!=other.my_rep ) { - if( my_rep ) { - cache_aligned_allocator >().deallocate(my_rep, 1); - my_rep = NULL; - } - if( other.my_rep ) { - my_rep = cache_aligned_allocator >().allocate(1); - new( my_rep ) concurrent_queue_iterator_rep( *other.my_rep ); - } - } - my_item = other.my_item; -} - -template -void concurrent_queue_iterator_base_v3::advance() { - __TBB_ASSERT( my_item, "attempt to increment iterator past end of queue" ); - size_t k = my_rep->head_counter; - const concurrent_queue_base_v3& queue = my_rep->my_queue; -#if TBB_USE_ASSERT - Value* tmp; - my_rep->get_item(tmp,k); - __TBB_ASSERT( my_item==tmp, NULL ); -#endif /* TBB_USE_ASSERT */ - size_t i = modulo_power_of_two( k/concurrent_queue_rep::n_queue, queue.my_rep->items_per_page ); - if( i==queue.my_rep->items_per_page-1 ) { - typename concurrent_queue_base_v3::page*& root = my_rep->array[concurrent_queue_rep::index(k)]; - root = root->next; - } - // advance k - my_rep->head_counter = ++k; - if( !my_rep->get_item(my_item, k) ) advance(); -} - -//! Similar to C++0x std::remove_cv -/** "tbb_" prefix added to avoid overload confusion with C++0x implementations. */ -template struct tbb_remove_cv {typedef T type;}; -template struct tbb_remove_cv {typedef T type;}; -template struct tbb_remove_cv {typedef T type;}; -template struct tbb_remove_cv {typedef T type;}; - -//! Meets requirements of a forward iterator for STL. -/** Value is either the T or const T type of the container. - @ingroup containers */ -template -class concurrent_queue_iterator: public concurrent_queue_iterator_base_v3::type>, - public std::iterator { -#if !__TBB_TEMPLATE_FRIENDS_BROKEN - template - friend class ::tbb::strict_ppl::concurrent_queue; -#else -public: -#endif - //! Construct iterator pointing to head of queue. - explicit concurrent_queue_iterator( const concurrent_queue_base_v3::type>& queue ) : - concurrent_queue_iterator_base_v3::type>(queue) - { - } - -public: - concurrent_queue_iterator() {} - - /** If Value==Container::value_type, then this routine is the copy constructor. - If Value==const Container::value_type, then this routine is a conversion constructor. */ - concurrent_queue_iterator( const concurrent_queue_iterator& other ) : - concurrent_queue_iterator_base_v3::type>(other) - {} - - //! Iterator assignment - concurrent_queue_iterator& operator=( const concurrent_queue_iterator& other ) { - this->assign(other); - return *this; - } - - //! Reference to current item - Value& operator*() const { - return *static_cast(this->my_item); - } - - Value* operator->() const {return &operator*();} - - //! Advance to next item in queue - concurrent_queue_iterator& operator++() { - this->advance(); - return *this; - } - - //! Post increment - Value* operator++(int) { - Value* result = &operator*(); - operator++(); - return result; - } -}; // concurrent_queue_iterator - - -template -bool operator==( const concurrent_queue_iterator& i, const concurrent_queue_iterator& j ) { - return i.my_item==j.my_item; -} - -template -bool operator!=( const concurrent_queue_iterator& i, const concurrent_queue_iterator& j ) { - return i.my_item!=j.my_item; -} - -} // namespace internal - -//! @endcond - -} // namespace strict_ppl - -//! @cond INTERNAL -namespace internal { - -class concurrent_queue_rep; -class concurrent_queue_iterator_rep; -class concurrent_queue_iterator_base_v3; -template class concurrent_queue_iterator; - -//! For internal use only. -/** Type-independent portion of concurrent_queue. - @ingroup containers */ -class concurrent_queue_base_v3: no_copy { -private: - //! Internal representation - concurrent_queue_rep* my_rep; - - friend class concurrent_queue_rep; - friend struct micro_queue; - friend class micro_queue_pop_finalizer; - friend class concurrent_queue_iterator_rep; - friend class concurrent_queue_iterator_base_v3; -protected: - //! Prefix on a page - struct page { - page* next; - uintptr_t mask; - }; - - //! Capacity of the queue - ptrdiff_t my_capacity; - - //! Always a power of 2 - size_t items_per_page; - - //! Size of an item - size_t item_size; - - enum copy_specifics { copy, move }; - -#if __TBB_PROTECTED_NESTED_CLASS_BROKEN -public: -#endif - template - struct padded_page: page { - //! Not defined anywhere - exists to quiet warnings. - padded_page(); - //! Not defined anywhere - exists to quiet warnings. - void operator=( const padded_page& ); - //! Must be last field. - T last; - }; - -private: - virtual void copy_item( page& dst, size_t index, const void* src ) = 0; - virtual void assign_and_destroy_item( void* dst, page& src, size_t index ) = 0; -protected: - __TBB_EXPORTED_METHOD concurrent_queue_base_v3( size_t item_size ); - virtual __TBB_EXPORTED_METHOD ~concurrent_queue_base_v3(); - - //! Enqueue item at tail of queue using copy operation - void __TBB_EXPORTED_METHOD internal_push( const void* src ); - - //! Dequeue item from head of queue - void __TBB_EXPORTED_METHOD internal_pop( void* dst ); - - //! Abort all pending queue operations - void __TBB_EXPORTED_METHOD internal_abort(); - - //! Attempt to enqueue item onto queue using copy operation - bool __TBB_EXPORTED_METHOD internal_push_if_not_full( const void* src ); - - //! Attempt to dequeue item from queue. - /** NULL if there was no item to dequeue. */ - bool __TBB_EXPORTED_METHOD internal_pop_if_present( void* dst ); - - //! Get size of queue - ptrdiff_t __TBB_EXPORTED_METHOD internal_size() const; - - //! Check if the queue is emtpy - bool __TBB_EXPORTED_METHOD internal_empty() const; - - //! Set the queue capacity - void __TBB_EXPORTED_METHOD internal_set_capacity( ptrdiff_t capacity, size_t element_size ); - - //! custom allocator - virtual page *allocate_page() = 0; - - //! custom de-allocator - virtual void deallocate_page( page *p ) = 0; - - //! free any remaining pages - /* note that the name may be misleading, but it remains so due to a historical accident. */ - void __TBB_EXPORTED_METHOD internal_finish_clear() ; - - //! throw an exception - void __TBB_EXPORTED_METHOD internal_throw_exception() const; - - //! copy internal representation - void __TBB_EXPORTED_METHOD assign( const concurrent_queue_base_v3& src ) ; - -#if __TBB_CPP11_RVALUE_REF_PRESENT - //! swap queues - void internal_swap( concurrent_queue_base_v3& src ) { - std::swap( my_capacity, src.my_capacity ); - std::swap( items_per_page, src.items_per_page ); - std::swap( item_size, src.item_size ); - std::swap( my_rep, src.my_rep ); - } -#endif /* __TBB_CPP11_RVALUE_REF_PRESENT */ - - //! Enqueues item at tail of queue using specified operation (copy or move) - void internal_insert_item( const void* src, copy_specifics op_type ); - - //! Attempts to enqueue at tail of queue using specified operation (copy or move) - bool internal_insert_if_not_full( const void* src, copy_specifics op_type ); - - //! Assigns one queue to another using specified operation (copy or move) - void internal_assign( const concurrent_queue_base_v3& src, copy_specifics op_type ); -private: - virtual void copy_page_item( page& dst, size_t dindex, const page& src, size_t sindex ) = 0; -}; - -//! For internal use only. -/** Backward compatible modification of concurrent_queue_base_v3 - @ingroup containers */ -class concurrent_queue_base_v8: public concurrent_queue_base_v3 { -protected: - concurrent_queue_base_v8( size_t item_sz ) : concurrent_queue_base_v3( item_sz ) {} - - //! move items - void __TBB_EXPORTED_METHOD move_content( concurrent_queue_base_v8& src ) ; - - //! Attempt to enqueue item onto queue using move operation - bool __TBB_EXPORTED_METHOD internal_push_move_if_not_full( const void* src ); - - //! Enqueue item at tail of queue using move operation - void __TBB_EXPORTED_METHOD internal_push_move( const void* src ); -private: - friend struct micro_queue; - virtual void move_page_item( page& dst, size_t dindex, const page& src, size_t sindex ) = 0; - virtual void move_item( page& dst, size_t index, const void* src ) = 0; -}; - -//! Type-independent portion of concurrent_queue_iterator. -/** @ingroup containers */ -class concurrent_queue_iterator_base_v3 { - //! concurrent_queue over which we are iterating. - /** NULL if one past last element in queue. */ - concurrent_queue_iterator_rep* my_rep; - - template - friend bool operator==( const concurrent_queue_iterator& i, const concurrent_queue_iterator& j ); - - template - friend bool operator!=( const concurrent_queue_iterator& i, const concurrent_queue_iterator& j ); - - void initialize( const concurrent_queue_base_v3& queue, size_t offset_of_data ); -protected: - //! Pointer to current item - void* my_item; - - //! Default constructor - concurrent_queue_iterator_base_v3() : my_rep(NULL), my_item(NULL) {} - - //! Copy constructor - concurrent_queue_iterator_base_v3( const concurrent_queue_iterator_base_v3& i ) : my_rep(NULL), my_item(NULL) { - assign(i); - } - - //! Obsolete entry point for constructing iterator pointing to head of queue. - /** Does not work correctly for SSE types. */ - __TBB_EXPORTED_METHOD concurrent_queue_iterator_base_v3( const concurrent_queue_base_v3& queue ); - - //! Construct iterator pointing to head of queue. - __TBB_EXPORTED_METHOD concurrent_queue_iterator_base_v3( const concurrent_queue_base_v3& queue, size_t offset_of_data ); - - //! Assignment - void __TBB_EXPORTED_METHOD assign( const concurrent_queue_iterator_base_v3& i ); - - //! Advance iterator one step towards tail of queue. - void __TBB_EXPORTED_METHOD advance(); - - //! Destructor - __TBB_EXPORTED_METHOD ~concurrent_queue_iterator_base_v3(); -}; - -typedef concurrent_queue_iterator_base_v3 concurrent_queue_iterator_base; - -//! Meets requirements of a forward iterator for STL. -/** Value is either the T or const T type of the container. - @ingroup containers */ -template -class concurrent_queue_iterator: public concurrent_queue_iterator_base, - public std::iterator { - -#if !__TBB_TEMPLATE_FRIENDS_BROKEN - template - friend class ::tbb::concurrent_bounded_queue; -#else -public: -#endif - - //! Construct iterator pointing to head of queue. - explicit concurrent_queue_iterator( const concurrent_queue_base_v3& queue ) : - concurrent_queue_iterator_base_v3(queue,__TBB_offsetof(concurrent_queue_base_v3::padded_page,last)) - { - } - -public: - concurrent_queue_iterator() {} - - /** If Value==Container::value_type, then this routine is the copy constructor. - If Value==const Container::value_type, then this routine is a conversion constructor. */ - concurrent_queue_iterator( const concurrent_queue_iterator& other ) : - concurrent_queue_iterator_base_v3(other) - {} - - //! Iterator assignment - concurrent_queue_iterator& operator=( const concurrent_queue_iterator& other ) { - assign(other); - return *this; - } - - //! Reference to current item - Value& operator*() const { - return *static_cast(my_item); - } - - Value* operator->() const {return &operator*();} - - //! Advance to next item in queue - concurrent_queue_iterator& operator++() { - advance(); - return *this; - } - - //! Post increment - Value* operator++(int) { - Value* result = &operator*(); - operator++(); - return result; - } -}; // concurrent_queue_iterator - - -template -bool operator==( const concurrent_queue_iterator& i, const concurrent_queue_iterator& j ) { - return i.my_item==j.my_item; -} - -template -bool operator!=( const concurrent_queue_iterator& i, const concurrent_queue_iterator& j ) { - return i.my_item!=j.my_item; -} - -} // namespace internal; - -//! @endcond - -} // namespace tbb - -#endif /* __TBB__concurrent_queue_impl_H */ diff --git a/inst/include/tbb/internal/_concurrent_unordered_impl.h b/inst/include/tbb/internal/_concurrent_unordered_impl.h deleted file mode 100644 index e7d324e0f..000000000 --- a/inst/include/tbb/internal/_concurrent_unordered_impl.h +++ /dev/null @@ -1,1528 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -/* Container implementations in this header are based on PPL implementations - provided by Microsoft. */ - -#ifndef __TBB__concurrent_unordered_impl_H -#define __TBB__concurrent_unordered_impl_H -#if !defined(__TBB_concurrent_unordered_map_H) && !defined(__TBB_concurrent_unordered_set_H) && !defined(__TBB_concurrent_hash_map_H) -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#include "../tbb_stddef.h" - -#if !TBB_USE_EXCEPTIONS && _MSC_VER - // Suppress "C++ exception handler used, but unwind semantics are not enabled" warning in STL headers - #pragma warning (push) - #pragma warning (disable: 4530) -#endif - -#include -#include // Need std::pair -#include // Need std::equal_to (in ../concurrent_unordered_*.h) -#include // For tbb_hasher -#include // Need std::memset -#include __TBB_STD_SWAP_HEADER - -#if !TBB_USE_EXCEPTIONS && _MSC_VER - #pragma warning (pop) -#endif - -#include "../atomic.h" -#include "../tbb_exception.h" -#include "../tbb_allocator.h" - -#if __TBB_INITIALIZER_LISTS_PRESENT - #include -#endif - -#include "_tbb_hash_compare_impl.h" - -namespace tbb { -namespace interface5 { -//! @cond INTERNAL -namespace internal { - -template -class split_ordered_list; -template -class concurrent_unordered_base; - -// Forward list iterators (without skipping dummy elements) -template -class flist_iterator : public std::iterator -{ - template - friend class split_ordered_list; - template - friend class concurrent_unordered_base; - template - friend class flist_iterator; - - typedef typename Solist::nodeptr_t nodeptr_t; -public: - typedef typename Solist::value_type value_type; - typedef typename Solist::difference_type difference_type; - typedef typename Solist::pointer pointer; - typedef typename Solist::reference reference; - - flist_iterator() : my_node_ptr(0) {} - flist_iterator( const flist_iterator &other ) - : my_node_ptr(other.my_node_ptr) {} - - reference operator*() const { return my_node_ptr->my_element; } - pointer operator->() const { return &**this; } - - flist_iterator& operator++() { - my_node_ptr = my_node_ptr->my_next; - return *this; - } - - flist_iterator operator++(int) { - flist_iterator tmp = *this; - ++*this; - return tmp; - } - -protected: - flist_iterator(nodeptr_t pnode) : my_node_ptr(pnode) {} - nodeptr_t get_node_ptr() const { return my_node_ptr; } - - nodeptr_t my_node_ptr; - - template - friend bool operator==( const flist_iterator &i, const flist_iterator &j ); - template - friend bool operator!=( const flist_iterator& i, const flist_iterator& j ); -}; - -template -bool operator==( const flist_iterator &i, const flist_iterator &j ) { - return i.my_node_ptr == j.my_node_ptr; -} -template -bool operator!=( const flist_iterator& i, const flist_iterator& j ) { - return i.my_node_ptr != j.my_node_ptr; -} - -// Split-order list iterators, needed to skip dummy elements -template -class solist_iterator : public flist_iterator -{ - typedef flist_iterator base_type; - typedef typename Solist::nodeptr_t nodeptr_t; - using base_type::get_node_ptr; - template - friend class split_ordered_list; - template - friend class solist_iterator; - template - friend bool operator==( const solist_iterator &i, const solist_iterator &j ); - template - friend bool operator!=( const solist_iterator& i, const solist_iterator& j ); - - const Solist *my_list_ptr; - solist_iterator(nodeptr_t pnode, const Solist *plist) : base_type(pnode), my_list_ptr(plist) {} - -public: - typedef typename Solist::value_type value_type; - typedef typename Solist::difference_type difference_type; - typedef typename Solist::pointer pointer; - typedef typename Solist::reference reference; - - solist_iterator() {} - solist_iterator(const solist_iterator &other ) - : base_type(other), my_list_ptr(other.my_list_ptr) {} - - reference operator*() const { - return this->base_type::operator*(); - } - - pointer operator->() const { - return (&**this); - } - - solist_iterator& operator++() { - do ++(*(base_type *)this); - while (get_node_ptr() != NULL && get_node_ptr()->is_dummy()); - - return (*this); - } - - solist_iterator operator++(int) { - solist_iterator tmp = *this; - do ++*this; - while (get_node_ptr() != NULL && get_node_ptr()->is_dummy()); - - return (tmp); - } -}; - -template -bool operator==( const solist_iterator &i, const solist_iterator &j ) { - return i.my_node_ptr == j.my_node_ptr && i.my_list_ptr == j.my_list_ptr; -} -template -bool operator!=( const solist_iterator& i, const solist_iterator& j ) { - return i.my_node_ptr != j.my_node_ptr || i.my_list_ptr != j.my_list_ptr; -} - -// Forward type and class definitions -typedef size_t sokey_t; - - -// Forward list in which elements are sorted in a split-order -template -class split_ordered_list -{ -public: - typedef split_ordered_list self_type; - typedef typename Allocator::template rebind::other allocator_type; - struct node; - typedef node *nodeptr_t; - - typedef typename allocator_type::size_type size_type; - typedef typename allocator_type::difference_type difference_type; - typedef typename allocator_type::pointer pointer; - typedef typename allocator_type::const_pointer const_pointer; - typedef typename allocator_type::reference reference; - typedef typename allocator_type::const_reference const_reference; - typedef typename allocator_type::value_type value_type; - - typedef solist_iterator const_iterator; - typedef solist_iterator iterator; - typedef flist_iterator raw_const_iterator; - typedef flist_iterator raw_iterator; - - // Node that holds the element in a split-ordered list - struct node : tbb::internal::no_assign - { - private: - // for compilers that try to generate default constructors though they are not needed. - node(); // VS 2008, 2010, 2012 - public: - // Initialize the node with the given order key - void init(sokey_t order_key) { - my_order_key = order_key; - my_next = NULL; - } - - // Return the order key (needed for hashing) - sokey_t get_order_key() const { // TODO: remove - return my_order_key; - } - - // Inserts the new element in the list in an atomic fashion - nodeptr_t atomic_set_next(nodeptr_t new_node, nodeptr_t current_node) - { - // Try to change the next pointer on the current element to a new element, only if it still points to the cached next - nodeptr_t exchange_node = tbb::internal::as_atomic(my_next).compare_and_swap(new_node, current_node); - - if (exchange_node == current_node) // TODO: why this branch? - { - // Operation succeeded, return the new node - return new_node; - } - else - { - // Operation failed, return the "interfering" node - return exchange_node; - } - } - - // Checks if this element in the list is a dummy, order enforcing node. Dummy nodes are used by buckets - // in the hash table to quickly index into the right subsection of the split-ordered list. - bool is_dummy() const { - return (my_order_key & 0x1) == 0; - } - - - nodeptr_t my_next; // Next element in the list - value_type my_element; // Element storage - sokey_t my_order_key; // Order key for this element - }; - - // Allocate a new node with the given order key; used to allocate dummy nodes - nodeptr_t create_node(sokey_t order_key) { - nodeptr_t pnode = my_node_allocator.allocate(1); - pnode->init(order_key); - return (pnode); - } - - // Allocate a new node with the given order key and value - template - nodeptr_t create_node(sokey_t order_key, __TBB_FORWARDING_REF(Arg) t, - /*AllowCreate=*/tbb::internal::true_type=tbb::internal::true_type()){ - nodeptr_t pnode = my_node_allocator.allocate(1); - - //TODO: use RAII scoped guard instead of explicit catch - __TBB_TRY { - new(static_cast(&pnode->my_element)) T(tbb::internal::forward(t)); - pnode->init(order_key); - } __TBB_CATCH(...) { - my_node_allocator.deallocate(pnode, 1); - __TBB_RETHROW(); - } - - return (pnode); - } - - // A helper to avoid excessive requiremens in internal_insert - template - nodeptr_t create_node(sokey_t, __TBB_FORWARDING_REF(Arg), - /*AllowCreate=*/tbb::internal::false_type){ - __TBB_ASSERT(false, "This compile-time helper should never get called"); - return nodeptr_t(); - } - - // Allocate a new node with the given parameters for constructing value - template - nodeptr_t create_node_v( __TBB_FORWARDING_REF(Args) __TBB_PARAMETER_PACK args){ - nodeptr_t pnode = my_node_allocator.allocate(1); - - //TODO: use RAII scoped guard instead of explicit catch - __TBB_TRY { - new(static_cast(&pnode->my_element)) T(__TBB_PACK_EXPANSION(tbb::internal::forward(args))); - } __TBB_CATCH(...) { - my_node_allocator.deallocate(pnode, 1); - __TBB_RETHROW(); - } - - return (pnode); - } - - split_ordered_list(allocator_type a = allocator_type()) - : my_node_allocator(a), my_element_count(0) - { - // Immediately allocate a dummy node with order key of 0. This node - // will always be the head of the list. - my_head = create_node(sokey_t(0)); - } - - ~split_ordered_list() - { - // Clear the list - clear(); - - // Remove the head element which is not cleared by clear() - nodeptr_t pnode = my_head; - my_head = NULL; - - __TBB_ASSERT(pnode != NULL && pnode->my_next == NULL, "Invalid head list node"); - - destroy_node(pnode); - } - - // Common forward list functions - - allocator_type get_allocator() const { - return (my_node_allocator); - } - - void clear() { - nodeptr_t pnext; - nodeptr_t pnode = my_head; - - __TBB_ASSERT(my_head != NULL, "Invalid head list node"); - pnext = pnode->my_next; - pnode->my_next = NULL; - pnode = pnext; - - while (pnode != NULL) - { - pnext = pnode->my_next; - destroy_node(pnode); - pnode = pnext; - } - - my_element_count = 0; - } - - // Returns a first non-dummy element in the SOL - iterator begin() { - return first_real_iterator(raw_begin()); - } - - // Returns a first non-dummy element in the SOL - const_iterator begin() const { - return first_real_iterator(raw_begin()); - } - - iterator end() { - return (iterator(0, this)); - } - - const_iterator end() const { - return (const_iterator(0, this)); - } - - const_iterator cbegin() const { - return (((const self_type *)this)->begin()); - } - - const_iterator cend() const { - return (((const self_type *)this)->end()); - } - - // Checks if the number of elements (non-dummy) is 0 - bool empty() const { - return (my_element_count == 0); - } - - // Returns the number of non-dummy elements in the list - size_type size() const { - return my_element_count; - } - - // Returns the maximum size of the list, determined by the allocator - size_type max_size() const { - return my_node_allocator.max_size(); - } - - // Swaps 'this' list with the passed in one - void swap(self_type& other) - { - if (this == &other) - { - // Nothing to do - return; - } - - std::swap(my_element_count, other.my_element_count); - std::swap(my_head, other.my_head); - } - - // Split-order list functions - - // Returns a first element in the SOL, which is always a dummy - raw_iterator raw_begin() { - return raw_iterator(my_head); - } - - // Returns a first element in the SOL, which is always a dummy - raw_const_iterator raw_begin() const { - return raw_const_iterator(my_head); - } - - raw_iterator raw_end() { - return raw_iterator(0); - } - - raw_const_iterator raw_end() const { - return raw_const_iterator(0); - } - - static sokey_t get_order_key(const raw_const_iterator& it) { - return it.get_node_ptr()->get_order_key(); - } - - static sokey_t get_safe_order_key(const raw_const_iterator& it) { - if( !it.get_node_ptr() ) return ~sokey_t(0); - return it.get_node_ptr()->get_order_key(); - } - - // Returns a public iterator version of the internal iterator. Public iterator must not - // be a dummy private iterator. - iterator get_iterator(raw_iterator it) { - __TBB_ASSERT(it.get_node_ptr() == NULL || !it.get_node_ptr()->is_dummy(), "Invalid user node (dummy)"); - return iterator(it.get_node_ptr(), this); - } - - // Returns a public iterator version of the internal iterator. Public iterator must not - // be a dummy private iterator. - const_iterator get_iterator(raw_const_iterator it) const { - __TBB_ASSERT(it.get_node_ptr() == NULL || !it.get_node_ptr()->is_dummy(), "Invalid user node (dummy)"); - return const_iterator(it.get_node_ptr(), this); - } - - // Returns a non-const version of the raw_iterator - raw_iterator get_iterator(raw_const_iterator it) { - return raw_iterator(it.get_node_ptr()); - } - - // Returns a non-const version of the iterator - static iterator get_iterator(const_iterator it) { - return iterator(it.my_node_ptr, it.my_list_ptr); - } - - // Returns a public iterator version of a first non-dummy internal iterator at or after - // the passed in internal iterator. - iterator first_real_iterator(raw_iterator it) - { - // Skip all dummy, internal only iterators - while (it != raw_end() && it.get_node_ptr()->is_dummy()) - ++it; - - return iterator(it.get_node_ptr(), this); - } - - // Returns a public iterator version of a first non-dummy internal iterator at or after - // the passed in internal iterator. - const_iterator first_real_iterator(raw_const_iterator it) const - { - // Skip all dummy, internal only iterators - while (it != raw_end() && it.get_node_ptr()->is_dummy()) - ++it; - - return const_iterator(it.get_node_ptr(), this); - } - - // Erase an element using the allocator - void destroy_node(nodeptr_t pnode) { - if (!pnode->is_dummy()) my_node_allocator.destroy(pnode); - my_node_allocator.deallocate(pnode, 1); - } - - // Try to insert a new element in the list. - // If insert fails, return the node that was inserted instead. - static nodeptr_t try_insert_atomic(nodeptr_t previous, nodeptr_t new_node, nodeptr_t current_node) { - new_node->my_next = current_node; - return previous->atomic_set_next(new_node, current_node); - } - - // Insert a new element between passed in iterators - std::pair try_insert(raw_iterator it, raw_iterator next, nodeptr_t pnode, size_type *new_count) - { - nodeptr_t inserted_node = try_insert_atomic(it.get_node_ptr(), pnode, next.get_node_ptr()); - - if (inserted_node == pnode) - { - // If the insert succeeded, check that the order is correct and increment the element count - check_range(it, next); - *new_count = tbb::internal::as_atomic(my_element_count).fetch_and_increment(); - return std::pair(iterator(pnode, this), true); - } - else - { - return std::pair(end(), false); - } - } - - // Insert a new dummy element, starting search at a parent dummy element - raw_iterator insert_dummy(raw_iterator it, sokey_t order_key) - { - raw_iterator last = raw_end(); - raw_iterator where = it; - - __TBB_ASSERT(where != last, "Invalid head node"); - - ++where; - - // Create a dummy element up front, even though it may be discarded (due to concurrent insertion) - nodeptr_t dummy_node = create_node(order_key); - - for (;;) - { - __TBB_ASSERT(it != last, "Invalid head list node"); - - // If the head iterator is at the end of the list, or past the point where this dummy - // node needs to be inserted, then try to insert it. - if (where == last || get_order_key(where) > order_key) - { - __TBB_ASSERT(get_order_key(it) < order_key, "Invalid node order in the list"); - - // Try to insert it in the right place - nodeptr_t inserted_node = try_insert_atomic(it.get_node_ptr(), dummy_node, where.get_node_ptr()); - - if (inserted_node == dummy_node) - { - // Insertion succeeded, check the list for order violations - check_range(it, where); - return raw_iterator(dummy_node); - } - else - { - // Insertion failed: either dummy node was inserted by another thread, or - // a real element was inserted at exactly the same place as dummy node. - // Proceed with the search from the previous location where order key was - // known to be larger (note: this is legal only because there is no safe - // concurrent erase operation supported). - where = it; - ++where; - continue; - } - } - else if (get_order_key(where) == order_key) - { - // Another dummy node with the same value found, discard the new one. - destroy_node(dummy_node); - return where; - } - - // Move the iterator forward - it = where; - ++where; - } - - } - - // This erase function can handle both real and dummy nodes - void erase_node(raw_iterator previous, raw_const_iterator& where) - { - nodeptr_t pnode = (where++).get_node_ptr(); - nodeptr_t prevnode = previous.get_node_ptr(); - __TBB_ASSERT(prevnode->my_next == pnode, "Erase must take consecutive iterators"); - prevnode->my_next = pnode->my_next; - - destroy_node(pnode); - } - - // Erase the element (previous node needs to be passed because this is a forward only list) - iterator erase_node(raw_iterator previous, const_iterator where) - { - raw_const_iterator it = where; - erase_node(previous, it); - my_element_count--; - - return get_iterator(first_real_iterator(it)); - } - - // Move all elements from the passed in split-ordered list to this one - void move_all(self_type& source) - { - raw_const_iterator first = source.raw_begin(); - raw_const_iterator last = source.raw_end(); - - if (first == last) - return; - - nodeptr_t previous_node = my_head; - raw_const_iterator begin_iterator = first++; - - // Move all elements one by one, including dummy ones - for (raw_const_iterator it = first; it != last;) - { - nodeptr_t pnode = it.get_node_ptr(); - - nodeptr_t dummy_node = pnode->is_dummy() ? create_node(pnode->get_order_key()) : create_node(pnode->get_order_key(), pnode->my_element); - previous_node = try_insert_atomic(previous_node, dummy_node, NULL); - __TBB_ASSERT(previous_node != NULL, "Insertion must succeed"); - raw_const_iterator where = it++; - source.erase_node(get_iterator(begin_iterator), where); - } - check_range(); - } - - -private: - //Need to setup private fields of split_ordered_list in move constructor and assignment of concurrent_unordered_base - template - friend class concurrent_unordered_base; - - // Check the list for order violations - void check_range( raw_iterator first, raw_iterator last ) - { -#if TBB_USE_ASSERT - for (raw_iterator it = first; it != last; ++it) - { - raw_iterator next = it; - ++next; - - __TBB_ASSERT(next == raw_end() || get_order_key(next) >= get_order_key(it), "!!! List order inconsistency !!!"); - } -#else - tbb::internal::suppress_unused_warning(first, last); -#endif - } - void check_range() - { -#if TBB_USE_ASSERT - check_range( raw_begin(), raw_end() ); -#endif - } - - typename allocator_type::template rebind::other my_node_allocator; // allocator object for nodes - size_type my_element_count; // Total item count, not counting dummy nodes - nodeptr_t my_head; // pointer to head node -}; - -#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) -#pragma warning(push) -#pragma warning(disable: 4127) // warning C4127: conditional expression is constant -#endif - -template -class concurrent_unordered_base : public Traits -{ -protected: - // Type definitions - typedef concurrent_unordered_base self_type; - typedef typename Traits::value_type value_type; - typedef typename Traits::key_type key_type; - typedef typename Traits::hash_compare hash_compare; - typedef typename Traits::allocator_type allocator_type; - typedef typename hash_compare::hasher hasher; - typedef typename hash_compare::key_equal key_equal; - typedef typename allocator_type::pointer pointer; - typedef typename allocator_type::const_pointer const_pointer; - typedef typename allocator_type::reference reference; - typedef typename allocator_type::const_reference const_reference; - typedef typename allocator_type::size_type size_type; - typedef typename allocator_type::difference_type difference_type; - typedef split_ordered_list solist_t; - typedef typename solist_t::nodeptr_t nodeptr_t; - // Iterators that walk the entire split-order list, including dummy nodes - typedef typename solist_t::raw_iterator raw_iterator; - typedef typename solist_t::raw_const_iterator raw_const_iterator; - typedef typename solist_t::iterator iterator; // TODO: restore const iterator for unordered_sets - typedef typename solist_t::const_iterator const_iterator; - typedef iterator local_iterator; - typedef const_iterator const_local_iterator; - using Traits::my_hash_compare; - using Traits::get_key; - using Traits::allow_multimapping; - - static const size_type initial_bucket_number = 8; // Initial number of buckets -private: - typedef std::pair pairii_t; - typedef std::pair paircc_t; - - static size_type const pointers_per_table = sizeof(size_type) * 8; // One bucket segment per bit - static const size_type initial_bucket_load = 4; // Initial maximum number of elements per bucket - - struct call_internal_clear_on_exit{ - concurrent_unordered_base* my_instance; - call_internal_clear_on_exit(concurrent_unordered_base* instance) : my_instance(instance) {} - void dismiss(){ my_instance = NULL;} - ~call_internal_clear_on_exit(){ - if (my_instance){ - my_instance->internal_clear(); - } - } - }; -protected: - // Constructors/Destructors - concurrent_unordered_base(size_type n_of_buckets = initial_bucket_number, - const hash_compare& hc = hash_compare(), const allocator_type& a = allocator_type()) - : Traits(hc), my_solist(a), - my_allocator(a), my_maximum_bucket_size((float) initial_bucket_load) - { - if( n_of_buckets == 0) ++n_of_buckets; - my_number_of_buckets = size_type(1)<<__TBB_Log2((uintptr_t)n_of_buckets*2-1); // round up to power of 2 - internal_init(); - } - - concurrent_unordered_base(const concurrent_unordered_base& right, const allocator_type& a) - : Traits(right.my_hash_compare), my_solist(a), my_allocator(a) - { - internal_init(); - internal_copy(right); - } - - concurrent_unordered_base(const concurrent_unordered_base& right) - : Traits(right.my_hash_compare), my_solist(right.get_allocator()), my_allocator(right.get_allocator()) - { - //FIXME:exception safety seems to be broken here - internal_init(); - internal_copy(right); - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - concurrent_unordered_base(concurrent_unordered_base&& right) - : Traits(right.my_hash_compare), my_solist(right.get_allocator()), my_allocator(right.get_allocator()) - { - internal_init(); - swap(right); - } - - concurrent_unordered_base(concurrent_unordered_base&& right, const allocator_type& a) - : Traits(right.my_hash_compare), my_solist(a), my_allocator(a) - { - call_internal_clear_on_exit clear_buckets_on_exception(this); - - internal_init(); - if (a == right.get_allocator()){ - this->swap(right); - }else{ - my_maximum_bucket_size = right.my_maximum_bucket_size; - my_number_of_buckets = right.my_number_of_buckets; - my_solist.my_element_count = right.my_solist.my_element_count; - - if (! right.my_solist.empty()){ - nodeptr_t previous_node = my_solist.my_head; - - // Move all elements one by one, including dummy ones - for (raw_const_iterator it = ++(right.my_solist.raw_begin()), last = right.my_solist.raw_end(); it != last; ++it) - { - const nodeptr_t pnode = it.get_node_ptr(); - nodeptr_t node; - if (pnode->is_dummy()) { - node = my_solist.create_node(pnode->get_order_key()); - size_type bucket = __TBB_ReverseBits(pnode->get_order_key()) % my_number_of_buckets; - set_bucket(bucket, node); - }else{ - node = my_solist.create_node(pnode->get_order_key(), std::move(pnode->my_element)); - } - - previous_node = my_solist.try_insert_atomic(previous_node, node, NULL); - __TBB_ASSERT(previous_node != NULL, "Insertion of node failed. Concurrent inserts in constructor ?"); - } - my_solist.check_range(); - } - } - - clear_buckets_on_exception.dismiss(); - } - -#endif // __TBB_CPP11_RVALUE_REF_PRESENT - - concurrent_unordered_base& operator=(const concurrent_unordered_base& right) { - if (this != &right) - internal_copy(right); - return (*this); - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - concurrent_unordered_base& operator=(concurrent_unordered_base&& other) - { - if(this != &other){ - typedef typename tbb::internal::allocator_traits::propagate_on_container_move_assignment pocma_t; - if(pocma_t::value || this->my_allocator == other.my_allocator) { - concurrent_unordered_base trash (std::move(*this)); - swap(other); - if (pocma_t::value) { - using std::swap; - //TODO: swapping allocators here may be a problem, replace with single direction moving - swap(this->my_solist.my_node_allocator, other.my_solist.my_node_allocator); - swap(this->my_allocator, other.my_allocator); - } - } else { - concurrent_unordered_base moved_copy(std::move(other),this->my_allocator); - this->swap(moved_copy); - } - } - return *this; - } - -#endif // __TBB_CPP11_RVALUE_REF_PRESENT - -#if __TBB_INITIALIZER_LISTS_PRESENT - //! assignment operator from initializer_list - concurrent_unordered_base& operator=(std::initializer_list il) - { - this->clear(); - this->insert(il.begin(),il.end()); - return (*this); - } -#endif // __TBB_INITIALIZER_LISTS_PRESENT - - - ~concurrent_unordered_base() { - // Delete all node segments - internal_clear(); - } - -public: - allocator_type get_allocator() const { - return my_solist.get_allocator(); - } - - // Size and capacity function - bool empty() const { - return my_solist.empty(); - } - - size_type size() const { - return my_solist.size(); - } - - size_type max_size() const { - return my_solist.max_size(); - } - - // Iterators - iterator begin() { - return my_solist.begin(); - } - - const_iterator begin() const { - return my_solist.begin(); - } - - iterator end() { - return my_solist.end(); - } - - const_iterator end() const { - return my_solist.end(); - } - - const_iterator cbegin() const { - return my_solist.cbegin(); - } - - const_iterator cend() const { - return my_solist.cend(); - } - - // Parallel traversal support - class const_range_type : tbb::internal::no_assign { - const concurrent_unordered_base &my_table; - raw_const_iterator my_begin_node; - raw_const_iterator my_end_node; - mutable raw_const_iterator my_midpoint_node; - public: - //! Type for size of a range - typedef typename concurrent_unordered_base::size_type size_type; - typedef typename concurrent_unordered_base::value_type value_type; - typedef typename concurrent_unordered_base::reference reference; - typedef typename concurrent_unordered_base::difference_type difference_type; - typedef typename concurrent_unordered_base::const_iterator iterator; - - //! True if range is empty. - bool empty() const {return my_begin_node == my_end_node;} - - //! True if range can be partitioned into two subranges. - bool is_divisible() const { - return my_midpoint_node != my_end_node; - } - //! Split range. - const_range_type( const_range_type &r, split ) : - my_table(r.my_table), my_end_node(r.my_end_node) - { - r.my_end_node = my_begin_node = r.my_midpoint_node; - __TBB_ASSERT( !empty(), "Splitting despite the range is not divisible" ); - __TBB_ASSERT( !r.empty(), "Splitting despite the range is not divisible" ); - set_midpoint(); - r.set_midpoint(); - } - //! Init range with container and grainsize specified - const_range_type( const concurrent_unordered_base &a_table ) : - my_table(a_table), my_begin_node(a_table.my_solist.begin()), - my_end_node(a_table.my_solist.end()) - { - set_midpoint(); - } - iterator begin() const { return my_table.my_solist.get_iterator(my_begin_node); } - iterator end() const { return my_table.my_solist.get_iterator(my_end_node); } - //! The grain size for this range. - size_type grainsize() const { return 1; } - - //! Set my_midpoint_node to point approximately half way between my_begin_node and my_end_node. - void set_midpoint() const { - if( my_begin_node == my_end_node ) // not divisible - my_midpoint_node = my_end_node; - else { - sokey_t begin_key = solist_t::get_safe_order_key(my_begin_node); - sokey_t end_key = solist_t::get_safe_order_key(my_end_node); - size_t mid_bucket = __TBB_ReverseBits( begin_key + (end_key-begin_key)/2 ) % my_table.my_number_of_buckets; - while ( !my_table.is_initialized(mid_bucket) ) mid_bucket = my_table.get_parent(mid_bucket); - if(__TBB_ReverseBits(mid_bucket) > begin_key) { - // found a dummy_node between begin and end - my_midpoint_node = my_table.my_solist.first_real_iterator(my_table.get_bucket( mid_bucket )); - } - else { - // didn't find a dummy node between begin and end. - my_midpoint_node = my_end_node; - } -#if TBB_USE_ASSERT - { - sokey_t mid_key = solist_t::get_safe_order_key(my_midpoint_node); - __TBB_ASSERT( begin_key < mid_key, "my_begin_node is after my_midpoint_node" ); - __TBB_ASSERT( mid_key <= end_key, "my_midpoint_node is after my_end_node" ); - } -#endif // TBB_USE_ASSERT - } - } - }; - - class range_type : public const_range_type { - public: - typedef typename concurrent_unordered_base::iterator iterator; - //! Split range. - range_type( range_type &r, split ) : const_range_type( r, split() ) {} - //! Init range with container and grainsize specified - range_type( const concurrent_unordered_base &a_table ) : const_range_type(a_table) {} - - iterator begin() const { return solist_t::get_iterator( const_range_type::begin() ); } - iterator end() const { return solist_t::get_iterator( const_range_type::end() ); } - }; - - range_type range() { - return range_type( *this ); - } - - const_range_type range() const { - return const_range_type( *this ); - } - - // Modifiers - std::pair insert(const value_type& value) { - return internal_insert(value); - } - - iterator insert(const_iterator, const value_type& value) { - // Ignore hint - return insert(value).first; - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - std::pair insert(value_type&& value) { - return internal_insert(std::move(value)); - } - - iterator insert(const_iterator, value_type&& value) { - // Ignore hint - return insert(std::move(value)).first; - } - -#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT - template - std::pair emplace(Args&&... args) { - nodeptr_t pnode = my_solist.create_node_v(tbb::internal::forward(args)...); - const sokey_t hashed_element_key = (sokey_t) my_hash_compare(get_key(pnode->my_element)); - const sokey_t order_key = split_order_key_regular(hashed_element_key); - pnode->init(order_key); - - return internal_insert(pnode->my_element, pnode); - } - - template - iterator emplace_hint(const_iterator, Args&&... args) { - // Ignore hint - return emplace(tbb::internal::forward(args)...).first; - } - -#endif // __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT -#endif // __TBB_CPP11_RVALUE_REF_PRESENT - - template - void insert(Iterator first, Iterator last) { - for (Iterator it = first; it != last; ++it) - insert(*it); - } - -#if __TBB_INITIALIZER_LISTS_PRESENT - //! Insert initializer list - void insert(std::initializer_list il) { - insert(il.begin(), il.end()); - } -#endif - - iterator unsafe_erase(const_iterator where) { - return internal_erase(where); - } - - iterator unsafe_erase(const_iterator first, const_iterator last) { - while (first != last) - unsafe_erase(first++); - return my_solist.get_iterator(first); - } - - size_type unsafe_erase(const key_type& key) { - pairii_t where = equal_range(key); - size_type item_count = internal_distance(where.first, where.second); - unsafe_erase(where.first, where.second); - return item_count; - } - - void swap(concurrent_unordered_base& right) { - if (this != &right) { - std::swap(my_hash_compare, right.my_hash_compare); // TODO: check what ADL meant here - my_solist.swap(right.my_solist); - internal_swap_buckets(right); - std::swap(my_number_of_buckets, right.my_number_of_buckets); - std::swap(my_maximum_bucket_size, right.my_maximum_bucket_size); - } - } - - // Observers - hasher hash_function() const { - return my_hash_compare.my_hash_object; - } - - key_equal key_eq() const { - return my_hash_compare.my_key_compare_object; - } - - void clear() { - // Clear list - my_solist.clear(); - - // Clear buckets - internal_clear(); - - // Initialize bucket 0 - __TBB_ASSERT(my_buckets[0] == NULL, NULL); - raw_iterator dummy_node = my_solist.raw_begin(); - set_bucket(0, dummy_node); - } - - // Lookup - iterator find(const key_type& key) { - return internal_find(key); - } - - const_iterator find(const key_type& key) const { - return const_cast(this)->internal_find(key); - } - - size_type count(const key_type& key) const { - if(allow_multimapping) { - paircc_t answer = equal_range(key); - size_type item_count = internal_distance(answer.first, answer.second); - return item_count; - } else { - return const_cast(this)->internal_find(key) == end()?0:1; - } - } - - std::pair equal_range(const key_type& key) { - return internal_equal_range(key); - } - - std::pair equal_range(const key_type& key) const { - return const_cast(this)->internal_equal_range(key); - } - - // Bucket interface - for debugging - size_type unsafe_bucket_count() const { - return my_number_of_buckets; - } - - size_type unsafe_max_bucket_count() const { - return segment_size(pointers_per_table-1); - } - - size_type unsafe_bucket_size(size_type bucket) { - size_type item_count = 0; - if (is_initialized(bucket)) { - raw_iterator it = get_bucket(bucket); - ++it; - for (; it != my_solist.raw_end() && !it.get_node_ptr()->is_dummy(); ++it) - ++item_count; - } - return item_count; - } - - size_type unsafe_bucket(const key_type& key) const { - sokey_t order_key = (sokey_t) my_hash_compare(key); - size_type bucket = order_key % my_number_of_buckets; - return bucket; - } - - // If the bucket is initialized, return a first non-dummy element in it - local_iterator unsafe_begin(size_type bucket) { - if (!is_initialized(bucket)) - return end(); - - raw_iterator it = get_bucket(bucket); - return my_solist.first_real_iterator(it); - } - - // If the bucket is initialized, return a first non-dummy element in it - const_local_iterator unsafe_begin(size_type bucket) const - { - if (!is_initialized(bucket)) - return end(); - - raw_const_iterator it = get_bucket(bucket); - return my_solist.first_real_iterator(it); - } - - // @REVIEW: Takes O(n) - // Returns the iterator after the last non-dummy element in the bucket - local_iterator unsafe_end(size_type bucket) - { - if (!is_initialized(bucket)) - return end(); - - raw_iterator it = get_bucket(bucket); - - // Find the end of the bucket, denoted by the dummy element - do ++it; - while(it != my_solist.raw_end() && !it.get_node_ptr()->is_dummy()); - - // Return the first real element past the end of the bucket - return my_solist.first_real_iterator(it); - } - - // @REVIEW: Takes O(n) - // Returns the iterator after the last non-dummy element in the bucket - const_local_iterator unsafe_end(size_type bucket) const - { - if (!is_initialized(bucket)) - return end(); - - raw_const_iterator it = get_bucket(bucket); - - // Find the end of the bucket, denoted by the dummy element - do ++it; - while(it != my_solist.raw_end() && !it.get_node_ptr()->is_dummy()); - - // Return the first real element past the end of the bucket - return my_solist.first_real_iterator(it); - } - - const_local_iterator unsafe_cbegin(size_type bucket) const { - return ((const self_type *) this)->unsafe_begin(bucket); - } - - const_local_iterator unsafe_cend(size_type bucket) const { - return ((const self_type *) this)->unsafe_end(bucket); - } - - // Hash policy - float load_factor() const { - return (float) size() / (float) unsafe_bucket_count(); - } - - float max_load_factor() const { - return my_maximum_bucket_size; - } - - void max_load_factor(float newmax) { - if (newmax != newmax || newmax < 0) - tbb::internal::throw_exception(tbb::internal::eid_invalid_load_factor); - my_maximum_bucket_size = newmax; - } - - // This function is a noop, because the underlying split-ordered list - // is already sorted, so an increase in the bucket number will be - // reflected next time this bucket is touched. - void rehash(size_type buckets) { - size_type current_buckets = my_number_of_buckets; - if (current_buckets >= buckets) - return; - my_number_of_buckets = size_type(1)<<__TBB_Log2((uintptr_t)buckets*2-1); // round up to power of 2 - } - -private: - - // Initialize the hash and keep the first bucket open - void internal_init() { - // Allocate an array of segment pointers - memset(my_buckets, 0, pointers_per_table * sizeof(void *)); - - // Initialize bucket 0 - raw_iterator dummy_node = my_solist.raw_begin(); - set_bucket(0, dummy_node); - } - - void internal_clear() { - for (size_type index = 0; index < pointers_per_table; ++index) { - if (my_buckets[index] != NULL) { - size_type sz = segment_size(index); - for (size_type index2 = 0; index2 < sz; ++index2) - my_allocator.destroy(&my_buckets[index][index2]); - my_allocator.deallocate(my_buckets[index], sz); - my_buckets[index] = 0; - } - } - } - - void internal_copy(const self_type& right) { - clear(); - - my_maximum_bucket_size = right.my_maximum_bucket_size; - my_number_of_buckets = right.my_number_of_buckets; - - __TBB_TRY { - insert(right.begin(), right.end()); - my_hash_compare = right.my_hash_compare; - } __TBB_CATCH(...) { - my_solist.clear(); - __TBB_RETHROW(); - } - } - - void internal_swap_buckets(concurrent_unordered_base& right) - { - // Swap all node segments - for (size_type index = 0; index < pointers_per_table; ++index) - { - raw_iterator * iterator_pointer = my_buckets[index]; - my_buckets[index] = right.my_buckets[index]; - right.my_buckets[index] = iterator_pointer; - } - } - - //TODO: why not use std::distance? - // Hash APIs - static size_type internal_distance(const_iterator first, const_iterator last) - { - size_type num = 0; - - for (const_iterator it = first; it != last; ++it) - ++num; - - return num; - } - - // Insert an element in the hash given its value - template - std::pair internal_insert(__TBB_FORWARDING_REF(ValueType) value, nodeptr_t pnode = NULL) - { - const key_type *pkey = &get_key(value); - sokey_t hash_key = (sokey_t) my_hash_compare(*pkey); - size_type new_count = 0; - sokey_t order_key = split_order_key_regular(hash_key); - raw_iterator previous = prepare_bucket(hash_key); - raw_iterator last = my_solist.raw_end(); - __TBB_ASSERT(previous != last, "Invalid head node"); - - // First node is a dummy node - for (raw_iterator where = previous;;) - { - ++where; - if (where == last || solist_t::get_order_key(where) > order_key || - // if multimapped, stop at the first item equal to us. - (allow_multimapping && solist_t::get_order_key(where) == order_key && - !my_hash_compare(get_key(*where), *pkey))) // TODO: fix negation - { - if (!pnode) { - pnode = my_solist.create_node(order_key, tbb::internal::forward(value), AllowCreate()); - // If the value was moved, the known reference to key might be invalid - pkey = &get_key(pnode->my_element); - } - - // Try to insert 'pnode' between 'previous' and 'where' - std::pair result = my_solist.try_insert(previous, where, pnode, &new_count); - - if (result.second) - { - // Insertion succeeded, adjust the table size, if needed - adjust_table_size(new_count, my_number_of_buckets); - return result; - } - else - { - // Insertion failed: either the same node was inserted by another thread, or - // another element was inserted at exactly the same place as this node. - // Proceed with the search from the previous location where order key was - // known to be larger (note: this is legal only because there is no safe - // concurrent erase operation supported). - where = previous; - continue; - } - } - else if (!allow_multimapping && solist_t::get_order_key(where) == order_key && - !my_hash_compare(get_key(*where), *pkey)) // TODO: fix negation - { // Element already in the list, return it - if (pnode) - my_solist.destroy_node(pnode); - return std::pair(my_solist.get_iterator(where), false); - } - // Move the iterator forward - previous = where; - } - } - - // Find the element in the split-ordered list - iterator internal_find(const key_type& key) - { - sokey_t hash_key = (sokey_t) my_hash_compare(key); - sokey_t order_key = split_order_key_regular(hash_key); - raw_iterator last = my_solist.raw_end(); - - for (raw_iterator it = prepare_bucket(hash_key); it != last; ++it) - { - if (solist_t::get_order_key(it) > order_key) - { - // If the order key is smaller than the current order key, the element - // is not in the hash. - return end(); - } - else if (solist_t::get_order_key(it) == order_key) - { - // The fact that order keys match does not mean that the element is found. - // Key function comparison has to be performed to check whether this is the - // right element. If not, keep searching while order key is the same. - if (!my_hash_compare(get_key(*it), key)) // TODO: fix negation - return my_solist.get_iterator(it); - } - } - - return end(); - } - - // Erase an element from the list. This is not a concurrency safe function. - iterator internal_erase(const_iterator it) - { - sokey_t hash_key = (sokey_t) my_hash_compare(get_key(*it)); - raw_iterator previous = prepare_bucket(hash_key); - raw_iterator last = my_solist.raw_end(); - __TBB_ASSERT(previous != last, "Invalid head node"); - - // First node is a dummy node - for (raw_iterator where = previous; ; previous = where) { - ++where; - if (where == last) - return end(); - else if (my_solist.get_iterator(where) == it) - return my_solist.erase_node(previous, it); - } - } - - // Return the [begin, end) pair of iterators with the same key values. - // This operation makes sense only if mapping is many-to-one. - pairii_t internal_equal_range(const key_type& key) - { - sokey_t hash_key = (sokey_t) my_hash_compare(key); - sokey_t order_key = split_order_key_regular(hash_key); - raw_iterator end_it = my_solist.raw_end(); - - for (raw_iterator it = prepare_bucket(hash_key); it != end_it; ++it) - { - if (solist_t::get_order_key(it) > order_key) - { - // There is no element with the given key - return pairii_t(end(), end()); - } - else if (solist_t::get_order_key(it) == order_key && - !my_hash_compare(get_key(*it), key)) // TODO: fix negation; also below - { - iterator first = my_solist.get_iterator(it); - iterator last = first; - do ++last; while( allow_multimapping && last != end() && !my_hash_compare(get_key(*last), key) ); - return pairii_t(first, last); - } - } - - return pairii_t(end(), end()); - } - - // Bucket APIs - void init_bucket(size_type bucket) - { - // Bucket 0 has no parent. - __TBB_ASSERT( bucket != 0, "The first bucket must always be initialized"); - - size_type parent_bucket = get_parent(bucket); - - // All parent_bucket buckets have to be initialized before this bucket is - if (!is_initialized(parent_bucket)) - init_bucket(parent_bucket); - - raw_iterator parent = get_bucket(parent_bucket); - - // Create a dummy first node in this bucket - raw_iterator dummy_node = my_solist.insert_dummy(parent, split_order_key_dummy(bucket)); - set_bucket(bucket, dummy_node); - } - - void adjust_table_size(size_type total_elements, size_type current_size) - { - // Grow the table by a factor of 2 if possible and needed - if ( ((float) total_elements / (float) current_size) > my_maximum_bucket_size ) - { - // Double the size of the hash only if size has not changed in between loads - my_number_of_buckets.compare_and_swap(2u*current_size, current_size); - //Simple "my_number_of_buckets.compare_and_swap( current_size<<1, current_size );" does not work for VC8 - //due to overzealous compiler warnings in /Wp64 mode - } - } - - size_type get_parent(size_type bucket) const - { - // Unsets bucket's most significant turned-on bit - size_type msb = __TBB_Log2((uintptr_t)bucket); - return bucket & ~(size_type(1) << msb); - } - - - // Dynamic sized array (segments) - //! @return segment index of given index in the array - static size_type segment_index_of( size_type index ) { - return size_type( __TBB_Log2( uintptr_t(index|1) ) ); - } - - //! @return the first array index of given segment - static size_type segment_base( size_type k ) { - return (size_type(1)< my_number_of_buckets; // Current table size - solist_t my_solist; // List where all the elements are kept - typename allocator_type::template rebind::other my_allocator; // Allocator object for segments - float my_maximum_bucket_size; // Maximum size of the bucket - atomic my_buckets[pointers_per_table]; // The segment table -}; -#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) -#pragma warning(pop) // warning 4127 is back -#endif - -} // namespace internal -//! @endcond -} // namespace interface5 -} // namespace tbb -#endif // __TBB__concurrent_unordered_impl_H diff --git a/inst/include/tbb/internal/_flow_graph_async_msg_impl.h b/inst/include/tbb/internal/_flow_graph_async_msg_impl.h deleted file mode 100644 index 7d2c3aefc..000000000 --- a/inst/include/tbb/internal/_flow_graph_async_msg_impl.h +++ /dev/null @@ -1,221 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB__flow_graph_async_msg_impl_H -#define __TBB__flow_graph_async_msg_impl_H - -#ifndef __TBB_flow_graph_H -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -// included in namespace tbb::flow::interfaceX (in flow_graph.h) - -template< typename T > class async_msg; - -namespace internal { - -template< typename T, typename = void > -struct async_helpers { - typedef async_msg async_type; - typedef T filtered_type; - - static const bool is_async_type = false; - - static const void* to_void_ptr(const T& t) { - return static_cast(&t); - } - - static void* to_void_ptr(T& t) { - return static_cast(&t); - } - - static const T& from_void_ptr(const void* p) { - return *static_cast(p); - } - - static T& from_void_ptr(void* p) { - return *static_cast(p); - } - - static task* try_put_task_wrapper_impl( receiver* const this_recv, const void *p, bool is_async ) { - if ( is_async ) { - // This (T) is NOT async and incoming 'A t' IS async - // Get data from async_msg - const async_msg& msg = async_helpers< async_msg >::from_void_ptr(p); - task* const new_task = msg.my_storage->subscribe(*this_recv); - // finalize() must be called after subscribe() because set() can be called in finalize() - // and 'this_recv' client must be subscribed by this moment - msg.finalize(); - return new_task; - } else { - // Incoming 't' is NOT async - return this_recv->try_put_task( from_void_ptr(p) ); - } - } -}; - -template< typename T > -struct async_helpers< T, typename std::enable_if< std::is_base_of, T>::value >::type > { - typedef T async_type; - typedef typename T::async_msg_data_type filtered_type; - - static const bool is_async_type = true; - - // Receiver-classes use const interfaces - static const void* to_void_ptr(const T& t) { - return static_cast( &static_cast&>(t) ); - } - - static void* to_void_ptr(T& t) { - return static_cast( &static_cast&>(t) ); - } - - // Sender-classes use non-const interfaces - static const T& from_void_ptr(const void* p) { - return *static_cast( static_cast*>(p) ); - } - - static T& from_void_ptr(void* p) { - return *static_cast( static_cast*>(p) ); - } - - // Used in receiver class - static task* try_put_task_wrapper_impl(receiver* const this_recv, const void *p, bool is_async) { - if ( is_async ) { - // Both are async - return this_recv->try_put_task( from_void_ptr(p) ); - } else { - // This (T) is async and incoming 'X t' is NOT async - // Create async_msg for X - const filtered_type& t = async_helpers::from_void_ptr(p); - const T msg(t); - return this_recv->try_put_task(msg); - } - } -}; - -template -class async_storage { -public: - typedef receiver async_storage_client; - - async_storage() { my_data_ready.store(false); } - - template - async_storage(C&& data) : my_data( std::forward(data) ) { - using namespace tbb::internal; - __TBB_STATIC_ASSERT( (is_same_type::type, typename strip::type>::value), "incoming type must be T" ); - - my_data_ready.store(true); - } - - template - bool set(C&& data) { - using namespace tbb::internal; - __TBB_STATIC_ASSERT( (is_same_type::type, typename strip::type>::value), "incoming type must be T" ); - - { - tbb::spin_mutex::scoped_lock locker(my_mutex); - - if (my_data_ready.load()) { - __TBB_ASSERT(false, "double set() call"); - return false; - } - - my_data = std::forward(data); - my_data_ready.store(true); - } - - // Thread sync is on my_data_ready flag - for (typename subscriber_list_type::iterator it = my_clients.begin(); it != my_clients.end(); ++it) { - (*it)->try_put(my_data); - } - - return true; - } - - task* subscribe(async_storage_client& client) { - if (! my_data_ready.load()) - { - tbb::spin_mutex::scoped_lock locker(my_mutex); - - if (! my_data_ready.load()) { -#if TBB_USE_ASSERT - for (typename subscriber_list_type::iterator it = my_clients.begin(); it != my_clients.end(); ++it) { - __TBB_ASSERT(*it != &client, "unexpected double subscription"); - } -#endif // TBB_USE_ASSERT - - // Subscribe - my_clients.push_back(&client); - return SUCCESSFULLY_ENQUEUED; - } - } - - __TBB_ASSERT(my_data_ready.load(), "data is NOT ready"); - return client.try_put_task(my_data); - } - -private: - tbb::spin_mutex my_mutex; - - tbb::atomic my_data_ready; - T my_data; - - typedef std::vector subscriber_list_type; - subscriber_list_type my_clients; -}; - -} // namespace internal - -template -class async_msg { - template< typename > friend class receiver; - template< typename, typename > friend struct internal::async_helpers; -public: - typedef T async_msg_data_type; - - async_msg() : my_storage(std::make_shared< internal::async_storage >()) {} - - async_msg(const T& t) : my_storage(std::make_shared< internal::async_storage >(t)) {} - - async_msg(T&& t) : my_storage(std::make_shared< internal::async_storage >( std::move(t) )) {} - - virtual ~async_msg() {} - - void set(const T& t) { - my_storage->set(t); - } - - void set(T&& t) { - my_storage->set( std::move(t) ); - } - -protected: - // Can be overridden in derived class to inform that - // async calculation chain is over - virtual void finalize() const {} - -private: - typedef std::shared_ptr< internal::async_storage > async_storage_ptr; - async_storage_ptr my_storage; -}; - -#endif // __TBB__flow_graph_async_msg_impl_H diff --git a/inst/include/tbb/internal/_flow_graph_body_impl.h b/inst/include/tbb/internal/_flow_graph_body_impl.h deleted file mode 100644 index 54f37411d..000000000 --- a/inst/include/tbb/internal/_flow_graph_body_impl.h +++ /dev/null @@ -1,320 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB__flow_graph_body_impl_H -#define __TBB__flow_graph_body_impl_H - -#ifndef __TBB_flow_graph_H -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -// included in namespace tbb::flow::interfaceX (in flow_graph.h) - -namespace internal { - -typedef tbb::internal::uint64_t tag_value; - -using tbb::internal::strip; - -namespace graph_policy_namespace { - - struct rejecting { }; - struct reserving { }; - struct queueing { }; - - // K == type of field used for key-matching. Each tag-matching port will be provided - // functor that, given an object accepted by the port, will return the - /// field of type K being used for matching. - template::type > > - struct key_matching { - typedef K key_type; - typedef typename strip::type base_key_type; - typedef KHash hash_compare_type; - }; - - // old tag_matching join's new specifier - typedef key_matching tag_matching; - -} // namespace graph_policy_namespace - -// -------------- function_body containers ---------------------- - -//! A functor that takes no input and generates a value of type Output -template< typename Output > -class source_body : tbb::internal::no_assign { -public: - virtual ~source_body() {} - virtual bool operator()(Output &output) = 0; - virtual source_body* clone() = 0; -}; - -//! The leaf for source_body -template< typename Output, typename Body> -class source_body_leaf : public source_body { -public: - source_body_leaf( const Body &_body ) : body(_body) { } - bool operator()(Output &output) __TBB_override { return body( output ); } - source_body_leaf* clone() __TBB_override { - return new source_body_leaf< Output, Body >(body); - } - Body get_body() { return body; } -private: - Body body; -}; - -//! A functor that takes an Input and generates an Output -template< typename Input, typename Output > -class function_body : tbb::internal::no_assign { -public: - virtual ~function_body() {} - virtual Output operator()(const Input &input) = 0; - virtual function_body* clone() = 0; -}; - -//! the leaf for function_body -template -class function_body_leaf : public function_body< Input, Output > { -public: - function_body_leaf( const B &_body ) : body(_body) { } - Output operator()(const Input &i) __TBB_override { return body(i); } - B get_body() { return body; } - function_body_leaf* clone() __TBB_override { - return new function_body_leaf< Input, Output, B >(body); - } -private: - B body; -}; - -//! the leaf for function_body specialized for Input and output of continue_msg -template -class function_body_leaf< continue_msg, continue_msg, B> : public function_body< continue_msg, continue_msg > { -public: - function_body_leaf( const B &_body ) : body(_body) { } - continue_msg operator()( const continue_msg &i ) __TBB_override { - body(i); - return i; - } - B get_body() { return body; } - function_body_leaf* clone() __TBB_override { - return new function_body_leaf< continue_msg, continue_msg, B >(body); - } -private: - B body; -}; - -//! the leaf for function_body specialized for Output of continue_msg -template -class function_body_leaf< Input, continue_msg, B> : public function_body< Input, continue_msg > { -public: - function_body_leaf( const B &_body ) : body(_body) { } - continue_msg operator()(const Input &i) __TBB_override { - body(i); - return continue_msg(); - } - B get_body() { return body; } - function_body_leaf* clone() __TBB_override { - return new function_body_leaf< Input, continue_msg, B >(body); - } -private: - B body; -}; - -//! the leaf for function_body specialized for Input of continue_msg -template -class function_body_leaf< continue_msg, Output, B > : public function_body< continue_msg, Output > { -public: - function_body_leaf( const B &_body ) : body(_body) { } - Output operator()(const continue_msg &i) __TBB_override { - return body(i); - } - B get_body() { return body; } - function_body_leaf* clone() __TBB_override { - return new function_body_leaf< continue_msg, Output, B >(body); - } -private: - B body; -}; - -//! function_body that takes an Input and a set of output ports -template -class multifunction_body : tbb::internal::no_assign { -public: - virtual ~multifunction_body () {} - virtual void operator()(const Input &/* input*/, OutputSet &/*oset*/) = 0; - virtual multifunction_body* clone() = 0; - virtual void* get_body_ptr() = 0; -}; - -//! leaf for multifunction. OutputSet can be a std::tuple or a vector. -template -class multifunction_body_leaf : public multifunction_body { -public: - multifunction_body_leaf(const B &_body) : body(_body) { } - void operator()(const Input &input, OutputSet &oset) __TBB_override { - body(input, oset); // body may explicitly put() to one or more of oset. - } - void* get_body_ptr() __TBB_override { return &body; } - multifunction_body_leaf* clone() __TBB_override { - return new multifunction_body_leaf(body); - } - -private: - B body; -}; - -// ------ function bodies for hash_buffers and key-matching joins. - -template -class type_to_key_function_body : tbb::internal::no_assign { - public: - virtual ~type_to_key_function_body() {} - virtual Output operator()(const Input &input) = 0; // returns an Output - virtual type_to_key_function_body* clone() = 0; -}; - -// specialization for ref output -template -class type_to_key_function_body : tbb::internal::no_assign { - public: - virtual ~type_to_key_function_body() {} - virtual const Output & operator()(const Input &input) = 0; // returns a const Output& - virtual type_to_key_function_body* clone() = 0; -}; - -template -class type_to_key_function_body_leaf : public type_to_key_function_body { -public: - type_to_key_function_body_leaf( const B &_body ) : body(_body) { } - Output operator()(const Input &i) __TBB_override { return body(i); } - B get_body() { return body; } - type_to_key_function_body_leaf* clone() __TBB_override { - return new type_to_key_function_body_leaf< Input, Output, B>(body); - } -private: - B body; -}; - -template -class type_to_key_function_body_leaf : public type_to_key_function_body< Input, Output&> { -public: - type_to_key_function_body_leaf( const B &_body ) : body(_body) { } - const Output& operator()(const Input &i) __TBB_override { - return body(i); - } - B get_body() { return body; } - type_to_key_function_body_leaf* clone() __TBB_override { - return new type_to_key_function_body_leaf< Input, Output&, B>(body); - } -private: - B body; -}; - -// --------------------------- end of function_body containers ------------------------ - -// --------------------------- node task bodies --------------------------------------- - -//! A task that calls a node's forward_task function -template< typename NodeType > -class forward_task_bypass : public task { - - NodeType &my_node; - -public: - - forward_task_bypass( NodeType &n ) : my_node(n) {} - - task *execute() __TBB_override { - task * new_task = my_node.forward_task(); - if (new_task == SUCCESSFULLY_ENQUEUED) new_task = NULL; - return new_task; - } -}; - -//! A task that calls a node's apply_body_bypass function, passing in an input of type Input -// return the task* unless it is SUCCESSFULLY_ENQUEUED, in which case return NULL -template< typename NodeType, typename Input > -class apply_body_task_bypass : public task { - - NodeType &my_node; - Input my_input; - -public: - - apply_body_task_bypass( NodeType &n, const Input &i ) : my_node(n), my_input(i) {} - - task *execute() __TBB_override { - task * next_task = my_node.apply_body_bypass( my_input ); - if(next_task == SUCCESSFULLY_ENQUEUED) next_task = NULL; - return next_task; - } -}; - -//! A task that calls a node's apply_body_bypass function with no input -template< typename NodeType > -class source_task_bypass : public task { - - NodeType &my_node; - -public: - - source_task_bypass( NodeType &n ) : my_node(n) {} - - task *execute() __TBB_override { - task *new_task = my_node.apply_body_bypass( ); - if(new_task == SUCCESSFULLY_ENQUEUED) return NULL; - return new_task; - } -}; - -// ------------------------ end of node task bodies ----------------------------------- - -//! An empty functor that takes an Input and returns a default constructed Output -template< typename Input, typename Output > -struct empty_body { - Output operator()( const Input & ) const { return Output(); } -}; - -template -class decrementer : public continue_receiver, tbb::internal::no_copy { - - T *my_node; - - task *execute() __TBB_override { - return my_node->decrement_counter(); - } - -protected: - - graph& graph_reference() __TBB_override { - return my_node->my_graph; - } - -public: - - typedef continue_msg input_type; - typedef continue_msg output_type; - decrementer( int number_of_predecessors = 0 ) : continue_receiver( number_of_predecessors ) { } - void set_owner( T *node ) { my_node = node; } -}; - -} // namespace internal - -#endif // __TBB__flow_graph_body_impl_H - diff --git a/inst/include/tbb/internal/_flow_graph_cache_impl.h b/inst/include/tbb/internal/_flow_graph_cache_impl.h deleted file mode 100644 index 7fac84e8a..000000000 --- a/inst/include/tbb/internal/_flow_graph_cache_impl.h +++ /dev/null @@ -1,562 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB__flow_graph_cache_impl_H -#define __TBB__flow_graph_cache_impl_H - -#ifndef __TBB_flow_graph_H -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -// included in namespace tbb::flow::interfaceX (in flow_graph.h) - -namespace internal { - -//! A node_cache maintains a std::queue of elements of type T. Each operation is protected by a lock. -template< typename T, typename M=spin_mutex > -class node_cache { - public: - - typedef size_t size_type; - - bool empty() { - typename mutex_type::scoped_lock lock( my_mutex ); - return internal_empty(); - } - - void add( T &n ) { - typename mutex_type::scoped_lock lock( my_mutex ); - internal_push(n); - } - - void remove( T &n ) { - typename mutex_type::scoped_lock lock( my_mutex ); - for ( size_t i = internal_size(); i != 0; --i ) { - T &s = internal_pop(); - if ( &s == &n ) return; // only remove one predecessor per request - internal_push(s); - } - } - - void clear() { - while( !my_q.empty()) (void)my_q.pop(); -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - my_built_predecessors.clear(); -#endif - } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - typedef edge_container built_predecessors_type; - built_predecessors_type &built_predecessors() { return my_built_predecessors; } - - typedef typename edge_container::edge_list_type predecessor_list_type; - void internal_add_built_predecessor( T &n ) { - typename mutex_type::scoped_lock lock( my_mutex ); - my_built_predecessors.add_edge(n); - } - - void internal_delete_built_predecessor( T &n ) { - typename mutex_type::scoped_lock lock( my_mutex ); - my_built_predecessors.delete_edge(n); - } - - void copy_predecessors( predecessor_list_type &v) { - typename mutex_type::scoped_lock lock( my_mutex ); - my_built_predecessors.copy_edges(v); - } - - size_t predecessor_count() { - typename mutex_type::scoped_lock lock(my_mutex); - return (size_t)(my_built_predecessors.edge_count()); - } -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ - -protected: - - typedef M mutex_type; - mutex_type my_mutex; - std::queue< T * > my_q; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - built_predecessors_type my_built_predecessors; -#endif - - // Assumes lock is held - inline bool internal_empty( ) { - return my_q.empty(); - } - - // Assumes lock is held - inline size_type internal_size( ) { - return my_q.size(); - } - - // Assumes lock is held - inline void internal_push( T &n ) { - my_q.push(&n); - } - - // Assumes lock is held - inline T &internal_pop() { - T *v = my_q.front(); - my_q.pop(); - return *v; - } - -}; - -//! A cache of predecessors that only supports try_get -template< typename T, typename M=spin_mutex > -#if __TBB_PREVIEW_ASYNC_MSG -// TODO: make predecessor_cache type T-independent when async_msg becomes regular feature -class predecessor_cache : public node_cache< untyped_sender, M > { -#else -class predecessor_cache : public node_cache< sender, M > { -#endif // __TBB_PREVIEW_ASYNC_MSG -public: - typedef M mutex_type; - typedef T output_type; -#if __TBB_PREVIEW_ASYNC_MSG - typedef untyped_sender predecessor_type; - typedef untyped_receiver successor_type; -#else - typedef sender predecessor_type; - typedef receiver successor_type; -#endif // __TBB_PREVIEW_ASYNC_MSG - - predecessor_cache( ) : my_owner( NULL ) { } - - void set_owner( successor_type *owner ) { my_owner = owner; } - - bool get_item( output_type &v ) { - - bool msg = false; - - do { - predecessor_type *src; - { - typename mutex_type::scoped_lock lock(this->my_mutex); - if ( this->internal_empty() ) { - break; - } - src = &this->internal_pop(); - } - - // Try to get from this sender - msg = src->try_get( v ); - - if (msg == false) { - // Relinquish ownership of the edge - if (my_owner) - src->register_successor( *my_owner ); - } else { - // Retain ownership of the edge - this->add(*src); - } - } while ( msg == false ); - return msg; - } - - // If we are removing arcs (rf_clear_edges), call clear() rather than reset(). - void reset() { - if (my_owner) { - for(;;) { - predecessor_type *src; - { - if (this->internal_empty()) break; - src = &this->internal_pop(); - } - src->register_successor( *my_owner ); - } - } - } - -protected: - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - using node_cache< predecessor_type, M >::my_built_predecessors; -#endif - successor_type *my_owner; -}; - -//! An cache of predecessors that supports requests and reservations -// TODO: make reservable_predecessor_cache type T-independent when async_msg becomes regular feature -template< typename T, typename M=spin_mutex > -class reservable_predecessor_cache : public predecessor_cache< T, M > { -public: - typedef M mutex_type; - typedef T output_type; -#if __TBB_PREVIEW_ASYNC_MSG - typedef untyped_sender predecessor_type; - typedef untyped_receiver successor_type; -#else - typedef sender predecessor_type; - typedef receiver successor_type; -#endif // __TBB_PREVIEW_ASYNC_MSG - - reservable_predecessor_cache( ) : reserved_src(NULL) { } - - bool - try_reserve( output_type &v ) { - bool msg = false; - - do { - { - typename mutex_type::scoped_lock lock(this->my_mutex); - if ( reserved_src || this->internal_empty() ) - return false; - - reserved_src = &this->internal_pop(); - } - - // Try to get from this sender - msg = reserved_src->try_reserve( v ); - - if (msg == false) { - typename mutex_type::scoped_lock lock(this->my_mutex); - // Relinquish ownership of the edge - reserved_src->register_successor( *this->my_owner ); - reserved_src = NULL; - } else { - // Retain ownership of the edge - this->add( *reserved_src ); - } - } while ( msg == false ); - - return msg; - } - - bool - try_release( ) { - reserved_src->try_release( ); - reserved_src = NULL; - return true; - } - - bool - try_consume( ) { - reserved_src->try_consume( ); - reserved_src = NULL; - return true; - } - - void reset( ) { - reserved_src = NULL; - predecessor_cache::reset( ); - } - - void clear() { - reserved_src = NULL; - predecessor_cache::clear(); - } - -private: - predecessor_type *reserved_src; -}; - - -//! An abstract cache of successors -// TODO: make successor_cache type T-independent when async_msg becomes regular feature -template -class successor_cache : tbb::internal::no_copy { -protected: - - typedef M mutex_type; - mutex_type my_mutex; - -#if __TBB_PREVIEW_ASYNC_MSG - typedef untyped_receiver successor_type; - typedef untyped_receiver *pointer_type; - typedef untyped_sender owner_type; -#else - typedef receiver successor_type; - typedef receiver *pointer_type; - typedef sender owner_type; -#endif // __TBB_PREVIEW_ASYNC_MSG - typedef std::list< pointer_type > successors_type; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - edge_container my_built_successors; -#endif - successors_type my_successors; - - owner_type *my_owner; - -public: -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - typedef typename edge_container::edge_list_type successor_list_type; - - edge_container &built_successors() { return my_built_successors; } - - void internal_add_built_successor( successor_type &r) { - typename mutex_type::scoped_lock l(my_mutex, true); - my_built_successors.add_edge( r ); - } - - void internal_delete_built_successor( successor_type &r) { - typename mutex_type::scoped_lock l(my_mutex, true); - my_built_successors.delete_edge(r); - } - - void copy_successors( successor_list_type &v) { - typename mutex_type::scoped_lock l(my_mutex, false); - my_built_successors.copy_edges(v); - } - - size_t successor_count() { - typename mutex_type::scoped_lock l(my_mutex,false); - return my_built_successors.edge_count(); - } - -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ - - successor_cache( ) : my_owner(NULL) {} - - void set_owner( owner_type *owner ) { my_owner = owner; } - - virtual ~successor_cache() {} - - void register_successor( successor_type &r ) { - typename mutex_type::scoped_lock l(my_mutex, true); - my_successors.push_back( &r ); - } - - void remove_successor( successor_type &r ) { - typename mutex_type::scoped_lock l(my_mutex, true); - for ( typename successors_type::iterator i = my_successors.begin(); - i != my_successors.end(); ++i ) { - if ( *i == & r ) { - my_successors.erase(i); - break; - } - } - } - - bool empty() { - typename mutex_type::scoped_lock l(my_mutex, false); - return my_successors.empty(); - } - - void clear() { - my_successors.clear(); -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - my_built_successors.clear(); -#endif - } - -#if !__TBB_PREVIEW_ASYNC_MSG - virtual task * try_put_task( const T &t ) = 0; -#endif // __TBB_PREVIEW_ASYNC_MSG - }; // successor_cache - -//! An abstract cache of successors, specialized to continue_msg -template<> -class successor_cache< continue_msg > : tbb::internal::no_copy { -protected: - - typedef spin_rw_mutex mutex_type; - mutex_type my_mutex; - -#if __TBB_PREVIEW_ASYNC_MSG - typedef untyped_receiver successor_type; - typedef untyped_receiver *pointer_type; -#else - typedef receiver successor_type; - typedef receiver *pointer_type; -#endif // __TBB_PREVIEW_ASYNC_MSG - typedef std::list< pointer_type > successors_type; - successors_type my_successors; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - edge_container my_built_successors; - typedef edge_container::edge_list_type successor_list_type; -#endif - - sender *my_owner; - -public: - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - - edge_container &built_successors() { return my_built_successors; } - - void internal_add_built_successor( successor_type &r) { - mutex_type::scoped_lock l(my_mutex, true); - my_built_successors.add_edge( r ); - } - - void internal_delete_built_successor( successor_type &r) { - mutex_type::scoped_lock l(my_mutex, true); - my_built_successors.delete_edge(r); - } - - void copy_successors( successor_list_type &v) { - mutex_type::scoped_lock l(my_mutex, false); - my_built_successors.copy_edges(v); - } - - size_t successor_count() { - mutex_type::scoped_lock l(my_mutex,false); - return my_built_successors.edge_count(); - } - -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ - - successor_cache( ) : my_owner(NULL) {} - - void set_owner( sender *owner ) { my_owner = owner; } - - virtual ~successor_cache() {} - - void register_successor( successor_type &r ) { - mutex_type::scoped_lock l(my_mutex, true); - my_successors.push_back( &r ); - if ( my_owner && r.is_continue_receiver() ) { - r.register_predecessor( *my_owner ); - } - } - - void remove_successor( successor_type &r ) { - mutex_type::scoped_lock l(my_mutex, true); - for ( successors_type::iterator i = my_successors.begin(); - i != my_successors.end(); ++i ) { - if ( *i == & r ) { - // TODO: Check if we need to test for continue_receiver before - // removing from r. - if ( my_owner ) - r.remove_predecessor( *my_owner ); - my_successors.erase(i); - break; - } - } - } - - bool empty() { - mutex_type::scoped_lock l(my_mutex, false); - return my_successors.empty(); - } - - void clear() { - my_successors.clear(); -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - my_built_successors.clear(); -#endif - } - -#if !__TBB_PREVIEW_ASYNC_MSG - virtual task * try_put_task( const continue_msg &t ) = 0; -#endif // __TBB_PREVIEW_ASYNC_MSG - -}; // successor_cache< continue_msg > - -//! A cache of successors that are broadcast to -// TODO: make broadcast_cache type T-independent when async_msg becomes regular feature -template -class broadcast_cache : public successor_cache { - typedef M mutex_type; - typedef typename successor_cache::successors_type successors_type; - -public: - - broadcast_cache( ) {} - - // as above, but call try_put_task instead, and return the last task we received (if any) -#if __TBB_PREVIEW_ASYNC_MSG - template - task * try_put_task( const X &t ) { -#else - task * try_put_task( const T &t ) __TBB_override { -#endif // __TBB_PREVIEW_ASYNC_MSG - task * last_task = NULL; - bool upgraded = true; - typename mutex_type::scoped_lock l(this->my_mutex, upgraded); - typename successors_type::iterator i = this->my_successors.begin(); - while ( i != this->my_successors.end() ) { - task *new_task = (*i)->try_put_task(t); - // workaround for icc bug - graph& graph_ref = (*i)->graph_reference(); - last_task = combine_tasks(graph_ref, last_task, new_task); // enqueue if necessary - if(new_task) { - ++i; - } - else { // failed - if ( (*i)->register_predecessor(*this->my_owner) ) { - if (!upgraded) { - l.upgrade_to_writer(); - upgraded = true; - } - i = this->my_successors.erase(i); - } else { - ++i; - } - } - } - return last_task; - } - -}; - -//! A cache of successors that are put in a round-robin fashion -// TODO: make round_robin_cache type T-independent when async_msg becomes regular feature -template -class round_robin_cache : public successor_cache { - typedef size_t size_type; - typedef M mutex_type; - typedef typename successor_cache::successors_type successors_type; - -public: - - round_robin_cache( ) {} - - size_type size() { - typename mutex_type::scoped_lock l(this->my_mutex, false); - return this->my_successors.size(); - } - -#if __TBB_PREVIEW_ASYNC_MSG - template - task * try_put_task( const X &t ) { -#else - task *try_put_task( const T &t ) __TBB_override { -#endif // __TBB_PREVIEW_ASYNC_MSG - bool upgraded = true; - typename mutex_type::scoped_lock l(this->my_mutex, upgraded); - typename successors_type::iterator i = this->my_successors.begin(); - while ( i != this->my_successors.end() ) { - task *new_task = (*i)->try_put_task(t); - if ( new_task ) { - return new_task; - } else { - if ( (*i)->register_predecessor(*this->my_owner) ) { - if (!upgraded) { - l.upgrade_to_writer(); - upgraded = true; - } - i = this->my_successors.erase(i); - } - else { - ++i; - } - } - } - return NULL; - } -}; - -} // namespace internal - -#endif // __TBB__flow_graph_cache_impl_H diff --git a/inst/include/tbb/internal/_flow_graph_impl.h b/inst/include/tbb/internal/_flow_graph_impl.h deleted file mode 100644 index 92278caf8..000000000 --- a/inst/include/tbb/internal/_flow_graph_impl.h +++ /dev/null @@ -1,839 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB__flow_graph_impl_H -#define __TBB__flow_graph_impl_H - -#ifndef __TBB_flow_graph_H -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -// included in namespace tbb::flow::interfaceX (in flow_graph.h) - -namespace internal { - - typedef tbb::internal::uint64_t tag_value; - - using tbb::internal::strip; - - namespace graph_policy_namespace { - - struct rejecting { }; - struct reserving { }; - struct queueing { }; - - // K == type of field used for key-matching. Each tag-matching port will be provided - // functor that, given an object accepted by the port, will return the - /// field of type K being used for matching. - template::type > > - struct key_matching { - typedef K key_type; - typedef typename strip::type base_key_type; - typedef KHash hash_compare_type; - }; - - // old tag_matching join's new specifier - typedef key_matching tag_matching; - } - -// -------------- function_body containers ---------------------- - - //! A functor that takes no input and generates a value of type Output - template< typename Output > - class source_body : tbb::internal::no_assign { - public: - virtual ~source_body() {} - virtual bool operator()(Output &output) = 0; - virtual source_body* clone() = 0; - }; - - //! The leaf for source_body - template< typename Output, typename Body> - class source_body_leaf : public source_body { - public: - source_body_leaf( const Body &_body ) : body(_body) { } - bool operator()(Output &output) __TBB_override { return body( output ); } - source_body_leaf* clone() __TBB_override { - return new source_body_leaf< Output, Body >(body); - } - Body get_body() { return body; } - private: - Body body; - }; - - //! A functor that takes an Input and generates an Output - template< typename Input, typename Output > - class function_body : tbb::internal::no_assign { - public: - virtual ~function_body() {} - virtual Output operator()(const Input &input) = 0; - virtual function_body* clone() = 0; - }; - - //! the leaf for function_body - template - class function_body_leaf : public function_body< Input, Output > { - public: - function_body_leaf( const B &_body ) : body(_body) { } - Output operator()(const Input &i) __TBB_override { return body(i); } - B get_body() { return body; } - function_body_leaf* clone() __TBB_override { - return new function_body_leaf< Input, Output, B >(body); - } - private: - B body; - }; - - //! the leaf for function_body specialized for Input and output of continue_msg - template - class function_body_leaf< continue_msg, continue_msg, B> : public function_body< continue_msg, continue_msg > { - public: - function_body_leaf( const B &_body ) : body(_body) { } - continue_msg operator()( const continue_msg &i ) __TBB_override { - body(i); - return i; - } - B get_body() { return body; } - function_body_leaf* clone() __TBB_override { - return new function_body_leaf< continue_msg, continue_msg, B >(body); - } - private: - B body; - }; - - //! the leaf for function_body specialized for Output of continue_msg - template - class function_body_leaf< Input, continue_msg, B> : public function_body< Input, continue_msg > { - public: - function_body_leaf( const B &_body ) : body(_body) { } - continue_msg operator()(const Input &i) __TBB_override { - body(i); - return continue_msg(); - } - B get_body() { return body; } - function_body_leaf* clone() __TBB_override { - return new function_body_leaf< Input, continue_msg, B >(body); - } - private: - B body; - }; - - //! the leaf for function_body specialized for Input of continue_msg - template - class function_body_leaf< continue_msg, Output, B > : public function_body< continue_msg, Output > { - public: - function_body_leaf( const B &_body ) : body(_body) { } - Output operator()(const continue_msg &i) __TBB_override { - return body(i); - } - B get_body() { return body; } - function_body_leaf* clone() __TBB_override { - return new function_body_leaf< continue_msg, Output, B >(body); - } - private: - B body; - }; - - //! function_body that takes an Input and a set of output ports - template - class multifunction_body : tbb::internal::no_assign { - public: - virtual ~multifunction_body () {} - virtual void operator()(const Input &/* input*/, OutputSet &/*oset*/) = 0; - virtual multifunction_body* clone() = 0; - virtual void* get_body_ptr() = 0; - }; - - //! leaf for multifunction. OutputSet can be a std::tuple or a vector. - template - class multifunction_body_leaf : public multifunction_body { - public: - multifunction_body_leaf(const B &_body) : body(_body) { } - void operator()(const Input &input, OutputSet &oset) __TBB_override { - body(input, oset); // body may explicitly put() to one or more of oset. - } - void* get_body_ptr() __TBB_override { return &body; } - multifunction_body_leaf* clone() __TBB_override { - return new multifunction_body_leaf(body); - } - - private: - B body; - }; - -// ------ function bodies for hash_buffers and key-matching joins. - -template -class type_to_key_function_body : tbb::internal::no_assign { - public: - virtual ~type_to_key_function_body() {} - virtual Output operator()(const Input &input) = 0; // returns an Output - virtual type_to_key_function_body* clone() = 0; -}; - -// specialization for ref output -template -class type_to_key_function_body : tbb::internal::no_assign { - public: - virtual ~type_to_key_function_body() {} - virtual const Output & operator()(const Input &input) = 0; // returns a const Output& - virtual type_to_key_function_body* clone() = 0; -}; - -template -class type_to_key_function_body_leaf : public type_to_key_function_body { -public: - type_to_key_function_body_leaf( const B &_body ) : body(_body) { } - Output operator()(const Input &i) __TBB_override { return body(i); } - B get_body() { return body; } - type_to_key_function_body_leaf* clone() __TBB_override { - return new type_to_key_function_body_leaf< Input, Output, B>(body); - } -private: - B body; -}; - -template -class type_to_key_function_body_leaf : public type_to_key_function_body< Input, Output&> { -public: - type_to_key_function_body_leaf( const B &_body ) : body(_body) { } - const Output& operator()(const Input &i) __TBB_override { - return body(i); - } - B get_body() { return body; } - type_to_key_function_body_leaf* clone() __TBB_override { - return new type_to_key_function_body_leaf< Input, Output&, B>(body); - } -private: - B body; -}; - -// --------------------------- end of function_body containers ------------------------ - -// --------------------------- node task bodies --------------------------------------- - - //! A task that calls a node's forward_task function - template< typename NodeType > - class forward_task_bypass : public task { - - NodeType &my_node; - - public: - - forward_task_bypass( NodeType &n ) : my_node(n) {} - - task *execute() __TBB_override { - task * new_task = my_node.forward_task(); - if (new_task == SUCCESSFULLY_ENQUEUED) new_task = NULL; - return new_task; - } - }; - - //! A task that calls a node's apply_body_bypass function, passing in an input of type Input - // return the task* unless it is SUCCESSFULLY_ENQUEUED, in which case return NULL - template< typename NodeType, typename Input > - class apply_body_task_bypass : public task { - - NodeType &my_node; - Input my_input; - - public: - - apply_body_task_bypass( NodeType &n, const Input &i ) : my_node(n), my_input(i) {} - - task *execute() __TBB_override { - task * next_task = my_node.apply_body_bypass( my_input ); - if(next_task == SUCCESSFULLY_ENQUEUED) next_task = NULL; - return next_task; - } - }; - - //! A task that calls a node's apply_body_bypass function with no input - template< typename NodeType > - class source_task_bypass : public task { - - NodeType &my_node; - - public: - - source_task_bypass( NodeType &n ) : my_node(n) {} - - task *execute() __TBB_override { - task *new_task = my_node.apply_body_bypass( ); - if(new_task == SUCCESSFULLY_ENQUEUED) return NULL; - return new_task; - } - }; - -// ------------------------ end of node task bodies ----------------------------------- - - //! An empty functor that takes an Input and returns a default constructed Output - template< typename Input, typename Output > - struct empty_body { - Output operator()( const Input & ) const { return Output(); } - }; - - //! A node_cache maintains a std::queue of elements of type T. Each operation is protected by a lock. - template< typename T, typename M=spin_mutex > - class node_cache { - public: - - typedef size_t size_type; - - bool empty() { - typename mutex_type::scoped_lock lock( my_mutex ); - return internal_empty(); - } - - void add( T &n ) { - typename mutex_type::scoped_lock lock( my_mutex ); - internal_push(n); - } - - void remove( T &n ) { - typename mutex_type::scoped_lock lock( my_mutex ); - for ( size_t i = internal_size(); i != 0; --i ) { - T &s = internal_pop(); - if ( &s == &n ) return; // only remove one predecessor per request - internal_push(s); - } - } - - void clear() { - while( !my_q.empty()) (void)my_q.pop(); -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - my_built_predecessors.clear(); -#endif - } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - typedef edge_container built_predecessors_type; - built_predecessors_type &built_predecessors() { return my_built_predecessors; } - - typedef typename edge_container::edge_list_type predecessor_list_type; - void internal_add_built_predecessor( T &n ) { - typename mutex_type::scoped_lock lock( my_mutex ); - my_built_predecessors.add_edge(n); - } - - void internal_delete_built_predecessor( T &n ) { - typename mutex_type::scoped_lock lock( my_mutex ); - my_built_predecessors.delete_edge(n); - } - - void copy_predecessors( predecessor_list_type &v) { - typename mutex_type::scoped_lock lock( my_mutex ); - my_built_predecessors.copy_edges(v); - } - - size_t predecessor_count() { - typename mutex_type::scoped_lock lock(my_mutex); - return (size_t)(my_built_predecessors.edge_count()); - } -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ - - protected: - - typedef M mutex_type; - mutex_type my_mutex; - std::queue< T * > my_q; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - built_predecessors_type my_built_predecessors; -#endif - - // Assumes lock is held - inline bool internal_empty( ) { - return my_q.empty(); - } - - // Assumes lock is held - inline size_type internal_size( ) { - return my_q.size(); - } - - // Assumes lock is held - inline void internal_push( T &n ) { - my_q.push(&n); - } - - // Assumes lock is held - inline T &internal_pop() { - T *v = my_q.front(); - my_q.pop(); - return *v; - } - - }; - - //! A cache of predecessors that only supports try_get - template< typename T, typename M=spin_mutex > -#if __TBB_PREVIEW_ASYNC_MSG - // TODO: make predecessor_cache type T-independent when async_msg becomes regular feature - class predecessor_cache : public node_cache< untyped_sender, M > { -#else - class predecessor_cache : public node_cache< sender, M > { -#endif // __TBB_PREVIEW_ASYNC_MSG - public: - typedef M mutex_type; - typedef T output_type; -#if __TBB_PREVIEW_ASYNC_MSG - typedef untyped_sender predecessor_type; - typedef untyped_receiver successor_type; -#else - typedef sender predecessor_type; - typedef receiver successor_type; -#endif // __TBB_PREVIEW_ASYNC_MSG - - predecessor_cache( ) : my_owner( NULL ) { } - - void set_owner( successor_type *owner ) { my_owner = owner; } - - bool get_item( output_type &v ) { - - bool msg = false; - - do { - predecessor_type *src; - { - typename mutex_type::scoped_lock lock(this->my_mutex); - if ( this->internal_empty() ) { - break; - } - src = &this->internal_pop(); - } - - // Try to get from this sender - msg = src->try_get( v ); - - if (msg == false) { - // Relinquish ownership of the edge - if (my_owner) - src->register_successor( *my_owner ); - } else { - // Retain ownership of the edge - this->add(*src); - } - } while ( msg == false ); - return msg; - } - - // If we are removing arcs (rf_clear_edges), call clear() rather than reset(). - void reset() { - if (my_owner) { - for(;;) { - predecessor_type *src; - { - if (this->internal_empty()) break; - src = &this->internal_pop(); - } - src->register_successor( *my_owner ); - } - } - } - - protected: - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - using node_cache< predecessor_type, M >::my_built_predecessors; -#endif - successor_type *my_owner; - }; - - //! An cache of predecessors that supports requests and reservations - // TODO: make reservable_predecessor_cache type T-independent when async_msg becomes regular feature - template< typename T, typename M=spin_mutex > - class reservable_predecessor_cache : public predecessor_cache< T, M > { - public: - typedef M mutex_type; - typedef T output_type; -#if __TBB_PREVIEW_ASYNC_MSG - typedef untyped_sender predecessor_type; - typedef untyped_receiver successor_type; -#else - typedef sender predecessor_type; - typedef receiver successor_type; -#endif // __TBB_PREVIEW_ASYNC_MSG - - reservable_predecessor_cache( ) : reserved_src(NULL) { } - - bool - try_reserve( output_type &v ) { - bool msg = false; - - do { - { - typename mutex_type::scoped_lock lock(this->my_mutex); - if ( reserved_src || this->internal_empty() ) - return false; - - reserved_src = &this->internal_pop(); - } - - // Try to get from this sender - msg = reserved_src->try_reserve( v ); - - if (msg == false) { - typename mutex_type::scoped_lock lock(this->my_mutex); - // Relinquish ownership of the edge - reserved_src->register_successor( *this->my_owner ); - reserved_src = NULL; - } else { - // Retain ownership of the edge - this->add( *reserved_src ); - } - } while ( msg == false ); - - return msg; - } - - bool - try_release( ) { - reserved_src->try_release( ); - reserved_src = NULL; - return true; - } - - bool - try_consume( ) { - reserved_src->try_consume( ); - reserved_src = NULL; - return true; - } - - void reset( ) { - reserved_src = NULL; - predecessor_cache::reset( ); - } - - void clear() { - reserved_src = NULL; - predecessor_cache::clear(); - } - - private: - predecessor_type *reserved_src; - }; - - - //! An abstract cache of successors - // TODO: make successor_cache type T-independent when async_msg becomes regular feature - template - class successor_cache : tbb::internal::no_copy { - protected: - - typedef M mutex_type; - mutex_type my_mutex; - -#if __TBB_PREVIEW_ASYNC_MSG - typedef untyped_receiver successor_type; - typedef untyped_receiver *pointer_type; - typedef untyped_sender owner_type; -#else - typedef receiver successor_type; - typedef receiver *pointer_type; - typedef sender owner_type; -#endif // __TBB_PREVIEW_ASYNC_MSG - typedef std::list< pointer_type > successors_type; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - edge_container my_built_successors; -#endif - successors_type my_successors; - - owner_type *my_owner; - - public: -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - typedef typename edge_container::edge_list_type successor_list_type; - - edge_container &built_successors() { return my_built_successors; } - - void internal_add_built_successor( successor_type &r) { - typename mutex_type::scoped_lock l(my_mutex, true); - my_built_successors.add_edge( r ); - } - - void internal_delete_built_successor( successor_type &r) { - typename mutex_type::scoped_lock l(my_mutex, true); - my_built_successors.delete_edge(r); - } - - void copy_successors( successor_list_type &v) { - typename mutex_type::scoped_lock l(my_mutex, false); - my_built_successors.copy_edges(v); - } - - size_t successor_count() { - typename mutex_type::scoped_lock l(my_mutex,false); - return my_built_successors.edge_count(); - } - -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ - - successor_cache( ) : my_owner(NULL) {} - - void set_owner( owner_type *owner ) { my_owner = owner; } - - virtual ~successor_cache() {} - - void register_successor( successor_type &r ) { - typename mutex_type::scoped_lock l(my_mutex, true); - my_successors.push_back( &r ); - } - - void remove_successor( successor_type &r ) { - typename mutex_type::scoped_lock l(my_mutex, true); - for ( typename successors_type::iterator i = my_successors.begin(); - i != my_successors.end(); ++i ) { - if ( *i == & r ) { - my_successors.erase(i); - break; - } - } - } - - bool empty() { - typename mutex_type::scoped_lock l(my_mutex, false); - return my_successors.empty(); - } - - void clear() { - my_successors.clear(); -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - my_built_successors.clear(); -#endif - } - -#if !__TBB_PREVIEW_ASYNC_MSG - virtual task * try_put_task( const T &t ) = 0; -#endif // __TBB_PREVIEW_ASYNC_MSG - }; // successor_cache - - //! An abstract cache of successors, specialized to continue_msg - template<> - class successor_cache< continue_msg > : tbb::internal::no_copy { - protected: - - typedef spin_rw_mutex mutex_type; - mutex_type my_mutex; - -#if __TBB_PREVIEW_ASYNC_MSG - typedef untyped_receiver successor_type; - typedef untyped_receiver *pointer_type; -#else - typedef receiver successor_type; - typedef receiver *pointer_type; -#endif // __TBB_PREVIEW_ASYNC_MSG - typedef std::list< pointer_type > successors_type; - successors_type my_successors; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - edge_container my_built_successors; - typedef edge_container::edge_list_type successor_list_type; -#endif - - sender *my_owner; - - public: - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - - edge_container &built_successors() { return my_built_successors; } - - void internal_add_built_successor( successor_type &r) { - mutex_type::scoped_lock l(my_mutex, true); - my_built_successors.add_edge( r ); - } - - void internal_delete_built_successor( successor_type &r) { - mutex_type::scoped_lock l(my_mutex, true); - my_built_successors.delete_edge(r); - } - - void copy_successors( successor_list_type &v) { - mutex_type::scoped_lock l(my_mutex, false); - my_built_successors.copy_edges(v); - } - - size_t successor_count() { - mutex_type::scoped_lock l(my_mutex,false); - return my_built_successors.edge_count(); - } - -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ - - successor_cache( ) : my_owner(NULL) {} - - void set_owner( sender *owner ) { my_owner = owner; } - - virtual ~successor_cache() {} - - void register_successor( successor_type &r ) { - mutex_type::scoped_lock l(my_mutex, true); - my_successors.push_back( &r ); - if ( my_owner && r.is_continue_receiver() ) { - r.register_predecessor( *my_owner ); - } - } - - void remove_successor( successor_type &r ) { - mutex_type::scoped_lock l(my_mutex, true); - for ( successors_type::iterator i = my_successors.begin(); - i != my_successors.end(); ++i ) { - if ( *i == & r ) { - // TODO: Check if we need to test for continue_receiver before - // removing from r. - if ( my_owner ) - r.remove_predecessor( *my_owner ); - my_successors.erase(i); - break; - } - } - } - - bool empty() { - mutex_type::scoped_lock l(my_mutex, false); - return my_successors.empty(); - } - - void clear() { - my_successors.clear(); -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - my_built_successors.clear(); -#endif - } - -#if !__TBB_PREVIEW_ASYNC_MSG - virtual task * try_put_task( const continue_msg &t ) = 0; -#endif // __TBB_PREVIEW_ASYNC_MSG - - }; // successor_cache< continue_msg > - - //! A cache of successors that are broadcast to - // TODO: make broadcast_cache type T-independent when async_msg becomes regular feature - template - class broadcast_cache : public successor_cache { - typedef M mutex_type; - typedef typename successor_cache::successors_type successors_type; - - public: - - broadcast_cache( ) {} - - // as above, but call try_put_task instead, and return the last task we received (if any) -#if __TBB_PREVIEW_ASYNC_MSG - template - task * try_put_task( const X &t ) { -#else - task * try_put_task( const T &t ) __TBB_override { -#endif // __TBB_PREVIEW_ASYNC_MSG - task * last_task = NULL; - bool upgraded = true; - typename mutex_type::scoped_lock l(this->my_mutex, upgraded); - typename successors_type::iterator i = this->my_successors.begin(); - while ( i != this->my_successors.end() ) { - task *new_task = (*i)->try_put_task(t); - last_task = combine_tasks(last_task, new_task); // enqueue if necessary - if(new_task) { - ++i; - } - else { // failed - if ( (*i)->register_predecessor(*this->my_owner) ) { - if (!upgraded) { - l.upgrade_to_writer(); - upgraded = true; - } - i = this->my_successors.erase(i); - } else { - ++i; - } - } - } - return last_task; - } - - }; - - //! A cache of successors that are put in a round-robin fashion - // TODO: make round_robin_cache type T-independent when async_msg becomes regular feature - template - class round_robin_cache : public successor_cache { - typedef size_t size_type; - typedef M mutex_type; - typedef typename successor_cache::successors_type successors_type; - - public: - - round_robin_cache( ) {} - - size_type size() { - typename mutex_type::scoped_lock l(this->my_mutex, false); - return this->my_successors.size(); - } - -#if __TBB_PREVIEW_ASYNC_MSG - template - task * try_put_task( const X &t ) { -#else - task *try_put_task( const T &t ) __TBB_override { -#endif // __TBB_PREVIEW_ASYNC_MSG - bool upgraded = true; - typename mutex_type::scoped_lock l(this->my_mutex, upgraded); - typename successors_type::iterator i = this->my_successors.begin(); - while ( i != this->my_successors.end() ) { - task *new_task = (*i)->try_put_task(t); - if ( new_task ) { - return new_task; - } else { - if ( (*i)->register_predecessor(*this->my_owner) ) { - if (!upgraded) { - l.upgrade_to_writer(); - upgraded = true; - } - i = this->my_successors.erase(i); - } - else { - ++i; - } - } - } - return NULL; - } - }; - - template - class decrementer : public continue_receiver, tbb::internal::no_copy { - - T *my_node; - - task *execute() __TBB_override { - return my_node->decrement_counter(); - } - - public: - - typedef continue_msg input_type; - typedef continue_msg output_type; - decrementer( int number_of_predecessors = 0 ) : continue_receiver( number_of_predecessors ) { } - void set_owner( T *node ) { my_node = node; } - }; - -} - -#endif // __TBB__flow_graph_impl_H - diff --git a/inst/include/tbb/internal/_flow_graph_indexer_impl.h b/inst/include/tbb/internal/_flow_graph_indexer_impl.h deleted file mode 100644 index 1fc6690c5..000000000 --- a/inst/include/tbb/internal/_flow_graph_indexer_impl.h +++ /dev/null @@ -1,478 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB__flow_graph_indexer_impl_H -#define __TBB__flow_graph_indexer_impl_H - -#ifndef __TBB_flow_graph_H -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#include "_flow_graph_types_impl.h" - -namespace internal { - - // Output of the indexer_node is a tbb::flow::tagged_msg, and will be of - // the form tagged_msg - // where the value of tag will indicate which result was put to the - // successor. - - template - task* do_try_put(const T &v, void *p) { - typename IndexerNodeBaseType::output_type o(K, v); - return reinterpret_cast(p)->try_put_task(&o); - } - - template - struct indexer_helper { - template - static inline void set_indexer_node_pointer(PortTuple &my_input, IndexerNodeBaseType *p) { - typedef typename tuple_element::type T; - task *(*indexer_node_put_task)(const T&, void *) = do_try_put; - tbb::flow::get(my_input).set_up(p, indexer_node_put_task); - indexer_helper::template set_indexer_node_pointer(my_input, p); - } - template - static inline void reset_inputs(InputTuple &my_input, reset_flags f) { - indexer_helper::reset_inputs(my_input, f); - tbb::flow::get(my_input).reset_receiver(f); - } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - template - static inline void extract(InputTuple &my_input) { - indexer_helper::extract(my_input); - tbb::flow::get(my_input).extract_receiver(); - } -#endif - }; - - template - struct indexer_helper { - template - static inline void set_indexer_node_pointer(PortTuple &my_input, IndexerNodeBaseType *p) { - typedef typename tuple_element<0, TupleTypes>::type T; - task *(*indexer_node_put_task)(const T&, void *) = do_try_put; - tbb::flow::get<0>(my_input).set_up(p, indexer_node_put_task); - } - template - static inline void reset_inputs(InputTuple &my_input, reset_flags f) { - tbb::flow::get<0>(my_input).reset_receiver(f); - } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - template - static inline void extract(InputTuple &my_input) { - tbb::flow::get<0>(my_input).extract_receiver(); - } -#endif - }; - - template - class indexer_input_port : public receiver { - private: - void* my_indexer_ptr; - typedef task* (* forward_function_ptr)(T const &, void* ); - forward_function_ptr my_try_put_task; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - spin_mutex my_pred_mutex; - typedef typename receiver::built_predecessors_type built_predecessors_type; - built_predecessors_type my_built_predecessors; -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ - public: -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - indexer_input_port() : my_pred_mutex() {} - indexer_input_port( const indexer_input_port & /*other*/ ) : receiver(), my_pred_mutex() { - } -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ - void set_up(void *p, forward_function_ptr f) { - my_indexer_ptr = p; - my_try_put_task = f; - } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - typedef typename receiver::predecessor_list_type predecessor_list_type; - typedef typename receiver::predecessor_type predecessor_type; - - built_predecessors_type &built_predecessors() __TBB_override { return my_built_predecessors; } - - size_t predecessor_count() __TBB_override { - spin_mutex::scoped_lock l(my_pred_mutex); - return my_built_predecessors.edge_count(); - } - void internal_add_built_predecessor(predecessor_type &p) __TBB_override { - spin_mutex::scoped_lock l(my_pred_mutex); - my_built_predecessors.add_edge(p); - } - void internal_delete_built_predecessor(predecessor_type &p) __TBB_override { - spin_mutex::scoped_lock l(my_pred_mutex); - my_built_predecessors.delete_edge(p); - } - void copy_predecessors( predecessor_list_type &v) __TBB_override { - spin_mutex::scoped_lock l(my_pred_mutex); - my_built_predecessors.copy_edges(v); - } -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ - protected: - template< typename R, typename B > friend class run_and_put_task; - template friend class internal::broadcast_cache; - template friend class internal::round_robin_cache; - task *try_put_task(const T &v) __TBB_override { - return my_try_put_task(v, my_indexer_ptr); - } - - public: -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - void reset_receiver(reset_flags f) __TBB_override { if(f&rf_clear_edges) my_built_predecessors.clear(); } -#else - void reset_receiver(reset_flags /*f*/) __TBB_override { } -#endif - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - void extract_receiver() { my_built_predecessors.receiver_extract(*this); } -#endif - }; - - template - class indexer_node_FE { - public: - static const int N = tbb::flow::tuple_size::value; - typedef OutputType output_type; - typedef InputTuple input_type; - - // Some versions of Intel C++ compiler fail to generate an implicit constructor for the class which has std::tuple as a member. - indexer_node_FE() : my_inputs() {} - - input_type &input_ports() { return my_inputs; } - protected: - input_type my_inputs; - }; - - //! indexer_node_base - template - class indexer_node_base : public graph_node, public indexer_node_FE, - public sender { - protected: - using graph_node::my_graph; - public: - static const size_t N = tbb::flow::tuple_size::value; - typedef OutputType output_type; - typedef StructTypes tuple_types; - typedef typename sender::successor_type successor_type; - typedef indexer_node_FE input_ports_type; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - typedef typename sender::built_successors_type built_successors_type; - typedef typename sender::successor_list_type successor_list_type; -#endif - - private: - // ----------- Aggregator ------------ - enum op_type { reg_succ, rem_succ, try__put_task -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - , add_blt_succ, del_blt_succ, - blt_succ_cnt, blt_succ_cpy -#endif - }; - typedef indexer_node_base class_type; - - class indexer_node_base_operation : public aggregated_operation { - public: - char type; - union { - output_type const *my_arg; - successor_type *my_succ; - task *bypass_t; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - size_t cnt_val; - successor_list_type *succv; -#endif - }; - indexer_node_base_operation(const output_type* e, op_type t) : - type(char(t)), my_arg(e) {} - indexer_node_base_operation(const successor_type &s, op_type t) : type(char(t)), - my_succ(const_cast(&s)) {} - indexer_node_base_operation(op_type t) : type(char(t)) {} - }; - - typedef internal::aggregating_functor handler_type; - friend class internal::aggregating_functor; - aggregator my_aggregator; - - void handle_operations(indexer_node_base_operation* op_list) { - indexer_node_base_operation *current; - while(op_list) { - current = op_list; - op_list = op_list->next; - switch(current->type) { - - case reg_succ: - my_successors.register_successor(*(current->my_succ)); - __TBB_store_with_release(current->status, SUCCEEDED); - break; - - case rem_succ: - my_successors.remove_successor(*(current->my_succ)); - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case try__put_task: { - current->bypass_t = my_successors.try_put_task(*(current->my_arg)); - __TBB_store_with_release(current->status, SUCCEEDED); // return of try_put_task actual return value - } - break; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - case add_blt_succ: - my_successors.internal_add_built_successor(*(current->my_succ)); - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case del_blt_succ: - my_successors.internal_delete_built_successor(*(current->my_succ)); - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case blt_succ_cnt: - current->cnt_val = my_successors.successor_count(); - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case blt_succ_cpy: - my_successors.copy_successors(*(current->succv)); - __TBB_store_with_release(current->status, SUCCEEDED); - break; -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ - } - } - } - // ---------- end aggregator ----------- - public: - indexer_node_base(graph& g) : graph_node(g), input_ports_type() { - indexer_helper::set_indexer_node_pointer(this->my_inputs, this); - my_successors.set_owner(this); - my_aggregator.initialize_handler(handler_type(this)); - } - - indexer_node_base(const indexer_node_base& other) : graph_node(other.my_graph), input_ports_type(), sender() { - indexer_helper::set_indexer_node_pointer(this->my_inputs, this); - my_successors.set_owner(this); - my_aggregator.initialize_handler(handler_type(this)); - } - - bool register_successor(successor_type &r) __TBB_override { - indexer_node_base_operation op_data(r, reg_succ); - my_aggregator.execute(&op_data); - return op_data.status == SUCCEEDED; - } - - bool remove_successor( successor_type &r) __TBB_override { - indexer_node_base_operation op_data(r, rem_succ); - my_aggregator.execute(&op_data); - return op_data.status == SUCCEEDED; - } - - task * try_put_task(output_type const *v) { // not a virtual method in this class - indexer_node_base_operation op_data(v, try__put_task); - my_aggregator.execute(&op_data); - return op_data.bypass_t; - } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - - built_successors_type &built_successors() __TBB_override { return my_successors.built_successors(); } - - void internal_add_built_successor( successor_type &r) __TBB_override { - indexer_node_base_operation op_data(r, add_blt_succ); - my_aggregator.execute(&op_data); - } - - void internal_delete_built_successor( successor_type &r) __TBB_override { - indexer_node_base_operation op_data(r, del_blt_succ); - my_aggregator.execute(&op_data); - } - - size_t successor_count() __TBB_override { - indexer_node_base_operation op_data(blt_succ_cnt); - my_aggregator.execute(&op_data); - return op_data.cnt_val; - } - - void copy_successors( successor_list_type &v) __TBB_override { - indexer_node_base_operation op_data(blt_succ_cpy); - op_data.succv = &v; - my_aggregator.execute(&op_data); - } - void extract() __TBB_override { - my_successors.built_successors().sender_extract(*this); - indexer_helper::extract(this->my_inputs); - } -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ - protected: - void reset_node(reset_flags f) __TBB_override { - if(f & rf_clear_edges) { - my_successors.clear(); - indexer_helper::reset_inputs(this->my_inputs,f); - } - } - - private: - broadcast_cache my_successors; - }; //indexer_node_base - - - template struct input_types; - - template - struct input_types<1, InputTuple> { - typedef typename tuple_element<0, InputTuple>::type first_type; - typedef typename internal::tagged_msg type; - }; - - template - struct input_types<2, InputTuple> { - typedef typename tuple_element<0, InputTuple>::type first_type; - typedef typename tuple_element<1, InputTuple>::type second_type; - typedef typename internal::tagged_msg type; - }; - - template - struct input_types<3, InputTuple> { - typedef typename tuple_element<0, InputTuple>::type first_type; - typedef typename tuple_element<1, InputTuple>::type second_type; - typedef typename tuple_element<2, InputTuple>::type third_type; - typedef typename internal::tagged_msg type; - }; - - template - struct input_types<4, InputTuple> { - typedef typename tuple_element<0, InputTuple>::type first_type; - typedef typename tuple_element<1, InputTuple>::type second_type; - typedef typename tuple_element<2, InputTuple>::type third_type; - typedef typename tuple_element<3, InputTuple>::type fourth_type; - typedef typename internal::tagged_msg type; - }; - - template - struct input_types<5, InputTuple> { - typedef typename tuple_element<0, InputTuple>::type first_type; - typedef typename tuple_element<1, InputTuple>::type second_type; - typedef typename tuple_element<2, InputTuple>::type third_type; - typedef typename tuple_element<3, InputTuple>::type fourth_type; - typedef typename tuple_element<4, InputTuple>::type fifth_type; - typedef typename internal::tagged_msg type; - }; - - template - struct input_types<6, InputTuple> { - typedef typename tuple_element<0, InputTuple>::type first_type; - typedef typename tuple_element<1, InputTuple>::type second_type; - typedef typename tuple_element<2, InputTuple>::type third_type; - typedef typename tuple_element<3, InputTuple>::type fourth_type; - typedef typename tuple_element<4, InputTuple>::type fifth_type; - typedef typename tuple_element<5, InputTuple>::type sixth_type; - typedef typename internal::tagged_msg type; - }; - - template - struct input_types<7, InputTuple> { - typedef typename tuple_element<0, InputTuple>::type first_type; - typedef typename tuple_element<1, InputTuple>::type second_type; - typedef typename tuple_element<2, InputTuple>::type third_type; - typedef typename tuple_element<3, InputTuple>::type fourth_type; - typedef typename tuple_element<4, InputTuple>::type fifth_type; - typedef typename tuple_element<5, InputTuple>::type sixth_type; - typedef typename tuple_element<6, InputTuple>::type seventh_type; - typedef typename internal::tagged_msg type; - }; - - - template - struct input_types<8, InputTuple> { - typedef typename tuple_element<0, InputTuple>::type first_type; - typedef typename tuple_element<1, InputTuple>::type second_type; - typedef typename tuple_element<2, InputTuple>::type third_type; - typedef typename tuple_element<3, InputTuple>::type fourth_type; - typedef typename tuple_element<4, InputTuple>::type fifth_type; - typedef typename tuple_element<5, InputTuple>::type sixth_type; - typedef typename tuple_element<6, InputTuple>::type seventh_type; - typedef typename tuple_element<7, InputTuple>::type eighth_type; - typedef typename internal::tagged_msg type; - }; - - - template - struct input_types<9, InputTuple> { - typedef typename tuple_element<0, InputTuple>::type first_type; - typedef typename tuple_element<1, InputTuple>::type second_type; - typedef typename tuple_element<2, InputTuple>::type third_type; - typedef typename tuple_element<3, InputTuple>::type fourth_type; - typedef typename tuple_element<4, InputTuple>::type fifth_type; - typedef typename tuple_element<5, InputTuple>::type sixth_type; - typedef typename tuple_element<6, InputTuple>::type seventh_type; - typedef typename tuple_element<7, InputTuple>::type eighth_type; - typedef typename tuple_element<8, InputTuple>::type nineth_type; - typedef typename internal::tagged_msg type; - }; - - template - struct input_types<10, InputTuple> { - typedef typename tuple_element<0, InputTuple>::type first_type; - typedef typename tuple_element<1, InputTuple>::type second_type; - typedef typename tuple_element<2, InputTuple>::type third_type; - typedef typename tuple_element<3, InputTuple>::type fourth_type; - typedef typename tuple_element<4, InputTuple>::type fifth_type; - typedef typename tuple_element<5, InputTuple>::type sixth_type; - typedef typename tuple_element<6, InputTuple>::type seventh_type; - typedef typename tuple_element<7, InputTuple>::type eighth_type; - typedef typename tuple_element<8, InputTuple>::type nineth_type; - typedef typename tuple_element<9, InputTuple>::type tenth_type; - typedef typename internal::tagged_msg type; - }; - - // type generators - template - struct indexer_types : public input_types::value, OutputTuple> { - static const int N = tbb::flow::tuple_size::value; - typedef typename input_types::type output_type; - typedef typename wrap_tuple_elements::type input_ports_type; - typedef internal::indexer_node_FE indexer_FE_type; - typedef internal::indexer_node_base indexer_base_type; - }; - - template - class unfolded_indexer_node : public indexer_types::indexer_base_type { - public: - typedef typename indexer_types::input_ports_type input_ports_type; - typedef OutputTuple tuple_types; - typedef typename indexer_types::output_type output_type; - private: - typedef typename indexer_types::indexer_base_type base_type; - public: - unfolded_indexer_node(graph& g) : base_type(g) {} - unfolded_indexer_node(const unfolded_indexer_node &other) : base_type(other) {} - }; - -} /* namespace internal */ - -#endif /* __TBB__flow_graph_indexer_impl_H */ diff --git a/inst/include/tbb/internal/_flow_graph_item_buffer_impl.h b/inst/include/tbb/internal/_flow_graph_item_buffer_impl.h deleted file mode 100644 index 85d2686dc..000000000 --- a/inst/include/tbb/internal/_flow_graph_item_buffer_impl.h +++ /dev/null @@ -1,288 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB__flow_graph_item_buffer_impl_H -#define __TBB__flow_graph_item_buffer_impl_H - -#ifndef __TBB_flow_graph_H -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#include "tbb/internal/_flow_graph_types_impl.h" // for aligned_pair - -// in namespace tbb::flow::interfaceX (included in _flow_graph_node_impl.h) - - //! Expandable buffer of items. The possible operations are push, pop, - //* tests for empty and so forth. No mutual exclusion is built in. - //* objects are constructed into and explicitly-destroyed. get_my_item gives - // a read-only reference to the item in the buffer. set_my_item may be called - // with either an empty or occupied slot. - - using internal::aligned_pair; - using internal::alignment_of; - -namespace internal { - - template > - class item_buffer { - public: - typedef T item_type; - enum buffer_item_state { no_item=0, has_item=1, reserved_item=2 }; - protected: - typedef size_t size_type; - typedef typename aligned_pair::type buffer_item_type; - typedef typename A::template rebind::other allocator_type; - - buffer_item_type *my_array; - size_type my_array_size; - static const size_type initial_buffer_size = 4; - size_type my_head; - size_type my_tail; - - bool buffer_empty() const { return my_head == my_tail; } - - buffer_item_type &item(size_type i) { - __TBB_ASSERT(!(size_type(&(my_array[i&(my_array_size-1)].second))%alignment_of::value),NULL); - __TBB_ASSERT(!(size_type(&(my_array[i&(my_array_size-1)].first))%alignment_of::value), NULL); - return my_array[i & (my_array_size - 1) ]; - } - - const buffer_item_type &item(size_type i) const { - __TBB_ASSERT(!(size_type(&(my_array[i&(my_array_size-1)].second))%alignment_of::value), NULL); - __TBB_ASSERT(!(size_type(&(my_array[i&(my_array_size-1)].first))%alignment_of::value), NULL); - return my_array[i & (my_array_size-1)]; - } - - bool my_item_valid(size_type i) const { return (i < my_tail) && (i >= my_head) && (item(i).second != no_item); } - bool my_item_reserved(size_type i) const { return item(i).second == reserved_item; } - - // object management in buffer - const item_type &get_my_item(size_t i) const { - __TBB_ASSERT(my_item_valid(i),"attempt to get invalid item"); - item_type *itm = (tbb::internal::punned_cast(&(item(i).first))); - return *(const item_type *)itm; - } - - // may be called with an empty slot or a slot that has already been constructed into. - void set_my_item(size_t i, const item_type &o) { - if(item(i).second != no_item) { - destroy_item(i); - } - new(&(item(i).first)) item_type(o); - item(i).second = has_item; - } - - // destructively-fetch an object from the buffer - void fetch_item(size_t i, item_type &o) { - __TBB_ASSERT(my_item_valid(i), "Trying to fetch an empty slot"); - o = get_my_item(i); // could have std::move assign semantics - destroy_item(i); - } - - // move an existing item from one slot to another. The moved-to slot must be unoccupied, - // the moved-from slot must exist and not be reserved. The after, from will be empty, - // to will be occupied but not reserved - void move_item(size_t to, size_t from) { - __TBB_ASSERT(!my_item_valid(to), "Trying to move to a non-empty slot"); - __TBB_ASSERT(my_item_valid(from), "Trying to move from an empty slot"); - set_my_item(to, get_my_item(from)); // could have std::move semantics - destroy_item(from); - - } - - // put an item in an empty slot. Return true if successful, else false - bool place_item(size_t here, const item_type &me) { -#if !TBB_DEPRECATED_SEQUENCER_DUPLICATES - if(my_item_valid(here)) return false; -#endif - set_my_item(here, me); - return true; - } - - // could be implemented with std::move semantics - void swap_items(size_t i, size_t j) { - __TBB_ASSERT(my_item_valid(i) && my_item_valid(j), "attempt to swap invalid item(s)"); - item_type temp = get_my_item(i); - set_my_item(i, get_my_item(j)); - set_my_item(j, temp); - } - - void destroy_item(size_type i) { - __TBB_ASSERT(my_item_valid(i), "destruction of invalid item"); - (tbb::internal::punned_cast(&(item(i).first)))->~item_type(); - item(i).second = no_item; - } - - // returns the front element - const item_type& front() const - { - __TBB_ASSERT(my_item_valid(my_head), "attempt to fetch head non-item"); - return get_my_item(my_head); - } - - // returns the back element - const item_type& back() const - { - __TBB_ASSERT(my_item_valid(my_tail - 1), "attempt to fetch head non-item"); - return get_my_item(my_tail - 1); - } - - // following methods are for reservation of the front of a bufffer. - void reserve_item(size_type i) { __TBB_ASSERT(my_item_valid(i) && !my_item_reserved(i), "item cannot be reserved"); item(i).second = reserved_item; } - void release_item(size_type i) { __TBB_ASSERT(my_item_reserved(i), "item is not reserved"); item(i).second = has_item; } - - void destroy_front() { destroy_item(my_head); ++my_head; } - void destroy_back() { destroy_item(my_tail-1); --my_tail; } - - // we have to be able to test against a new tail value without changing my_tail - // grow_array doesn't work if we change my_tail when the old array is too small - size_type size(size_t new_tail = 0) { return (new_tail ? new_tail : my_tail) - my_head; } - size_type capacity() { return my_array_size; } - // sequencer_node does not use this method, so we don't - // need a version that passes in the new_tail value. - bool buffer_full() { return size() >= capacity(); } - - //! Grows the internal array. - void grow_my_array( size_t minimum_size ) { - // test that we haven't made the structure inconsistent. - __TBB_ASSERT(capacity() >= my_tail - my_head, "total items exceed capacity"); - size_type new_size = my_array_size ? 2*my_array_size : initial_buffer_size; - while( new_sizeback(); - destroy_back(); - return true; - } - - bool pop_front(item_type &v) { - if(!my_item_valid(my_head)) { - return false; - } - v = this->front(); - destroy_front(); - return true; - } - - // This is used both for reset and for grow_my_array. In the case of grow_my_array - // we want to retain the values of the head and tail. - void clean_up_buffer(bool reset_pointers) { - if (my_array) { - for( size_type i=my_head; i > - class reservable_item_buffer : public item_buffer { - protected: - using item_buffer::my_item_valid; - using item_buffer::my_head; - - public: - reservable_item_buffer() : item_buffer(), my_reserved(false) {} - void reset() {my_reserved = false; item_buffer::reset(); } - protected: - - bool reserve_front(T &v) { - if(my_reserved || !my_item_valid(this->my_head)) return false; - my_reserved = true; - // reserving the head - v = this->front(); - this->reserve_item(this->my_head); - return true; - } - - void consume_front() { - __TBB_ASSERT(my_reserved, "Attempt to consume a non-reserved item"); - this->destroy_front(); - my_reserved = false; - } - - void release_front() { - __TBB_ASSERT(my_reserved, "Attempt to release a non-reserved item"); - this->release_item(this->my_head); - my_reserved = false; - } - - bool my_reserved; - }; - -} // namespace internal - -#endif // __TBB__flow_graph_item_buffer_impl_H diff --git a/inst/include/tbb/internal/_flow_graph_join_impl.h b/inst/include/tbb/internal/_flow_graph_join_impl.h deleted file mode 100644 index 4999bef76..000000000 --- a/inst/include/tbb/internal/_flow_graph_join_impl.h +++ /dev/null @@ -1,1991 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB__flow_graph_join_impl_H -#define __TBB__flow_graph_join_impl_H - -#ifndef __TBB_flow_graph_H -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -namespace internal { - - struct forwarding_base { - forwarding_base(graph &g) : graph_pointer(&g) {} - virtual ~forwarding_base() {} - // decrement_port_count may create a forwarding task. If we cannot handle the task - // ourselves, ask decrement_port_count to deal with it. - virtual task * decrement_port_count(bool handle_task) = 0; - virtual void increment_port_count() = 0; - // moved here so input ports can queue tasks - graph* graph_pointer; - }; - - // specialization that lets us keep a copy of the current_key for building results. - // KeyType can be a reference type. - template - struct matching_forwarding_base :public forwarding_base { - typedef typename tbb::internal::strip::type current_key_type; - matching_forwarding_base(graph &g) : forwarding_base(g) { } - virtual task * increment_key_count(current_key_type const & /*t*/, bool /*handle_task*/) = 0; // {return NULL;} - current_key_type current_key; // so ports can refer to FE's desired items - }; - - template< int N > - struct join_helper { - - template< typename TupleType, typename PortType > - static inline void set_join_node_pointer(TupleType &my_input, PortType *port) { - tbb::flow::get( my_input ).set_join_node_pointer(port); - join_helper::set_join_node_pointer( my_input, port ); - } - template< typename TupleType > - static inline void consume_reservations( TupleType &my_input ) { - tbb::flow::get( my_input ).consume(); - join_helper::consume_reservations( my_input ); - } - - template< typename TupleType > - static inline void release_my_reservation( TupleType &my_input ) { - tbb::flow::get( my_input ).release(); - } - - template - static inline void release_reservations( TupleType &my_input) { - join_helper::release_reservations(my_input); - release_my_reservation(my_input); - } - - template< typename InputTuple, typename OutputTuple > - static inline bool reserve( InputTuple &my_input, OutputTuple &out) { - if ( !tbb::flow::get( my_input ).reserve( tbb::flow::get( out ) ) ) return false; - if ( !join_helper::reserve( my_input, out ) ) { - release_my_reservation( my_input ); - return false; - } - return true; - } - - template - static inline bool get_my_item( InputTuple &my_input, OutputTuple &out) { - bool res = tbb::flow::get(my_input).get_item(tbb::flow::get(out) ); // may fail - return join_helper::get_my_item(my_input, out) && res; // do get on other inputs before returning - } - - template - static inline bool get_items(InputTuple &my_input, OutputTuple &out) { - return get_my_item(my_input, out); - } - - template - static inline void reset_my_port(InputTuple &my_input) { - join_helper::reset_my_port(my_input); - tbb::flow::get(my_input).reset_port(); - } - - template - static inline void reset_ports(InputTuple& my_input) { - reset_my_port(my_input); - } - - template - static inline void set_key_functors(InputTuple &my_input, KeyFuncTuple &my_key_funcs) { - tbb::flow::get(my_input).set_my_key_func(tbb::flow::get(my_key_funcs)); - tbb::flow::get(my_key_funcs) = NULL; - join_helper::set_key_functors(my_input, my_key_funcs); - } - - template< typename KeyFuncTuple> - static inline void copy_key_functors(KeyFuncTuple &my_inputs, KeyFuncTuple &other_inputs) { - if(tbb::flow::get(other_inputs).get_my_key_func()) { - tbb::flow::get(my_inputs).set_my_key_func(tbb::flow::get(other_inputs).get_my_key_func()->clone()); - } - join_helper::copy_key_functors(my_inputs, other_inputs); - } - - template - static inline void reset_inputs(InputTuple &my_input, reset_flags f) { - join_helper::reset_inputs(my_input, f); - tbb::flow::get(my_input).reset_receiver(f); - } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - template - static inline void extract_inputs(InputTuple &my_input) { - join_helper::extract_inputs(my_input); - tbb::flow::get(my_input).extract_receiver(); - } -#endif - }; // join_helper - - template< > - struct join_helper<1> { - - template< typename TupleType, typename PortType > - static inline void set_join_node_pointer(TupleType &my_input, PortType *port) { - tbb::flow::get<0>( my_input ).set_join_node_pointer(port); - } - - template< typename TupleType > - static inline void consume_reservations( TupleType &my_input ) { - tbb::flow::get<0>( my_input ).consume(); - } - - template< typename TupleType > - static inline void release_my_reservation( TupleType &my_input ) { - tbb::flow::get<0>( my_input ).release(); - } - - template - static inline void release_reservations( TupleType &my_input) { - release_my_reservation(my_input); - } - - template< typename InputTuple, typename OutputTuple > - static inline bool reserve( InputTuple &my_input, OutputTuple &out) { - return tbb::flow::get<0>( my_input ).reserve( tbb::flow::get<0>( out ) ); - } - - template - static inline bool get_my_item( InputTuple &my_input, OutputTuple &out) { - return tbb::flow::get<0>(my_input).get_item(tbb::flow::get<0>(out)); - } - - template - static inline bool get_items(InputTuple &my_input, OutputTuple &out) { - return get_my_item(my_input, out); - } - - template - static inline void reset_my_port(InputTuple &my_input) { - tbb::flow::get<0>(my_input).reset_port(); - } - - template - static inline void reset_ports(InputTuple& my_input) { - reset_my_port(my_input); - } - - template - static inline void set_key_functors(InputTuple &my_input, KeyFuncTuple &my_key_funcs) { - tbb::flow::get<0>(my_input).set_my_key_func(tbb::flow::get<0>(my_key_funcs)); - tbb::flow::get<0>(my_key_funcs) = NULL; - } - - template< typename KeyFuncTuple> - static inline void copy_key_functors(KeyFuncTuple &my_inputs, KeyFuncTuple &other_inputs) { - if(tbb::flow::get<0>(other_inputs).get_my_key_func()) { - tbb::flow::get<0>(my_inputs).set_my_key_func(tbb::flow::get<0>(other_inputs).get_my_key_func()->clone()); - } - } - template - static inline void reset_inputs(InputTuple &my_input, reset_flags f) { - tbb::flow::get<0>(my_input).reset_receiver(f); - } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - template - static inline void extract_inputs(InputTuple &my_input) { - tbb::flow::get<0>(my_input).extract_receiver(); - } -#endif - }; // join_helper<1> - - //! The two-phase join port - template< typename T > - class reserving_port : public receiver { - public: - typedef T input_type; - typedef typename receiver::predecessor_type predecessor_type; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - typedef typename receiver::predecessor_list_type predecessor_list_type; - typedef typename receiver::built_predecessors_type built_predecessors_type; -#endif - private: - // ----------- Aggregator ------------ - enum op_type { reg_pred, rem_pred, res_item, rel_res, con_res -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - , add_blt_pred, del_blt_pred, blt_pred_cnt, blt_pred_cpy -#endif - }; - enum op_stat {WAIT=0, SUCCEEDED, FAILED}; - typedef reserving_port class_type; - - class reserving_port_operation : public aggregated_operation { - public: - char type; - union { - T *my_arg; - predecessor_type *my_pred; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - size_t cnt_val; - predecessor_list_type *plist; -#endif - }; - reserving_port_operation(const T& e, op_type t) : - type(char(t)), my_arg(const_cast(&e)) {} - reserving_port_operation(const predecessor_type &s, op_type t) : type(char(t)), - my_pred(const_cast(&s)) {} - reserving_port_operation(op_type t) : type(char(t)) {} - }; - - typedef internal::aggregating_functor handler_type; - friend class internal::aggregating_functor; - aggregator my_aggregator; - - void handle_operations(reserving_port_operation* op_list) { - reserving_port_operation *current; - bool no_predecessors; - while(op_list) { - current = op_list; - op_list = op_list->next; - switch(current->type) { - case reg_pred: - no_predecessors = my_predecessors.empty(); - my_predecessors.add(*(current->my_pred)); - if ( no_predecessors ) { - (void) my_join->decrement_port_count(true); // may try to forward - } - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case rem_pred: - my_predecessors.remove(*(current->my_pred)); - if(my_predecessors.empty()) my_join->increment_port_count(); - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case res_item: - if ( reserved ) { - __TBB_store_with_release(current->status, FAILED); - } - else if ( my_predecessors.try_reserve( *(current->my_arg) ) ) { - reserved = true; - __TBB_store_with_release(current->status, SUCCEEDED); - } else { - if ( my_predecessors.empty() ) { - my_join->increment_port_count(); - } - __TBB_store_with_release(current->status, FAILED); - } - break; - case rel_res: - reserved = false; - my_predecessors.try_release( ); - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case con_res: - reserved = false; - my_predecessors.try_consume( ); - __TBB_store_with_release(current->status, SUCCEEDED); - break; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - case add_blt_pred: - my_predecessors.internal_add_built_predecessor(*(current->my_pred)); - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case del_blt_pred: - my_predecessors.internal_delete_built_predecessor(*(current->my_pred)); - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case blt_pred_cnt: - current->cnt_val = my_predecessors.predecessor_count(); - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case blt_pred_cpy: - my_predecessors.copy_predecessors(*(current->plist)); - __TBB_store_with_release(current->status, SUCCEEDED); - break; -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ - } - } - } - - protected: - template< typename R, typename B > friend class run_and_put_task; - template friend class internal::broadcast_cache; - template friend class internal::round_robin_cache; - task *try_put_task( const T & ) __TBB_override { - return NULL; - } - - public: - - //! Constructor - reserving_port() : reserved(false) { - my_join = NULL; - my_predecessors.set_owner( this ); - my_aggregator.initialize_handler(handler_type(this)); - } - - // copy constructor - reserving_port(const reserving_port& /* other */) : receiver() { - reserved = false; - my_join = NULL; - my_predecessors.set_owner( this ); - my_aggregator.initialize_handler(handler_type(this)); - } - - void set_join_node_pointer(forwarding_base *join) { - my_join = join; - } - - //! Add a predecessor - bool register_predecessor( predecessor_type &src ) __TBB_override { - reserving_port_operation op_data(src, reg_pred); - my_aggregator.execute(&op_data); - return op_data.status == SUCCEEDED; - } - - //! Remove a predecessor - bool remove_predecessor( predecessor_type &src ) __TBB_override { - reserving_port_operation op_data(src, rem_pred); - my_aggregator.execute(&op_data); - return op_data.status == SUCCEEDED; - } - - //! Reserve an item from the port - bool reserve( T &v ) { - reserving_port_operation op_data(v, res_item); - my_aggregator.execute(&op_data); - return op_data.status == SUCCEEDED; - } - - //! Release the port - void release( ) { - reserving_port_operation op_data(rel_res); - my_aggregator.execute(&op_data); - } - - //! Complete use of the port - void consume( ) { - reserving_port_operation op_data(con_res); - my_aggregator.execute(&op_data); - } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - built_predecessors_type &built_predecessors() __TBB_override { return my_predecessors.built_predecessors(); } - void internal_add_built_predecessor(predecessor_type &src) __TBB_override { - reserving_port_operation op_data(src, add_blt_pred); - my_aggregator.execute(&op_data); - } - - void internal_delete_built_predecessor(predecessor_type &src) __TBB_override { - reserving_port_operation op_data(src, del_blt_pred); - my_aggregator.execute(&op_data); - } - - size_t predecessor_count() __TBB_override { - reserving_port_operation op_data(blt_pred_cnt); - my_aggregator.execute(&op_data); - return op_data.cnt_val; - } - - void copy_predecessors(predecessor_list_type &l) __TBB_override { - reserving_port_operation op_data(blt_pred_cpy); - op_data.plist = &l; - my_aggregator.execute(&op_data); - } - - void extract_receiver() { - my_predecessors.built_predecessors().receiver_extract(*this); - } - -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ - - void reset_receiver( reset_flags f) __TBB_override { - if(f & rf_clear_edges) my_predecessors.clear(); - else - my_predecessors.reset(); - reserved = false; - __TBB_ASSERT(!(f&rf_clear_edges) || my_predecessors.empty(), "port edges not removed"); - } - - private: - forwarding_base *my_join; - reservable_predecessor_cache< T, null_mutex > my_predecessors; - bool reserved; - }; // reserving_port - - //! queueing join_port - template - class queueing_port : public receiver, public item_buffer { - public: - typedef T input_type; - typedef typename receiver::predecessor_type predecessor_type; - typedef queueing_port class_type; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - typedef typename receiver::built_predecessors_type built_predecessors_type; - typedef typename receiver::predecessor_list_type predecessor_list_type; -#endif - - // ----------- Aggregator ------------ - private: - enum op_type { get__item, res_port, try__put_task -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - , add_blt_pred, del_blt_pred, blt_pred_cnt, blt_pred_cpy -#endif - }; - enum op_stat {WAIT=0, SUCCEEDED, FAILED}; - - class queueing_port_operation : public aggregated_operation { - public: - char type; - T my_val; - T *my_arg; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - predecessor_type *pred; - size_t cnt_val; - predecessor_list_type *plist; -#endif - task * bypass_t; - // constructor for value parameter - queueing_port_operation(const T& e, op_type t) : - type(char(t)), my_val(e) - , bypass_t(NULL) - {} - // constructor for pointer parameter - queueing_port_operation(const T* p, op_type t) : - type(char(t)), my_arg(const_cast(p)) - , bypass_t(NULL) - {} - // constructor with no parameter - queueing_port_operation(op_type t) : type(char(t)) - , bypass_t(NULL) - {} - }; - - typedef internal::aggregating_functor handler_type; - friend class internal::aggregating_functor; - aggregator my_aggregator; - - void handle_operations(queueing_port_operation* op_list) { - queueing_port_operation *current; - bool was_empty; - while(op_list) { - current = op_list; - op_list = op_list->next; - switch(current->type) { - case try__put_task: { - task *rtask = NULL; - was_empty = this->buffer_empty(); - this->push_back(current->my_val); - if (was_empty) rtask = my_join->decrement_port_count(false); - else - rtask = SUCCESSFULLY_ENQUEUED; - current->bypass_t = rtask; - __TBB_store_with_release(current->status, SUCCEEDED); - } - break; - case get__item: - if(!this->buffer_empty()) { - *(current->my_arg) = this->front(); - __TBB_store_with_release(current->status, SUCCEEDED); - } - else { - __TBB_store_with_release(current->status, FAILED); - } - break; - case res_port: - __TBB_ASSERT(this->my_item_valid(this->my_head), "No item to reset"); - this->destroy_front(); - if(this->my_item_valid(this->my_head)) { - (void)my_join->decrement_port_count(true); - } - __TBB_store_with_release(current->status, SUCCEEDED); - break; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - case add_blt_pred: - my_built_predecessors.add_edge(*(current->pred)); - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case del_blt_pred: - my_built_predecessors.delete_edge(*(current->pred)); - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case blt_pred_cnt: - current->cnt_val = my_built_predecessors.edge_count(); - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case blt_pred_cpy: - my_built_predecessors.copy_edges(*(current->plist)); - __TBB_store_with_release(current->status, SUCCEEDED); - break; -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ - } - } - } - // ------------ End Aggregator --------------- - - protected: - template< typename R, typename B > friend class run_and_put_task; - template friend class internal::broadcast_cache; - template friend class internal::round_robin_cache; - task *try_put_task(const T &v) __TBB_override { - queueing_port_operation op_data(v, try__put_task); - my_aggregator.execute(&op_data); - __TBB_ASSERT(op_data.status == SUCCEEDED || !op_data.bypass_t, "inconsistent return from aggregator"); - if(!op_data.bypass_t) return SUCCESSFULLY_ENQUEUED; - return op_data.bypass_t; - } - - public: - - //! Constructor - queueing_port() : item_buffer() { - my_join = NULL; - my_aggregator.initialize_handler(handler_type(this)); - } - - //! copy constructor - queueing_port(const queueing_port& /* other */) : receiver(), item_buffer() { - my_join = NULL; - my_aggregator.initialize_handler(handler_type(this)); - } - - //! record parent for tallying available items - void set_join_node_pointer(forwarding_base *join) { - my_join = join; - } - - bool get_item( T &v ) { - queueing_port_operation op_data(&v, get__item); - my_aggregator.execute(&op_data); - return op_data.status == SUCCEEDED; - } - - // reset_port is called when item is accepted by successor, but - // is initiated by join_node. - void reset_port() { - queueing_port_operation op_data(res_port); - my_aggregator.execute(&op_data); - return; - } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - built_predecessors_type &built_predecessors() __TBB_override { return my_built_predecessors; } - - void internal_add_built_predecessor(predecessor_type &p) __TBB_override { - queueing_port_operation op_data(add_blt_pred); - op_data.pred = &p; - my_aggregator.execute(&op_data); - } - - void internal_delete_built_predecessor(predecessor_type &p) __TBB_override { - queueing_port_operation op_data(del_blt_pred); - op_data.pred = &p; - my_aggregator.execute(&op_data); - } - - size_t predecessor_count() __TBB_override { - queueing_port_operation op_data(blt_pred_cnt); - my_aggregator.execute(&op_data); - return op_data.cnt_val; - } - - void copy_predecessors(predecessor_list_type &l) __TBB_override { - queueing_port_operation op_data(blt_pred_cpy); - op_data.plist = &l; - my_aggregator.execute(&op_data); - } - - void extract_receiver() { - item_buffer::reset(); - my_built_predecessors.receiver_extract(*this); - } -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ - - void reset_receiver(reset_flags f) __TBB_override { - tbb::internal::suppress_unused_warning(f); - item_buffer::reset(); -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - if (f & rf_clear_edges) - my_built_predecessors.clear(); -#endif - } - - private: - forwarding_base *my_join; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - edge_container my_built_predecessors; -#endif - }; // queueing_port - -#include "_flow_graph_tagged_buffer_impl.h" - - template - struct count_element { - K my_key; - size_t my_value; - }; - - // method to access the key in the counting table - // the ref has already been removed from K - template< typename K > - struct key_to_count_functor { - typedef count_element table_item_type; - const K& operator()(const table_item_type& v) { return v.my_key; } - }; - - // the ports can have only one template parameter. We wrap the types needed in - // a traits type - template< class TraitsType > - class key_matching_port : - public receiver, - public hash_buffer< typename TraitsType::K, typename TraitsType::T, typename TraitsType::TtoK, - typename TraitsType::KHash > { - public: - typedef TraitsType traits; - typedef key_matching_port class_type; - typedef typename TraitsType::T input_type; - typedef typename TraitsType::K key_type; - typedef typename tbb::internal::strip::type noref_key_type; - typedef typename receiver::predecessor_type predecessor_type; - typedef typename TraitsType::TtoK type_to_key_func_type; - typedef typename TraitsType::KHash hash_compare_type; - typedef hash_buffer< key_type, input_type, type_to_key_func_type, hash_compare_type > buffer_type; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - typedef typename receiver::built_predecessors_type built_predecessors_type; - typedef typename receiver::predecessor_list_type predecessor_list_type; -#endif - private: -// ----------- Aggregator ------------ - private: - enum op_type { try__put, get__item, res_port -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - , add_blt_pred, del_blt_pred, blt_pred_cnt, blt_pred_cpy -#endif - }; - enum op_stat {WAIT=0, SUCCEEDED, FAILED}; - - class key_matching_port_operation : public aggregated_operation { - public: - char type; - input_type my_val; - input_type *my_arg; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - predecessor_type *pred; - size_t cnt_val; - predecessor_list_type *plist; -#endif - // constructor for value parameter - key_matching_port_operation(const input_type& e, op_type t) : - type(char(t)), my_val(e) {} - // constructor for pointer parameter - key_matching_port_operation(const input_type* p, op_type t) : - type(char(t)), my_arg(const_cast(p)) {} - // constructor with no parameter - key_matching_port_operation(op_type t) : type(char(t)) {} - }; - - typedef internal::aggregating_functor handler_type; - friend class internal::aggregating_functor; - aggregator my_aggregator; - - void handle_operations(key_matching_port_operation* op_list) { - key_matching_port_operation *current; - while(op_list) { - current = op_list; - op_list = op_list->next; - switch(current->type) { - case try__put: { - bool was_inserted = this->insert_with_key(current->my_val); - // return failure if a duplicate insertion occurs - __TBB_store_with_release(current->status, was_inserted ? SUCCEEDED : FAILED); - } - break; - case get__item: - // use current_key from FE for item - if(!this->find_with_key(my_join->current_key, *(current->my_arg))) { - __TBB_ASSERT(false, "Failed to find item corresponding to current_key."); - } - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case res_port: - // use current_key from FE for item - this->delete_with_key(my_join->current_key); - __TBB_store_with_release(current->status, SUCCEEDED); - break; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - case add_blt_pred: - my_built_predecessors.add_edge(*(current->pred)); - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case del_blt_pred: - my_built_predecessors.delete_edge(*(current->pred)); - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case blt_pred_cnt: - current->cnt_val = my_built_predecessors.edge_count(); - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case blt_pred_cpy: - my_built_predecessors.copy_edges(*(current->plist)); - __TBB_store_with_release(current->status, SUCCEEDED); - break; -#endif - } - } - } -// ------------ End Aggregator --------------- - protected: - template< typename R, typename B > friend class run_and_put_task; - template friend class internal::broadcast_cache; - template friend class internal::round_robin_cache; - task *try_put_task(const input_type& v) __TBB_override { - key_matching_port_operation op_data(v, try__put); - task *rtask = NULL; - my_aggregator.execute(&op_data); - if(op_data.status == SUCCEEDED) { - rtask = my_join->increment_key_count((*(this->get_key_func()))(v), false); // may spawn - // rtask has to reflect the return status of the try_put - if(!rtask) rtask = SUCCESSFULLY_ENQUEUED; - } - return rtask; - } - - public: - - key_matching_port() : receiver(), buffer_type() { - my_join = NULL; - my_aggregator.initialize_handler(handler_type(this)); - } - - // copy constructor - key_matching_port(const key_matching_port& /*other*/) : receiver(), buffer_type() { - my_join = NULL; - my_aggregator.initialize_handler(handler_type(this)); - } - - ~key_matching_port() { } - - void set_join_node_pointer(forwarding_base *join) { - my_join = dynamic_cast*>(join); - } - - void set_my_key_func(type_to_key_func_type *f) { this->set_key_func(f); } - - type_to_key_func_type* get_my_key_func() { return this->get_key_func(); } - - bool get_item( input_type &v ) { - // aggregator uses current_key from FE for Key - key_matching_port_operation op_data(&v, get__item); - my_aggregator.execute(&op_data); - return op_data.status == SUCCEEDED; - } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - built_predecessors_type &built_predecessors() __TBB_override { return my_built_predecessors; } - - void internal_add_built_predecessor(predecessor_type &p) __TBB_override { - key_matching_port_operation op_data(add_blt_pred); - op_data.pred = &p; - my_aggregator.execute(&op_data); - } - - void internal_delete_built_predecessor(predecessor_type &p) __TBB_override { - key_matching_port_operation op_data(del_blt_pred); - op_data.pred = &p; - my_aggregator.execute(&op_data); - } - - size_t predecessor_count() __TBB_override { - key_matching_port_operation op_data(blt_pred_cnt); - my_aggregator.execute(&op_data); - return op_data.cnt_val; - } - - void copy_predecessors(predecessor_list_type &l) __TBB_override { - key_matching_port_operation op_data(blt_pred_cpy); - op_data.plist = &l; - my_aggregator.execute(&op_data); - } -#endif - - // reset_port is called when item is accepted by successor, but - // is initiated by join_node. - void reset_port() { - key_matching_port_operation op_data(res_port); - my_aggregator.execute(&op_data); - return; - } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - void extract_receiver() { - buffer_type::reset(); - my_built_predecessors.receiver_extract(*this); - } -#endif - void reset_receiver(reset_flags f ) __TBB_override { - tbb::internal::suppress_unused_warning(f); - buffer_type::reset(); -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - if (f & rf_clear_edges) - my_built_predecessors.clear(); -#endif - } - - private: - // my_join forwarding base used to count number of inputs that - // received key. - matching_forwarding_base *my_join; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - edge_container my_built_predecessors; -#endif - }; // key_matching_port - - using namespace graph_policy_namespace; - - template - class join_node_base; - - //! join_node_FE : implements input port policy - template - class join_node_FE; - - template - class join_node_FE : public forwarding_base { - public: - static const int N = tbb::flow::tuple_size::value; - typedef OutputTuple output_type; - typedef InputTuple input_type; - typedef join_node_base base_node_type; // for forwarding - - join_node_FE(graph &g) : forwarding_base(g), my_node(NULL) { - ports_with_no_inputs = N; - join_helper::set_join_node_pointer(my_inputs, this); - } - - join_node_FE(const join_node_FE& other) : forwarding_base(*(other.forwarding_base::graph_pointer)), my_node(NULL) { - ports_with_no_inputs = N; - join_helper::set_join_node_pointer(my_inputs, this); - } - - void set_my_node(base_node_type *new_my_node) { my_node = new_my_node; } - - void increment_port_count() __TBB_override { - ++ports_with_no_inputs; - } - - // if all input_ports have predecessors, spawn forward to try and consume tuples - task * decrement_port_count(bool handle_task) __TBB_override { - if(ports_with_no_inputs.fetch_and_decrement() == 1) { - if(this->graph_pointer->is_active()) { - task *rtask = new ( task::allocate_additional_child_of( *(this->graph_pointer->root_task()) ) ) - forward_task_bypass(*my_node); - if(!handle_task) return rtask; - FLOW_SPAWN(*rtask); - } - } - return NULL; - } - - input_type &input_ports() { return my_inputs; } - - protected: - - void reset( reset_flags f) { - // called outside of parallel contexts - ports_with_no_inputs = N; - join_helper::reset_inputs(my_inputs, f); - } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - void extract( ) { - // called outside of parallel contexts - ports_with_no_inputs = N; - join_helper::extract_inputs(my_inputs); - } -#endif - - // all methods on input ports should be called under mutual exclusion from join_node_base. - - bool tuple_build_may_succeed() { - return !ports_with_no_inputs; - } - - bool try_to_make_tuple(output_type &out) { - if(ports_with_no_inputs) return false; - return join_helper::reserve(my_inputs, out); - } - - void tuple_accepted() { - join_helper::consume_reservations(my_inputs); - } - void tuple_rejected() { - join_helper::release_reservations(my_inputs); - } - - input_type my_inputs; - base_node_type *my_node; - atomic ports_with_no_inputs; - }; // join_node_FE - - template - class join_node_FE : public forwarding_base { - public: - static const int N = tbb::flow::tuple_size::value; - typedef OutputTuple output_type; - typedef InputTuple input_type; - typedef join_node_base base_node_type; // for forwarding - - join_node_FE(graph &g) : forwarding_base(g), my_node(NULL) { - ports_with_no_items = N; - join_helper::set_join_node_pointer(my_inputs, this); - } - - join_node_FE(const join_node_FE& other) : forwarding_base(*(other.forwarding_base::graph_pointer)), my_node(NULL) { - ports_with_no_items = N; - join_helper::set_join_node_pointer(my_inputs, this); - } - - // needed for forwarding - void set_my_node(base_node_type *new_my_node) { my_node = new_my_node; } - - void reset_port_count() { - ports_with_no_items = N; - } - - // if all input_ports have items, spawn forward to try and consume tuples - task * decrement_port_count(bool handle_task) __TBB_override - { - if(ports_with_no_items.fetch_and_decrement() == 1) { - if(this->graph_pointer->is_active()) { - task *rtask = new ( task::allocate_additional_child_of( *(this->graph_pointer->root_task()) ) ) - forward_task_bypass (*my_node); - if(!handle_task) return rtask; - FLOW_SPAWN( *rtask); - } - } - return NULL; - } - - void increment_port_count() __TBB_override { __TBB_ASSERT(false, NULL); } // should never be called - - input_type &input_ports() { return my_inputs; } - - protected: - - void reset( reset_flags f) { - reset_port_count(); - join_helper::reset_inputs(my_inputs, f ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - void extract() { - reset_port_count(); - join_helper::extract_inputs(my_inputs); - } -#endif - // all methods on input ports should be called under mutual exclusion from join_node_base. - - bool tuple_build_may_succeed() { - return !ports_with_no_items; - } - - bool try_to_make_tuple(output_type &out) { - if(ports_with_no_items) return false; - return join_helper::get_items(my_inputs, out); - } - - void tuple_accepted() { - reset_port_count(); - join_helper::reset_ports(my_inputs); - } - void tuple_rejected() { - // nothing to do. - } - - input_type my_inputs; - base_node_type *my_node; - atomic ports_with_no_items; - }; // join_node_FE - - // key_matching join front-end. - template - class join_node_FE, InputTuple, OutputTuple> : public matching_forwarding_base, - // buffer of key value counts - public hash_buffer< // typedefed below to key_to_count_buffer_type - typename tbb::internal::strip::type&, // force ref type on K - count_element::type>, - internal::type_to_key_function_body< - count_element::type>, - typename tbb::internal::strip::type& >, - KHash >, - // buffer of output items - public item_buffer { - public: - static const int N = tbb::flow::tuple_size::value; - typedef OutputTuple output_type; - typedef InputTuple input_type; - typedef K key_type; - typedef typename tbb::internal::strip::type unref_key_type; - typedef KHash key_hash_compare; - // must use K without ref. - typedef count_element count_element_type; - // method that lets us refer to the key of this type. - typedef key_to_count_functor key_to_count_func; - typedef internal::type_to_key_function_body< count_element_type, unref_key_type&> TtoK_function_body_type; - typedef internal::type_to_key_function_body_leaf TtoK_function_body_leaf_type; - // this is the type of the special table that keeps track of the number of discrete - // elements corresponding to each key that we've seen. - typedef hash_buffer< unref_key_type&, count_element_type, TtoK_function_body_type, key_hash_compare > - key_to_count_buffer_type; - typedef item_buffer output_buffer_type; - typedef join_node_base, InputTuple, OutputTuple> base_node_type; // for forwarding - typedef matching_forwarding_base forwarding_base_type; - -// ----------- Aggregator ------------ - // the aggregator is only needed to serialize the access to the hash table. - // and the output_buffer_type base class - private: - enum op_type { res_count, inc_count, may_succeed, try_make }; - enum op_stat {WAIT=0, SUCCEEDED, FAILED}; - typedef join_node_FE, InputTuple, OutputTuple> class_type; - - class key_matching_FE_operation : public aggregated_operation { - public: - char type; - unref_key_type my_val; - output_type* my_output; - task *bypass_t; - bool enqueue_task; - // constructor for value parameter - key_matching_FE_operation(const unref_key_type& e , bool q_task , op_type t) : type(char(t)), my_val(e), - my_output(NULL), bypass_t(NULL), enqueue_task(q_task) {} - key_matching_FE_operation(output_type *p, op_type t) : type(char(t)), my_output(p), bypass_t(NULL), - enqueue_task(true) {} - // constructor with no parameter - key_matching_FE_operation(op_type t) : type(char(t)), my_output(NULL), bypass_t(NULL), enqueue_task(true) {} - }; - - typedef internal::aggregating_functor handler_type; - friend class internal::aggregating_functor; - aggregator my_aggregator; - - // called from aggregator, so serialized - // returns a task pointer if the a task would have been enqueued but we asked that - // it be returned. Otherwise returns NULL. - task * fill_output_buffer(unref_key_type &t, bool should_enqueue, bool handle_task) { - output_type l_out; - task *rtask = NULL; - bool do_fwd = should_enqueue && this->buffer_empty() && this->graph_pointer->is_active(); - this->current_key = t; - this->delete_with_key(this->current_key); // remove the key - if(join_helper::get_items(my_inputs, l_out)) { // <== call back - this->push_back(l_out); - if(do_fwd) { // we enqueue if receiving an item from predecessor, not if successor asks for item - rtask = new ( task::allocate_additional_child_of( *(this->graph_pointer->root_task()) ) ) - forward_task_bypass(*my_node); - if(handle_task) { - FLOW_SPAWN(*rtask); - rtask = NULL; - } - do_fwd = false; - } - // retire the input values - join_helper::reset_ports(my_inputs); // <== call back - } - else { - __TBB_ASSERT(false, "should have had something to push"); - } - return rtask; - } - - void handle_operations(key_matching_FE_operation* op_list) { - key_matching_FE_operation *current; - while(op_list) { - current = op_list; - op_list = op_list->next; - switch(current->type) { - case res_count: // called from BE - { - this->destroy_front(); - __TBB_store_with_release(current->status, SUCCEEDED); - } - break; - case inc_count: { // called from input ports - count_element_type *p = 0; - unref_key_type &t = current->my_val; - bool do_enqueue = current->enqueue_task; - if(!(this->find_ref_with_key(t,p))) { - count_element_type ev; - ev.my_key = t; - ev.my_value = 0; - this->insert_with_key(ev); - if(!(this->find_ref_with_key(t,p))) { - __TBB_ASSERT(false, "should find key after inserting it"); - } - } - if(++(p->my_value) == size_t(N)) { - task *rtask = fill_output_buffer(t, true, do_enqueue); - __TBB_ASSERT(!rtask || !do_enqueue, "task should not be returned"); - current->bypass_t = rtask; - } - } - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case may_succeed: // called from BE - __TBB_store_with_release(current->status, this->buffer_empty() ? FAILED : SUCCEEDED); - break; - case try_make: // called from BE - if(this->buffer_empty()) { - __TBB_store_with_release(current->status, FAILED); - } - else { - *(current->my_output) = this->front(); - __TBB_store_with_release(current->status, SUCCEEDED); - } - break; - } - } - } -// ------------ End Aggregator --------------- - - public: - template - join_node_FE(graph &g, FunctionTuple &TtoK_funcs) : forwarding_base_type(g), my_node(NULL) { - join_helper::set_join_node_pointer(my_inputs, this); - join_helper::set_key_functors(my_inputs, TtoK_funcs); - my_aggregator.initialize_handler(handler_type(this)); - TtoK_function_body_type *cfb = new TtoK_function_body_leaf_type(key_to_count_func()); - this->set_key_func(cfb); - } - - join_node_FE(const join_node_FE& other) : forwarding_base_type(*(other.forwarding_base_type::graph_pointer)), key_to_count_buffer_type(), - output_buffer_type() { - my_node = NULL; - join_helper::set_join_node_pointer(my_inputs, this); - join_helper::copy_key_functors(my_inputs, const_cast(other.my_inputs)); - my_aggregator.initialize_handler(handler_type(this)); - TtoK_function_body_type *cfb = new TtoK_function_body_leaf_type(key_to_count_func()); - this->set_key_func(cfb); - } - - // needed for forwarding - void set_my_node(base_node_type *new_my_node) { my_node = new_my_node; } - - void reset_port_count() { // called from BE - key_matching_FE_operation op_data(res_count); - my_aggregator.execute(&op_data); - return; - } - - // if all input_ports have items, spawn forward to try and consume tuples - // return a task if we are asked and did create one. - task *increment_key_count(unref_key_type const & t, bool handle_task) __TBB_override { // called from input_ports - key_matching_FE_operation op_data(t, handle_task, inc_count); - my_aggregator.execute(&op_data); - return op_data.bypass_t; - } - - task *decrement_port_count(bool /*handle_task*/) __TBB_override { __TBB_ASSERT(false, NULL); return NULL; } - - void increment_port_count() __TBB_override { __TBB_ASSERT(false, NULL); } // should never be called - - input_type &input_ports() { return my_inputs; } - - protected: - - void reset( reset_flags f ) { - // called outside of parallel contexts - join_helper::reset_inputs(my_inputs, f); - - key_to_count_buffer_type::reset(); - output_buffer_type::reset(); - } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - void extract() { - // called outside of parallel contexts - join_helper::extract_inputs(my_inputs); - key_to_count_buffer_type::reset(); // have to reset the tag counts - output_buffer_type::reset(); // also the queue of outputs - // my_node->current_tag = NO_TAG; - } -#endif - // all methods on input ports should be called under mutual exclusion from join_node_base. - - bool tuple_build_may_succeed() { // called from back-end - key_matching_FE_operation op_data(may_succeed); - my_aggregator.execute(&op_data); - return op_data.status == SUCCEEDED; - } - - // cannot lock while calling back to input_ports. current_key will only be set - // and reset under the aggregator, so it will remain consistent. - bool try_to_make_tuple(output_type &out) { - key_matching_FE_operation op_data(&out,try_make); - my_aggregator.execute(&op_data); - return op_data.status == SUCCEEDED; - } - - void tuple_accepted() { - reset_port_count(); // reset current_key after ports reset. - } - - void tuple_rejected() { - // nothing to do. - } - - input_type my_inputs; // input ports - base_node_type *my_node; - }; // join_node_FE, InputTuple, OutputTuple> - - //! join_node_base - template - class join_node_base : public graph_node, public join_node_FE, - public sender { - protected: - using graph_node::my_graph; - public: - typedef OutputTuple output_type; - - typedef typename sender::successor_type successor_type; - typedef join_node_FE input_ports_type; - using input_ports_type::tuple_build_may_succeed; - using input_ports_type::try_to_make_tuple; - using input_ports_type::tuple_accepted; - using input_ports_type::tuple_rejected; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - typedef typename sender::built_successors_type built_successors_type; - typedef typename sender::successor_list_type successor_list_type; -#endif - - private: - // ----------- Aggregator ------------ - enum op_type { reg_succ, rem_succ, try__get, do_fwrd, do_fwrd_bypass -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - , add_blt_succ, del_blt_succ, blt_succ_cnt, blt_succ_cpy -#endif - }; - enum op_stat {WAIT=0, SUCCEEDED, FAILED}; - typedef join_node_base class_type; - - class join_node_base_operation : public aggregated_operation { - public: - char type; - union { - output_type *my_arg; - successor_type *my_succ; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - size_t cnt_val; - successor_list_type *slist; -#endif - }; - task *bypass_t; - join_node_base_operation(const output_type& e, op_type t) : type(char(t)), - my_arg(const_cast(&e)), bypass_t(NULL) {} - join_node_base_operation(const successor_type &s, op_type t) : type(char(t)), - my_succ(const_cast(&s)), bypass_t(NULL) {} - join_node_base_operation(op_type t) : type(char(t)), bypass_t(NULL) {} - }; - - typedef internal::aggregating_functor handler_type; - friend class internal::aggregating_functor; - bool forwarder_busy; - aggregator my_aggregator; - - void handle_operations(join_node_base_operation* op_list) { - join_node_base_operation *current; - while(op_list) { - current = op_list; - op_list = op_list->next; - switch(current->type) { - case reg_succ: { - my_successors.register_successor(*(current->my_succ)); - if(tuple_build_may_succeed() && !forwarder_busy && this->graph_node::my_graph.is_active()) { - task *rtask = new ( task::allocate_additional_child_of(*(this->graph_node::my_graph.root_task())) ) - forward_task_bypass - >(*this); - FLOW_SPAWN(*rtask); - forwarder_busy = true; - } - __TBB_store_with_release(current->status, SUCCEEDED); - } - break; - case rem_succ: - my_successors.remove_successor(*(current->my_succ)); - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case try__get: - if(tuple_build_may_succeed()) { - if(try_to_make_tuple(*(current->my_arg))) { - tuple_accepted(); - __TBB_store_with_release(current->status, SUCCEEDED); - } - else __TBB_store_with_release(current->status, FAILED); - } - else __TBB_store_with_release(current->status, FAILED); - break; - case do_fwrd_bypass: { - bool build_succeeded; - task *last_task = NULL; - output_type out; - if(tuple_build_may_succeed()) { // checks output queue of FE - do { - build_succeeded = try_to_make_tuple(out); // fetch front_end of queue - if(build_succeeded) { - task *new_task = my_successors.try_put_task(out); - last_task = combine_tasks(last_task, new_task); - if(new_task) { - tuple_accepted(); - } - else { - tuple_rejected(); - build_succeeded = false; - } - } - } while(build_succeeded); - } - current->bypass_t = last_task; - __TBB_store_with_release(current->status, SUCCEEDED); - forwarder_busy = false; - } - break; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - case add_blt_succ: - my_successors.internal_add_built_successor(*(current->my_succ)); - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case del_blt_succ: - my_successors.internal_delete_built_successor(*(current->my_succ)); - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case blt_succ_cnt: - current->cnt_val = my_successors.successor_count(); - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case blt_succ_cpy: - my_successors.copy_successors(*(current->slist)); - __TBB_store_with_release(current->status, SUCCEEDED); - break; -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ - } - } - } - // ---------- end aggregator ----------- - public: - join_node_base(graph &g) : graph_node(g), input_ports_type(g), forwarder_busy(false) { - my_successors.set_owner(this); - input_ports_type::set_my_node(this); - my_aggregator.initialize_handler(handler_type(this)); - } - - join_node_base(const join_node_base& other) : - graph_node(other.graph_node::my_graph), input_ports_type(other), - sender(), forwarder_busy(false), my_successors() { - my_successors.set_owner(this); - input_ports_type::set_my_node(this); - my_aggregator.initialize_handler(handler_type(this)); - } - - template - join_node_base(graph &g, FunctionTuple f) : graph_node(g), input_ports_type(g, f), forwarder_busy(false) { - my_successors.set_owner(this); - input_ports_type::set_my_node(this); - my_aggregator.initialize_handler(handler_type(this)); - } - - bool register_successor(successor_type &r) __TBB_override { - join_node_base_operation op_data(r, reg_succ); - my_aggregator.execute(&op_data); - return op_data.status == SUCCEEDED; - } - - bool remove_successor( successor_type &r) __TBB_override { - join_node_base_operation op_data(r, rem_succ); - my_aggregator.execute(&op_data); - return op_data.status == SUCCEEDED; - } - - bool try_get( output_type &v) __TBB_override { - join_node_base_operation op_data(v, try__get); - my_aggregator.execute(&op_data); - return op_data.status == SUCCEEDED; - } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - built_successors_type &built_successors() __TBB_override { return my_successors.built_successors(); } - - void internal_add_built_successor( successor_type &r) __TBB_override { - join_node_base_operation op_data(r, add_blt_succ); - my_aggregator.execute(&op_data); - } - - void internal_delete_built_successor( successor_type &r) __TBB_override { - join_node_base_operation op_data(r, del_blt_succ); - my_aggregator.execute(&op_data); - } - - size_t successor_count() __TBB_override { - join_node_base_operation op_data(blt_succ_cnt); - my_aggregator.execute(&op_data); - return op_data.cnt_val; - } - - void copy_successors(successor_list_type &l) __TBB_override { - join_node_base_operation op_data(blt_succ_cpy); - op_data.slist = &l; - my_aggregator.execute(&op_data); - } -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - void extract() __TBB_override { - input_ports_type::extract(); - my_successors.built_successors().sender_extract(*this); - } -#endif - - protected: - - void reset_node(reset_flags f) __TBB_override { - input_ports_type::reset(f); - if(f & rf_clear_edges) my_successors.clear(); - } - - private: - broadcast_cache my_successors; - - friend class forward_task_bypass< join_node_base >; - task *forward_task() { - join_node_base_operation op_data(do_fwrd_bypass); - my_aggregator.execute(&op_data); - return op_data.bypass_t; - } - - }; // join_node_base - - // join base class type generator - template class PT, typename OutputTuple, typename JP> - struct join_base { - typedef typename internal::join_node_base::type, OutputTuple> type; - }; - - template - struct join_base > { - typedef key_matching key_traits_type; - typedef K key_type; - typedef KHash key_hash_compare; - typedef typename internal::join_node_base< key_traits_type, - // ports type - typename wrap_key_tuple_elements::type, - OutputTuple > type; - }; - - //! unfolded_join_node : passes input_ports_type to join_node_base. We build the input port type - // using tuple_element. The class PT is the port type (reserving_port, queueing_port, key_matching_port) - // and should match the typename. - - template class PT, typename OutputTuple, typename JP> - class unfolded_join_node : public join_base::type { - public: - typedef typename wrap_tuple_elements::type input_ports_type; - typedef OutputTuple output_type; - private: - typedef join_node_base base_type; - public: - unfolded_join_node(graph &g) : base_type(g) {} - unfolded_join_node(const unfolded_join_node &other) : base_type(other) {} - }; - -#if __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING - template - struct key_from_message_body { - K operator()(const T& t) const { - using tbb::flow::key_from_message; - return key_from_message(t); - } - }; - // Adds const to reference type - template - struct key_from_message_body { - const K& operator()(const T& t) const { - using tbb::flow::key_from_message; - return key_from_message(t); - } - }; -#endif /* __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING */ - // key_matching unfolded_join_node. This must be a separate specialization because the constructors - // differ. - - template - class unfolded_join_node<2,key_matching_port,OutputTuple,key_matching > : public - join_base<2,key_matching_port,OutputTuple,key_matching >::type { - typedef typename tbb::flow::tuple_element<0, OutputTuple>::type T0; - typedef typename tbb::flow::tuple_element<1, OutputTuple>::type T1; - public: - typedef typename wrap_key_tuple_elements<2,key_matching_port,key_matching,OutputTuple>::type input_ports_type; - typedef OutputTuple output_type; - private: - typedef join_node_base, input_ports_type, output_type > base_type; - typedef typename internal::type_to_key_function_body *f0_p; - typedef typename internal::type_to_key_function_body *f1_p; - typedef typename tbb::flow::tuple< f0_p, f1_p > func_initializer_type; - public: -#if __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING - unfolded_join_node(graph &g) : base_type(g, - func_initializer_type( - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()) - ) ) { - } -#endif /* __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING */ - template - unfolded_join_node(graph &g, Body0 body0, Body1 body1) : base_type(g, - func_initializer_type( - new internal::type_to_key_function_body_leaf(body0), - new internal::type_to_key_function_body_leaf(body1) - ) ) { - __TBB_STATIC_ASSERT(tbb::flow::tuple_size::value == 2, "wrong number of body initializers"); - } - unfolded_join_node(const unfolded_join_node &other) : base_type(other) {} - }; - - template - class unfolded_join_node<3,key_matching_port,OutputTuple,key_matching > : public - join_base<3,key_matching_port,OutputTuple,key_matching >::type { - typedef typename tbb::flow::tuple_element<0, OutputTuple>::type T0; - typedef typename tbb::flow::tuple_element<1, OutputTuple>::type T1; - typedef typename tbb::flow::tuple_element<2, OutputTuple>::type T2; - public: - typedef typename wrap_key_tuple_elements<3,key_matching_port,key_matching,OutputTuple>::type input_ports_type; - typedef OutputTuple output_type; - private: - typedef join_node_base, input_ports_type, output_type > base_type; - typedef typename internal::type_to_key_function_body *f0_p; - typedef typename internal::type_to_key_function_body *f1_p; - typedef typename internal::type_to_key_function_body *f2_p; - typedef typename tbb::flow::tuple< f0_p, f1_p, f2_p > func_initializer_type; - public: -#if __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING - unfolded_join_node(graph &g) : base_type(g, - func_initializer_type( - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()) - ) ) { - } -#endif /* __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING */ - template - unfolded_join_node(graph &g, Body0 body0, Body1 body1, Body2 body2) : base_type(g, - func_initializer_type( - new internal::type_to_key_function_body_leaf(body0), - new internal::type_to_key_function_body_leaf(body1), - new internal::type_to_key_function_body_leaf(body2) - ) ) { - __TBB_STATIC_ASSERT(tbb::flow::tuple_size::value == 3, "wrong number of body initializers"); - } - unfolded_join_node(const unfolded_join_node &other) : base_type(other) {} - }; - - template - class unfolded_join_node<4,key_matching_port,OutputTuple,key_matching > : public - join_base<4,key_matching_port,OutputTuple,key_matching >::type { - typedef typename tbb::flow::tuple_element<0, OutputTuple>::type T0; - typedef typename tbb::flow::tuple_element<1, OutputTuple>::type T1; - typedef typename tbb::flow::tuple_element<2, OutputTuple>::type T2; - typedef typename tbb::flow::tuple_element<3, OutputTuple>::type T3; - public: - typedef typename wrap_key_tuple_elements<4,key_matching_port,key_matching,OutputTuple>::type input_ports_type; - typedef OutputTuple output_type; - private: - typedef join_node_base, input_ports_type, output_type > base_type; - typedef typename internal::type_to_key_function_body *f0_p; - typedef typename internal::type_to_key_function_body *f1_p; - typedef typename internal::type_to_key_function_body *f2_p; - typedef typename internal::type_to_key_function_body *f3_p; - typedef typename tbb::flow::tuple< f0_p, f1_p, f2_p, f3_p > func_initializer_type; - public: -#if __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING - unfolded_join_node(graph &g) : base_type(g, - func_initializer_type( - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()) - ) ) { - } -#endif /* __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING */ - template - unfolded_join_node(graph &g, Body0 body0, Body1 body1, Body2 body2, Body3 body3) : base_type(g, - func_initializer_type( - new internal::type_to_key_function_body_leaf(body0), - new internal::type_to_key_function_body_leaf(body1), - new internal::type_to_key_function_body_leaf(body2), - new internal::type_to_key_function_body_leaf(body3) - ) ) { - __TBB_STATIC_ASSERT(tbb::flow::tuple_size::value == 4, "wrong number of body initializers"); - } - unfolded_join_node(const unfolded_join_node &other) : base_type(other) {} - }; - - template - class unfolded_join_node<5,key_matching_port,OutputTuple,key_matching > : public - join_base<5,key_matching_port,OutputTuple,key_matching >::type { - typedef typename tbb::flow::tuple_element<0, OutputTuple>::type T0; - typedef typename tbb::flow::tuple_element<1, OutputTuple>::type T1; - typedef typename tbb::flow::tuple_element<2, OutputTuple>::type T2; - typedef typename tbb::flow::tuple_element<3, OutputTuple>::type T3; - typedef typename tbb::flow::tuple_element<4, OutputTuple>::type T4; - public: - typedef typename wrap_key_tuple_elements<5,key_matching_port,key_matching,OutputTuple>::type input_ports_type; - typedef OutputTuple output_type; - private: - typedef join_node_base , input_ports_type, output_type > base_type; - typedef typename internal::type_to_key_function_body *f0_p; - typedef typename internal::type_to_key_function_body *f1_p; - typedef typename internal::type_to_key_function_body *f2_p; - typedef typename internal::type_to_key_function_body *f3_p; - typedef typename internal::type_to_key_function_body *f4_p; - typedef typename tbb::flow::tuple< f0_p, f1_p, f2_p, f3_p, f4_p > func_initializer_type; - public: -#if __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING - unfolded_join_node(graph &g) : base_type(g, - func_initializer_type( - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()) - ) ) { - } -#endif /* __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING */ - template - unfolded_join_node(graph &g, Body0 body0, Body1 body1, Body2 body2, Body3 body3, Body4 body4) : base_type(g, - func_initializer_type( - new internal::type_to_key_function_body_leaf(body0), - new internal::type_to_key_function_body_leaf(body1), - new internal::type_to_key_function_body_leaf(body2), - new internal::type_to_key_function_body_leaf(body3), - new internal::type_to_key_function_body_leaf(body4) - ) ) { - __TBB_STATIC_ASSERT(tbb::flow::tuple_size::value == 5, "wrong number of body initializers"); - } - unfolded_join_node(const unfolded_join_node &other) : base_type(other) {} - }; - -#if __TBB_VARIADIC_MAX >= 6 - template - class unfolded_join_node<6,key_matching_port,OutputTuple,key_matching > : public - join_base<6,key_matching_port,OutputTuple,key_matching >::type { - typedef typename tbb::flow::tuple_element<0, OutputTuple>::type T0; - typedef typename tbb::flow::tuple_element<1, OutputTuple>::type T1; - typedef typename tbb::flow::tuple_element<2, OutputTuple>::type T2; - typedef typename tbb::flow::tuple_element<3, OutputTuple>::type T3; - typedef typename tbb::flow::tuple_element<4, OutputTuple>::type T4; - typedef typename tbb::flow::tuple_element<5, OutputTuple>::type T5; - public: - typedef typename wrap_key_tuple_elements<6,key_matching_port,key_matching,OutputTuple>::type input_ports_type; - typedef OutputTuple output_type; - private: - typedef join_node_base , input_ports_type, output_type > base_type; - typedef typename internal::type_to_key_function_body *f0_p; - typedef typename internal::type_to_key_function_body *f1_p; - typedef typename internal::type_to_key_function_body *f2_p; - typedef typename internal::type_to_key_function_body *f3_p; - typedef typename internal::type_to_key_function_body *f4_p; - typedef typename internal::type_to_key_function_body *f5_p; - typedef typename tbb::flow::tuple< f0_p, f1_p, f2_p, f3_p, f4_p, f5_p > func_initializer_type; - public: -#if __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING - unfolded_join_node(graph &g) : base_type(g, - func_initializer_type( - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()) - ) ) { - } -#endif /* __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING */ - template - unfolded_join_node(graph &g, Body0 body0, Body1 body1, Body2 body2, Body3 body3, Body4 body4, Body5 body5) - : base_type(g, func_initializer_type( - new internal::type_to_key_function_body_leaf(body0), - new internal::type_to_key_function_body_leaf(body1), - new internal::type_to_key_function_body_leaf(body2), - new internal::type_to_key_function_body_leaf(body3), - new internal::type_to_key_function_body_leaf(body4), - new internal::type_to_key_function_body_leaf(body5) - ) ) { - __TBB_STATIC_ASSERT(tbb::flow::tuple_size::value == 6, "wrong number of body initializers"); - } - unfolded_join_node(const unfolded_join_node &other) : base_type(other) {} - }; -#endif - -#if __TBB_VARIADIC_MAX >= 7 - template - class unfolded_join_node<7,key_matching_port,OutputTuple,key_matching > : public - join_base<7,key_matching_port,OutputTuple,key_matching >::type { - typedef typename tbb::flow::tuple_element<0, OutputTuple>::type T0; - typedef typename tbb::flow::tuple_element<1, OutputTuple>::type T1; - typedef typename tbb::flow::tuple_element<2, OutputTuple>::type T2; - typedef typename tbb::flow::tuple_element<3, OutputTuple>::type T3; - typedef typename tbb::flow::tuple_element<4, OutputTuple>::type T4; - typedef typename tbb::flow::tuple_element<5, OutputTuple>::type T5; - typedef typename tbb::flow::tuple_element<6, OutputTuple>::type T6; - public: - typedef typename wrap_key_tuple_elements<7,key_matching_port,key_matching,OutputTuple>::type input_ports_type; - typedef OutputTuple output_type; - private: - typedef join_node_base , input_ports_type, output_type > base_type; - typedef typename internal::type_to_key_function_body *f0_p; - typedef typename internal::type_to_key_function_body *f1_p; - typedef typename internal::type_to_key_function_body *f2_p; - typedef typename internal::type_to_key_function_body *f3_p; - typedef typename internal::type_to_key_function_body *f4_p; - typedef typename internal::type_to_key_function_body *f5_p; - typedef typename internal::type_to_key_function_body *f6_p; - typedef typename tbb::flow::tuple< f0_p, f1_p, f2_p, f3_p, f4_p, f5_p, f6_p > func_initializer_type; - public: -#if __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING - unfolded_join_node(graph &g) : base_type(g, - func_initializer_type( - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()) - ) ) { - } -#endif /* __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING */ - template - unfolded_join_node(graph &g, Body0 body0, Body1 body1, Body2 body2, Body3 body3, Body4 body4, - Body5 body5, Body6 body6) : base_type(g, func_initializer_type( - new internal::type_to_key_function_body_leaf(body0), - new internal::type_to_key_function_body_leaf(body1), - new internal::type_to_key_function_body_leaf(body2), - new internal::type_to_key_function_body_leaf(body3), - new internal::type_to_key_function_body_leaf(body4), - new internal::type_to_key_function_body_leaf(body5), - new internal::type_to_key_function_body_leaf(body6) - ) ) { - __TBB_STATIC_ASSERT(tbb::flow::tuple_size::value == 7, "wrong number of body initializers"); - } - unfolded_join_node(const unfolded_join_node &other) : base_type(other) {} - }; -#endif - -#if __TBB_VARIADIC_MAX >= 8 - template - class unfolded_join_node<8,key_matching_port,OutputTuple,key_matching > : public - join_base<8,key_matching_port,OutputTuple,key_matching >::type { - typedef typename tbb::flow::tuple_element<0, OutputTuple>::type T0; - typedef typename tbb::flow::tuple_element<1, OutputTuple>::type T1; - typedef typename tbb::flow::tuple_element<2, OutputTuple>::type T2; - typedef typename tbb::flow::tuple_element<3, OutputTuple>::type T3; - typedef typename tbb::flow::tuple_element<4, OutputTuple>::type T4; - typedef typename tbb::flow::tuple_element<5, OutputTuple>::type T5; - typedef typename tbb::flow::tuple_element<6, OutputTuple>::type T6; - typedef typename tbb::flow::tuple_element<7, OutputTuple>::type T7; - public: - typedef typename wrap_key_tuple_elements<8,key_matching_port,key_matching,OutputTuple>::type input_ports_type; - typedef OutputTuple output_type; - private: - typedef join_node_base , input_ports_type, output_type > base_type; - typedef typename internal::type_to_key_function_body *f0_p; - typedef typename internal::type_to_key_function_body *f1_p; - typedef typename internal::type_to_key_function_body *f2_p; - typedef typename internal::type_to_key_function_body *f3_p; - typedef typename internal::type_to_key_function_body *f4_p; - typedef typename internal::type_to_key_function_body *f5_p; - typedef typename internal::type_to_key_function_body *f6_p; - typedef typename internal::type_to_key_function_body *f7_p; - typedef typename tbb::flow::tuple< f0_p, f1_p, f2_p, f3_p, f4_p, f5_p, f6_p, f7_p > func_initializer_type; - public: -#if __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING - unfolded_join_node(graph &g) : base_type(g, - func_initializer_type( - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()) - ) ) { - } -#endif /* __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING */ - template - unfolded_join_node(graph &g, Body0 body0, Body1 body1, Body2 body2, Body3 body3, Body4 body4, - Body5 body5, Body6 body6, Body7 body7) : base_type(g, func_initializer_type( - new internal::type_to_key_function_body_leaf(body0), - new internal::type_to_key_function_body_leaf(body1), - new internal::type_to_key_function_body_leaf(body2), - new internal::type_to_key_function_body_leaf(body3), - new internal::type_to_key_function_body_leaf(body4), - new internal::type_to_key_function_body_leaf(body5), - new internal::type_to_key_function_body_leaf(body6), - new internal::type_to_key_function_body_leaf(body7) - ) ) { - __TBB_STATIC_ASSERT(tbb::flow::tuple_size::value == 8, "wrong number of body initializers"); - } - unfolded_join_node(const unfolded_join_node &other) : base_type(other) {} - }; -#endif - -#if __TBB_VARIADIC_MAX >= 9 - template - class unfolded_join_node<9,key_matching_port,OutputTuple,key_matching > : public - join_base<9,key_matching_port,OutputTuple,key_matching >::type { - typedef typename tbb::flow::tuple_element<0, OutputTuple>::type T0; - typedef typename tbb::flow::tuple_element<1, OutputTuple>::type T1; - typedef typename tbb::flow::tuple_element<2, OutputTuple>::type T2; - typedef typename tbb::flow::tuple_element<3, OutputTuple>::type T3; - typedef typename tbb::flow::tuple_element<4, OutputTuple>::type T4; - typedef typename tbb::flow::tuple_element<5, OutputTuple>::type T5; - typedef typename tbb::flow::tuple_element<6, OutputTuple>::type T6; - typedef typename tbb::flow::tuple_element<7, OutputTuple>::type T7; - typedef typename tbb::flow::tuple_element<8, OutputTuple>::type T8; - public: - typedef typename wrap_key_tuple_elements<9,key_matching_port,key_matching,OutputTuple>::type input_ports_type; - typedef OutputTuple output_type; - private: - typedef join_node_base , input_ports_type, output_type > base_type; - typedef typename internal::type_to_key_function_body *f0_p; - typedef typename internal::type_to_key_function_body *f1_p; - typedef typename internal::type_to_key_function_body *f2_p; - typedef typename internal::type_to_key_function_body *f3_p; - typedef typename internal::type_to_key_function_body *f4_p; - typedef typename internal::type_to_key_function_body *f5_p; - typedef typename internal::type_to_key_function_body *f6_p; - typedef typename internal::type_to_key_function_body *f7_p; - typedef typename internal::type_to_key_function_body *f8_p; - typedef typename tbb::flow::tuple< f0_p, f1_p, f2_p, f3_p, f4_p, f5_p, f6_p, f7_p, f8_p > func_initializer_type; - public: -#if __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING - unfolded_join_node(graph &g) : base_type(g, - func_initializer_type( - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()) - ) ) { - } -#endif /* __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING */ - template - unfolded_join_node(graph &g, Body0 body0, Body1 body1, Body2 body2, Body3 body3, Body4 body4, - Body5 body5, Body6 body6, Body7 body7, Body8 body8) : base_type(g, func_initializer_type( - new internal::type_to_key_function_body_leaf(body0), - new internal::type_to_key_function_body_leaf(body1), - new internal::type_to_key_function_body_leaf(body2), - new internal::type_to_key_function_body_leaf(body3), - new internal::type_to_key_function_body_leaf(body4), - new internal::type_to_key_function_body_leaf(body5), - new internal::type_to_key_function_body_leaf(body6), - new internal::type_to_key_function_body_leaf(body7), - new internal::type_to_key_function_body_leaf(body8) - ) ) { - __TBB_STATIC_ASSERT(tbb::flow::tuple_size::value == 9, "wrong number of body initializers"); - } - unfolded_join_node(const unfolded_join_node &other) : base_type(other) {} - }; -#endif - -#if __TBB_VARIADIC_MAX >= 10 - template - class unfolded_join_node<10,key_matching_port,OutputTuple,key_matching > : public - join_base<10,key_matching_port,OutputTuple,key_matching >::type { - typedef typename tbb::flow::tuple_element<0, OutputTuple>::type T0; - typedef typename tbb::flow::tuple_element<1, OutputTuple>::type T1; - typedef typename tbb::flow::tuple_element<2, OutputTuple>::type T2; - typedef typename tbb::flow::tuple_element<3, OutputTuple>::type T3; - typedef typename tbb::flow::tuple_element<4, OutputTuple>::type T4; - typedef typename tbb::flow::tuple_element<5, OutputTuple>::type T5; - typedef typename tbb::flow::tuple_element<6, OutputTuple>::type T6; - typedef typename tbb::flow::tuple_element<7, OutputTuple>::type T7; - typedef typename tbb::flow::tuple_element<8, OutputTuple>::type T8; - typedef typename tbb::flow::tuple_element<9, OutputTuple>::type T9; - public: - typedef typename wrap_key_tuple_elements<10,key_matching_port,key_matching,OutputTuple>::type input_ports_type; - typedef OutputTuple output_type; - private: - typedef join_node_base , input_ports_type, output_type > base_type; - typedef typename internal::type_to_key_function_body *f0_p; - typedef typename internal::type_to_key_function_body *f1_p; - typedef typename internal::type_to_key_function_body *f2_p; - typedef typename internal::type_to_key_function_body *f3_p; - typedef typename internal::type_to_key_function_body *f4_p; - typedef typename internal::type_to_key_function_body *f5_p; - typedef typename internal::type_to_key_function_body *f6_p; - typedef typename internal::type_to_key_function_body *f7_p; - typedef typename internal::type_to_key_function_body *f8_p; - typedef typename internal::type_to_key_function_body *f9_p; - typedef typename tbb::flow::tuple< f0_p, f1_p, f2_p, f3_p, f4_p, f5_p, f6_p, f7_p, f8_p, f9_p > func_initializer_type; - public: -#if __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING - unfolded_join_node(graph &g) : base_type(g, - func_initializer_type( - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()) - ) ) { - } -#endif /* __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING */ - template - unfolded_join_node(graph &g, Body0 body0, Body1 body1, Body2 body2, Body3 body3, Body4 body4, - Body5 body5, Body6 body6, Body7 body7, Body8 body8, Body9 body9) : base_type(g, func_initializer_type( - new internal::type_to_key_function_body_leaf(body0), - new internal::type_to_key_function_body_leaf(body1), - new internal::type_to_key_function_body_leaf(body2), - new internal::type_to_key_function_body_leaf(body3), - new internal::type_to_key_function_body_leaf(body4), - new internal::type_to_key_function_body_leaf(body5), - new internal::type_to_key_function_body_leaf(body6), - new internal::type_to_key_function_body_leaf(body7), - new internal::type_to_key_function_body_leaf(body8), - new internal::type_to_key_function_body_leaf(body9) - ) ) { - __TBB_STATIC_ASSERT(tbb::flow::tuple_size::value == 10, "wrong number of body initializers"); - } - unfolded_join_node(const unfolded_join_node &other) : base_type(other) {} - }; -#endif - - //! templated function to refer to input ports of the join node - template - typename tbb::flow::tuple_element::type &input_port(JNT &jn) { - return tbb::flow::get(jn.input_ports()); - } - -} -#endif // __TBB__flow_graph_join_impl_H - diff --git a/inst/include/tbb/internal/_flow_graph_node_impl.h b/inst/include/tbb/internal/_flow_graph_node_impl.h deleted file mode 100644 index eeff72bcf..000000000 --- a/inst/include/tbb/internal/_flow_graph_node_impl.h +++ /dev/null @@ -1,798 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB__flow_graph_node_impl_H -#define __TBB__flow_graph_node_impl_H - -#ifndef __TBB_flow_graph_H -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#include "_flow_graph_item_buffer_impl.h" - -//! @cond INTERNAL -namespace internal { - - using tbb::internal::aggregated_operation; - using tbb::internal::aggregating_functor; - using tbb::internal::aggregator; - - template< typename T, typename A > - class function_input_queue : public item_buffer { - public: - bool empty() const { - return this->buffer_empty(); - } - - const T& front() const { - return this->item_buffer::front(); - } - - bool pop( T& t ) { - return this->pop_front( t ); - } - - void pop() { - this->destroy_front(); - } - - bool push( T& t ) { - return this->push_back( t ); - } - }; - - //! Input and scheduling for a function node that takes a type Input as input - // The only up-ref is apply_body_impl, which should implement the function - // call and any handling of the result. - template< typename Input, typename A, typename ImplType > - class function_input_base : public receiver, tbb::internal::no_assign { - enum op_type {reg_pred, rem_pred, app_body, try_fwd, tryput_bypass, app_body_bypass -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - , add_blt_pred, del_blt_pred, - blt_pred_cnt, blt_pred_cpy // create vector copies of preds and succs -#endif - }; - typedef function_input_base class_type; - - public: - - //! The input type of this receiver - typedef Input input_type; - typedef typename receiver::predecessor_type predecessor_type; - typedef predecessor_cache predecessor_cache_type; - typedef function_input_queue input_queue_type; - typedef typename A::template rebind< input_queue_type >::other queue_allocator_type; - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - typedef typename predecessor_cache_type::built_predecessors_type built_predecessors_type; - typedef typename receiver::predecessor_list_type predecessor_list_type; -#endif - - //! Constructor for function_input_base - function_input_base( graph &g, size_t max_concurrency, input_queue_type *q = NULL) - : my_graph_ptr(&g), my_max_concurrency(max_concurrency), my_concurrency(0), - my_queue(q), forwarder_busy(false) { - my_predecessors.set_owner(this); - my_aggregator.initialize_handler(handler_type(this)); - } - - //! Copy constructor - function_input_base( const function_input_base& src, input_queue_type *q = NULL) : - receiver(), tbb::internal::no_assign(), - my_graph_ptr(src.my_graph_ptr), my_max_concurrency(src.my_max_concurrency), - my_concurrency(0), my_queue(q), forwarder_busy(false) - { - my_predecessors.set_owner(this); - my_aggregator.initialize_handler(handler_type(this)); - } - - //! Destructor - // The queue is allocated by the constructor for {multi}function_node. - // TODO: pass the graph_buffer_policy to the base so it can allocate the queue instead. - // This would be an interface-breaking change. - virtual ~function_input_base() { - if ( my_queue ) delete my_queue; - } - - //! Put to the node, returning a task if available - task * try_put_task( const input_type &t ) __TBB_override { - if ( my_max_concurrency == 0 ) { - return create_body_task( t ); - } else { - operation_type op_data(t, tryput_bypass); - my_aggregator.execute(&op_data); - if(op_data.status == internal::SUCCEEDED) { - return op_data.bypass_t; - } - return NULL; - } - } - - //! Adds src to the list of cached predecessors. - bool register_predecessor( predecessor_type &src ) __TBB_override { - operation_type op_data(reg_pred); - op_data.r = &src; - my_aggregator.execute(&op_data); - return true; - } - - //! Removes src from the list of cached predecessors. - bool remove_predecessor( predecessor_type &src ) __TBB_override { - operation_type op_data(rem_pred); - op_data.r = &src; - my_aggregator.execute(&op_data); - return true; - } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - //! Adds to list of predecessors added by make_edge - void internal_add_built_predecessor( predecessor_type &src) __TBB_override { - operation_type op_data(add_blt_pred); - op_data.r = &src; - my_aggregator.execute(&op_data); - } - - //! removes from to list of predecessors (used by remove_edge) - void internal_delete_built_predecessor( predecessor_type &src) __TBB_override { - operation_type op_data(del_blt_pred); - op_data.r = &src; - my_aggregator.execute(&op_data); - } - - size_t predecessor_count() __TBB_override { - operation_type op_data(blt_pred_cnt); - my_aggregator.execute(&op_data); - return op_data.cnt_val; - } - - void copy_predecessors(predecessor_list_type &v) __TBB_override { - operation_type op_data(blt_pred_cpy); - op_data.predv = &v; - my_aggregator.execute(&op_data); - } - - built_predecessors_type &built_predecessors() __TBB_override { - return my_predecessors.built_predecessors(); - } -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ - - protected: - - void reset_function_input_base( reset_flags f) { - my_concurrency = 0; - if(my_queue) { - my_queue->reset(); - } - reset_receiver(f); - forwarder_busy = false; - } - - graph* my_graph_ptr; - const size_t my_max_concurrency; - size_t my_concurrency; - input_queue_type *my_queue; - predecessor_cache my_predecessors; - - void reset_receiver( reset_flags f) __TBB_override { - if( f & rf_clear_edges) my_predecessors.clear(); - else - my_predecessors.reset(); - __TBB_ASSERT(!(f & rf_clear_edges) || my_predecessors.empty(), "function_input_base reset failed"); - } - - private: - - friend class apply_body_task_bypass< class_type, input_type >; - friend class forward_task_bypass< class_type >; - - class operation_type : public aggregated_operation< operation_type > { - public: - char type; - union { - input_type *elem; - predecessor_type *r; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - size_t cnt_val; - predecessor_list_type *predv; -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ - }; - tbb::task *bypass_t; - operation_type(const input_type& e, op_type t) : - type(char(t)), elem(const_cast(&e)) {} - operation_type(op_type t) : type(char(t)), r(NULL) {} - }; - - bool forwarder_busy; - typedef internal::aggregating_functor handler_type; - friend class internal::aggregating_functor; - aggregator< handler_type, operation_type > my_aggregator; - - task* create_and_spawn_task(bool spawn) { - task* new_task = NULL; - if(my_queue) { - if(!my_queue->empty()) { - ++my_concurrency; - new_task = create_body_task(my_queue->front()); - - my_queue->pop(); - } - } - else { - input_type i; - if(my_predecessors.get_item(i)) { - ++my_concurrency; - new_task = create_body_task(i); - } - } - //! Spawns a task that applies a body - // task == NULL => g.reset(), which shouldn't occur in concurrent context - if(spawn && new_task) { - FLOW_SPAWN(*new_task); - new_task = SUCCESSFULLY_ENQUEUED; - } - - return new_task; - } - void handle_operations(operation_type *op_list) { - operation_type *tmp; - while (op_list) { - tmp = op_list; - op_list = op_list->next; - switch (tmp->type) { - case reg_pred: - my_predecessors.add(*(tmp->r)); - __TBB_store_with_release(tmp->status, SUCCEEDED); - if (!forwarder_busy) { - forwarder_busy = true; - spawn_forward_task(); - } - break; - case rem_pred: - my_predecessors.remove(*(tmp->r)); - __TBB_store_with_release(tmp->status, SUCCEEDED); - break; - case app_body: - __TBB_ASSERT(my_max_concurrency != 0, NULL); - --my_concurrency; - __TBB_store_with_release(tmp->status, SUCCEEDED); - if (my_concurrencybypass_t = NULL; - __TBB_ASSERT(my_max_concurrency != 0, NULL); - --my_concurrency; - if(my_concurrencybypass_t = create_and_spawn_task(/*spawn=*/false); - - __TBB_store_with_release(tmp->status, SUCCEEDED); - } - break; - case tryput_bypass: internal_try_put_task(tmp); break; - case try_fwd: internal_forward(tmp); break; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - case add_blt_pred: { - my_predecessors.internal_add_built_predecessor(*(tmp->r)); - __TBB_store_with_release(tmp->status, SUCCEEDED); - } - break; - case del_blt_pred: - my_predecessors.internal_delete_built_predecessor(*(tmp->r)); - __TBB_store_with_release(tmp->status, SUCCEEDED); - break; - case blt_pred_cnt: - tmp->cnt_val = my_predecessors.predecessor_count(); - __TBB_store_with_release(tmp->status, SUCCEEDED); - break; - case blt_pred_cpy: - my_predecessors.copy_predecessors( *(tmp->predv) ); - __TBB_store_with_release(tmp->status, SUCCEEDED); - break; -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ - } - } - } - - //! Put to the node, but return the task instead of enqueueing it - void internal_try_put_task(operation_type *op) { - __TBB_ASSERT(my_max_concurrency != 0, NULL); - if (my_concurrency < my_max_concurrency) { - ++my_concurrency; - task * new_task = create_body_task(*(op->elem)); - op->bypass_t = new_task; - __TBB_store_with_release(op->status, SUCCEEDED); - } else if ( my_queue && my_queue->push(*(op->elem)) ) { - op->bypass_t = SUCCESSFULLY_ENQUEUED; - __TBB_store_with_release(op->status, SUCCEEDED); - } else { - op->bypass_t = NULL; - __TBB_store_with_release(op->status, FAILED); - } - } - - //! Tries to spawn bodies if available and if concurrency allows - void internal_forward(operation_type *op) { - op->bypass_t = NULL; - if (my_concurrency < my_max_concurrency || !my_max_concurrency) - op->bypass_t = create_and_spawn_task(/*spawn=*/false); - if(op->bypass_t) - __TBB_store_with_release(op->status, SUCCEEDED); - else { - forwarder_busy = false; - __TBB_store_with_release(op->status, FAILED); - } - } - - //! Applies the body to the provided input - // then decides if more work is available - task * apply_body_bypass( input_type &i ) { - task * new_task = static_cast(this)->apply_body_impl_bypass(i); - if ( my_max_concurrency != 0 ) { - operation_type op_data(app_body_bypass); // tries to pop an item or get_item, enqueues another apply_body - my_aggregator.execute(&op_data); - tbb::task *ttask = op_data.bypass_t; - new_task = combine_tasks(new_task, ttask); - } - return new_task; - } - - //! allocates a task to apply a body - inline task * create_body_task( const input_type &input ) { - - return (my_graph_ptr->is_active()) ? - new(task::allocate_additional_child_of(*(my_graph_ptr->root_task()))) - apply_body_task_bypass < class_type, input_type >(*this, input) : - NULL; - } - - //! This is executed by an enqueued task, the "forwarder" - task *forward_task() { - operation_type op_data(try_fwd); - task *rval = NULL; - do { - op_data.status = WAIT; - my_aggregator.execute(&op_data); - if(op_data.status == SUCCEEDED) { - tbb::task *ttask = op_data.bypass_t; - rval = combine_tasks(rval, ttask); - } - } while (op_data.status == SUCCEEDED); - return rval; - } - - inline task *create_forward_task() { - return (my_graph_ptr->is_active()) ? - new(task::allocate_additional_child_of(*(my_graph_ptr->root_task()))) forward_task_bypass< class_type >(*this) : - NULL; - } - - //! Spawns a task that calls forward() - inline void spawn_forward_task() { - task* tp = create_forward_task(); - if(tp) { - FLOW_SPAWN(*tp); - } - } - }; // function_input_base - - //! Implements methods for a function node that takes a type Input as input and sends - // a type Output to its successors. - template< typename Input, typename Output, typename A> - class function_input : public function_input_base > { - public: - typedef Input input_type; - typedef Output output_type; - typedef function_body function_body_type; - typedef function_input my_class; - typedef function_input_base base_type; - typedef function_input_queue input_queue_type; - - // constructor - template - function_input( graph &g, size_t max_concurrency, Body& body, input_queue_type *q = NULL ) : - base_type(g, max_concurrency, q), - my_body( new internal::function_body_leaf< input_type, output_type, Body>(body) ), - my_init_body( new internal::function_body_leaf< input_type, output_type, Body>(body) ) { - } - - //! Copy constructor - function_input( const function_input& src, input_queue_type *q = NULL ) : - base_type(src, q), - my_body( src.my_init_body->clone() ), - my_init_body(src.my_init_body->clone() ) { - } - - ~function_input() { - delete my_body; - delete my_init_body; - } - - template< typename Body > - Body copy_function_object() { - function_body_type &body_ref = *this->my_body; - return dynamic_cast< internal::function_body_leaf & >(body_ref).get_body(); - } - - task * apply_body_impl_bypass( const input_type &i) { -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - // There is an extra copied needed to capture the - // body execution without the try_put - tbb::internal::fgt_begin_body( my_body ); - output_type v = (*my_body)(i); - tbb::internal::fgt_end_body( my_body ); - task * new_task = successors().try_put_task( v ); -#else - task * new_task = successors().try_put_task( (*my_body)(i) ); -#endif - return new_task; - } - - protected: - - void reset_function_input(reset_flags f) { - base_type::reset_function_input_base(f); - if(f & rf_reset_bodies) { - function_body_type *tmp = my_init_body->clone(); - delete my_body; - my_body = tmp; - } - } - - function_body_type *my_body; - function_body_type *my_init_body; - virtual broadcast_cache &successors() = 0; - - }; // function_input - - - // helper templates to clear the successor edges of the output ports of an multifunction_node - template struct clear_element { - template static void clear_this(P &p) { - (void)tbb::flow::get(p).successors().clear(); - clear_element::clear_this(p); - } - template static bool this_empty(P &p) { - if(tbb::flow::get(p).successors().empty()) - return clear_element::this_empty(p); - return false; - } - }; - - template<> struct clear_element<1> { - template static void clear_this(P &p) { - (void)tbb::flow::get<0>(p).successors().clear(); - } - template static bool this_empty(P &p) { - return tbb::flow::get<0>(p).successors().empty(); - } - }; - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - // helper templates to extract the output ports of an multifunction_node from graph - template struct extract_element { - template static void extract_this(P &p) { - (void)tbb::flow::get(p).successors().built_successors().sender_extract(tbb::flow::get(p)); - extract_element::extract_this(p); - } - }; - - template<> struct extract_element<1> { - template static void extract_this(P &p) { - (void)tbb::flow::get<0>(p).successors().built_successors().sender_extract(tbb::flow::get<0>(p)); - } - }; -#endif - - //! Implements methods for a function node that takes a type Input as input - // and has a tuple of output ports specified. - template< typename Input, typename OutputPortSet, typename A> - class multifunction_input : public function_input_base > { - public: - static const int N = tbb::flow::tuple_size::value; - typedef Input input_type; - typedef OutputPortSet output_ports_type; - typedef multifunction_body multifunction_body_type; - typedef multifunction_input my_class; - typedef function_input_base base_type; - typedef function_input_queue input_queue_type; - - // constructor - template - multifunction_input( - graph &g, - size_t max_concurrency, - Body& body, - input_queue_type *q = NULL ) : - base_type(g, max_concurrency, q), - my_body( new internal::multifunction_body_leaf(body) ), - my_init_body( new internal::multifunction_body_leaf(body) ) { - } - - //! Copy constructor - multifunction_input( const multifunction_input& src, input_queue_type *q = NULL ) : - base_type(src, q), - my_body( src.my_init_body->clone() ), - my_init_body(src.my_init_body->clone() ) { - } - - ~multifunction_input() { - delete my_body; - delete my_init_body; - } - - template< typename Body > - Body copy_function_object() { - multifunction_body_type &body_ref = *this->my_body; - return *static_cast(dynamic_cast< internal::multifunction_body_leaf & >(body_ref).get_body_ptr()); - } - - // for multifunction nodes we do not have a single successor as such. So we just tell - // the task we were successful. - task * apply_body_impl_bypass( const input_type &i) { - tbb::internal::fgt_begin_body( my_body ); - (*my_body)(i, my_output_ports); - tbb::internal::fgt_end_body( my_body ); - task * new_task = SUCCESSFULLY_ENQUEUED; - return new_task; - } - - output_ports_type &output_ports(){ return my_output_ports; } - - protected: -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - void extract() { - extract_element::extract_this(my_output_ports); - } -#endif - - void reset(reset_flags f) { - base_type::reset_function_input_base(f); - if(f & rf_clear_edges)clear_element::clear_this(my_output_ports); - if(f & rf_reset_bodies) { - multifunction_body_type *tmp = my_init_body->clone(); - delete my_body; - my_body = tmp; - } - __TBB_ASSERT(!(f & rf_clear_edges) || clear_element::this_empty(my_output_ports), "multifunction_node reset failed"); - } - - multifunction_body_type *my_body; - multifunction_body_type *my_init_body; - output_ports_type my_output_ports; - - }; // multifunction_input - - // template to refer to an output port of a multifunction_node - template - typename tbb::flow::tuple_element::type &output_port(MOP &op) { - return tbb::flow::get(op.output_ports()); - } - -// helper structs for split_node - template - struct emit_element { - template - static void emit_this(const T &t, P &p) { - (void)tbb::flow::get(p).try_put(tbb::flow::get(t)); - emit_element::emit_this(t,p); - } - }; - - template<> - struct emit_element<1> { - template - static void emit_this(const T &t, P &p) { - (void)tbb::flow::get<0>(p).try_put(tbb::flow::get<0>(t)); - } - }; - - //! Implements methods for an executable node that takes continue_msg as input - template< typename Output > - class continue_input : public continue_receiver { - public: - - //! The input type of this receiver - typedef continue_msg input_type; - - //! The output type of this receiver - typedef Output output_type; - typedef function_body function_body_type; - - template< typename Body > - continue_input( graph &g, Body& body ) - : my_graph_ptr(&g), - my_body( new internal::function_body_leaf< input_type, output_type, Body>(body) ), - my_init_body( new internal::function_body_leaf< input_type, output_type, Body>(body) ) { } - - template< typename Body > - continue_input( graph &g, int number_of_predecessors, Body& body ) - : continue_receiver( number_of_predecessors ), my_graph_ptr(&g), - my_body( new internal::function_body_leaf< input_type, output_type, Body>(body) ), - my_init_body( new internal::function_body_leaf< input_type, output_type, Body>(body) ) - { } - - continue_input( const continue_input& src ) : continue_receiver(src), - my_graph_ptr(src.my_graph_ptr), - my_body( src.my_init_body->clone() ), - my_init_body( src.my_init_body->clone() ) {} - - ~continue_input() { - delete my_body; - delete my_init_body; - } - - template< typename Body > - Body copy_function_object() { - function_body_type &body_ref = *my_body; - return dynamic_cast< internal::function_body_leaf & >(body_ref).get_body(); - } - - void reset_receiver( reset_flags f) __TBB_override { - continue_receiver::reset_receiver(f); - if(f & rf_reset_bodies) { - function_body_type *tmp = my_init_body->clone(); - delete my_body; - my_body = tmp; - } - } - - protected: - - graph* my_graph_ptr; - function_body_type *my_body; - function_body_type *my_init_body; - - virtual broadcast_cache &successors() = 0; - - friend class apply_body_task_bypass< continue_input< Output >, continue_msg >; - - //! Applies the body to the provided input - task *apply_body_bypass( input_type ) { -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - // There is an extra copied needed to capture the - // body execution without the try_put - tbb::internal::fgt_begin_body( my_body ); - output_type v = (*my_body)( continue_msg() ); - tbb::internal::fgt_end_body( my_body ); - return successors().try_put_task( v ); -#else - return successors().try_put_task( (*my_body)( continue_msg() ) ); -#endif - } - - //! Spawns a task that applies the body - task *execute( ) __TBB_override { - return (my_graph_ptr->is_active()) ? - new ( task::allocate_additional_child_of( *(my_graph_ptr->root_task()) ) ) - apply_body_task_bypass< continue_input< Output >, continue_msg >( *this, continue_msg() ) : - NULL; - } - - }; // continue_input - - //! Implements methods for both executable and function nodes that puts Output to its successors - template< typename Output > - class function_output : public sender { - public: - - template friend struct clear_element; - typedef Output output_type; - typedef typename sender::successor_type successor_type; - typedef broadcast_cache broadcast_cache_type; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - typedef typename sender::built_successors_type built_successors_type; - typedef typename sender::successor_list_type successor_list_type; -#endif - - function_output() { my_successors.set_owner(this); } - function_output(const function_output & /*other*/) : sender() { - my_successors.set_owner(this); - } - - //! Adds a new successor to this node - bool register_successor( successor_type &r ) __TBB_override { - successors().register_successor( r ); - return true; - } - - //! Removes a successor from this node - bool remove_successor( successor_type &r ) __TBB_override { - successors().remove_successor( r ); - return true; - } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - built_successors_type &built_successors() __TBB_override { return successors().built_successors(); } - - - void internal_add_built_successor( successor_type &r) __TBB_override { - successors().internal_add_built_successor( r ); - } - - void internal_delete_built_successor( successor_type &r) __TBB_override { - successors().internal_delete_built_successor( r ); - } - - size_t successor_count() __TBB_override { - return successors().successor_count(); - } - - void copy_successors( successor_list_type &v) __TBB_override { - successors().copy_successors(v); - } -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ - - // for multifunction_node. The function_body that implements - // the node will have an input and an output tuple of ports. To put - // an item to a successor, the body should - // - // get(output_ports).try_put(output_value); - // - // if task pointer is returned will always spawn and return true, else - // return value will be bool returned from successors.try_put. - task *try_put_task(const output_type &i) { // not a virtual method in this class - return my_successors.try_put_task(i); - } - - broadcast_cache_type &successors() { return my_successors; } - protected: - broadcast_cache_type my_successors; - - }; // function_output - - template< typename Output > - class multifunction_output : public function_output { - public: - typedef Output output_type; - typedef function_output base_type; - using base_type::my_successors; - - multifunction_output() : base_type() {my_successors.set_owner(this);} - multifunction_output( const multifunction_output &/*other*/) : base_type() { my_successors.set_owner(this); } - - bool try_put(const output_type &i) { - task *res = my_successors.try_put_task(i); - if(!res) return false; - if(res != SUCCESSFULLY_ENQUEUED) FLOW_SPAWN(*res); - return true; - } - }; // multifunction_output - -//composite_node -#if TBB_PREVIEW_FLOW_GRAPH_TRACE && __TBB_FLOW_GRAPH_CPP11_FEATURES - template - void add_nodes_impl(CompositeType*, bool) {} - - template< typename CompositeType, typename NodeType1, typename... NodeTypes > - void add_nodes_impl(CompositeType *c_node, bool visible, const NodeType1& n1, const NodeTypes&... n) { - void *addr = const_cast(&n1); - - if(visible) - tbb::internal::itt_relation_add( tbb::internal::ITT_DOMAIN_FLOW, c_node, tbb::internal::FLOW_NODE, tbb::internal::__itt_relation_is_parent_of, addr, tbb::internal::FLOW_NODE ); - else - tbb::internal::itt_relation_add( tbb::internal::ITT_DOMAIN_FLOW, addr, tbb::internal::FLOW_NODE, tbb::internal::__itt_relation_is_child_of, c_node, tbb::internal::FLOW_NODE ); - add_nodes_impl(c_node, visible, n...); - } -#endif - -} // internal - -#endif // __TBB__flow_graph_node_impl_H diff --git a/inst/include/tbb/internal/_flow_graph_streaming_node.h b/inst/include/tbb/internal/_flow_graph_streaming_node.h deleted file mode 100644 index 493f76e80..000000000 --- a/inst/include/tbb/internal/_flow_graph_streaming_node.h +++ /dev/null @@ -1,745 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_flow_graph_streaming_H -#define __TBB_flow_graph_streaming_H - -#ifndef __TBB_flow_graph_H -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#if __TBB_PREVIEW_STREAMING_NODE - -// Included in namespace tbb::flow::interfaceX (in flow_graph.h) - -namespace internal { - -template -struct port_ref_impl { - // "+1" since the port_ref range is a closed interval (includes its endpoints). - static const int size = N2 - N1 + 1; -}; - -} // internal - -// The purpose of the port_ref_impl is the pretty syntax: the deduction of a compile-time constant is processed from the return type. -// So it is possible to use this helper without parentheses, e.g. "port_ref<0>". -template -internal::port_ref_impl port_ref() { - return internal::port_ref_impl(); -}; - -namespace internal { - -template -struct num_arguments { - static const int value = 1; -}; - -template -struct num_arguments(*)()> { - static const int value = port_ref_impl::size; -}; - -template -struct num_arguments> { - static const int value = port_ref_impl::size; -}; - -template -void ignore_return_values( Args&&... ) {} - -template -T or_return_values( T&& t ) { return t; } -template -T or_return_values( T&& t, Rest&&... rest ) { - return t | or_return_values( std::forward(rest)... ); -} - -template -struct key_from_policy { - typedef size_t type; - typedef std::false_type is_key_matching; -}; - -template -struct key_from_policy< key_matching > { - typedef Key type; - typedef std::true_type is_key_matching; -}; - -template -struct key_from_policy< key_matching > { - typedef const Key &type; - typedef std::true_type is_key_matching; -}; - -template -class streaming_device_with_key { - Device my_device; - typename std::decay::type my_key; -public: - // TODO: investigate why default constructor is required - streaming_device_with_key() {} - streaming_device_with_key( const Device& d, Key k ) : my_device( d ), my_key( k ) {} - Key key() const { return my_key; } - const Device& device() const { return my_device; } -}; - -// --------- Kernel argument helpers --------- // -template -struct is_port_ref_impl { - typedef std::false_type type; -}; - -template -struct is_port_ref_impl< port_ref_impl > { - typedef std::true_type type; -}; - -template -struct is_port_ref_impl< port_ref_impl( * )() > { - typedef std::true_type type; -}; - -template -struct is_port_ref { - typedef typename is_port_ref_impl< typename tbb::internal::strip::type >::type type; -}; - -template -struct convert_and_call_impl; - -template -struct convert_and_call_impl { - static const size_t my_delta = 1; // Index 0 contains device - - template - static void doit(F& f, Tuple& t, A1& a1, Args1&... args1, Args2&... args2) { - convert_and_call_impl::doit_impl(typename is_port_ref::type(), f, t, a1, args1..., args2...); - } - template - static void doit_impl(std::false_type, F& f, Tuple& t, A1& a1, Args1&... args1, Args2&... args2) { - convert_and_call_impl::doit(f, t, args1..., args2..., a1); - } - template - static void doit_impl(std::true_type x, F& f, Tuple& t, port_ref_impl, Args1&... args1, Args2&... args2) { - convert_and_call_impl, Args1...>::doit_impl(x, f, t, port_ref(), args1..., - args2..., std::get(t)); - } - template - static void doit_impl(std::true_type, F& f, Tuple& t, port_ref_impl, Args1&... args1, Args2&... args2) { - convert_and_call_impl::doit(f, t, args1..., args2..., std::get(t)); - } - - template - static void doit_impl(std::true_type x, F& f, Tuple& t, port_ref_impl(* fn)(), Args1&... args1, Args2&... args2) { - doit_impl(x, f, t, fn(), args1..., args2...); - } - template - static void doit_impl(std::true_type x, F& f, Tuple& t, port_ref_impl(* fn)(), Args1&... args1, Args2&... args2) { - doit_impl(x, f, t, fn(), args1..., args2...); - } -}; - -template <> -struct convert_and_call_impl<> { - template - static void doit(F& f, Tuple&, Args2&... args2) { - f(args2...); - } -}; -// ------------------------------------------- // - -template -struct streaming_node_traits { - // Do not use 'using' instead of 'struct' because Microsoft Visual C++ 12.0 fails to compile. - template - struct async_msg_type { - typedef typename StreamFactory::template async_msg_type type; - }; - - typedef tuple< typename async_msg_type::type... > input_tuple; - typedef input_tuple output_tuple; - typedef tuple< streaming_device_with_key< typename StreamFactory::device_type, typename key_from_policy::type >, - typename async_msg_type::type... > kernel_input_tuple; - - // indexer_node parameters pack expansion workaround for VS2013 for streaming_node - typedef indexer_node< typename async_msg_type::type... > indexer_node_type; -}; - -// Default empty implementation -template -class kernel_executor_helper { - typedef typename StreamFactory::device_type device_type; - typedef typename StreamFactory::kernel_type kernel_type; - typedef KernelInputTuple kernel_input_tuple; -protected: - template - void enqueue_kernel_impl( kernel_input_tuple&, StreamFactory& factory, device_type device, const kernel_type& kernel, Args&... args ) const { - factory.send_kernel( device, kernel, args... ); - } -}; - -// Implementation for StreamFactory supporting range -template -class kernel_executor_helper::type > { - typedef typename StreamFactory::device_type device_type; - typedef typename StreamFactory::kernel_type kernel_type; - typedef KernelInputTuple kernel_input_tuple; - - typedef typename StreamFactory::range_type range_type; - - // Container for randge. It can contain either port references or real range. - struct range_wrapper { - virtual range_type get_range( const kernel_input_tuple &ip ) const = 0; - virtual range_wrapper *clone() const = 0; - virtual ~range_wrapper() {} - }; - - struct range_value : public range_wrapper { - range_value( const range_type& value ) : my_value(value) {} - - range_value( range_type&& value ) : my_value(std::move(value)) {} - - range_type get_range( const kernel_input_tuple & ) const __TBB_override { - return my_value; - } - - range_wrapper *clone() const __TBB_override { - return new range_value(my_value); - } - private: - range_type my_value; - }; - - template - struct range_mapper : public range_wrapper { - range_mapper() {} - - range_type get_range( const kernel_input_tuple &ip ) const __TBB_override { - // "+1" since get<0>(ip) is StreamFactory::device. - return get(ip).data(false); - } - - range_wrapper *clone() const __TBB_override { - return new range_mapper; - } - }; - -protected: - template - void enqueue_kernel_impl( kernel_input_tuple& ip, StreamFactory& factory, device_type device, const kernel_type& kernel, Args&... args ) const { - __TBB_ASSERT(my_range_wrapper, "Range is not set. Call set_range() before running streaming_node."); - factory.send_kernel( device, kernel, my_range_wrapper->get_range(ip), args... ); - } - -public: - kernel_executor_helper() : my_range_wrapper(NULL) {} - - kernel_executor_helper(const kernel_executor_helper& executor) : my_range_wrapper(executor.my_range_wrapper ? executor.my_range_wrapper->clone() : NULL) {} - - kernel_executor_helper(kernel_executor_helper&& executor) : my_range_wrapper(executor.my_range_wrapper) { - // Set moving holder mappers to NULL to prevent double deallocation - executor.my_range_wrapper = NULL; - } - - ~kernel_executor_helper() { - if (my_range_wrapper) delete my_range_wrapper; - } - - void set_range(const range_type& work_size) { - my_range_wrapper = new range_value(work_size); - } - - void set_range(range_type&& work_size) { - my_range_wrapper = new range_value(std::move(work_size)); - } - - template - void set_range(port_ref_impl) { - my_range_wrapper = new range_mapper; - } - - template - void set_range(port_ref_impl(*)()) { - my_range_wrapper = new range_mapper; - } - -private: - range_wrapper* my_range_wrapper; -}; - -} // internal - -/* -/---------------------------------------- streaming_node ------------------------------------\ -| | -| /--------------\ /----------------------\ /-----------\ /----------------------\ | -| | | | (device_with_key) O---O | | | | -| | | | | | | | | | -O---O indexer_node O---O device_selector_node O---O join_node O---O kernel_node O---O -| | | | (multifunction_node) | | | | (multifunction_node) | | -O---O | | O---O | | O---O -| \--------------/ \----------------------/ \-----------/ \----------------------/ | -| | -\--------------------------------------------------------------------------------------------/ -*/ -template -class streaming_node; - -template -class streaming_node< tuple, JP, StreamFactory > - : public composite_node < typename internal::streaming_node_traits::input_tuple, - typename internal::streaming_node_traits::output_tuple > - , public internal::kernel_executor_helper< StreamFactory, typename internal::streaming_node_traits::kernel_input_tuple > -{ - typedef typename internal::streaming_node_traits::input_tuple input_tuple; - typedef typename internal::streaming_node_traits::output_tuple output_tuple; - typedef typename internal::key_from_policy::type key_type; -protected: - typedef typename StreamFactory::device_type device_type; - typedef typename StreamFactory::kernel_type kernel_type; -private: - typedef internal::streaming_device_with_key device_with_key_type; - typedef composite_node base_type; - static const size_t NUM_INPUTS = tuple_size::value; - static const size_t NUM_OUTPUTS = tuple_size::value; - - typedef typename internal::make_sequence::type input_sequence; - typedef typename internal::make_sequence::type output_sequence; - - typedef typename internal::streaming_node_traits::indexer_node_type indexer_node_type; - typedef typename indexer_node_type::output_type indexer_node_output_type; - typedef typename internal::streaming_node_traits::kernel_input_tuple kernel_input_tuple; - typedef multifunction_node device_selector_node; - typedef multifunction_node kernel_multifunction_node; - - template - typename base_type::input_ports_type get_input_ports( internal::sequence ) { - return std::tie( internal::input_port( my_indexer_node )... ); - } - - template - typename base_type::output_ports_type get_output_ports( internal::sequence ) { - return std::tie( internal::output_port( my_kernel_node )... ); - } - - typename base_type::input_ports_type get_input_ports() { - return get_input_ports( input_sequence() ); - } - - typename base_type::output_ports_type get_output_ports() { - return get_output_ports( output_sequence() ); - } - - template - int make_Nth_edge() { - make_edge( internal::output_port( my_device_selector_node ), internal::input_port( my_join_node ) ); - return 0; - } - - template - void make_edges( internal::sequence ) { - make_edge( my_indexer_node, my_device_selector_node ); - make_edge( my_device_selector_node, my_join_node ); - internal::ignore_return_values( make_Nth_edge()... ); - make_edge( my_join_node, my_kernel_node ); - } - - void make_edges() { - make_edges( input_sequence() ); - } - - class device_selector_base { - public: - virtual void operator()( const indexer_node_output_type &v, typename device_selector_node::output_ports_type &op ) = 0; - virtual device_selector_base *clone( streaming_node &n ) const = 0; - virtual ~device_selector_base() {} - }; - - template - class device_selector : public device_selector_base, tbb::internal::no_assign { - public: - device_selector( UserFunctor uf, streaming_node &n, StreamFactory &f ) - : my_dispatch_funcs( create_dispatch_funcs( input_sequence() ) ) - , my_user_functor( uf ), my_node(n), my_factory( f ) - { - my_port_epoches.fill( 0 ); - } - - void operator()( const indexer_node_output_type &v, typename device_selector_node::output_ports_type &op ) __TBB_override { - (this->*my_dispatch_funcs[ v.tag() ])( my_port_epoches[ v.tag() ], v, op ); - __TBB_ASSERT( (tbb::internal::is_same_type::is_key_matching, std::false_type>::value) - || my_port_epoches[v.tag()] == 0, "Epoch is changed when key matching is requested" ); - } - - device_selector_base *clone( streaming_node &n ) const __TBB_override { - return new device_selector( my_user_functor, n, my_factory ); - } - private: - typedef void(device_selector::*send_and_put_fn_type)(size_t &, const indexer_node_output_type &, typename device_selector_node::output_ports_type &); - typedef std::array < send_and_put_fn_type, NUM_INPUTS > dispatch_funcs_type; - - template - static dispatch_funcs_type create_dispatch_funcs( internal::sequence ) { - dispatch_funcs_type dispatch = { { &device_selector::send_and_put_impl... } }; - return dispatch; - } - - template - key_type get_key( std::false_type, const T &, size_t &epoch ) { - __TBB_STATIC_ASSERT( (tbb::internal::is_same_type::value), "" ); - return epoch++; - } - - template - key_type get_key( std::true_type, const T &t, size_t &/*epoch*/ ) { - using tbb::flow::key_from_message; - return key_from_message( t ); - } - - template - void send_and_put_impl( size_t &epoch, const indexer_node_output_type &v, typename device_selector_node::output_ports_type &op ) { - typedef typename tuple_element::type::output_type elem_type; - elem_type e = internal::cast_to( v ); - device_type device = get_device( get_key( typename internal::key_from_policy::is_key_matching(), e, epoch ), get<0>( op ) ); - my_factory.send_data( device, e ); - get( op ).try_put( e ); - } - - template< typename DevicePort > - device_type get_device( key_type key, DevicePort& dp ) { - typename std::unordered_map::type, epoch_desc>::iterator it = my_devices.find( key ); - if ( it == my_devices.end() ) { - device_type d = my_user_functor( my_factory ); - std::tie( it, std::ignore ) = my_devices.insert( std::make_pair( key, d ) ); - bool res = dp.try_put( device_with_key_type( d, key ) ); - __TBB_ASSERT_EX( res, NULL ); - my_node.notify_new_device( d ); - } - epoch_desc &e = it->second; - device_type d = e.my_device; - if ( ++e.my_request_number == NUM_INPUTS ) my_devices.erase( it ); - return d; - } - - struct epoch_desc { - epoch_desc(device_type d ) : my_device( d ), my_request_number( 0 ) {} - device_type my_device; - size_t my_request_number; - }; - - std::unordered_map::type, epoch_desc> my_devices; - std::array my_port_epoches; - dispatch_funcs_type my_dispatch_funcs; - UserFunctor my_user_functor; - streaming_node &my_node; - StreamFactory &my_factory; - }; - - class device_selector_body { - public: - device_selector_body( device_selector_base *d ) : my_device_selector( d ) {} - - void operator()( const indexer_node_output_type &v, typename device_selector_node::output_ports_type &op ) { - (*my_device_selector)(v, op); - } - private: - device_selector_base *my_device_selector; - }; - - class args_storage_base : tbb::internal::no_copy { - public: - typedef typename kernel_multifunction_node::output_ports_type output_ports_type; - - virtual void enqueue( kernel_input_tuple &ip, output_ports_type &op, const streaming_node &n ) = 0; - virtual void send( device_type d ) = 0; - virtual args_storage_base *clone() const = 0; - virtual ~args_storage_base () {} - - protected: - args_storage_base( const kernel_type& kernel, StreamFactory &f ) - : my_kernel( kernel ), my_factory( f ) - {} - - args_storage_base( const args_storage_base &k ) - : my_kernel( k.my_kernel ), my_factory( k.my_factory ) - {} - - const kernel_type my_kernel; - StreamFactory &my_factory; - }; - - template - class args_storage : public args_storage_base { - typedef typename args_storage_base::output_ports_type output_ports_type; - - // ---------- Update events helpers ---------- // - template - bool do_try_put( const kernel_input_tuple& ip, output_ports_type &op ) const { - const auto& t = get( ip ); - auto &port = get( op ); - return port.try_put( t ); - } - - template - bool do_try_put( const kernel_input_tuple& ip, output_ports_type &op, internal::sequence ) const { - return internal::or_return_values( do_try_put( ip, op )... ); - } - - // ------------------------------------------- // - class run_kernel_func : tbb::internal::no_assign { - public: - run_kernel_func( kernel_input_tuple &ip, const streaming_node &node, const args_storage& storage ) - : my_kernel_func( ip, node, storage, get<0>(ip).device() ) {} - - // It is immpossible to use Args... because a function pointer cannot be casted to a function reference implicitly. - // Allow the compiler to deduce types for function pointers automatically. - template - void operator()( FnArgs&... args ) { - internal::convert_and_call_impl::doit( my_kernel_func, my_kernel_func.my_ip, args... ); - } - private: - struct kernel_func : tbb::internal::no_copy { - kernel_input_tuple &my_ip; - const streaming_node &my_node; - const args_storage& my_storage; - device_type my_device; - - kernel_func( kernel_input_tuple &ip, const streaming_node &node, const args_storage& storage, device_type device ) - : my_ip( ip ), my_node( node ), my_storage( storage ), my_device( device ) - {} - - template - void operator()( FnArgs&... args ) { - my_node.enqueue_kernel( my_ip, my_storage.my_factory, my_device, my_storage.my_kernel, args... ); - } - } my_kernel_func; - }; - - template - class run_finalize_func : tbb::internal::no_assign { - public: - run_finalize_func( kernel_input_tuple &ip, StreamFactory &factory, FinalizeFn fn ) - : my_ip( ip ), my_finalize_func( factory, get<0>(ip).device(), fn ) {} - - // It is immpossible to use Args... because a function pointer cannot be casted to a function reference implicitly. - // Allow the compiler to deduce types for function pointers automatically. - template - void operator()( FnArgs&... args ) { - internal::convert_and_call_impl::doit( my_finalize_func, my_ip, args... ); - } - private: - kernel_input_tuple &my_ip; - - struct finalize_func : tbb::internal::no_assign { - StreamFactory &my_factory; - device_type my_device; - FinalizeFn my_fn; - - finalize_func( StreamFactory &factory, device_type device, FinalizeFn fn ) - : my_factory(factory), my_device(device), my_fn(fn) {} - - template - void operator()( FnArgs&... args ) { - my_factory.finalize( my_device, my_fn, args... ); - } - } my_finalize_func; - }; - - template - static run_finalize_func make_run_finalize_func( kernel_input_tuple &ip, StreamFactory &factory, FinalizeFn fn ) { - return run_finalize_func( ip, factory, fn ); - } - - class send_func : tbb::internal::no_assign { - public: - send_func( StreamFactory &factory, device_type d ) - : my_factory(factory), my_device( d ) {} - - template - void operator()( FnArgs&... args ) { - my_factory.send_data( my_device, args... ); - } - private: - StreamFactory &my_factory; - device_type my_device; - }; - - public: - args_storage( const kernel_type& kernel, StreamFactory &f, Args&&... args ) - : args_storage_base( kernel, f ) - , my_args_pack( std::forward(args)... ) - {} - - args_storage( const args_storage &k ) : args_storage_base( k ), my_args_pack( k.my_args_pack ) {} - - args_storage( const args_storage_base &k, Args&&... args ) : args_storage_base( k ), my_args_pack( std::forward(args)... ) {} - - void enqueue( kernel_input_tuple &ip, output_ports_type &op, const streaming_node &n ) __TBB_override { - // Make const qualified args_pack (from non-const) - const args_pack_type& const_args_pack = my_args_pack; - // factory.enqure_kernel() gets - // - 'ip' tuple elements by reference and updates it (and 'ip') with dependencies - // - arguments (from my_args_pack) by const-reference via const_args_pack - tbb::internal::call( run_kernel_func( ip, n, *this ), const_args_pack ); - - if (! do_try_put( ip, op, input_sequence() ) ) { - graph& g = n.my_graph; - // No one message was passed to successors so set a callback to extend the graph lifetime until the kernel completion. - g.increment_wait_count(); - - // factory.finalize() gets - // - 'ip' tuple elements by reference, so 'ip' might be changed - // - arguments (from my_args_pack) by const-reference via const_args_pack - tbb::internal::call( make_run_finalize_func(ip, this->my_factory, [&g] { - g.decrement_wait_count(); - }), const_args_pack ); - } - } - - void send( device_type d ) __TBB_override { - // factory.send() gets arguments by reference and updates these arguments with dependencies - // (it gets but usually ignores port_ref-s) - tbb::internal::call( send_func( this->my_factory, d ), my_args_pack ); - } - - args_storage_base *clone() const __TBB_override { - // Create new args_storage with copying constructor. - return new args_storage( *this ); - } - - private: - typedef tbb::internal::stored_pack args_pack_type; - args_pack_type my_args_pack; - }; - - // Body for kernel_multifunction_node. - class kernel_body : tbb::internal::no_assign { - public: - kernel_body( const streaming_node &node ) : my_node( node ) {} - - void operator()( kernel_input_tuple ip, typename args_storage_base::output_ports_type &op ) { - __TBB_ASSERT( (my_node.my_args_storage != NULL), "No arguments storage" ); - // 'ip' is passed by value to create local copy for updating inside enqueue_kernel() - my_node.my_args_storage->enqueue( ip, op, my_node ); - } - private: - const streaming_node &my_node; - }; - - template ::type > - struct wrap_to_async { - typedef T type; // Keep port_ref as it is - }; - - template - struct wrap_to_async { - typedef typename StreamFactory::template async_msg_type< typename tbb::internal::strip::type > type; - }; - - template - args_storage_base *make_args_storage(const args_storage_base& storage, Args&&... args) const { - // In this variadic template convert all simple types 'T' into 'async_msg_type' - return new args_storage(storage, std::forward(args)...); - } - - void notify_new_device( device_type d ) { - my_args_storage->send( d ); - } - - template - void enqueue_kernel( kernel_input_tuple& ip, StreamFactory& factory, device_type device, const kernel_type& kernel, Args&... args ) const { - this->enqueue_kernel_impl( ip, factory, device, kernel, args... ); - } - -public: - template - streaming_node( graph &g, const kernel_type& kernel, DeviceSelector d, StreamFactory &f ) - : base_type( g ) - , my_indexer_node( g ) - , my_device_selector( new device_selector( d, *this, f ) ) - , my_device_selector_node( g, serial, device_selector_body( my_device_selector ) ) - , my_join_node( g ) - , my_kernel_node( g, serial, kernel_body( *this ) ) - // By default, streaming_node maps all its ports to the kernel arguments on a one-to-one basis. - , my_args_storage( make_args_storage( args_storage<>(kernel, f), port_ref<0, NUM_INPUTS - 1>() ) ) - { - base_type::set_external_ports( get_input_ports(), get_output_ports() ); - make_edges(); - } - - streaming_node( const streaming_node &node ) - : base_type( node.my_graph ) - , my_indexer_node( node.my_indexer_node ) - , my_device_selector( node.my_device_selector->clone( *this ) ) - , my_device_selector_node( node.my_graph, serial, device_selector_body( my_device_selector ) ) - , my_join_node( node.my_join_node ) - , my_kernel_node( node.my_graph, serial, kernel_body( *this ) ) - , my_args_storage( node.my_args_storage->clone() ) - { - base_type::set_external_ports( get_input_ports(), get_output_ports() ); - make_edges(); - } - - streaming_node( streaming_node &&node ) - : base_type( node.my_graph ) - , my_indexer_node( std::move( node.my_indexer_node ) ) - , my_device_selector( node.my_device_selector->clone(*this) ) - , my_device_selector_node( node.my_graph, serial, device_selector_body( my_device_selector ) ) - , my_join_node( std::move( node.my_join_node ) ) - , my_kernel_node( node.my_graph, serial, kernel_body( *this ) ) - , my_args_storage( node.my_args_storage ) - { - base_type::set_external_ports( get_input_ports(), get_output_ports() ); - make_edges(); - // Set moving node mappers to NULL to prevent double deallocation. - node.my_args_storage = NULL; - } - - ~streaming_node() { - if ( my_args_storage ) delete my_args_storage; - if ( my_device_selector ) delete my_device_selector; - } - - template - void set_args( Args&&... args ) { - // Copy the base class of args_storage and create new storage for "Args...". - args_storage_base * const new_args_storage = make_args_storage( *my_args_storage, typename wrap_to_async::type(std::forward(args))...); - delete my_args_storage; - my_args_storage = new_args_storage; - } - -protected: - void reset_node( reset_flags = rf_reset_protocol ) __TBB_override { __TBB_ASSERT( false, "Not implemented yet" ); } - -private: - indexer_node_type my_indexer_node; - device_selector_base *my_device_selector; - device_selector_node my_device_selector_node; - join_node my_join_node; - kernel_multifunction_node my_kernel_node; - - args_storage_base *my_args_storage; -}; - -#endif // __TBB_PREVIEW_STREAMING_NODE -#endif // __TBB_flow_graph_streaming_H diff --git a/inst/include/tbb/internal/_flow_graph_tagged_buffer_impl.h b/inst/include/tbb/internal/_flow_graph_tagged_buffer_impl.h deleted file mode 100644 index 46755fe07..000000000 --- a/inst/include/tbb/internal/_flow_graph_tagged_buffer_impl.h +++ /dev/null @@ -1,253 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -// a hash table buffer that can expand, and can support as many deletions as -// additions, list-based, with elements of list held in array (for destruction -// management), multiplicative hashing (like ets). No synchronization built-in. -// - -#ifndef __TBB__flow_graph_hash_buffer_impl_H -#define __TBB__flow_graph_hash_buffer_impl_H - -#ifndef __TBB_flow_graph_H -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -// included in namespace tbb::flow::interfaceX::internal - -// elements in the table are a simple list; we need pointer to next element to -// traverse the chain -template -struct buffer_element_type { - // the second parameter below is void * because we can't forward-declare the type - // itself, so we just reinterpret_cast below. - typedef typename aligned_pair::type type; -}; - -template - < - typename Key, // type of key within ValueType - typename ValueType, - typename ValueToKey, // abstract method that returns "const Key" or "const Key&" given ValueType - typename HashCompare, // has hash and equal - typename Allocator=tbb::cache_aligned_allocator< typename aligned_pair::type > - > -class hash_buffer : public HashCompare { -public: - static const size_t INITIAL_SIZE = 8; // initial size of the hash pointer table - typedef ValueType value_type; - typedef typename buffer_element_type< value_type >::type element_type; - typedef value_type *pointer_type; - typedef element_type *list_array_type; // array we manage manually - typedef list_array_type *pointer_array_type; - typedef typename Allocator::template rebind::other pointer_array_allocator_type; - typedef typename Allocator::template rebind::other elements_array_allocator; - typedef typename tbb::internal::strip::type Knoref; - -private: - ValueToKey *my_key; - size_t my_size; - size_t nelements; - pointer_array_type pointer_array; // pointer_array[my_size] - list_array_type elements_array; // elements_array[my_size / 2] - element_type* free_list; - - size_t mask() { return my_size - 1; } - - void set_up_free_list( element_type **p_free_list, list_array_type la, size_t sz) { - for(size_t i=0; i < sz - 1; ++i ) { // construct free list - la[i].second = &(la[i+1]); - } - la[sz-1].second = NULL; - *p_free_list = (element_type *)&(la[0]); - } - - // cleanup for exceptions - struct DoCleanup { - pointer_array_type *my_pa; - list_array_type *my_elements; - size_t my_size; - - DoCleanup(pointer_array_type &pa, list_array_type &my_els, size_t sz) : - my_pa(&pa), my_elements(&my_els), my_size(sz) { } - ~DoCleanup() { - if(my_pa) { - size_t dont_care = 0; - internal_free_buffer(*my_pa, *my_elements, my_size, dont_care); - } - } - }; - - // exception-safety requires we do all the potentially-throwing operations first - void grow_array() { - size_t new_size = my_size*2; - size_t new_nelements = nelements; // internal_free_buffer zeroes this - list_array_type new_elements_array = NULL; - pointer_array_type new_pointer_array = NULL; - list_array_type new_free_list = NULL; - { - DoCleanup my_cleanup(new_pointer_array, new_elements_array, new_size); - new_elements_array = elements_array_allocator().allocate(my_size); - new_pointer_array = pointer_array_allocator_type().allocate(new_size); - for(size_t i=0; i < new_size; ++i) new_pointer_array[i] = NULL; - set_up_free_list(&new_free_list, new_elements_array, my_size ); - - for(size_t i=0; i < my_size; ++i) { - for( element_type* op = pointer_array[i]; op; op = (element_type *)(op->second)) { - value_type *ov = reinterpret_cast(&(op->first)); - // could have std::move semantics - internal_insert_with_key(new_pointer_array, new_size, new_free_list, *ov); - } - } - my_cleanup.my_pa = NULL; - my_cleanup.my_elements = NULL; - } - - internal_free_buffer(pointer_array, elements_array, my_size, nelements); - free_list = new_free_list; - pointer_array = new_pointer_array; - elements_array = new_elements_array; - my_size = new_size; - nelements = new_nelements; - } - - // v should have perfect forwarding if std::move implemented. - // we use this method to move elements in grow_array, so can't use class fields - void internal_insert_with_key( element_type **p_pointer_array, size_t p_sz, list_array_type &p_free_list, - const value_type &v) { - size_t l_mask = p_sz-1; - __TBB_ASSERT(my_key, "Error: value-to-key functor not provided"); - size_t h = this->hash((*my_key)(v)) & l_mask; - __TBB_ASSERT(p_free_list, "Error: free list not set up."); - element_type* my_elem = p_free_list; p_free_list = (element_type *)(p_free_list->second); - (void) new(&(my_elem->first)) value_type(v); - my_elem->second = p_pointer_array[h]; - p_pointer_array[h] = my_elem; - } - - void internal_initialize_buffer() { - pointer_array = pointer_array_allocator_type().allocate(my_size); - for(size_t i = 0; i < my_size; ++i) pointer_array[i] = NULL; - elements_array = elements_array_allocator().allocate(my_size / 2); - set_up_free_list(&free_list, elements_array, my_size / 2); - } - - // made static so an enclosed class can use to properly dispose of the internals - static void internal_free_buffer( pointer_array_type &pa, list_array_type &el, size_t &sz, size_t &ne ) { - if(pa) { - for(size_t i = 0; i < sz; ++i ) { - element_type *p_next; - for( element_type *p = pa[i]; p; p = p_next) { - p_next = (element_type *)p->second; - internal::punned_cast(&(p->first))->~value_type(); - } - } - pointer_array_allocator_type().deallocate(pa, sz); - pa = NULL; - } - // Separate test (if allocation of pa throws, el may be allocated. - // but no elements will be constructed.) - if(el) { - elements_array_allocator().deallocate(el, sz / 2); - el = NULL; - } - sz = INITIAL_SIZE; - ne = 0; - } - -public: - hash_buffer() : my_key(NULL), my_size(INITIAL_SIZE), nelements(0) { - internal_initialize_buffer(); - } - - ~hash_buffer() { - internal_free_buffer(pointer_array, elements_array, my_size, nelements); - if(my_key) delete my_key; - } - - void reset() { - internal_free_buffer(pointer_array, elements_array, my_size, nelements); - internal_initialize_buffer(); - } - - // Take ownership of func object allocated with new. - // This method is only used internally, so can't be misused by user. - void set_key_func(ValueToKey *vtk) { my_key = vtk; } - // pointer is used to clone() - ValueToKey* get_key_func() { return my_key; } - - bool insert_with_key(const value_type &v) { - pointer_type p = NULL; - __TBB_ASSERT(my_key, "Error: value-to-key functor not provided"); - if(find_ref_with_key((*my_key)(v), p)) { - p->~value_type(); - (void) new(p) value_type(v); // copy-construct into the space - return false; - } - ++nelements; - if(nelements*2 > my_size) grow_array(); - internal_insert_with_key(pointer_array, my_size, free_list, v); - return true; - } - - // returns true and sets v to array element if found, else returns false. - bool find_ref_with_key(const Knoref& k, pointer_type &v) { - size_t i = this->hash(k) & mask(); - for(element_type* p = pointer_array[i]; p; p = (element_type *)(p->second)) { - pointer_type pv = reinterpret_cast(&(p->first)); - __TBB_ASSERT(my_key, "Error: value-to-key functor not provided"); - if(this->equal((*my_key)(*pv), k)) { - v = pv; - return true; - } - } - return false; - } - - bool find_with_key( const Knoref& k, value_type &v) { - value_type *p; - if(find_ref_with_key(k, p)) { - v = *p; - return true; - } - else - return false; - } - - void delete_with_key(const Knoref& k) { - size_t h = this->hash(k) & mask(); - element_type* prev = NULL; - for(element_type* p = pointer_array[h]; p; prev = p, p = (element_type *)(p->second)) { - value_type *vp = reinterpret_cast(&(p->first)); - __TBB_ASSERT(my_key, "Error: value-to-key functor not provided"); - if(this->equal((*my_key)(*vp), k)) { - vp->~value_type(); - if(prev) prev->second = p->second; - else pointer_array[h] = (element_type *)(p->second); - p->second = free_list; - free_list = p; - --nelements; - return; - } - } - __TBB_ASSERT(false, "key not found for delete"); - } -}; -#endif // __TBB__flow_graph_hash_buffer_impl_H diff --git a/inst/include/tbb/internal/_flow_graph_trace_impl.h b/inst/include/tbb/internal/_flow_graph_trace_impl.h deleted file mode 100644 index b798888df..000000000 --- a/inst/include/tbb/internal/_flow_graph_trace_impl.h +++ /dev/null @@ -1,238 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef _FGT_GRAPH_TRACE_IMPL_H -#define _FGT_GRAPH_TRACE_IMPL_H - -#include "../tbb_profiling.h" - -namespace tbb { - namespace internal { - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - -static inline void fgt_internal_create_input_port( void *node, void *p, string_index name_index ) { - itt_make_task_group( ITT_DOMAIN_FLOW, p, FLOW_INPUT_PORT, node, FLOW_NODE, name_index ); -} - -static inline void fgt_internal_create_output_port( void *node, void *p, string_index name_index ) { - itt_make_task_group( ITT_DOMAIN_FLOW, p, FLOW_OUTPUT_PORT, node, FLOW_NODE, name_index ); -} - -template -void register_input_port(void *node, tbb::flow::receiver* port, string_index name_index) { - //TODO: Make fgt_internal_create_input_port a function template? - fgt_internal_create_input_port( node, port, name_index); -} - -template < typename PortsTuple, int N > -struct fgt_internal_input_helper { - static void register_port( void *node, PortsTuple &ports ) { - register_input_port( node, &(tbb::flow::get(ports)), static_cast(FLOW_INPUT_PORT_0 + N - 1) ); - fgt_internal_input_helper::register_port( node, ports ); - } -}; - -template < typename PortsTuple > -struct fgt_internal_input_helper { - static void register_port( void *node, PortsTuple &ports ) { - register_input_port( node, &(tbb::flow::get<0>(ports)), FLOW_INPUT_PORT_0 ); - } -}; - -template -void register_output_port(void *node, tbb::flow::sender* port, string_index name_index) { - //TODO: Make fgt_internal_create_output_port a function template? - fgt_internal_create_output_port( node, static_cast(port), name_index); -} - -template < typename PortsTuple, int N > -struct fgt_internal_output_helper { - static void register_port( void *node, PortsTuple &ports ) { - register_output_port( node, &(tbb::flow::get(ports)), static_cast(FLOW_OUTPUT_PORT_0 + N - 1) ); - fgt_internal_output_helper::register_port( node, ports ); - } -}; - -template < typename PortsTuple > -struct fgt_internal_output_helper { - static void register_port( void *node, PortsTuple &ports ) { - register_output_port( node, &(tbb::flow::get<0>(ports)), FLOW_OUTPUT_PORT_0 ); - } -}; - -template< typename NodeType > -void fgt_multioutput_node_desc( const NodeType *node, const char *desc ) { - void *addr = (void *)( static_cast< tbb::flow::receiver< typename NodeType::input_type > * >(const_cast< NodeType *>(node)) ); - itt_metadata_str_add( ITT_DOMAIN_FLOW, addr, FLOW_NODE, FLOW_OBJECT_NAME, desc ); -} - -template< typename NodeType > -void fgt_multiinput_multioutput_node_desc( const NodeType *node, const char *desc ) { - void *addr = const_cast(node); - itt_metadata_str_add( ITT_DOMAIN_FLOW, addr, FLOW_NODE, FLOW_OBJECT_NAME, desc ); -} - -template< typename NodeType > -static inline void fgt_node_desc( const NodeType *node, const char *desc ) { - void *addr = (void *)( static_cast< tbb::flow::sender< typename NodeType::output_type > * >(const_cast< NodeType *>(node)) ); - itt_metadata_str_add( ITT_DOMAIN_FLOW, addr, FLOW_NODE, FLOW_OBJECT_NAME, desc ); -} - -static inline void fgt_graph_desc( void *g, const char *desc ) { - itt_metadata_str_add( ITT_DOMAIN_FLOW, g, FLOW_GRAPH, FLOW_OBJECT_NAME, desc ); -} - -static inline void fgt_body( void *node, void *body ) { - itt_relation_add( ITT_DOMAIN_FLOW, body, FLOW_BODY, __itt_relation_is_child_of, node, FLOW_NODE ); -} - -template< int N, typename PortsTuple > -static inline void fgt_multioutput_node( string_index t, void *g, void *input_port, PortsTuple &ports ) { - itt_make_task_group( ITT_DOMAIN_FLOW, input_port, FLOW_NODE, g, FLOW_GRAPH, t ); - fgt_internal_create_input_port( input_port, input_port, FLOW_INPUT_PORT_0 ); - fgt_internal_output_helper::register_port( input_port, ports ); -} - -template< int N, typename PortsTuple > -static inline void fgt_multioutput_node_with_body( string_index t, void *g, void *input_port, PortsTuple &ports, void *body ) { - itt_make_task_group( ITT_DOMAIN_FLOW, input_port, FLOW_NODE, g, FLOW_GRAPH, t ); - fgt_internal_create_input_port( input_port, input_port, FLOW_INPUT_PORT_0 ); - fgt_internal_output_helper::register_port( input_port, ports ); - fgt_body( input_port, body ); -} - -template< int N, typename PortsTuple > -static inline void fgt_multiinput_node( string_index t, void *g, PortsTuple &ports, void *output_port) { - itt_make_task_group( ITT_DOMAIN_FLOW, output_port, FLOW_NODE, g, FLOW_GRAPH, t ); - fgt_internal_create_output_port( output_port, output_port, FLOW_OUTPUT_PORT_0 ); - fgt_internal_input_helper::register_port( output_port, ports ); -} - -static inline void fgt_node( string_index t, void *g, void *output_port ) { - itt_make_task_group( ITT_DOMAIN_FLOW, output_port, FLOW_NODE, g, FLOW_GRAPH, t ); - fgt_internal_create_output_port( output_port, output_port, FLOW_OUTPUT_PORT_0 ); -} - -static inline void fgt_node_with_body( string_index t, void *g, void *output_port, void *body ) { - itt_make_task_group( ITT_DOMAIN_FLOW, output_port, FLOW_NODE, g, FLOW_GRAPH, t ); - fgt_internal_create_output_port( output_port, output_port, FLOW_OUTPUT_PORT_0 ); - fgt_body( output_port, body ); -} - - -static inline void fgt_node( string_index t, void *g, void *input_port, void *output_port ) { - fgt_node( t, g, output_port ); - fgt_internal_create_input_port( output_port, input_port, FLOW_INPUT_PORT_0 ); -} - -static inline void fgt_node_with_body( string_index t, void *g, void *input_port, void *output_port, void *body ) { - fgt_node_with_body( t, g, output_port, body ); - fgt_internal_create_input_port( output_port, input_port, FLOW_INPUT_PORT_0 ); -} - - -static inline void fgt_node( string_index t, void *g, void *input_port, void *decrement_port, void *output_port ) { - fgt_node( t, g, input_port, output_port ); - fgt_internal_create_input_port( output_port, decrement_port, FLOW_INPUT_PORT_1 ); -} - -static inline void fgt_make_edge( void *output_port, void *input_port ) { - itt_relation_add( ITT_DOMAIN_FLOW, output_port, FLOW_OUTPUT_PORT, __itt_relation_is_predecessor_to, input_port, FLOW_INPUT_PORT); -} - -static inline void fgt_remove_edge( void *output_port, void *input_port ) { - itt_relation_add( ITT_DOMAIN_FLOW, output_port, FLOW_OUTPUT_PORT, __itt_relation_is_sibling_of, input_port, FLOW_INPUT_PORT); -} - -static inline void fgt_graph( void *g ) { - itt_make_task_group( ITT_DOMAIN_FLOW, g, FLOW_GRAPH, NULL, FLOW_NULL, FLOW_GRAPH ); -} - -static inline void fgt_begin_body( void *body ) { - itt_task_begin( ITT_DOMAIN_FLOW, body, FLOW_BODY, NULL, FLOW_NULL, FLOW_BODY ); -} - -static inline void fgt_end_body( void * ) { - itt_task_end( ITT_DOMAIN_FLOW ); -} - -static inline void fgt_async_try_put_begin( void *node, void *port ) { - itt_task_begin( ITT_DOMAIN_FLOW, port, FLOW_OUTPUT_PORT, node, FLOW_NODE, FLOW_OUTPUT_PORT ); -} - -static inline void fgt_async_try_put_end( void *, void * ) { - itt_task_end( ITT_DOMAIN_FLOW ); -} - -static inline void fgt_async_reserve( void *node, void *graph ) { - itt_region_begin( ITT_DOMAIN_FLOW, node, FLOW_NODE, graph, FLOW_GRAPH, FLOW_NULL ); -} - -static inline void fgt_async_commit( void *node, void *graph ) { - itt_region_end( ITT_DOMAIN_FLOW, node, FLOW_NODE ); -} - -#else // TBB_PREVIEW_FLOW_GRAPH_TRACE - -static inline void fgt_graph( void * /*g*/ ) { } - -template< typename NodeType > -static inline void fgt_multioutput_node_desc( const NodeType * /*node*/, const char * /*desc*/ ) { } - -template< typename NodeType > -static inline void fgt_node_desc( const NodeType * /*node*/, const char * /*desc*/ ) { } - -static inline void fgt_graph_desc( void * /*g*/, const char * /*desc*/ ) { } - -static inline void fgt_body( void * /*node*/, void * /*body*/ ) { } - -template< int N, typename PortsTuple > -static inline void fgt_multioutput_node( string_index /*t*/, void * /*g*/, void * /*input_port*/, PortsTuple & /*ports*/ ) { } - -template< int N, typename PortsTuple > -static inline void fgt_multioutput_node_with_body( string_index /*t*/, void * /*g*/, void * /*input_port*/, PortsTuple & /*ports*/, void * /*body*/ ) { } - -template< int N, typename PortsTuple > -static inline void fgt_multiinput_node( string_index /*t*/, void * /*g*/, PortsTuple & /*ports*/, void * /*output_port*/ ) { } - -static inline void fgt_node( string_index /*t*/, void * /*g*/, void * /*output_port*/ ) { } -static inline void fgt_node( string_index /*t*/, void * /*g*/, void * /*input_port*/, void * /*output_port*/ ) { } -static inline void fgt_node( string_index /*t*/, void * /*g*/, void * /*input_port*/, void * /*decrement_port*/, void * /*output_port*/ ) { } - -static inline void fgt_node_with_body( string_index /*t*/, void * /*g*/, void * /*output_port*/, void * /*body*/ ) { } -static inline void fgt_node_with_body( string_index /*t*/, void * /*g*/, void * /*input_port*/, void * /*output_port*/, void * /*body*/ ) { } - -static inline void fgt_make_edge( void * /*output_port*/, void * /*input_port*/ ) { } -static inline void fgt_remove_edge( void * /*output_port*/, void * /*input_port*/ ) { } - -static inline void fgt_begin_body( void * /*body*/ ) { } -static inline void fgt_end_body( void * /*body*/) { } -static inline void fgt_async_try_put_begin( void * /*node*/, void * /*port*/ ) { } -static inline void fgt_async_try_put_end( void * /*node*/ , void * /*port*/ ) { } -static inline void fgt_async_reserve( void * /*node*/, void * /*graph*/ ) { } -static inline void fgt_async_commit( void * /*node*/, void * /*graph*/ ) { } - -#endif // TBB_PREVIEW_FLOW_GRAPH_TRACE - - } // namespace internal -} // namespace tbb - -#endif diff --git a/inst/include/tbb/internal/_flow_graph_types_impl.h b/inst/include/tbb/internal/_flow_graph_types_impl.h deleted file mode 100644 index 73b5f5479..000000000 --- a/inst/include/tbb/internal/_flow_graph_types_impl.h +++ /dev/null @@ -1,713 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB__flow_graph_types_impl_H -#define __TBB__flow_graph_types_impl_H - -#ifndef __TBB_flow_graph_H -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -// included in namespace tbb::flow::interfaceX - -namespace internal { - - // the change to key_matching (adding a K and KHash template parameter, making it a class) - // means we have to pass this data to the key_matching_port. All the ports have only one - // template parameter, so we have to wrap the following types in a trait: - // - // . K == key_type - // . KHash == hash and compare for Key - // . TtoK == function_body that given an object of T, returns its K - // . T == type accepted by port, and stored in the hash table - // - // The port will have an additional parameter on node construction, which is a function_body - // that accepts a const T& and returns a K which is the field in T which is its K. - template - struct KeyTrait { - typedef Kp K; - typedef Tp T; - typedef internal::type_to_key_function_body TtoK; - typedef KHashp KHash; - }; - -// wrap each element of a tuple in a template, and make a tuple of the result. - template class PT, typename TypeTuple> - struct wrap_tuple_elements; - - // A wrapper that generates the traits needed for each port of a key-matching join, - // and the type of the tuple of input ports. - template class PT, typename KeyTraits, typename TypeTuple> - struct wrap_key_tuple_elements; - - template class PT, typename TypeTuple> - struct wrap_tuple_elements<1, PT, TypeTuple> { - typedef typename tbb::flow::tuple< - PT::type> > - type; - }; - - template class PT, typename KeyTraits, typename TypeTuple> - struct wrap_key_tuple_elements<1, PT, KeyTraits, TypeTuple > { - typedef typename KeyTraits::key_type K; - typedef typename KeyTraits::hash_compare_type KHash; - typedef KeyTrait::type> KeyTrait0; - typedef typename tbb::flow::tuple< PT > type; - }; - - template class PT, typename TypeTuple> - struct wrap_tuple_elements<2, PT, TypeTuple> { - typedef typename tbb::flow::tuple< - PT::type>, - PT::type> > - type; - }; - - template class PT, typename KeyTraits, typename TypeTuple> - struct wrap_key_tuple_elements<2, PT, KeyTraits, TypeTuple> { - typedef typename KeyTraits::key_type K; - typedef typename KeyTraits::hash_compare_type KHash; - typedef KeyTrait::type> KeyTrait0; - typedef KeyTrait::type> KeyTrait1; - typedef typename tbb::flow::tuple< PT, PT > type; - }; - - template class PT, typename TypeTuple> - struct wrap_tuple_elements<3, PT, TypeTuple> { - typedef typename tbb::flow::tuple< - PT::type>, - PT::type>, - PT::type> > - type; - }; - - template class PT, typename KeyTraits, typename TypeTuple> - struct wrap_key_tuple_elements<3, PT, KeyTraits, TypeTuple> { - typedef typename KeyTraits::key_type K; - typedef typename KeyTraits::hash_compare_type KHash; - typedef KeyTrait::type> KeyTrait0; - typedef KeyTrait::type> KeyTrait1; - typedef KeyTrait::type> KeyTrait2; - typedef typename tbb::flow::tuple< PT, PT, PT > type; - }; - - template class PT, typename TypeTuple> - struct wrap_tuple_elements<4, PT, TypeTuple> { - typedef typename tbb::flow::tuple< - PT::type>, - PT::type>, - PT::type>, - PT::type> > - type; - }; - - template class PT, typename KeyTraits, typename TypeTuple> - struct wrap_key_tuple_elements<4, PT, KeyTraits, TypeTuple> { - typedef typename KeyTraits::key_type K; - typedef typename KeyTraits::hash_compare_type KHash; - typedef KeyTrait::type> KeyTrait0; - typedef KeyTrait::type> KeyTrait1; - typedef KeyTrait::type> KeyTrait2; - typedef KeyTrait::type> KeyTrait3; - typedef typename tbb::flow::tuple< PT, PT, PT, - PT > type; - }; - - template class PT, typename TypeTuple> - struct wrap_tuple_elements<5, PT, TypeTuple> { - typedef typename tbb::flow::tuple< - PT::type>, - PT::type>, - PT::type>, - PT::type>, - PT::type> > - type; - }; - - template class PT, typename KeyTraits, typename TypeTuple> - struct wrap_key_tuple_elements<5, PT, KeyTraits, TypeTuple> { - typedef typename KeyTraits::key_type K; - typedef typename KeyTraits::hash_compare_type KHash; - typedef KeyTrait::type> KeyTrait0; - typedef KeyTrait::type> KeyTrait1; - typedef KeyTrait::type> KeyTrait2; - typedef KeyTrait::type> KeyTrait3; - typedef KeyTrait::type> KeyTrait4; - typedef typename tbb::flow::tuple< PT, PT, PT, - PT, PT > type; - }; - -#if __TBB_VARIADIC_MAX >= 6 - template class PT, typename TypeTuple> - struct wrap_tuple_elements<6, PT, TypeTuple> { - typedef typename tbb::flow::tuple< - PT::type>, - PT::type>, - PT::type>, - PT::type>, - PT::type>, - PT::type> > - type; - }; - - template class PT, typename KeyTraits, typename TypeTuple> - struct wrap_key_tuple_elements<6, PT, KeyTraits, TypeTuple> { - typedef typename KeyTraits::key_type K; - typedef typename KeyTraits::hash_compare_type KHash; - typedef KeyTrait::type> KeyTrait0; - typedef KeyTrait::type> KeyTrait1; - typedef KeyTrait::type> KeyTrait2; - typedef KeyTrait::type> KeyTrait3; - typedef KeyTrait::type> KeyTrait4; - typedef KeyTrait::type> KeyTrait5; - typedef typename tbb::flow::tuple< PT, PT, PT, PT, - PT, PT > type; - }; -#endif - -#if __TBB_VARIADIC_MAX >= 7 - template class PT, typename TypeTuple> - struct wrap_tuple_elements<7, PT, TypeTuple> { - typedef typename tbb::flow::tuple< - PT::type>, - PT::type>, - PT::type>, - PT::type>, - PT::type>, - PT::type>, - PT::type> > - type; - }; - - template class PT, typename KeyTraits, typename TypeTuple> - struct wrap_key_tuple_elements<7, PT, KeyTraits, TypeTuple> { - typedef typename KeyTraits::key_type K; - typedef typename KeyTraits::hash_compare_type KHash; - typedef KeyTrait::type> KeyTrait0; - typedef KeyTrait::type> KeyTrait1; - typedef KeyTrait::type> KeyTrait2; - typedef KeyTrait::type> KeyTrait3; - typedef KeyTrait::type> KeyTrait4; - typedef KeyTrait::type> KeyTrait5; - typedef KeyTrait::type> KeyTrait6; - typedef typename tbb::flow::tuple< PT, PT, PT, PT, - PT, PT, PT > type; - }; -#endif - -#if __TBB_VARIADIC_MAX >= 8 - template class PT, typename TypeTuple> - struct wrap_tuple_elements<8, PT, TypeTuple> { - typedef typename tbb::flow::tuple< - PT::type>, - PT::type>, - PT::type>, - PT::type>, - PT::type>, - PT::type>, - PT::type>, - PT::type> > - type; - }; - - template class PT, typename KeyTraits, typename TypeTuple> - struct wrap_key_tuple_elements<8, PT, KeyTraits, TypeTuple> { - typedef typename KeyTraits::key_type K; - typedef typename KeyTraits::hash_compare_type KHash; - typedef KeyTrait::type> KeyTrait0; - typedef KeyTrait::type> KeyTrait1; - typedef KeyTrait::type> KeyTrait2; - typedef KeyTrait::type> KeyTrait3; - typedef KeyTrait::type> KeyTrait4; - typedef KeyTrait::type> KeyTrait5; - typedef KeyTrait::type> KeyTrait6; - typedef KeyTrait::type> KeyTrait7; - typedef typename tbb::flow::tuple< PT, PT, PT, PT, - PT, PT, PT, PT > type; - }; -#endif - -#if __TBB_VARIADIC_MAX >= 9 - template class PT, typename TypeTuple> - struct wrap_tuple_elements<9, PT, TypeTuple> { - typedef typename tbb::flow::tuple< - PT::type>, - PT::type>, - PT::type>, - PT::type>, - PT::type>, - PT::type>, - PT::type>, - PT::type>, - PT::type> > - type; - }; - - template class PT, typename KeyTraits, typename TypeTuple> - struct wrap_key_tuple_elements<9, PT, KeyTraits, TypeTuple> { - typedef typename KeyTraits::key_type K; - typedef typename KeyTraits::hash_compare_type KHash; - typedef KeyTrait::type> KeyTrait0; - typedef KeyTrait::type> KeyTrait1; - typedef KeyTrait::type> KeyTrait2; - typedef KeyTrait::type> KeyTrait3; - typedef KeyTrait::type> KeyTrait4; - typedef KeyTrait::type> KeyTrait5; - typedef KeyTrait::type> KeyTrait6; - typedef KeyTrait::type> KeyTrait7; - typedef KeyTrait::type> KeyTrait8; - typedef typename tbb::flow::tuple< PT, PT, PT, PT, - PT, PT, PT, PT, PT > type; - }; -#endif - -#if __TBB_VARIADIC_MAX >= 10 - template class PT, typename TypeTuple> - struct wrap_tuple_elements<10, PT, TypeTuple> { - typedef typename tbb::flow::tuple< - PT::type>, - PT::type>, - PT::type>, - PT::type>, - PT::type>, - PT::type>, - PT::type>, - PT::type>, - PT::type>, - PT::type> > - type; - }; - - template class PT, typename KeyTraits, typename TypeTuple> - struct wrap_key_tuple_elements<10, PT, KeyTraits, TypeTuple> { - typedef typename KeyTraits::key_type K; - typedef typename KeyTraits::hash_compare_type KHash; - typedef KeyTrait::type> KeyTrait0; - typedef KeyTrait::type> KeyTrait1; - typedef KeyTrait::type> KeyTrait2; - typedef KeyTrait::type> KeyTrait3; - typedef KeyTrait::type> KeyTrait4; - typedef KeyTrait::type> KeyTrait5; - typedef KeyTrait::type> KeyTrait6; - typedef KeyTrait::type> KeyTrait7; - typedef KeyTrait::type> KeyTrait8; - typedef KeyTrait::type> KeyTrait9; - typedef typename tbb::flow::tuple< PT, PT, PT, PT, - PT, PT, PT, PT, PT, - PT > type; - }; -#endif - -#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT - template< int... S > class sequence {}; - - template< int N, int... S > - struct make_sequence : make_sequence < N - 1, N - 1, S... > {}; - - template< int... S > - struct make_sequence < 0, S... > { - typedef sequence type; - }; -#endif /* __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT */ - -#if __TBB_INITIALIZER_LISTS_PRESENT - // Until C++14 std::initializer_list does not guarantee life time of contained objects. - template - class initializer_list_wrapper { - public: - typedef T value_type; - typedef const T& reference; - typedef const T& const_reference; - typedef size_t size_type; - - typedef T* iterator; - typedef const T* const_iterator; - - initializer_list_wrapper( std::initializer_list il ) __TBB_NOEXCEPT( true ) : my_begin( static_cast(malloc( il.size()*sizeof( T ) )) ) { - iterator dst = my_begin; - for ( typename std::initializer_list::const_iterator src = il.begin(); src != il.end(); ++src ) - new (dst++) T( *src ); - my_end = dst; - } - - initializer_list_wrapper( const initializer_list_wrapper& ilw ) __TBB_NOEXCEPT( true ) : my_begin( static_cast(malloc( ilw.size()*sizeof( T ) )) ) { - iterator dst = my_begin; - for ( typename std::initializer_list::const_iterator src = ilw.begin(); src != ilw.end(); ++src ) - new (dst++) T( *src ); - my_end = dst; - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - initializer_list_wrapper( initializer_list_wrapper&& ilw ) __TBB_NOEXCEPT( true ) : my_begin( ilw.my_begin ), my_end( ilw.my_end ) { - ilw.my_begin = ilw.my_end = NULL; - } -#endif /* __TBB_CPP11_RVALUE_REF_PRESENT */ - - ~initializer_list_wrapper() { - if ( my_begin ) - free( my_begin ); - } - - const_iterator begin() const __TBB_NOEXCEPT(true) { return my_begin; } - const_iterator end() const __TBB_NOEXCEPT(true) { return my_end; } - size_t size() const __TBB_NOEXCEPT(true) { return (size_t)(my_end - my_begin); } - - private: - iterator my_begin; - iterator my_end; - }; -#endif /* __TBB_INITIALIZER_LISTS_PRESENT */ - -//! type mimicking std::pair but with trailing fill to ensure each element of an array -//* will have the correct alignment - template - struct type_plus_align { - char first[sizeof(T1)]; - T2 second; - char fill1[REM]; - }; - - template - struct type_plus_align { - char first[sizeof(T1)]; - T2 second; - }; - - template struct alignment_of { - typedef struct { char t; U padded; } test_alignment; - static const size_t value = sizeof(test_alignment) - sizeof(U); - }; - - // T1, T2 are actual types stored. The space defined for T1 in the type returned - // is a char array of the correct size. Type T2 should be trivially-constructible, - // T1 must be explicitly managed. - template - struct aligned_pair { - static const size_t t1_align = alignment_of::value; - static const size_t t2_align = alignment_of::value; - typedef type_plus_align just_pair; - static const size_t max_align = t1_align < t2_align ? t2_align : t1_align; - static const size_t extra_bytes = sizeof(just_pair) % max_align; - static const size_t remainder = extra_bytes ? max_align - extra_bytes : 0; - public: - typedef type_plus_align type; - }; // aligned_pair - -// support for variant type -// type we use when we're not storing a value -struct default_constructed { }; - -// type which contains another type, tests for what type is contained, and references to it. -// internal::Wrapper -// void CopyTo( void *newSpace) : builds a Wrapper copy of itself in newSpace - -// struct to allow us to copy and test the type of objects -struct WrapperBase { - virtual ~WrapperBase() {} - virtual void CopyTo(void* /*newSpace*/) const { } -}; - -// Wrapper contains a T, with the ability to test what T is. The Wrapper can be -// constructed from a T, can be copy-constructed from another Wrapper, and can be -// examined via value(), but not modified. -template -struct Wrapper: public WrapperBase { - typedef T value_type; - typedef T* pointer_type; -private: - T value_space; -public: - const value_type &value() const { return value_space; } - -private: - Wrapper(); - - // on exception will ensure the Wrapper will contain only a trivially-constructed object - struct _unwind_space { - pointer_type space; - _unwind_space(pointer_type p) : space(p) {} - ~_unwind_space() { - if(space) (void) new (space) Wrapper(default_constructed()); - } - }; -public: - explicit Wrapper( const T& other ) : value_space(other) { } - explicit Wrapper(const Wrapper& other) : value_space(other.value_space) { } - - void CopyTo(void* newSpace) const __TBB_override { - _unwind_space guard((pointer_type)newSpace); - (void) new(newSpace) Wrapper(value_space); - guard.space = NULL; - } - ~Wrapper() { } -}; - -// specialization for array objects -template -struct Wrapper : public WrapperBase { - typedef T value_type; - typedef T* pointer_type; - // space must be untyped. - typedef T ArrayType[N]; -private: - // The space is not of type T[N] because when copy-constructing, it would be - // default-initialized and then copied to in some fashion, resulting in two - // constructions and one destruction per element. If the type is char[ ], we - // placement new into each element, resulting in one construction per element. - static const size_t space_size = sizeof(ArrayType) / sizeof(char); - char value_space[space_size]; - - - // on exception will ensure the already-built objects will be destructed - // (the value_space is a char array, so it is already trivially-destructible.) - struct _unwind_class { - pointer_type space; - int already_built; - _unwind_class(pointer_type p) : space(p), already_built(0) {} - ~_unwind_class() { - if(space) { - for(size_t i = already_built; i > 0 ; --i ) space[i-1].~value_type(); - (void) new(space) Wrapper(default_constructed()); - } - } - }; -public: - const ArrayType &value() const { - char *vp = const_cast(value_space); - return reinterpret_cast(*vp); - } - -private: - Wrapper(); -public: - // have to explicitly construct because other decays to a const value_type* - explicit Wrapper(const ArrayType& other) { - _unwind_class guard((pointer_type)value_space); - pointer_type vp = reinterpret_cast(&value_space); - for(size_t i = 0; i < N; ++i ) { - (void) new(vp++) value_type(other[i]); - ++(guard.already_built); - } - guard.space = NULL; - } - explicit Wrapper(const Wrapper& other) : WrapperBase() { - // we have to do the heavy lifting to copy contents - _unwind_class guard((pointer_type)value_space); - pointer_type dp = reinterpret_cast(value_space); - pointer_type sp = reinterpret_cast(const_cast(other.value_space)); - for(size_t i = 0; i < N; ++i, ++dp, ++sp) { - (void) new(dp) value_type(*sp); - ++(guard.already_built); - } - guard.space = NULL; - } - - void CopyTo(void* newSpace) const __TBB_override { - (void) new(newSpace) Wrapper(*this); // exceptions handled in copy constructor - } - - ~Wrapper() { - // have to destroy explicitly in reverse order - pointer_type vp = reinterpret_cast(&value_space); - for(size_t i = N; i > 0 ; --i ) vp[i-1].~value_type(); - } -}; - -// given a tuple, return the type of the element that has the maximum alignment requirement. -// Given a tuple and that type, return the number of elements of the object with the max -// alignment requirement that is at least as big as the largest object in the tuple. - -template struct pick_one; -template struct pick_one { typedef T1 type; }; -template struct pick_one { typedef T2 type; }; - -template< template class Selector, typename T1, typename T2 > -struct pick_max { - typedef typename pick_one< (Selector::value > Selector::value), T1, T2 >::type type; -}; - -template struct size_of { static const int value = sizeof(T); }; - -template< size_t N, class Tuple, template class Selector > struct pick_tuple_max { - typedef typename pick_tuple_max::type LeftMaxType; - typedef typename tbb::flow::tuple_element::type ThisType; - typedef typename pick_max::type type; -}; - -template< class Tuple, template class Selector > struct pick_tuple_max<0, Tuple, Selector> { - typedef typename tbb::flow::tuple_element<0, Tuple>::type type; -}; - -// is the specified type included in a tuple? -template -struct is_element_of { - typedef typename tbb::flow::tuple_element::type T_i; - static const bool value = tbb::internal::is_same_type::value || is_element_of::value; -}; - -template -struct is_element_of { - typedef typename tbb::flow::tuple_element<0, Tuple>::type T_i; - static const bool value = tbb::internal::is_same_type::value; -}; - -// allow the construction of types that are listed tuple. If a disallowed type -// construction is written, a method involving this type is created. The -// type has no definition, so a syntax error is generated. -template struct ERROR_Type_Not_allowed_In_Tagged_Msg_Not_Member_Of_Tuple; - -template struct do_if; -template -struct do_if { - static void construct(void *mySpace, const T& x) { - (void) new(mySpace) Wrapper(x); - } -}; -template -struct do_if { - static void construct(void * /*mySpace*/, const T& x) { - // This method is instantiated when the type T does not match any of the - // element types in the Tuple in variant. - ERROR_Type_Not_allowed_In_Tagged_Msg_Not_Member_Of_Tuple::bad_type(x); - } -}; - -// Tuple tells us the allowed types that variant can hold. It determines the alignment of the space in -// Wrapper, and how big Wrapper is. -// -// the object can only be tested for type, and a read-only reference can be fetched by cast_to(). - -using tbb::internal::punned_cast; -struct tagged_null_type {}; -template -class tagged_msg { - typedef tbb::flow::tuple= 6 - , T5 - #endif - #if __TBB_VARIADIC_MAX >= 7 - , T6 - #endif - #if __TBB_VARIADIC_MAX >= 8 - , T7 - #endif - #if __TBB_VARIADIC_MAX >= 9 - , T8 - #endif - #if __TBB_VARIADIC_MAX >= 10 - , T9 - #endif - > Tuple; - -private: - class variant { - static const size_t N = tbb::flow::tuple_size::value; - typedef typename pick_tuple_max::type AlignType; - typedef typename pick_tuple_max::type MaxSizeType; - static const size_t MaxNBytes = (sizeof(Wrapper)+sizeof(AlignType)-1); - static const size_t MaxNElements = MaxNBytes/sizeof(AlignType); - typedef typename tbb::aligned_space SpaceType; - SpaceType my_space; - static const size_t MaxSize = sizeof(SpaceType); - - public: - variant() { (void) new(&my_space) Wrapper(default_constructed()); } - - template - variant( const T& x ) { - do_if::value>::construct(&my_space,x); - } - - variant(const variant& other) { - const WrapperBase * h = punned_cast(&(other.my_space)); - h->CopyTo(&my_space); - } - - // assignment must destroy and re-create the Wrapper type, as there is no way - // to create a Wrapper-to-Wrapper assign even if we find they agree in type. - void operator=( const variant& rhs ) { - if(&rhs != this) { - WrapperBase *h = punned_cast(&my_space); - h->~WrapperBase(); - const WrapperBase *ch = punned_cast(&(rhs.my_space)); - ch->CopyTo(&my_space); - } - } - - template - const U& variant_cast_to() const { - const Wrapper *h = dynamic_cast*>(punned_cast(&my_space)); - if(!h) { - tbb::internal::throw_exception(tbb::internal::eid_bad_tagged_msg_cast); - } - return h->value(); - } - template - bool variant_is_a() const { return dynamic_cast*>(punned_cast(&my_space)) != NULL; } - - bool variant_is_default_constructed() const {return variant_is_a();} - - ~variant() { - WrapperBase *h = punned_cast(&my_space); - h->~WrapperBase(); - } - }; //class variant - - TagType my_tag; - variant my_msg; - -public: - tagged_msg(): my_tag(TagType(~0)), my_msg(){} - - template - tagged_msg(T const &index, R const &value) : my_tag(index), my_msg(value) {} - - #if __TBB_CONST_REF_TO_ARRAY_TEMPLATE_PARAM_BROKEN - template - tagged_msg(T const &index, R (&value)[N]) : my_tag(index), my_msg(value) {} - #endif - - void set_tag(TagType const &index) {my_tag = index;} - TagType tag() const {return my_tag;} - - template - const V& cast_to() const {return my_msg.template variant_cast_to();} - - template - bool is_a() const {return my_msg.template variant_is_a();} - - bool is_default_constructed() const {return my_msg.variant_is_default_constructed();} -}; //class tagged_msg - -// template to simplify cast and test for tagged_msg in template contexts -template -const V& cast_to(T const &t) { return t.template cast_to(); } - -template -bool is_a(T const &t) { return t.template is_a(); } - -enum op_stat { WAIT = 0, SUCCEEDED, FAILED }; - -} // namespace internal - -#endif /* __TBB__flow_graph_types_impl_H */ diff --git a/inst/include/tbb/internal/_mutex_padding.h b/inst/include/tbb/internal/_mutex_padding.h deleted file mode 100644 index 6c1d9b59e..000000000 --- a/inst/include/tbb/internal/_mutex_padding.h +++ /dev/null @@ -1,102 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_mutex_padding_H -#define __TBB_mutex_padding_H - -// wrapper for padding mutexes to be alone on a cache line, without requiring they be allocated -// from a pool. Because we allow them to be defined anywhere they must be two cache lines in size. - - -namespace tbb { -namespace interface7 { -namespace internal { - -static const size_t cache_line_size = 64; - -// Pad a mutex to occupy a number of full cache lines sufficient to avoid false sharing -// with other data; space overhead is up to 2*cache_line_size-1. -template class padded_mutex; - -template -class padded_mutex : tbb::internal::mutex_copy_deprecated_and_disabled { - typedef long pad_type; - pad_type my_pad[((sizeof(Mutex)+cache_line_size-1)/cache_line_size+1)*cache_line_size/sizeof(pad_type)]; - - Mutex *impl() { return (Mutex *)((uintptr_t(this)|(cache_line_size-1))+1);} - -public: - static const bool is_rw_mutex = Mutex::is_rw_mutex; - static const bool is_recursive_mutex = Mutex::is_recursive_mutex; - static const bool is_fair_mutex = Mutex::is_fair_mutex; - - padded_mutex() { new(impl()) Mutex(); } - ~padded_mutex() { impl()->~Mutex(); } - - //! Represents acquisition of a mutex. - class scoped_lock : tbb::internal::no_copy { - typename Mutex::scoped_lock my_scoped_lock; - public: - scoped_lock() : my_scoped_lock() {} - scoped_lock( padded_mutex& m ) : my_scoped_lock(*m.impl()) { } - ~scoped_lock() { } - - void acquire( padded_mutex& m ) { my_scoped_lock.acquire(*m.impl()); } - bool try_acquire( padded_mutex& m ) { return my_scoped_lock.try_acquire(*m.impl()); } - void release() { my_scoped_lock.release(); } - }; -}; - -template -class padded_mutex : tbb::internal::mutex_copy_deprecated_and_disabled { - typedef long pad_type; - pad_type my_pad[((sizeof(Mutex)+cache_line_size-1)/cache_line_size+1)*cache_line_size/sizeof(pad_type)]; - - Mutex *impl() { return (Mutex *)((uintptr_t(this)|(cache_line_size-1))+1);} - -public: - static const bool is_rw_mutex = Mutex::is_rw_mutex; - static const bool is_recursive_mutex = Mutex::is_recursive_mutex; - static const bool is_fair_mutex = Mutex::is_fair_mutex; - - padded_mutex() { new(impl()) Mutex(); } - ~padded_mutex() { impl()->~Mutex(); } - - //! Represents acquisition of a mutex. - class scoped_lock : tbb::internal::no_copy { - typename Mutex::scoped_lock my_scoped_lock; - public: - scoped_lock() : my_scoped_lock() {} - scoped_lock( padded_mutex& m, bool write = true ) : my_scoped_lock(*m.impl(),write) { } - ~scoped_lock() { } - - void acquire( padded_mutex& m, bool write = true ) { my_scoped_lock.acquire(*m.impl(),write); } - bool try_acquire( padded_mutex& m, bool write = true ) { return my_scoped_lock.try_acquire(*m.impl(),write); } - bool upgrade_to_writer() { return my_scoped_lock.upgrade_to_writer(); } - bool downgrade_to_reader() { return my_scoped_lock.downgrade_to_reader(); } - void release() { my_scoped_lock.release(); } - }; -}; - -} // namespace internal -} // namespace interface7 -} // namespace tbb - -#endif /* __TBB_mutex_padding_H */ diff --git a/inst/include/tbb/internal/_range_iterator.h b/inst/include/tbb/internal/_range_iterator.h deleted file mode 100644 index 5ebc42e89..000000000 --- a/inst/include/tbb/internal/_range_iterator.h +++ /dev/null @@ -1,70 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_range_iterator_H -#define __TBB_range_iterator_H - -#include "../tbb_stddef.h" - -#if __TBB_CPP11_STD_BEGIN_END_PRESENT && __TBB_CPP11_AUTO_PRESENT && __TBB_CPP11_DECLTYPE_PRESENT - #include -#endif - -namespace tbb { - // iterators to first and last elements of container - namespace internal { - -#if __TBB_CPP11_STD_BEGIN_END_PRESENT && __TBB_CPP11_AUTO_PRESENT && __TBB_CPP11_DECLTYPE_PRESENT - using std::begin; - using std::end; - template - auto first(Container& c)-> decltype(begin(c)) {return begin(c);} - - template - auto first(const Container& c)-> decltype(begin(c)) {return begin(c);} - - template - auto last(Container& c)-> decltype(begin(c)) {return end(c);} - - template - auto last(const Container& c)-> decltype(begin(c)) {return end(c);} -#else - template - typename Container::iterator first(Container& c) {return c.begin();} - - template - typename Container::const_iterator first(const Container& c) {return c.begin();} - - template - typename Container::iterator last(Container& c) {return c.end();} - - template - typename Container::const_iterator last(const Container& c) {return c.end();} -#endif - - template - T* first(T (&arr) [size]) {return arr;} - - template - T* last(T (&arr) [size]) {return arr + size;} - } //namespace internal -} //namespace tbb - -#endif // __TBB_range_iterator_H diff --git a/inst/include/tbb/internal/_tbb_hash_compare_impl.h b/inst/include/tbb/internal/_tbb_hash_compare_impl.h deleted file mode 100644 index 6381e2dc8..000000000 --- a/inst/include/tbb/internal/_tbb_hash_compare_impl.h +++ /dev/null @@ -1,109 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -// must be included outside namespaces. -#ifndef __TBB_tbb_hash_compare_impl_H -#define __TBB_tbb_hash_compare_impl_H - -#include - -namespace tbb { -namespace interface5 { -namespace internal { - -// Template class for hash compare -template -class hash_compare -{ -public: - typedef Hasher hasher; - typedef Key_equality key_equal; - - hash_compare() {} - - hash_compare(Hasher a_hasher) : my_hash_object(a_hasher) {} - - hash_compare(Hasher a_hasher, Key_equality a_keyeq) : my_hash_object(a_hasher), my_key_compare_object(a_keyeq) {} - - size_t operator()(const Key& key) const { - return ((size_t)my_hash_object(key)); - } - - bool operator()(const Key& key1, const Key& key2) const { - // TODO: get rid of the result invertion - return (!my_key_compare_object(key1, key2)); - } - - Hasher my_hash_object; // The hash object - Key_equality my_key_compare_object; // The equality comparator object -}; - -//! Hash multiplier -static const size_t hash_multiplier = tbb::internal::select_size_t_constant<2654435769U, 11400714819323198485ULL>::value; - -} // namespace internal - -//! Hasher functions -template -inline size_t tbb_hasher( const T& t ) { - return static_cast( t ) * internal::hash_multiplier; -} -template -inline size_t tbb_hasher( P* ptr ) { - size_t const h = reinterpret_cast( ptr ); - return (h >> 3) ^ h; -} -template -inline size_t tbb_hasher( const std::basic_string& s ) { - size_t h = 0; - for( const E* c = s.c_str(); *c; ++c ) - h = static_cast(*c) ^ (h * internal::hash_multiplier); - return h; -} -template -inline size_t tbb_hasher( const std::pair& p ) { - return tbb_hasher(p.first) ^ tbb_hasher(p.second); -} - -} // namespace interface5 -using interface5::tbb_hasher; - -// Template class for hash compare -template -class tbb_hash -{ -public: - tbb_hash() {} - - size_t operator()(const Key& key) const - { - return tbb_hasher(key); - } -}; - -//! hash_compare that is default argument for concurrent_hash_map -template -struct tbb_hash_compare { - static size_t hash( const Key& a ) { return tbb_hasher(a); } - static bool equal( const Key& a, const Key& b ) { return a == b; } -}; - -} // namespace tbb -#endif /* __TBB_tbb_hash_compare_impl_H */ diff --git a/inst/include/tbb/internal/_tbb_trace_impl.h b/inst/include/tbb/internal/_tbb_trace_impl.h deleted file mode 100644 index e7eb5784f..000000000 --- a/inst/include/tbb/internal/_tbb_trace_impl.h +++ /dev/null @@ -1,60 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef _FGT_TBB_TRACE_IMPL_H -#define _FGT_TBB_TRACE_IMPL_H - -#include "../tbb_profiling.h" - -namespace tbb { - namespace internal { - -#if TBB_PREVIEW_ALGORITHM_TRACE - - static inline void fgt_algorithm( string_index t, void *algorithm, void *parent ) { - itt_make_task_group( ITT_DOMAIN_FLOW, algorithm, FGT_ALGORITHM, parent, FGT_ALGORITHM, t ); - } - static inline void fgt_begin_algorithm( string_index t, void *algorithm ) { - itt_task_begin( ITT_DOMAIN_FLOW, algorithm, FGT_ALGORITHM, NULL, FLOW_NULL, t ); - } - static inline void fgt_end_algorithm( void * ) { - itt_task_end( ITT_DOMAIN_FLOW ); - } - static inline void fgt_alg_begin_body( string_index t, void *body, void *algorithm ) { - itt_task_begin( ITT_DOMAIN_FLOW, body, FLOW_BODY, algorithm, FGT_ALGORITHM, t ); - } - static inline void fgt_alg_end_body( void * ) { - itt_task_end( ITT_DOMAIN_FLOW ); - } - -#else // TBB_PREVIEW_ALGORITHM_TRACE - - static inline void fgt_algorithm( string_index /*t*/, void * /*algorithm*/, void * /*parent*/ ) { } - static inline void fgt_begin_algorithm( string_index /*t*/, void * /*algorithm*/ ) { } - static inline void fgt_end_algorithm( void * ) { } - static inline void fgt_alg_begin_body( string_index /*t*/, void * /*body*/, void * /*algorithm*/ ) { } - static inline void fgt_alg_end_body( void * ) { } - -#endif // TBB_PREVIEW_ALGORITHM_TRACEE - - } // namespace internal -} // namespace tbb - -#endif diff --git a/inst/include/tbb/internal/_tbb_windef.h b/inst/include/tbb/internal/_tbb_windef.h deleted file mode 100644 index e798dee49..000000000 --- a/inst/include/tbb/internal/_tbb_windef.h +++ /dev/null @@ -1,73 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_tbb_windef_H -#error Do not #include this internal file directly; use public TBB headers instead. -#endif /* __TBB_tbb_windef_H */ - -// Check that the target Windows version has all API calls requried for TBB. -// Do not increase the version in condition beyond 0x0500 without prior discussion! -#if defined(_WIN32_WINNT) && _WIN32_WINNT<0x0501 -#error TBB is unable to run on old Windows versions; _WIN32_WINNT must be 0x0501 or greater. -#endif - -#if !defined(_MT) -#error TBB requires linkage with multithreaded C/C++ runtime library. \ - Choose multithreaded DLL runtime in project settings, or use /MD[d] compiler switch. -#endif - -// Workaround for the problem with MVSC headers failing to define namespace std -namespace std { - using ::size_t; using ::ptrdiff_t; -} - -#define __TBB_STRING_AUX(x) #x -#define __TBB_STRING(x) __TBB_STRING_AUX(x) - -// Default setting of TBB_USE_DEBUG -#ifdef TBB_USE_DEBUG -# if TBB_USE_DEBUG -# if !defined(_DEBUG) -# pragma message(__FILE__ "(" __TBB_STRING(__LINE__) ") : Warning: Recommend using /MDd if compiling with TBB_USE_DEBUG!=0") -# endif -# else -# if defined(_DEBUG) -# pragma message(__FILE__ "(" __TBB_STRING(__LINE__) ") : Warning: Recommend using /MD if compiling with TBB_USE_DEBUG==0") -# endif -# endif -#endif - -#if (__TBB_BUILD || __TBBMALLOC_BUILD) && !defined(__TBB_NO_IMPLICIT_LINKAGE) -#define __TBB_NO_IMPLICIT_LINKAGE 1 -#endif - -#if _MSC_VER - #if !__TBB_NO_IMPLICIT_LINKAGE - #ifdef __TBB_LIB_NAME - #pragma comment(lib, __TBB_STRING(__TBB_LIB_NAME)) - #else - #ifdef _DEBUG - #pragma comment(lib, "tbb_debug.lib") - #else - #pragma comment(lib, "tbb.lib") - #endif - #endif - #endif -#endif diff --git a/inst/include/tbb/internal/_template_helpers.h b/inst/include/tbb/internal/_template_helpers.h deleted file mode 100644 index 1e0abbe87..000000000 --- a/inst/include/tbb/internal/_template_helpers.h +++ /dev/null @@ -1,158 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_template_helpers_H -#define __TBB_template_helpers_H - -#include -#include - -namespace tbb { namespace internal { - -//! Enables one or the other code branches -template struct enable_if {}; -template struct enable_if { typedef T type; }; - -//! Strips its template type argument from cv- and ref-qualifiers -template struct strip { typedef T type; }; -template struct strip { typedef T type; }; -template struct strip { typedef T type; }; -template struct strip { typedef T type; }; -template struct strip { typedef T type; }; -template struct strip { typedef T type; }; -template struct strip { typedef T type; }; -template struct strip { typedef T type; }; -//! Specialization for function pointers -template struct strip { typedef T(*type)(); }; -#if __TBB_CPP11_RVALUE_REF_PRESENT -template struct strip { typedef T type; }; -template struct strip { typedef T type; }; -template struct strip { typedef T type; }; -template struct strip { typedef T type; }; -#endif -//! Specialization for arrays converts to a corresponding pointer -template struct strip { typedef T* type; }; -template struct strip { typedef const T* type; }; -template struct strip { typedef volatile T* type; }; -template struct strip { typedef const volatile T* type; }; - -//! Detects whether two given types are the same -template struct is_same_type { static const bool value = false; }; -template struct is_same_type { static const bool value = true; }; - -template struct is_ref { static const bool value = false; }; -template struct is_ref { static const bool value = true; }; - -#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT -//! std::void_t internal implementation (to avoid GCC < 4.7 "template aliases" absence) -template struct void_t { typedef void type; }; -#endif - -#if __TBB_CPP11_RVALUE_REF_PRESENT && __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT - -//! Allows to store a function parameter pack as a variable and later pass it to another function -template< typename... Types > -struct stored_pack; - -template<> -struct stored_pack<> -{ - typedef stored_pack<> pack_type; - stored_pack() {} - - // Friend front-end functions - template< typename F, typename Pack > friend void call( F&& f, Pack&& p ); - template< typename Ret, typename F, typename Pack > friend Ret call_and_return( F&& f, Pack&& p ); - -protected: - // Ideally, ref-qualified non-static methods would be used, - // but that would greatly reduce the set of compilers where it works. - template< typename Ret, typename F, typename... Preceding > - static Ret call( F&& f, const pack_type& /*pack*/, Preceding&&... params ) { - return std::forward(f)( std::forward(params)... ); - } - template< typename Ret, typename F, typename... Preceding > - static Ret call( F&& f, pack_type&& /*pack*/, Preceding&&... params ) { - return std::forward(f)( std::forward(params)... ); - } -}; - -template< typename T, typename... Types > -struct stored_pack : stored_pack -{ - typedef stored_pack pack_type; - typedef stored_pack pack_remainder; - // Since lifetime of original values is out of control, copies should be made. - // Thus references should be stripped away from the deduced type. - typename strip::type leftmost_value; - - // Here rvalue references act in the same way as forwarding references, - // as long as class template parameters were deduced via forwarding references. - stored_pack( T&& t, Types&&... types ) - : pack_remainder(std::forward(types)...), leftmost_value(std::forward(t)) {} - - // Friend front-end functions - template< typename F, typename Pack > friend void call( F&& f, Pack&& p ); - template< typename Ret, typename F, typename Pack > friend Ret call_and_return( F&& f, Pack&& p ); - -protected: - template< typename Ret, typename F, typename... Preceding > - static Ret call( F&& f, pack_type& pack, Preceding&&... params ) { - return pack_remainder::template call( - std::forward(f), static_cast(pack), - std::forward(params)... , pack.leftmost_value - ); - } - template< typename Ret, typename F, typename... Preceding > - static Ret call( F&& f, const pack_type& pack, Preceding&&... params ) { - return pack_remainder::template call( - std::forward(f), static_cast(pack), - std::forward(params)... , pack.leftmost_value - ); - } - template< typename Ret, typename F, typename... Preceding > - static Ret call( F&& f, pack_type&& pack, Preceding&&... params ) { - return pack_remainder::template call( - std::forward(f), static_cast(pack), - std::forward(params)... , std::move(pack.leftmost_value) - ); - } -}; - -//! Calls the given function with arguments taken from a stored_pack -template< typename F, typename Pack > -void call( F&& f, Pack&& p ) { - strip::type::template call( std::forward(f), std::forward(p) ); -} - -template< typename Ret, typename F, typename Pack > -Ret call_and_return( F&& f, Pack&& p ) { - return strip::type::template call( std::forward(f), std::forward(p) ); -} - -template< typename... Types > -stored_pack save_pack( Types&&... types ) { - return stored_pack( std::forward(types)... ); -} - -#endif /* __TBB_CPP11_RVALUE_REF_PRESENT && __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT */ -} } // namespace internal, namespace tbb - -#endif /* __TBB_template_helpers_H */ diff --git a/inst/include/tbb/internal/_x86_eliding_mutex_impl.h b/inst/include/tbb/internal/_x86_eliding_mutex_impl.h deleted file mode 100644 index ef5f92239..000000000 --- a/inst/include/tbb/internal/_x86_eliding_mutex_impl.h +++ /dev/null @@ -1,148 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB__x86_eliding_mutex_impl_H -#define __TBB__x86_eliding_mutex_impl_H - -#ifndef __TBB_spin_mutex_H -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#if ( __TBB_x86_32 || __TBB_x86_64 ) - -namespace tbb { -namespace interface7 { -namespace internal { - -template -class padded_mutex; - -//! An eliding lock that occupies a single byte. -/** A x86_eliding_mutex is an HLE-enabled spin mutex. It is recommended to - put the mutex on a cache line that is not shared by the data it protects. - It should be used for locking short critical sections where the lock is - contended but the data it protects are not. If zero-initialized, the - mutex is considered unheld. - @ingroup synchronization */ -class x86_eliding_mutex : tbb::internal::mutex_copy_deprecated_and_disabled { - //! 0 if lock is released, 1 if lock is acquired. - __TBB_atomic_flag flag; - - friend class padded_mutex; - -public: - //! Construct unacquired lock. - /** Equivalent to zero-initialization of *this. */ - x86_eliding_mutex() : flag(0) {} - -// bug in gcc 3.x.x causes syntax error in spite of the friend declaration above. -// Make the scoped_lock public in that case. -#if __TBB_USE_X86_ELIDING_MUTEX || __TBB_GCC_VERSION < 40000 -#else - // by default we will not provide the scoped_lock interface. The user - // should use the padded version of the mutex. scoped_lock is used in - // padded_mutex template. -private: -#endif - // scoped_lock in padded_mutex<> is the interface to use. - //! Represents acquisition of a mutex. - class scoped_lock : tbb::internal::no_copy { - private: - //! Points to currently held mutex, or NULL if no lock is held. - x86_eliding_mutex* my_mutex; - - public: - //! Construct without acquiring a mutex. - scoped_lock() : my_mutex(NULL) {} - - //! Construct and acquire lock on a mutex. - scoped_lock( x86_eliding_mutex& m ) : my_mutex(NULL) { acquire(m); } - - //! Acquire lock. - void acquire( x86_eliding_mutex& m ) { - __TBB_ASSERT( !my_mutex, "already holding a lock" ); - - my_mutex=&m; - my_mutex->lock(); - } - - //! Try acquiring lock (non-blocking) - /** Return true if lock acquired; false otherwise. */ - bool try_acquire( x86_eliding_mutex& m ) { - __TBB_ASSERT( !my_mutex, "already holding a lock" ); - - bool result = m.try_lock(); - if( result ) { - my_mutex = &m; - } - return result; - } - - //! Release lock - void release() { - __TBB_ASSERT( my_mutex, "release on scoped_lock that is not holding a lock" ); - - my_mutex->unlock(); - my_mutex = NULL; - } - - //! Destroy lock. If holding a lock, releases the lock first. - ~scoped_lock() { - if( my_mutex ) { - release(); - } - } - }; -#if __TBB_USE_X86_ELIDING_MUTEX || __TBB_GCC_VERSION < 40000 -#else -public: -#endif /* __TBB_USE_X86_ELIDING_MUTEX */ - - // Mutex traits - static const bool is_rw_mutex = false; - static const bool is_recursive_mutex = false; - static const bool is_fair_mutex = false; - - // ISO C++0x compatibility methods - - //! Acquire lock - void lock() { - __TBB_LockByteElided(flag); - } - - //! Try acquiring lock (non-blocking) - /** Return true if lock acquired; false otherwise. */ - bool try_lock() { - return __TBB_TryLockByteElided(flag); - } - - //! Release lock - void unlock() { - __TBB_UnlockByteElided( flag ); - } -}; // end of x86_eliding_mutex - -} // namespace internal -} // namespace interface7 -} // namespace tbb - -#endif /* ( __TBB_x86_32 || __TBB_x86_64 ) */ - -#endif /* __TBB__x86_eliding_mutex_impl_H */ diff --git a/inst/include/tbb/internal/_x86_rtm_rw_mutex_impl.h b/inst/include/tbb/internal/_x86_rtm_rw_mutex_impl.h deleted file mode 100644 index b08c23313..000000000 --- a/inst/include/tbb/internal/_x86_rtm_rw_mutex_impl.h +++ /dev/null @@ -1,225 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB__x86_rtm_rw_mutex_impl_H -#define __TBB__x86_rtm_rw_mutex_impl_H - -#ifndef __TBB_spin_rw_mutex_H -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#if __TBB_TSX_AVAILABLE - -#include "../tbb_stddef.h" -#include "../tbb_machine.h" -#include "../tbb_profiling.h" -#include "../spin_rw_mutex.h" - -namespace tbb { -namespace interface8 { -namespace internal { - -enum RTM_type { - RTM_not_in_mutex, - RTM_transacting_reader, - RTM_transacting_writer, - RTM_real_reader, - RTM_real_writer -}; - -static const unsigned long speculation_granularity = 64; - -//! Fast, unfair, spinning speculation-enabled reader-writer lock with backoff and -// writer-preference -/** @ingroup synchronization */ -class x86_rtm_rw_mutex: private spin_rw_mutex { -#if __TBB_USE_X86_RTM_RW_MUTEX || __TBB_GCC_VERSION < 40000 -// bug in gcc 3.x.x causes syntax error in spite of the friend declaration below. -// Make the scoped_lock public in that case. -public: -#else -private: -#endif - friend class interface7::internal::padded_mutex; - class scoped_lock; // should be private - friend class scoped_lock; -private: - //! @cond INTERNAL - - //! Internal construct unacquired mutex. - void __TBB_EXPORTED_METHOD internal_construct(); - - //! Internal acquire write lock. - // only_speculate == true if we're doing a try_lock, else false. - void __TBB_EXPORTED_METHOD internal_acquire_writer(x86_rtm_rw_mutex::scoped_lock&, bool only_speculate=false); - - //! Internal acquire read lock. - // only_speculate == true if we're doing a try_lock, else false. - void __TBB_EXPORTED_METHOD internal_acquire_reader(x86_rtm_rw_mutex::scoped_lock&, bool only_speculate=false); - - //! Internal upgrade reader to become a writer. - bool __TBB_EXPORTED_METHOD internal_upgrade( x86_rtm_rw_mutex::scoped_lock& ); - - //! Out of line code for downgrading a writer to a reader. - bool __TBB_EXPORTED_METHOD internal_downgrade( x86_rtm_rw_mutex::scoped_lock& ); - - //! Internal try_acquire write lock. - bool __TBB_EXPORTED_METHOD internal_try_acquire_writer( x86_rtm_rw_mutex::scoped_lock& ); - - //! Internal release lock. - void __TBB_EXPORTED_METHOD internal_release( x86_rtm_rw_mutex::scoped_lock& ); - - static x86_rtm_rw_mutex* internal_get_mutex( const spin_rw_mutex::scoped_lock& lock ) - { - return static_cast( lock.internal_get_mutex() ); - } - static void internal_set_mutex( spin_rw_mutex::scoped_lock& lock, spin_rw_mutex* mtx ) - { - lock.internal_set_mutex( mtx ); - } - //! @endcond -public: - //! Construct unacquired mutex. - x86_rtm_rw_mutex() { - w_flag = false; -#if TBB_USE_THREADING_TOOLS - internal_construct(); -#endif - } - -#if TBB_USE_ASSERT - //! Empty destructor. - ~x86_rtm_rw_mutex() {} -#endif /* TBB_USE_ASSERT */ - - // Mutex traits - static const bool is_rw_mutex = true; - static const bool is_recursive_mutex = false; - static const bool is_fair_mutex = false; - -#if __TBB_USE_X86_RTM_RW_MUTEX || __TBB_GCC_VERSION < 40000 -#else - // by default we will not provide the scoped_lock interface. The user - // should use the padded version of the mutex. scoped_lock is used in - // padded_mutex template. -private: -#endif - //! The scoped locking pattern - /** It helps to avoid the common problem of forgetting to release lock. - It also nicely provides the "node" for queuing locks. */ - // Speculation-enabled scoped lock for spin_rw_mutex - // The idea is to be able to reuse the acquire/release methods of spin_rw_mutex - // and its scoped lock wherever possible. The only way to use a speculative lock is to use - // a scoped_lock. (because transaction_state must be local) - - class scoped_lock : tbb::internal::no_copy { - friend class x86_rtm_rw_mutex; - spin_rw_mutex::scoped_lock my_scoped_lock; - - RTM_type transaction_state; - - public: - //! Construct lock that has not acquired a mutex. - /** Equivalent to zero-initialization of *this. */ - scoped_lock() : my_scoped_lock(), transaction_state(RTM_not_in_mutex) { - } - - //! Acquire lock on given mutex. - scoped_lock( x86_rtm_rw_mutex& m, bool write = true ) : my_scoped_lock(), - transaction_state(RTM_not_in_mutex) { - acquire(m, write); - } - - //! Release lock (if lock is held). - ~scoped_lock() { - if(transaction_state != RTM_not_in_mutex) release(); - } - - //! Acquire lock on given mutex. - void acquire( x86_rtm_rw_mutex& m, bool write = true ) { - if( write ) m.internal_acquire_writer(*this); - else m.internal_acquire_reader(*this); - } - - //! Release lock - void release() { - x86_rtm_rw_mutex* mutex = x86_rtm_rw_mutex::internal_get_mutex(my_scoped_lock); - __TBB_ASSERT( mutex, "lock is not acquired" ); - __TBB_ASSERT( transaction_state!=RTM_not_in_mutex, "lock is not acquired" ); - return mutex->internal_release(*this); - } - - //! Upgrade reader to become a writer. - /** Returns whether the upgrade happened without releasing and re-acquiring the lock */ - bool upgrade_to_writer() { - x86_rtm_rw_mutex* mutex = x86_rtm_rw_mutex::internal_get_mutex(my_scoped_lock); - __TBB_ASSERT( mutex, "lock is not acquired" ); - __TBB_ASSERT( transaction_state==RTM_transacting_reader || transaction_state==RTM_real_reader, "Invalid state for upgrade" ); - return mutex->internal_upgrade(*this); - } - - //! Downgrade writer to become a reader. - /** Returns whether the downgrade happened without releasing and re-acquiring the lock */ - bool downgrade_to_reader() { - x86_rtm_rw_mutex* mutex = x86_rtm_rw_mutex::internal_get_mutex(my_scoped_lock); - __TBB_ASSERT( mutex, "lock is not acquired" ); - __TBB_ASSERT( transaction_state==RTM_transacting_writer || transaction_state==RTM_real_writer, "Invalid state for downgrade" ); - return mutex->internal_downgrade(*this); - } - - //! Attempt to acquire mutex. - /** returns true if successful. */ - bool try_acquire( x86_rtm_rw_mutex& m, bool write = true ) { -#if TBB_USE_ASSERT - x86_rtm_rw_mutex* mutex = x86_rtm_rw_mutex::internal_get_mutex(my_scoped_lock); - __TBB_ASSERT( !mutex, "lock is already acquired" ); -#endif - // have to assign m to our mutex. - // cannot set the mutex, because try_acquire in spin_rw_mutex depends on it being NULL. - if(write) return m.internal_try_acquire_writer(*this); - // speculatively acquire the lock. If this fails, do try_acquire on the spin_rw_mutex. - m.internal_acquire_reader(*this, /*only_speculate=*/true); - if(transaction_state == RTM_transacting_reader) return true; - if( my_scoped_lock.try_acquire(m, false)) { - transaction_state = RTM_real_reader; - return true; - } - return false; - } - - }; // class x86_rtm_rw_mutex::scoped_lock - - // ISO C++0x compatibility methods not provided because we cannot maintain - // state about whether a thread is in a transaction. - -private: - char pad[speculation_granularity-sizeof(spin_rw_mutex)]; // padding - - // If true, writer holds the spin_rw_mutex. - tbb::atomic w_flag; // want this on a separate cache line - -}; // x86_rtm_rw_mutex - -} // namespace internal -} // namespace interface8 -} // namespace tbb - -#endif /* __TBB_TSX_AVAILABLE */ -#endif /* __TBB__x86_rtm_rw_mutex_impl_H */ diff --git a/inst/include/tbb/machine/gcc_armv7.h b/inst/include/tbb/machine/gcc_armv7.h deleted file mode 100644 index 642c14fe2..000000000 --- a/inst/include/tbb/machine/gcc_armv7.h +++ /dev/null @@ -1,217 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -/* - Platform isolation layer for the ARMv7-a architecture. -*/ - -#ifndef __TBB_machine_H -#error Do not include this file directly; include tbb_machine.h instead -#endif - -//TODO: is ARMv7 is the only version ever to support? -#if !(__ARM_ARCH_7A__) -#error compilation requires an ARMv7-a architecture. -#endif - -#include -#include - -#define __TBB_WORDSIZE 4 - -// Traditionally ARM is little-endian. -// Note that, since only the layout of aligned 32-bit words is of interest, -// any apparent PDP-endianness of 32-bit words at half-word alignment or -// any little-endian ordering of big-endian 32-bit words in 64-bit quantities -// may be disregarded for this setting. -#if __BIG_ENDIAN__ || (defined(__BYTE_ORDER__) && __BYTE_ORDER__==__ORDER_BIG_ENDIAN__) - #define __TBB_ENDIANNESS __TBB_ENDIAN_BIG -#elif __LITTLE_ENDIAN__ || (defined(__BYTE_ORDER__) && __BYTE_ORDER__==__ORDER_LITTLE_ENDIAN__) - #define __TBB_ENDIANNESS __TBB_ENDIAN_LITTLE -#elif defined(__BYTE_ORDER__) - #define __TBB_ENDIANNESS __TBB_ENDIAN_UNSUPPORTED -#else - #define __TBB_ENDIANNESS __TBB_ENDIAN_DETECT -#endif - - -#define __TBB_compiler_fence() __asm__ __volatile__("": : :"memory") -#define __TBB_full_memory_fence() __asm__ __volatile__("dmb ish": : :"memory") -#define __TBB_control_consistency_helper() __TBB_full_memory_fence() -#define __TBB_acquire_consistency_helper() __TBB_full_memory_fence() -#define __TBB_release_consistency_helper() __TBB_full_memory_fence() - -//-------------------------------------------------- -// Compare and swap -//-------------------------------------------------- - -/** - * Atomic CAS for 32 bit values, if *ptr==comparand, then *ptr=value, returns *ptr - * @param ptr pointer to value in memory to be swapped with value if *ptr==comparand - * @param value value to assign *ptr to if *ptr==comparand - * @param comparand value to compare with *ptr - * @return value originally in memory at ptr, regardless of success -*/ -static inline int32_t __TBB_machine_cmpswp4(volatile void *ptr, int32_t value, int32_t comparand ) -{ - int32_t oldval, res; - - __TBB_full_memory_fence(); - - do { - __asm__ __volatile__( - "ldrex %1, [%3]\n" - "mov %0, #0\n" - "cmp %1, %4\n" - "it eq\n" - "strexeq %0, %5, [%3]\n" - : "=&r" (res), "=&r" (oldval), "+Qo" (*(volatile int32_t*)ptr) - : "r" ((volatile int32_t *)ptr), "Ir" (comparand), "r" (value) - : "cc"); - } while (res); - - __TBB_full_memory_fence(); - - return oldval; -} - -/** - * Atomic CAS for 64 bit values, if *ptr==comparand, then *ptr=value, returns *ptr - * @param ptr pointer to value in memory to be swapped with value if *ptr==comparand - * @param value value to assign *ptr to if *ptr==comparand - * @param comparand value to compare with *ptr - * @return value originally in memory at ptr, regardless of success - */ -static inline int64_t __TBB_machine_cmpswp8(volatile void *ptr, int64_t value, int64_t comparand ) -{ - int64_t oldval; - int32_t res; - - __TBB_full_memory_fence(); - - do { - __asm__ __volatile__( - "mov %0, #0\n" - "ldrexd %1, %H1, [%3]\n" - "cmp %1, %4\n" - "it eq\n" - "cmpeq %H1, %H4\n" - "it eq\n" - "strexdeq %0, %5, %H5, [%3]" - : "=&r" (res), "=&r" (oldval), "+Qo" (*(volatile int64_t*)ptr) - : "r" ((volatile int64_t *)ptr), "r" (comparand), "r" (value) - : "cc"); - } while (res); - - __TBB_full_memory_fence(); - - return oldval; -} - -static inline int32_t __TBB_machine_fetchadd4(volatile void* ptr, int32_t addend) -{ - unsigned long tmp; - int32_t result, tmp2; - - __TBB_full_memory_fence(); - - __asm__ __volatile__( -"1: ldrex %0, [%4]\n" -" add %3, %0, %5\n" -" strex %1, %3, [%4]\n" -" cmp %1, #0\n" -" bne 1b\n" - : "=&r" (result), "=&r" (tmp), "+Qo" (*(volatile int32_t*)ptr), "=&r"(tmp2) - : "r" ((volatile int32_t *)ptr), "Ir" (addend) - : "cc"); - - __TBB_full_memory_fence(); - - return result; -} - -static inline int64_t __TBB_machine_fetchadd8(volatile void *ptr, int64_t addend) -{ - unsigned long tmp; - int64_t result, tmp2; - - __TBB_full_memory_fence(); - - __asm__ __volatile__( -"1: ldrexd %0, %H0, [%4]\n" -" adds %3, %0, %5\n" -" adc %H3, %H0, %H5\n" -" strexd %1, %3, %H3, [%4]\n" -" cmp %1, #0\n" -" bne 1b" - : "=&r" (result), "=&r" (tmp), "+Qo" (*(volatile int64_t*)ptr), "=&r"(tmp2) - : "r" ((volatile int64_t *)ptr), "r" (addend) - : "cc"); - - - __TBB_full_memory_fence(); - - return result; -} - -inline void __TBB_machine_pause (int32_t delay ) -{ - while(delay>0) - { - __TBB_compiler_fence(); - delay--; - } -} - -namespace tbb { -namespace internal { - template - struct machine_load_store_relaxed { - static inline T load ( const volatile T& location ) { - const T value = location; - - /* - * An extra memory barrier is required for errata #761319 - * Please see http://infocenter.arm.com/help/topic/com.arm.doc.uan0004a - */ - __TBB_acquire_consistency_helper(); - return value; - } - - static inline void store ( volatile T& location, T value ) { - location = value; - } - }; -}} // namespaces internal, tbb - -// Machine specific atomic operations - -#define __TBB_CompareAndSwap4(P,V,C) __TBB_machine_cmpswp4(P,V,C) -#define __TBB_CompareAndSwap8(P,V,C) __TBB_machine_cmpswp8(P,V,C) -#define __TBB_Pause(V) __TBB_machine_pause(V) - -// Use generics for some things -#define __TBB_USE_GENERIC_PART_WORD_CAS 1 -#define __TBB_USE_GENERIC_PART_WORD_FETCH_ADD 1 -#define __TBB_USE_GENERIC_PART_WORD_FETCH_STORE 1 -#define __TBB_USE_GENERIC_FETCH_STORE 1 -#define __TBB_USE_GENERIC_HALF_FENCED_LOAD_STORE 1 -#define __TBB_USE_GENERIC_DWORD_LOAD_STORE 1 -#define __TBB_USE_GENERIC_SEQUENTIAL_CONSISTENCY_LOAD_STORE 1 diff --git a/inst/include/tbb/machine/gcc_generic.h b/inst/include/tbb/machine/gcc_generic.h deleted file mode 100644 index 5fc2a9011..000000000 --- a/inst/include/tbb/machine/gcc_generic.h +++ /dev/null @@ -1,184 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#if !defined(__TBB_machine_H) || defined(__TBB_machine_gcc_generic_H) -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#define __TBB_machine_gcc_generic_H - -#include -#include - -#define __TBB_WORDSIZE __SIZEOF_POINTER__ - -#if __TBB_GCC_64BIT_ATOMIC_BUILTINS_BROKEN - #define __TBB_64BIT_ATOMICS 0 -#endif - -/** FPU control setting not available for non-Intel architectures on Android **/ -#if __ANDROID__ && __TBB_generic_arch - #define __TBB_CPU_CTL_ENV_PRESENT 0 -#endif - -// __BYTE_ORDER__ is used in accordance with http://gcc.gnu.org/onlinedocs/cpp/Common-Predefined-Macros.html, -// but __BIG_ENDIAN__ or __LITTLE_ENDIAN__ may be more commonly found instead. -#if __BIG_ENDIAN__ || (defined(__BYTE_ORDER__) && __BYTE_ORDER__==__ORDER_BIG_ENDIAN__) - #define __TBB_ENDIANNESS __TBB_ENDIAN_BIG -#elif __LITTLE_ENDIAN__ || (defined(__BYTE_ORDER__) && __BYTE_ORDER__==__ORDER_LITTLE_ENDIAN__) - #define __TBB_ENDIANNESS __TBB_ENDIAN_LITTLE -#elif defined(__BYTE_ORDER__) - #define __TBB_ENDIANNESS __TBB_ENDIAN_UNSUPPORTED -#else - #define __TBB_ENDIANNESS __TBB_ENDIAN_DETECT -#endif - -#if __TBB_GCC_VERSION < 40700 -// Use __sync_* builtins - -/** As this generic implementation has absolutely no information about underlying - hardware, its performance most likely will be sub-optimal because of full memory - fence usages where a more lightweight synchronization means (or none at all) - could suffice. Thus if you use this header to enable TBB on a new platform, - consider forking it and relaxing below helpers as appropriate. **/ -#define __TBB_acquire_consistency_helper() __sync_synchronize() -#define __TBB_release_consistency_helper() __sync_synchronize() -#define __TBB_full_memory_fence() __sync_synchronize() -#define __TBB_control_consistency_helper() __sync_synchronize() - -#define __TBB_MACHINE_DEFINE_ATOMICS(S,T) \ -inline T __TBB_machine_cmpswp##S( volatile void *ptr, T value, T comparand ) { \ - return __sync_val_compare_and_swap(reinterpret_cast(ptr),comparand,value); \ -} \ -inline T __TBB_machine_fetchadd##S( volatile void *ptr, T value ) { \ - return __sync_fetch_and_add(reinterpret_cast(ptr),value); \ -} - -#define __TBB_USE_GENERIC_FETCH_STORE 1 - -#else -// __TBB_GCC_VERSION >= 40700; use __atomic_* builtins available since gcc 4.7 - -#define __TBB_compiler_fence() __asm__ __volatile__("": : :"memory") -// Acquire and release fence intrinsics in GCC might miss compiler fence. -// Adding it at both sides of an intrinsic, as we do not know what reordering can be made. -#define __TBB_acquire_consistency_helper() __TBB_compiler_fence(); __atomic_thread_fence(__ATOMIC_ACQUIRE); __TBB_compiler_fence() -#define __TBB_release_consistency_helper() __TBB_compiler_fence(); __atomic_thread_fence(__ATOMIC_RELEASE); __TBB_compiler_fence() -#define __TBB_full_memory_fence() __atomic_thread_fence(__ATOMIC_SEQ_CST) -#define __TBB_control_consistency_helper() __TBB_acquire_consistency_helper() - -#define __TBB_MACHINE_DEFINE_ATOMICS(S,T) \ -inline T __TBB_machine_cmpswp##S( volatile void *ptr, T value, T comparand ) { \ - (void)__atomic_compare_exchange_n(reinterpret_cast(ptr), &comparand, value, \ - false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); \ - return comparand; \ -} \ -inline T __TBB_machine_fetchadd##S( volatile void *ptr, T value ) { \ - return __atomic_fetch_add(reinterpret_cast(ptr), value, __ATOMIC_SEQ_CST); \ -} \ -inline T __TBB_machine_fetchstore##S( volatile void *ptr, T value ) { \ - return __atomic_exchange_n(reinterpret_cast(ptr), value, __ATOMIC_SEQ_CST); \ -} - -#endif // __TBB_GCC_VERSION < 40700 - -__TBB_MACHINE_DEFINE_ATOMICS(1,int8_t) -__TBB_MACHINE_DEFINE_ATOMICS(2,int16_t) -__TBB_MACHINE_DEFINE_ATOMICS(4,int32_t) -__TBB_MACHINE_DEFINE_ATOMICS(8,int64_t) - -#undef __TBB_MACHINE_DEFINE_ATOMICS - -namespace tbb{ namespace internal { namespace gcc_builtins { - inline int clz(unsigned int x){ return __builtin_clz(x);}; - inline int clz(unsigned long int x){ return __builtin_clzl(x);}; - inline int clz(unsigned long long int x){ return __builtin_clzll(x);}; -}}} -//gcc __builtin_clz builtin count _number_ of leading zeroes -static inline intptr_t __TBB_machine_lg( uintptr_t x ) { - return sizeof(x)*8 - tbb::internal::gcc_builtins::clz(x) -1 ; -} - - -typedef unsigned char __TBB_Flag; -typedef __TBB_atomic __TBB_Flag __TBB_atomic_flag; - -#if __TBB_GCC_VERSION < 40700 -// Use __sync_* builtins - -static inline void __TBB_machine_or( volatile void *ptr, uintptr_t addend ) { - __sync_fetch_and_or(reinterpret_cast(ptr),addend); -} - -static inline void __TBB_machine_and( volatile void *ptr, uintptr_t addend ) { - __sync_fetch_and_and(reinterpret_cast(ptr),addend); -} - -inline bool __TBB_machine_try_lock_byte( __TBB_atomic_flag &flag ) { - return __sync_lock_test_and_set(&flag,1)==0; -} - -inline void __TBB_machine_unlock_byte( __TBB_atomic_flag &flag ) { - __sync_lock_release(&flag); -} - -#else -// __TBB_GCC_VERSION >= 40700; use __atomic_* builtins available since gcc 4.7 - -static inline void __TBB_machine_or( volatile void *ptr, uintptr_t addend ) { - __atomic_fetch_or(reinterpret_cast(ptr),addend,__ATOMIC_SEQ_CST); -} - -static inline void __TBB_machine_and( volatile void *ptr, uintptr_t addend ) { - __atomic_fetch_and(reinterpret_cast(ptr),addend,__ATOMIC_SEQ_CST); -} - -inline bool __TBB_machine_try_lock_byte( __TBB_atomic_flag &flag ) { - return !__atomic_test_and_set(&flag,__ATOMIC_ACQUIRE); -} - -inline void __TBB_machine_unlock_byte( __TBB_atomic_flag &flag ) { - __atomic_clear(&flag,__ATOMIC_RELEASE); -} - -#endif // __TBB_GCC_VERSION < 40700 - -// Machine specific atomic operations -#define __TBB_AtomicOR(P,V) __TBB_machine_or(P,V) -#define __TBB_AtomicAND(P,V) __TBB_machine_and(P,V) - -#define __TBB_TryLockByte __TBB_machine_try_lock_byte -#define __TBB_UnlockByte __TBB_machine_unlock_byte - -// Definition of other functions -#define __TBB_Log2(V) __TBB_machine_lg(V) - -// TODO: implement with __atomic_* builtins where available -#define __TBB_USE_GENERIC_HALF_FENCED_LOAD_STORE 1 -#define __TBB_USE_GENERIC_RELAXED_LOAD_STORE 1 -#define __TBB_USE_GENERIC_SEQUENTIAL_CONSISTENCY_LOAD_STORE 1 - -#if __TBB_WORDSIZE==4 - #define __TBB_USE_GENERIC_DWORD_LOAD_STORE 1 -#endif - -#if __TBB_x86_32 || __TBB_x86_64 -#include "gcc_itsx.h" -#endif diff --git a/inst/include/tbb/machine/gcc_ia32_common.h b/inst/include/tbb/machine/gcc_ia32_common.h deleted file mode 100644 index f5efc375b..000000000 --- a/inst/include/tbb/machine/gcc_ia32_common.h +++ /dev/null @@ -1,112 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_machine_gcc_ia32_common_H -#define __TBB_machine_gcc_ia32_common_H - -//TODO: Add a higher-level function, e.g. tbb::internal::log2(), into tbb_stddef.h, which -//uses __TBB_Log2 and contains the assert and remove the assert from here and all other -//platform-specific headers. -//TODO: Check if use of gcc intrinsic gives a better chance for cross call optimizations -template -static inline intptr_t __TBB_machine_lg( T x ) { - __TBB_ASSERT(x>0, "The logarithm of a non-positive value is undefined."); - uintptr_t j, i = x; - __asm__("bsr %1,%0" : "=r"(j) : "r"(i)); - return j; -} -#define __TBB_Log2(V) __TBB_machine_lg(V) - -#ifndef __TBB_Pause -//TODO: check if raising a ratio of pause instructions to loop control instructions -//(via e.g. loop unrolling) gives any benefit for HT. E.g, the current implementation -//does about 2 CPU-consuming instructions for every pause instruction. Perhaps for -//high pause counts it should use an unrolled loop to raise the ratio, and thus free -//up more integer cycles for the other hyperthread. On the other hand, if the loop is -//unrolled too far, it won't fit in the core's loop cache, and thus take away -//instruction decode slots from the other hyperthread. - -//TODO: check if use of gcc __builtin_ia32_pause intrinsic gives a "some how" better performing code -static inline void __TBB_machine_pause( int32_t delay ) { - for (int32_t i = 0; i < delay; i++) { - __asm__ __volatile__("pause;"); - } - return; -} -#define __TBB_Pause(V) __TBB_machine_pause(V) -#endif /* !__TBB_Pause */ - -namespace tbb { namespace internal { typedef uint64_t machine_tsc_t; } } -static inline tbb::internal::machine_tsc_t __TBB_machine_time_stamp() { -#if __INTEL_COMPILER - return _rdtsc(); -#else - tbb::internal::uint32_t hi, lo; - __asm__ __volatile__("rdtsc" : "=d"(hi), "=a"(lo)); - return (tbb::internal::machine_tsc_t( hi ) << 32) | lo; -#endif -} -#define __TBB_time_stamp() __TBB_machine_time_stamp() - -// API to retrieve/update FPU control setting -#ifndef __TBB_CPU_CTL_ENV_PRESENT -#define __TBB_CPU_CTL_ENV_PRESENT 1 -namespace tbb { -namespace internal { -class cpu_ctl_env { -private: - int mxcsr; - short x87cw; - static const int MXCSR_CONTROL_MASK = ~0x3f; /* all except last six status bits */ -public: - bool operator!=( const cpu_ctl_env& ctl ) const { return mxcsr != ctl.mxcsr || x87cw != ctl.x87cw; } - void get_env() { - #if __TBB_ICC_12_0_INL_ASM_FSTCW_BROKEN - cpu_ctl_env loc_ctl; - __asm__ __volatile__ ( - "stmxcsr %0\n\t" - "fstcw %1" - : "=m"(loc_ctl.mxcsr), "=m"(loc_ctl.x87cw) - ); - *this = loc_ctl; - #else - __asm__ __volatile__ ( - "stmxcsr %0\n\t" - "fstcw %1" - : "=m"(mxcsr), "=m"(x87cw) - ); - #endif - mxcsr &= MXCSR_CONTROL_MASK; - } - void set_env() const { - __asm__ __volatile__ ( - "ldmxcsr %0\n\t" - "fldcw %1" - : : "m"(mxcsr), "m"(x87cw) - ); - } -}; -} // namespace internal -} // namespace tbb -#endif /* !__TBB_CPU_CTL_ENV_PRESENT */ - -#include "gcc_itsx.h" - -#endif /* __TBB_machine_gcc_ia32_common_H */ diff --git a/inst/include/tbb/machine/gcc_itsx.h b/inst/include/tbb/machine/gcc_itsx.h deleted file mode 100644 index caa35441f..000000000 --- a/inst/include/tbb/machine/gcc_itsx.h +++ /dev/null @@ -1,123 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#if !defined(__TBB_machine_H) || defined(__TBB_machine_gcc_itsx_H) -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#define __TBB_machine_gcc_itsx_H - -#define __TBB_OP_XACQUIRE 0xF2 -#define __TBB_OP_XRELEASE 0xF3 -#define __TBB_OP_LOCK 0xF0 - -#define __TBB_STRINGIZE_INTERNAL(arg) #arg -#define __TBB_STRINGIZE(arg) __TBB_STRINGIZE_INTERNAL(arg) - -#ifdef __TBB_x86_64 -#define __TBB_r_out "=r" -#else -#define __TBB_r_out "=q" -#endif - -inline static uint8_t __TBB_machine_try_lock_elided( volatile uint8_t* lk ) -{ - uint8_t value = 1; - __asm__ volatile (".byte " __TBB_STRINGIZE(__TBB_OP_XACQUIRE)"; lock; xchgb %0, %1;" - : __TBB_r_out(value), "=m"(*lk) : "0"(value), "m"(*lk) : "memory" ); - return uint8_t(value^1); -} - -inline static void __TBB_machine_try_lock_elided_cancel() -{ - // 'pause' instruction aborts HLE/RTM transactions - __asm__ volatile ("pause\n" : : : "memory" ); -} - -inline static void __TBB_machine_unlock_elided( volatile uint8_t* lk ) -{ - __asm__ volatile (".byte " __TBB_STRINGIZE(__TBB_OP_XRELEASE)"; movb $0, %0" - : "=m"(*lk) : "m"(*lk) : "memory" ); -} - -#if __TBB_TSX_INTRINSICS_PRESENT -#include - -#define __TBB_machine_is_in_transaction _xtest -#define __TBB_machine_begin_transaction _xbegin -#define __TBB_machine_end_transaction _xend -#define __TBB_machine_transaction_conflict_abort() _xabort(0xff) - -#else - -/*! - * Check if the instruction is executed in a transaction or not - */ -inline static bool __TBB_machine_is_in_transaction() -{ - int8_t res = 0; -#if __TBB_x86_32 - __asm__ volatile (".byte 0x0F; .byte 0x01; .byte 0xD6;\n" - "setz %0" : "=q"(res) : : "memory" ); -#else - __asm__ volatile (".byte 0x0F; .byte 0x01; .byte 0xD6;\n" - "setz %0" : "=r"(res) : : "memory" ); -#endif - return res==0; -} - -/*! - * Enter speculative execution mode. - * @return -1 on success - * abort cause ( or 0 ) on abort - */ -inline static uint32_t __TBB_machine_begin_transaction() -{ - uint32_t res = ~uint32_t(0); // success value - __asm__ volatile ("1: .byte 0xC7; .byte 0xF8;\n" // XBEGIN - " .long 2f-1b-6\n" // 2f-1b == difference in addresses of start - // of XBEGIN and the MOVL - // 2f - 1b - 6 == that difference minus the size of the - // XBEGIN instruction. This is the abort offset to - // 2: below. - " jmp 3f\n" // success (leave -1 in res) - "2: movl %%eax,%0\n" // store failure code in res - "3:" - :"=r"(res):"0"(res):"memory","%eax"); - return res; -} - -/*! - * Attempt to commit/end transaction - */ -inline static void __TBB_machine_end_transaction() -{ - __asm__ volatile (".byte 0x0F; .byte 0x01; .byte 0xD5" :::"memory"); // XEND -} - -/* - * aborts with code 0xFF (lock already held) - */ -inline static void __TBB_machine_transaction_conflict_abort() -{ - __asm__ volatile (".byte 0xC6; .byte 0xF8; .byte 0xFF" :::"memory"); -} - -#endif /* __TBB_TSX_INTRINSICS_PRESENT */ diff --git a/inst/include/tbb/machine/ibm_aix51.h b/inst/include/tbb/machine/ibm_aix51.h deleted file mode 100644 index a905b4e12..000000000 --- a/inst/include/tbb/machine/ibm_aix51.h +++ /dev/null @@ -1,70 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -// TODO: revise by comparing with mac_ppc.h - -#if !defined(__TBB_machine_H) || defined(__TBB_machine_ibm_aix51_H) -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#define __TBB_machine_ibm_aix51_H - -#define __TBB_WORDSIZE 8 -#define __TBB_ENDIANNESS __TBB_ENDIAN_BIG // assumption based on operating system - -#include -#include -#include - -extern "C" { -int32_t __TBB_machine_cas_32 (volatile void* ptr, int32_t value, int32_t comparand); -int64_t __TBB_machine_cas_64 (volatile void* ptr, int64_t value, int64_t comparand); -void __TBB_machine_flush (); -void __TBB_machine_lwsync (); -void __TBB_machine_isync (); -} - -// Mapping of old entry point names retained for the sake of backward binary compatibility -#define __TBB_machine_cmpswp4 __TBB_machine_cas_32 -#define __TBB_machine_cmpswp8 __TBB_machine_cas_64 - -#define __TBB_Yield() sched_yield() - -#define __TBB_USE_GENERIC_PART_WORD_CAS 1 -#define __TBB_USE_GENERIC_FETCH_ADD 1 -#define __TBB_USE_GENERIC_FETCH_STORE 1 -#define __TBB_USE_GENERIC_HALF_FENCED_LOAD_STORE 1 -#define __TBB_USE_GENERIC_RELAXED_LOAD_STORE 1 -#define __TBB_USE_GENERIC_SEQUENTIAL_CONSISTENCY_LOAD_STORE 1 - -#if __GNUC__ - #define __TBB_control_consistency_helper() __asm__ __volatile__( "isync": : :"memory") - #define __TBB_acquire_consistency_helper() __asm__ __volatile__("lwsync": : :"memory") - #define __TBB_release_consistency_helper() __asm__ __volatile__("lwsync": : :"memory") - #define __TBB_full_memory_fence() __asm__ __volatile__( "sync": : :"memory") -#else - // IBM C++ Compiler does not support inline assembly - // TODO: Since XL 9.0 or earlier GCC syntax is supported. Replace with more - // lightweight implementation (like in mac_ppc.h) - #define __TBB_control_consistency_helper() __TBB_machine_isync () - #define __TBB_acquire_consistency_helper() __TBB_machine_lwsync () - #define __TBB_release_consistency_helper() __TBB_machine_lwsync () - #define __TBB_full_memory_fence() __TBB_machine_flush () -#endif diff --git a/inst/include/tbb/machine/icc_generic.h b/inst/include/tbb/machine/icc_generic.h deleted file mode 100644 index 048630007..000000000 --- a/inst/include/tbb/machine/icc_generic.h +++ /dev/null @@ -1,262 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#if !defined(__TBB_machine_H) || defined(__TBB_machine_icc_generic_H) -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#if ! __TBB_ICC_BUILTIN_ATOMICS_PRESENT - #error "Intel C++ Compiler of at least 12.0 version is needed to use ICC intrinsics port" -#endif - -#define __TBB_machine_icc_generic_H - -//ICC mimics the "native" target compiler -#if _MSC_VER - #include "msvc_ia32_common.h" -#else - #include "gcc_ia32_common.h" -#endif - -//TODO: Make __TBB_WORDSIZE macro optional for ICC intrinsics port. -//As compiler intrinsics are used for all the operations it is possible to do. - -#if __TBB_x86_32 - #define __TBB_WORDSIZE 4 -#else - #define __TBB_WORDSIZE 8 -#endif -#define __TBB_ENDIANNESS __TBB_ENDIAN_LITTLE - -//__TBB_compiler_fence() defined just in case, as it seems not to be used on its own anywhere else -#ifndef __TBB_compiler_fence -#if _MSC_VER - //TODO: any way to use same intrinsics on windows and linux? - #pragma intrinsic(_ReadWriteBarrier) - #define __TBB_compiler_fence() _ReadWriteBarrier() -#else - #define __TBB_compiler_fence() __asm__ __volatile__("": : :"memory") -#endif -#endif - -#ifndef __TBB_full_memory_fence -#if _MSC_VER - //TODO: any way to use same intrinsics on windows and linux? - #pragma intrinsic(_mm_mfence) - #define __TBB_full_memory_fence() _mm_mfence() -#else - #define __TBB_full_memory_fence() __asm__ __volatile__("mfence": : :"memory") -#endif -#endif - -#ifndef __TBB_control_consistency_helper -#define __TBB_control_consistency_helper() __TBB_compiler_fence() -#endif - -namespace tbb { namespace internal { -//TODO: is there any way to reuse definition of memory_order enum from ICC instead of copy paste. -//however it seems unlikely that ICC will silently change exact enum values, as they are defined -//in the ISO exactly like this. -//TODO: add test that exact values of the enum are same as in the ISO C++11 -typedef enum memory_order { - memory_order_relaxed, memory_order_consume, memory_order_acquire, - memory_order_release, memory_order_acq_rel, memory_order_seq_cst -} memory_order; - -namespace icc_intrinsics_port { - template - T convert_argument(T value){ - return value; - } - //The overload below is needed to have explicit conversion of pointer to void* in argument list. - //compiler bug? - //TODO: add according broken macro and recheck with ICC 13.0 if the overload is still needed - template - void* convert_argument(T* value){ - return (void*)value; - } -} -//TODO: code below is a bit repetitive, consider simplifying it -template -struct machine_load_store { - static T load_with_acquire ( const volatile T& location ) { - return __atomic_load_explicit(&location, memory_order_acquire); - } - static void store_with_release ( volatile T &location, T value ) { - __atomic_store_explicit(&location, icc_intrinsics_port::convert_argument(value), memory_order_release); - } -}; - -template -struct machine_load_store_relaxed { - static inline T load ( const T& location ) { - return __atomic_load_explicit(&location, memory_order_relaxed); - } - static inline void store ( T& location, T value ) { - __atomic_store_explicit(&location, icc_intrinsics_port::convert_argument(value), memory_order_relaxed); - } -}; - -template -struct machine_load_store_seq_cst { - static T load ( const volatile T& location ) { - return __atomic_load_explicit(&location, memory_order_seq_cst); - } - - static void store ( volatile T &location, T value ) { - __atomic_store_explicit(&location, value, memory_order_seq_cst); - } -}; - -}} // namespace tbb::internal - -namespace tbb{ namespace internal { namespace icc_intrinsics_port{ - typedef enum memory_order_map { - relaxed = memory_order_relaxed, - acquire = memory_order_acquire, - release = memory_order_release, - full_fence= memory_order_seq_cst - } memory_order_map; -}}}// namespace tbb::internal - -#define __TBB_MACHINE_DEFINE_ATOMICS(S,T,M) \ -inline T __TBB_machine_cmpswp##S##M( volatile void *ptr, T value, T comparand ) { \ - __atomic_compare_exchange_strong_explicit( \ - (T*)ptr \ - ,&comparand \ - ,value \ - , tbb::internal::icc_intrinsics_port::M \ - , tbb::internal::icc_intrinsics_port::M); \ - return comparand; \ -} \ - \ -inline T __TBB_machine_fetchstore##S##M(volatile void *ptr, T value) { \ - return __atomic_exchange_explicit((T*)ptr, value, tbb::internal::icc_intrinsics_port::M); \ -} \ - \ -inline T __TBB_machine_fetchadd##S##M(volatile void *ptr, T value) { \ - return __atomic_fetch_add_explicit((T*)ptr, value, tbb::internal::icc_intrinsics_port::M); \ -} \ - -__TBB_MACHINE_DEFINE_ATOMICS(1,tbb::internal::int8_t, full_fence) -__TBB_MACHINE_DEFINE_ATOMICS(1,tbb::internal::int8_t, acquire) -__TBB_MACHINE_DEFINE_ATOMICS(1,tbb::internal::int8_t, release) -__TBB_MACHINE_DEFINE_ATOMICS(1,tbb::internal::int8_t, relaxed) - -__TBB_MACHINE_DEFINE_ATOMICS(2,tbb::internal::int16_t, full_fence) -__TBB_MACHINE_DEFINE_ATOMICS(2,tbb::internal::int16_t, acquire) -__TBB_MACHINE_DEFINE_ATOMICS(2,tbb::internal::int16_t, release) -__TBB_MACHINE_DEFINE_ATOMICS(2,tbb::internal::int16_t, relaxed) - -__TBB_MACHINE_DEFINE_ATOMICS(4,tbb::internal::int32_t, full_fence) -__TBB_MACHINE_DEFINE_ATOMICS(4,tbb::internal::int32_t, acquire) -__TBB_MACHINE_DEFINE_ATOMICS(4,tbb::internal::int32_t, release) -__TBB_MACHINE_DEFINE_ATOMICS(4,tbb::internal::int32_t, relaxed) - -__TBB_MACHINE_DEFINE_ATOMICS(8,tbb::internal::int64_t, full_fence) -__TBB_MACHINE_DEFINE_ATOMICS(8,tbb::internal::int64_t, acquire) -__TBB_MACHINE_DEFINE_ATOMICS(8,tbb::internal::int64_t, release) -__TBB_MACHINE_DEFINE_ATOMICS(8,tbb::internal::int64_t, relaxed) - - -#undef __TBB_MACHINE_DEFINE_ATOMICS - -#define __TBB_USE_FENCED_ATOMICS 1 - -namespace tbb { namespace internal { -#if __TBB_FORCE_64BIT_ALIGNMENT_BROKEN -__TBB_MACHINE_DEFINE_LOAD8_GENERIC_FENCED(full_fence) -__TBB_MACHINE_DEFINE_STORE8_GENERIC_FENCED(full_fence) - -__TBB_MACHINE_DEFINE_LOAD8_GENERIC_FENCED(acquire) -__TBB_MACHINE_DEFINE_STORE8_GENERIC_FENCED(release) - -__TBB_MACHINE_DEFINE_LOAD8_GENERIC_FENCED(relaxed) -__TBB_MACHINE_DEFINE_STORE8_GENERIC_FENCED(relaxed) - -template -struct machine_load_store { - static T load_with_acquire ( const volatile T& location ) { - if( tbb::internal::is_aligned(&location,8)) { - return __atomic_load_explicit(&location, memory_order_acquire); - } else { - return __TBB_machine_generic_load8acquire(&location); - } - } - static void store_with_release ( volatile T &location, T value ) { - if( tbb::internal::is_aligned(&location,8)) { - __atomic_store_explicit(&location, icc_intrinsics_port::convert_argument(value), memory_order_release); - } else { - return __TBB_machine_generic_store8release(&location,value); - } - } -}; - -template -struct machine_load_store_relaxed { - static T load( const volatile T& location ) { - if( tbb::internal::is_aligned(&location,8)) { - return __atomic_load_explicit(&location, memory_order_relaxed); - } else { - return __TBB_machine_generic_load8relaxed(&location); - } - } - static void store( volatile T &location, T value ) { - if( tbb::internal::is_aligned(&location,8)) { - __atomic_store_explicit(&location, icc_intrinsics_port::convert_argument(value), memory_order_relaxed); - } else { - return __TBB_machine_generic_store8relaxed(&location,value); - } - } -}; - -template -struct machine_load_store_seq_cst { - static T load ( const volatile T& location ) { - if( tbb::internal::is_aligned(&location,8)) { - return __atomic_load_explicit(&location, memory_order_seq_cst); - } else { - return __TBB_machine_generic_load8full_fence(&location); - } - - } - - static void store ( volatile T &location, T value ) { - if( tbb::internal::is_aligned(&location,8)) { - __atomic_store_explicit(&location, value, memory_order_seq_cst); - } else { - return __TBB_machine_generic_store8full_fence(&location,value); - } - - } -}; - -#endif -}} // namespace tbb::internal -template -inline void __TBB_machine_OR( T *operand, T addend ) { - __atomic_fetch_or_explicit(operand, addend, tbb::internal::memory_order_seq_cst); -} - -template -inline void __TBB_machine_AND( T *operand, T addend ) { - __atomic_fetch_and_explicit(operand, addend, tbb::internal::memory_order_seq_cst); -} - diff --git a/inst/include/tbb/machine/linux_common.h b/inst/include/tbb/machine/linux_common.h deleted file mode 100644 index 4d2d355b1..000000000 --- a/inst/include/tbb/machine/linux_common.h +++ /dev/null @@ -1,84 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_machine_H -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#include -#define __TBB_Yield() sched_yield() - -#include -/* Futex definitions */ -#include - -#if defined(SYS_futex) - -#define __TBB_USE_FUTEX 1 -#include -#include -// Unfortunately, some versions of Linux do not have a header that defines FUTEX_WAIT and FUTEX_WAKE. - -#ifdef FUTEX_WAIT -#define __TBB_FUTEX_WAIT FUTEX_WAIT -#else -#define __TBB_FUTEX_WAIT 0 -#endif - -#ifdef FUTEX_WAKE -#define __TBB_FUTEX_WAKE FUTEX_WAKE -#else -#define __TBB_FUTEX_WAKE 1 -#endif - -#ifndef __TBB_ASSERT -#error machine specific headers must be included after tbb_stddef.h -#endif - -namespace tbb { - -namespace internal { - -inline int futex_wait( void *futex, int comparand ) { - int r = syscall( SYS_futex,futex,__TBB_FUTEX_WAIT,comparand,NULL,NULL,0 ); -#if TBB_USE_ASSERT - int e = errno; - __TBB_ASSERT( r==0||r==EWOULDBLOCK||(r==-1&&(e==EAGAIN||e==EINTR)), "futex_wait failed." ); -#endif /* TBB_USE_ASSERT */ - return r; -} - -inline int futex_wakeup_one( void *futex ) { - int r = ::syscall( SYS_futex,futex,__TBB_FUTEX_WAKE,1,NULL,NULL,0 ); - __TBB_ASSERT( r==0||r==1, "futex_wakeup_one: more than one thread woken up?" ); - return r; -} - -inline int futex_wakeup_all( void *futex ) { - int r = ::syscall( SYS_futex,futex,__TBB_FUTEX_WAKE,INT_MAX,NULL,NULL,0 ); - __TBB_ASSERT( r>=0, "futex_wakeup_all: error in waking up threads" ); - return r; -} - -} /* namespace internal */ - -} /* namespace tbb */ - -#endif /* SYS_futex */ diff --git a/inst/include/tbb/machine/linux_ia32.h b/inst/include/tbb/machine/linux_ia32.h deleted file mode 100644 index 932d343d1..000000000 --- a/inst/include/tbb/machine/linux_ia32.h +++ /dev/null @@ -1,232 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#if !defined(__TBB_machine_H) || defined(__TBB_machine_linux_ia32_H) -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#define __TBB_machine_linux_ia32_H - -#include -#include "gcc_ia32_common.h" - -#define __TBB_WORDSIZE 4 -#define __TBB_ENDIANNESS __TBB_ENDIAN_LITTLE - -#define __TBB_compiler_fence() __asm__ __volatile__("": : :"memory") -#define __TBB_control_consistency_helper() __TBB_compiler_fence() -#define __TBB_acquire_consistency_helper() __TBB_compiler_fence() -#define __TBB_release_consistency_helper() __TBB_compiler_fence() -#define __TBB_full_memory_fence() __asm__ __volatile__("mfence": : :"memory") - -#if __TBB_ICC_ASM_VOLATILE_BROKEN -#define __TBB_VOLATILE -#else -#define __TBB_VOLATILE volatile -#endif - -#define __TBB_MACHINE_DEFINE_ATOMICS(S,T,X,R) \ -static inline T __TBB_machine_cmpswp##S (volatile void *ptr, T value, T comparand ) \ -{ \ - T result; \ - \ - __asm__ __volatile__("lock\ncmpxchg" X " %2,%1" \ - : "=a"(result), "=m"(*(__TBB_VOLATILE T*)ptr) \ - : "q"(value), "0"(comparand), "m"(*(__TBB_VOLATILE T*)ptr) \ - : "memory"); \ - return result; \ -} \ - \ -static inline T __TBB_machine_fetchadd##S(volatile void *ptr, T addend) \ -{ \ - T result; \ - __asm__ __volatile__("lock\nxadd" X " %0,%1" \ - : R (result), "=m"(*(__TBB_VOLATILE T*)ptr) \ - : "0"(addend), "m"(*(__TBB_VOLATILE T*)ptr) \ - : "memory"); \ - return result; \ -} \ - \ -static inline T __TBB_machine_fetchstore##S(volatile void *ptr, T value) \ -{ \ - T result; \ - __asm__ __volatile__("lock\nxchg" X " %0,%1" \ - : R (result), "=m"(*(__TBB_VOLATILE T*)ptr) \ - : "0"(value), "m"(*(__TBB_VOLATILE T*)ptr) \ - : "memory"); \ - return result; \ -} \ - -__TBB_MACHINE_DEFINE_ATOMICS(1,int8_t,"","=q") -__TBB_MACHINE_DEFINE_ATOMICS(2,int16_t,"","=r") -__TBB_MACHINE_DEFINE_ATOMICS(4,int32_t,"l","=r") - -#if __INTEL_COMPILER -#pragma warning( push ) -// reference to EBX in a function requiring stack alignment -#pragma warning( disable: 998 ) -#endif - -#if __TBB_GCC_CAS8_BUILTIN_INLINING_BROKEN -#define __TBB_IA32_CAS8_NOINLINE __attribute__ ((noinline)) -#else -#define __TBB_IA32_CAS8_NOINLINE -#endif - -static inline __TBB_IA32_CAS8_NOINLINE int64_t __TBB_machine_cmpswp8 (volatile void *ptr, int64_t value, int64_t comparand ) { -//TODO: remove the extra part of condition once __TBB_GCC_BUILTIN_ATOMICS_PRESENT is lowered to gcc version 4.1.2 -#if (__TBB_GCC_BUILTIN_ATOMICS_PRESENT || (__TBB_GCC_VERSION >= 40102)) && !__TBB_GCC_64BIT_ATOMIC_BUILTINS_BROKEN - return __sync_val_compare_and_swap( reinterpret_cast(ptr), comparand, value ); -#else /* !__TBB_GCC_BUILTIN_ATOMICS_PRESENT */ - //TODO: look like ICC 13.0 has some issues with this code, investigate it more deeply - int64_t result; - union { - int64_t i64; - int32_t i32[2]; - }; - i64 = value; -#if __PIC__ - /* compiling position-independent code */ - // EBX register preserved for compliance with position-independent code rules on IA32 - int32_t tmp; - __asm__ __volatile__ ( - "movl %%ebx,%2\n\t" - "movl %5,%%ebx\n\t" -#if __GNUC__==3 - "lock\n\t cmpxchg8b %1\n\t" -#else - "lock\n\t cmpxchg8b (%3)\n\t" -#endif - "movl %2,%%ebx" - : "=A"(result) - , "=m"(*(__TBB_VOLATILE int64_t *)ptr) - , "=m"(tmp) -#if __GNUC__==3 - : "m"(*(__TBB_VOLATILE int64_t *)ptr) -#else - : "SD"(ptr) -#endif - , "0"(comparand) - , "m"(i32[0]), "c"(i32[1]) - : "memory" -#if __INTEL_COMPILER - ,"ebx" -#endif - ); -#else /* !__PIC__ */ - __asm__ __volatile__ ( - "lock\n\t cmpxchg8b %1\n\t" - : "=A"(result), "=m"(*(__TBB_VOLATILE int64_t *)ptr) - : "m"(*(__TBB_VOLATILE int64_t *)ptr) - , "0"(comparand) - , "b"(i32[0]), "c"(i32[1]) - : "memory" - ); -#endif /* __PIC__ */ - return result; -#endif /* !__TBB_GCC_BUILTIN_ATOMICS_PRESENT */ -} - -#undef __TBB_IA32_CAS8_NOINLINE - -#if __INTEL_COMPILER -#pragma warning( pop ) -#endif // warning 998 is back - -static inline void __TBB_machine_or( volatile void *ptr, uint32_t addend ) { - __asm__ __volatile__("lock\norl %1,%0" : "=m"(*(__TBB_VOLATILE uint32_t *)ptr) : "r"(addend), "m"(*(__TBB_VOLATILE uint32_t *)ptr) : "memory"); -} - -static inline void __TBB_machine_and( volatile void *ptr, uint32_t addend ) { - __asm__ __volatile__("lock\nandl %1,%0" : "=m"(*(__TBB_VOLATILE uint32_t *)ptr) : "r"(addend), "m"(*(__TBB_VOLATILE uint32_t *)ptr) : "memory"); -} - -//TODO: Check if it possible and profitable for IA-32 architecture on (Linux* and Windows*) -//to use of 64-bit load/store via floating point registers together with full fence -//for sequentially consistent load/store, instead of CAS. - -#if __clang__ -#define __TBB_fildq "fildll" -#define __TBB_fistpq "fistpll" -#else -#define __TBB_fildq "fildq" -#define __TBB_fistpq "fistpq" -#endif - -static inline int64_t __TBB_machine_aligned_load8 (const volatile void *ptr) { - __TBB_ASSERT(tbb::internal::is_aligned(ptr,8),"__TBB_machine_aligned_load8 should be used with 8 byte aligned locations only \n"); - int64_t result; - __asm__ __volatile__ ( __TBB_fildq " %1\n\t" - __TBB_fistpq " %0" : "=m"(result) : "m"(*(const __TBB_VOLATILE uint64_t*)ptr) : "memory" ); - return result; -} - -static inline void __TBB_machine_aligned_store8 (volatile void *ptr, int64_t value ) { - __TBB_ASSERT(tbb::internal::is_aligned(ptr,8),"__TBB_machine_aligned_store8 should be used with 8 byte aligned locations only \n"); - // Aligned store - __asm__ __volatile__ ( __TBB_fildq " %1\n\t" - __TBB_fistpq " %0" : "=m"(*(__TBB_VOLATILE int64_t*)ptr) : "m"(value) : "memory" ); -} - -static inline int64_t __TBB_machine_load8 (const volatile void *ptr) { -#if __TBB_FORCE_64BIT_ALIGNMENT_BROKEN - if( tbb::internal::is_aligned(ptr,8)) { -#endif - return __TBB_machine_aligned_load8(ptr); -#if __TBB_FORCE_64BIT_ALIGNMENT_BROKEN - } else { - // Unaligned load - return __TBB_machine_cmpswp8(const_cast(ptr),0,0); - } -#endif -} - -//! Handles misaligned 8-byte store -/** Defined in tbb_misc.cpp */ -extern "C" void __TBB_machine_store8_slow( volatile void *ptr, int64_t value ); -extern "C" void __TBB_machine_store8_slow_perf_warning( volatile void *ptr ); - -static inline void __TBB_machine_store8(volatile void *ptr, int64_t value) { -#if __TBB_FORCE_64BIT_ALIGNMENT_BROKEN - if( tbb::internal::is_aligned(ptr,8)) { -#endif - __TBB_machine_aligned_store8(ptr,value); -#if __TBB_FORCE_64BIT_ALIGNMENT_BROKEN - } else { - // Unaligned store -#if TBB_USE_PERFORMANCE_WARNINGS - __TBB_machine_store8_slow_perf_warning(ptr); -#endif /* TBB_USE_PERFORMANCE_WARNINGS */ - __TBB_machine_store8_slow(ptr,value); - } -#endif -} - -// Machine specific atomic operations -#define __TBB_AtomicOR(P,V) __TBB_machine_or(P,V) -#define __TBB_AtomicAND(P,V) __TBB_machine_and(P,V) - -#define __TBB_USE_GENERIC_DWORD_FETCH_ADD 1 -#define __TBB_USE_GENERIC_DWORD_FETCH_STORE 1 -#define __TBB_USE_FETCHSTORE_AS_FULL_FENCED_STORE 1 -#define __TBB_USE_GENERIC_HALF_FENCED_LOAD_STORE 1 -#define __TBB_USE_GENERIC_RELAXED_LOAD_STORE 1 -#define __TBB_USE_GENERIC_SEQUENTIAL_CONSISTENCY_LOAD_STORE 1 - diff --git a/inst/include/tbb/machine/linux_ia64.h b/inst/include/tbb/machine/linux_ia64.h deleted file mode 100644 index f47722880..000000000 --- a/inst/include/tbb/machine/linux_ia64.h +++ /dev/null @@ -1,181 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#if !defined(__TBB_machine_H) || defined(__TBB_machine_linux_ia64_H) -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#define __TBB_machine_linux_ia64_H - -#include -#include - -#define __TBB_WORDSIZE 8 -#define __TBB_ENDIANNESS __TBB_ENDIAN_LITTLE - -#if __INTEL_COMPILER - #define __TBB_compiler_fence() - #define __TBB_control_consistency_helper() __TBB_compiler_fence() - #define __TBB_acquire_consistency_helper() - #define __TBB_release_consistency_helper() - #define __TBB_full_memory_fence() __mf() -#else - #define __TBB_compiler_fence() __asm__ __volatile__("": : :"memory") - #define __TBB_control_consistency_helper() __TBB_compiler_fence() - // Even though GCC imbues volatile loads with acquire semantics, it sometimes moves - // loads over the acquire fence. The following helpers stop such incorrect code motion. - #define __TBB_acquire_consistency_helper() __TBB_compiler_fence() - #define __TBB_release_consistency_helper() __TBB_compiler_fence() - #define __TBB_full_memory_fence() __asm__ __volatile__("mf": : :"memory") -#endif /* !__INTEL_COMPILER */ - -// Most of the functions will be in a .s file -// TODO: revise dynamic_link, memory pools and etc. if the library dependency is removed. - -extern "C" { - int8_t __TBB_machine_fetchadd1__TBB_full_fence (volatile void *ptr, int8_t addend); - int8_t __TBB_machine_fetchadd1acquire(volatile void *ptr, int8_t addend); - int8_t __TBB_machine_fetchadd1release(volatile void *ptr, int8_t addend); - - int16_t __TBB_machine_fetchadd2__TBB_full_fence (volatile void *ptr, int16_t addend); - int16_t __TBB_machine_fetchadd2acquire(volatile void *ptr, int16_t addend); - int16_t __TBB_machine_fetchadd2release(volatile void *ptr, int16_t addend); - - int32_t __TBB_machine_fetchadd4__TBB_full_fence (volatile void *ptr, int32_t value); - int32_t __TBB_machine_fetchadd4acquire(volatile void *ptr, int32_t addend); - int32_t __TBB_machine_fetchadd4release(volatile void *ptr, int32_t addend); - - int64_t __TBB_machine_fetchadd8__TBB_full_fence (volatile void *ptr, int64_t value); - int64_t __TBB_machine_fetchadd8acquire(volatile void *ptr, int64_t addend); - int64_t __TBB_machine_fetchadd8release(volatile void *ptr, int64_t addend); - - int8_t __TBB_machine_fetchstore1__TBB_full_fence (volatile void *ptr, int8_t value); - int8_t __TBB_machine_fetchstore1acquire(volatile void *ptr, int8_t value); - int8_t __TBB_machine_fetchstore1release(volatile void *ptr, int8_t value); - - int16_t __TBB_machine_fetchstore2__TBB_full_fence (volatile void *ptr, int16_t value); - int16_t __TBB_machine_fetchstore2acquire(volatile void *ptr, int16_t value); - int16_t __TBB_machine_fetchstore2release(volatile void *ptr, int16_t value); - - int32_t __TBB_machine_fetchstore4__TBB_full_fence (volatile void *ptr, int32_t value); - int32_t __TBB_machine_fetchstore4acquire(volatile void *ptr, int32_t value); - int32_t __TBB_machine_fetchstore4release(volatile void *ptr, int32_t value); - - int64_t __TBB_machine_fetchstore8__TBB_full_fence (volatile void *ptr, int64_t value); - int64_t __TBB_machine_fetchstore8acquire(volatile void *ptr, int64_t value); - int64_t __TBB_machine_fetchstore8release(volatile void *ptr, int64_t value); - - int8_t __TBB_machine_cmpswp1__TBB_full_fence (volatile void *ptr, int8_t value, int8_t comparand); - int8_t __TBB_machine_cmpswp1acquire(volatile void *ptr, int8_t value, int8_t comparand); - int8_t __TBB_machine_cmpswp1release(volatile void *ptr, int8_t value, int8_t comparand); - - int16_t __TBB_machine_cmpswp2__TBB_full_fence (volatile void *ptr, int16_t value, int16_t comparand); - int16_t __TBB_machine_cmpswp2acquire(volatile void *ptr, int16_t value, int16_t comparand); - int16_t __TBB_machine_cmpswp2release(volatile void *ptr, int16_t value, int16_t comparand); - - int32_t __TBB_machine_cmpswp4__TBB_full_fence (volatile void *ptr, int32_t value, int32_t comparand); - int32_t __TBB_machine_cmpswp4acquire(volatile void *ptr, int32_t value, int32_t comparand); - int32_t __TBB_machine_cmpswp4release(volatile void *ptr, int32_t value, int32_t comparand); - - int64_t __TBB_machine_cmpswp8__TBB_full_fence (volatile void *ptr, int64_t value, int64_t comparand); - int64_t __TBB_machine_cmpswp8acquire(volatile void *ptr, int64_t value, int64_t comparand); - int64_t __TBB_machine_cmpswp8release(volatile void *ptr, int64_t value, int64_t comparand); - - int64_t __TBB_machine_lg(uint64_t value); - void __TBB_machine_pause(int32_t delay); - bool __TBB_machine_trylockbyte( volatile unsigned char &ptr ); - int64_t __TBB_machine_lockbyte( volatile unsigned char &ptr ); - - //! Retrieves the current RSE backing store pointer. IA64 specific. - void* __TBB_get_bsp(); - - int32_t __TBB_machine_load1_relaxed(const void *ptr); - int32_t __TBB_machine_load2_relaxed(const void *ptr); - int32_t __TBB_machine_load4_relaxed(const void *ptr); - int64_t __TBB_machine_load8_relaxed(const void *ptr); - - void __TBB_machine_store1_relaxed(void *ptr, int32_t value); - void __TBB_machine_store2_relaxed(void *ptr, int32_t value); - void __TBB_machine_store4_relaxed(void *ptr, int32_t value); - void __TBB_machine_store8_relaxed(void *ptr, int64_t value); -} // extern "C" - -// Mapping old entry points to the names corresponding to the new full_fence identifier. -#define __TBB_machine_fetchadd1full_fence __TBB_machine_fetchadd1__TBB_full_fence -#define __TBB_machine_fetchadd2full_fence __TBB_machine_fetchadd2__TBB_full_fence -#define __TBB_machine_fetchadd4full_fence __TBB_machine_fetchadd4__TBB_full_fence -#define __TBB_machine_fetchadd8full_fence __TBB_machine_fetchadd8__TBB_full_fence -#define __TBB_machine_fetchstore1full_fence __TBB_machine_fetchstore1__TBB_full_fence -#define __TBB_machine_fetchstore2full_fence __TBB_machine_fetchstore2__TBB_full_fence -#define __TBB_machine_fetchstore4full_fence __TBB_machine_fetchstore4__TBB_full_fence -#define __TBB_machine_fetchstore8full_fence __TBB_machine_fetchstore8__TBB_full_fence -#define __TBB_machine_cmpswp1full_fence __TBB_machine_cmpswp1__TBB_full_fence -#define __TBB_machine_cmpswp2full_fence __TBB_machine_cmpswp2__TBB_full_fence -#define __TBB_machine_cmpswp4full_fence __TBB_machine_cmpswp4__TBB_full_fence -#define __TBB_machine_cmpswp8full_fence __TBB_machine_cmpswp8__TBB_full_fence - -// Mapping relaxed operations to the entry points implementing them. -/** On IA64 RMW operations implicitly have acquire semantics. Thus one cannot - actually have completely relaxed RMW operation here. **/ -#define __TBB_machine_fetchadd1relaxed __TBB_machine_fetchadd1acquire -#define __TBB_machine_fetchadd2relaxed __TBB_machine_fetchadd2acquire -#define __TBB_machine_fetchadd4relaxed __TBB_machine_fetchadd4acquire -#define __TBB_machine_fetchadd8relaxed __TBB_machine_fetchadd8acquire -#define __TBB_machine_fetchstore1relaxed __TBB_machine_fetchstore1acquire -#define __TBB_machine_fetchstore2relaxed __TBB_machine_fetchstore2acquire -#define __TBB_machine_fetchstore4relaxed __TBB_machine_fetchstore4acquire -#define __TBB_machine_fetchstore8relaxed __TBB_machine_fetchstore8acquire -#define __TBB_machine_cmpswp1relaxed __TBB_machine_cmpswp1acquire -#define __TBB_machine_cmpswp2relaxed __TBB_machine_cmpswp2acquire -#define __TBB_machine_cmpswp4relaxed __TBB_machine_cmpswp4acquire -#define __TBB_machine_cmpswp8relaxed __TBB_machine_cmpswp8acquire - -#define __TBB_MACHINE_DEFINE_ATOMICS(S,V) \ - template \ - struct machine_load_store_relaxed { \ - static inline T load ( const T& location ) { \ - return (T)__TBB_machine_load##S##_relaxed(&location); \ - } \ - static inline void store ( T& location, T value ) { \ - __TBB_machine_store##S##_relaxed(&location, (V)value); \ - } \ - } - -namespace tbb { -namespace internal { - __TBB_MACHINE_DEFINE_ATOMICS(1,int8_t); - __TBB_MACHINE_DEFINE_ATOMICS(2,int16_t); - __TBB_MACHINE_DEFINE_ATOMICS(4,int32_t); - __TBB_MACHINE_DEFINE_ATOMICS(8,int64_t); -}} // namespaces internal, tbb - -#undef __TBB_MACHINE_DEFINE_ATOMICS - -#define __TBB_USE_FENCED_ATOMICS 1 -#define __TBB_USE_GENERIC_HALF_FENCED_LOAD_STORE 1 -#define __TBB_USE_GENERIC_SEQUENTIAL_CONSISTENCY_LOAD_STORE 1 - -// Definition of Lock functions -#define __TBB_TryLockByte(P) __TBB_machine_trylockbyte(P) -#define __TBB_LockByte(P) __TBB_machine_lockbyte(P) - -// Definition of other utility functions -#define __TBB_Pause(V) __TBB_machine_pause(V) -#define __TBB_Log2(V) __TBB_machine_lg(V) diff --git a/inst/include/tbb/machine/linux_intel64.h b/inst/include/tbb/machine/linux_intel64.h deleted file mode 100644 index 02153c2a4..000000000 --- a/inst/include/tbb/machine/linux_intel64.h +++ /dev/null @@ -1,96 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#if !defined(__TBB_machine_H) || defined(__TBB_machine_linux_intel64_H) -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#define __TBB_machine_linux_intel64_H - -#include -#include "gcc_ia32_common.h" - -#define __TBB_WORDSIZE 8 -#define __TBB_ENDIANNESS __TBB_ENDIAN_LITTLE - -#define __TBB_compiler_fence() __asm__ __volatile__("": : :"memory") -#define __TBB_control_consistency_helper() __TBB_compiler_fence() -#define __TBB_acquire_consistency_helper() __TBB_compiler_fence() -#define __TBB_release_consistency_helper() __TBB_compiler_fence() - -#ifndef __TBB_full_memory_fence -#define __TBB_full_memory_fence() __asm__ __volatile__("mfence": : :"memory") -#endif - -#define __TBB_MACHINE_DEFINE_ATOMICS(S,T,X) \ -static inline T __TBB_machine_cmpswp##S (volatile void *ptr, T value, T comparand ) \ -{ \ - T result; \ - \ - __asm__ __volatile__("lock\ncmpxchg" X " %2,%1" \ - : "=a"(result), "=m"(*(volatile T*)ptr) \ - : "q"(value), "0"(comparand), "m"(*(volatile T*)ptr) \ - : "memory"); \ - return result; \ -} \ - \ -static inline T __TBB_machine_fetchadd##S(volatile void *ptr, T addend) \ -{ \ - T result; \ - __asm__ __volatile__("lock\nxadd" X " %0,%1" \ - : "=r"(result),"=m"(*(volatile T*)ptr) \ - : "0"(addend), "m"(*(volatile T*)ptr) \ - : "memory"); \ - return result; \ -} \ - \ -static inline T __TBB_machine_fetchstore##S(volatile void *ptr, T value) \ -{ \ - T result; \ - __asm__ __volatile__("lock\nxchg" X " %0,%1" \ - : "=r"(result),"=m"(*(volatile T*)ptr) \ - : "0"(value), "m"(*(volatile T*)ptr) \ - : "memory"); \ - return result; \ -} \ - -__TBB_MACHINE_DEFINE_ATOMICS(1,int8_t,"") -__TBB_MACHINE_DEFINE_ATOMICS(2,int16_t,"") -__TBB_MACHINE_DEFINE_ATOMICS(4,int32_t,"") -__TBB_MACHINE_DEFINE_ATOMICS(8,int64_t,"q") - -#undef __TBB_MACHINE_DEFINE_ATOMICS - -static inline void __TBB_machine_or( volatile void *ptr, uint64_t value ) { - __asm__ __volatile__("lock\norq %1,%0" : "=m"(*(volatile uint64_t*)ptr) : "r"(value), "m"(*(volatile uint64_t*)ptr) : "memory"); -} - -static inline void __TBB_machine_and( volatile void *ptr, uint64_t value ) { - __asm__ __volatile__("lock\nandq %1,%0" : "=m"(*(volatile uint64_t*)ptr) : "r"(value), "m"(*(volatile uint64_t*)ptr) : "memory"); -} - -#define __TBB_AtomicOR(P,V) __TBB_machine_or(P,V) -#define __TBB_AtomicAND(P,V) __TBB_machine_and(P,V) - -#define __TBB_USE_FETCHSTORE_AS_FULL_FENCED_STORE 1 -#define __TBB_USE_GENERIC_HALF_FENCED_LOAD_STORE 1 -#define __TBB_USE_GENERIC_RELAXED_LOAD_STORE 1 -#define __TBB_USE_GENERIC_SEQUENTIAL_CONSISTENCY_LOAD_STORE 1 - diff --git a/inst/include/tbb/machine/mac_ppc.h b/inst/include/tbb/machine/mac_ppc.h deleted file mode 100644 index 13f387b40..000000000 --- a/inst/include/tbb/machine/mac_ppc.h +++ /dev/null @@ -1,313 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#if !defined(__TBB_machine_H) || defined(__TBB_machine_gcc_power_H) -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#define __TBB_machine_gcc_power_H - -#include -#include - -// TODO: rename to gcc_power.h? -// This file is for Power Architecture with compilers supporting GNU inline-assembler syntax (currently GNU g++ and IBM XL). -// Note that XL V9.0 (sometimes?) has trouble dealing with empty input and/or clobber lists, so they should be avoided. - -#if __powerpc64__ || __ppc64__ - // IBM XL documents __powerpc64__ (and __PPC64__). - // Apple documents __ppc64__ (with __ppc__ only on 32-bit). - #define __TBB_WORDSIZE 8 -#else - #define __TBB_WORDSIZE 4 -#endif - -// Traditionally Power Architecture is big-endian. -// Little-endian could be just an address manipulation (compatibility with TBB not verified), -// or normal little-endian (on more recent systems). Embedded PowerPC systems may support -// page-specific endianness, but then one endianness must be hidden from TBB so that it still sees only one. -#if __BIG_ENDIAN__ || (defined(__BYTE_ORDER__) && __BYTE_ORDER__==__ORDER_BIG_ENDIAN__) - #define __TBB_ENDIANNESS __TBB_ENDIAN_BIG -#elif __LITTLE_ENDIAN__ || (defined(__BYTE_ORDER__) && __BYTE_ORDER__==__ORDER_LITTLE_ENDIAN__) - #define __TBB_ENDIANNESS __TBB_ENDIAN_LITTLE -#elif defined(__BYTE_ORDER__) - #define __TBB_ENDIANNESS __TBB_ENDIAN_UNSUPPORTED -#else - #define __TBB_ENDIANNESS __TBB_ENDIAN_DETECT -#endif - -// On Power Architecture, (lock-free) 64-bit atomics require 64-bit hardware: -#if __TBB_WORDSIZE==8 - // Do not change the following definition, because TBB itself will use 64-bit atomics in 64-bit builds. - #define __TBB_64BIT_ATOMICS 1 -#elif __bgp__ - // Do not change the following definition, because this is known 32-bit hardware. - #define __TBB_64BIT_ATOMICS 0 -#else - // To enable 64-bit atomics in 32-bit builds, set the value below to 1 instead of 0. - // You must make certain that the program will only use them on actual 64-bit hardware - // (which typically means that the entire program is only executed on such hardware), - // because their implementation involves machine instructions that are illegal elsewhere. - // The setting can be chosen independently per compilation unit, - // which also means that TBB itself does not need to be rebuilt. - // Alternatively (but only for the current architecture and TBB version), - // override the default as a predefined macro when invoking the compiler. - #ifndef __TBB_64BIT_ATOMICS - #define __TBB_64BIT_ATOMICS 0 - #endif -#endif - -inline int32_t __TBB_machine_cmpswp4 (volatile void *ptr, int32_t value, int32_t comparand ) -{ - int32_t result; - - __asm__ __volatile__("sync\n" - "0:\n\t" - "lwarx %[res],0,%[ptr]\n\t" /* load w/ reservation */ - "cmpw %[res],%[cmp]\n\t" /* compare against comparand */ - "bne- 1f\n\t" /* exit if not same */ - "stwcx. %[val],0,%[ptr]\n\t" /* store new value */ - "bne- 0b\n" /* retry if reservation lost */ - "1:\n\t" /* the exit */ - "isync" - : [res]"=&r"(result) - , "+m"(* (int32_t*) ptr) /* redundant with "memory" */ - : [ptr]"r"(ptr) - , [val]"r"(value) - , [cmp]"r"(comparand) - : "memory" /* compiler full fence */ - , "cr0" /* clobbered by cmp and/or stwcx. */ - ); - return result; -} - -#if __TBB_WORDSIZE==8 - -inline int64_t __TBB_machine_cmpswp8 (volatile void *ptr, int64_t value, int64_t comparand ) -{ - int64_t result; - __asm__ __volatile__("sync\n" - "0:\n\t" - "ldarx %[res],0,%[ptr]\n\t" /* load w/ reservation */ - "cmpd %[res],%[cmp]\n\t" /* compare against comparand */ - "bne- 1f\n\t" /* exit if not same */ - "stdcx. %[val],0,%[ptr]\n\t" /* store new value */ - "bne- 0b\n" /* retry if reservation lost */ - "1:\n\t" /* the exit */ - "isync" - : [res]"=&r"(result) - , "+m"(* (int64_t*) ptr) /* redundant with "memory" */ - : [ptr]"r"(ptr) - , [val]"r"(value) - , [cmp]"r"(comparand) - : "memory" /* compiler full fence */ - , "cr0" /* clobbered by cmp and/or stdcx. */ - ); - return result; -} - -#elif __TBB_64BIT_ATOMICS /* && __TBB_WORDSIZE==4 */ - -inline int64_t __TBB_machine_cmpswp8 (volatile void *ptr, int64_t value, int64_t comparand ) -{ - int64_t result; - int64_t value_register, comparand_register, result_register; // dummy variables to allocate registers - __asm__ __volatile__("sync\n\t" - "ld %[val],%[valm]\n\t" - "ld %[cmp],%[cmpm]\n" - "0:\n\t" - "ldarx %[res],0,%[ptr]\n\t" /* load w/ reservation */ - "cmpd %[res],%[cmp]\n\t" /* compare against comparand */ - "bne- 1f\n\t" /* exit if not same */ - "stdcx. %[val],0,%[ptr]\n\t" /* store new value */ - "bne- 0b\n" /* retry if reservation lost */ - "1:\n\t" /* the exit */ - "std %[res],%[resm]\n\t" - "isync" - : [resm]"=m"(result) - , [res] "=&r"( result_register) - , [val] "=&r"( value_register) - , [cmp] "=&r"(comparand_register) - , "+m"(* (int64_t*) ptr) /* redundant with "memory" */ - : [ptr] "r"(ptr) - , [valm]"m"(value) - , [cmpm]"m"(comparand) - : "memory" /* compiler full fence */ - , "cr0" /* clobbered by cmpd and/or stdcx. */ - ); - return result; -} - -#endif /* __TBB_WORDSIZE==4 && __TBB_64BIT_ATOMICS */ - -#define __TBB_MACHINE_DEFINE_LOAD_STORE(S,ldx,stx,cmpx) \ - template \ - struct machine_load_store { \ - static inline T load_with_acquire(const volatile T& location) { \ - T result; \ - __asm__ __volatile__(ldx " %[res],0(%[ptr])\n" \ - "0:\n\t" \ - cmpx " %[res],%[res]\n\t" \ - "bne- 0b\n\t" \ - "isync" \ - : [res]"=r"(result) \ - : [ptr]"b"(&location) /* cannot use register 0 here */ \ - , "m"(location) /* redundant with "memory" */ \ - : "memory" /* compiler acquire fence */ \ - , "cr0" /* clobbered by cmpw/cmpd */); \ - return result; \ - } \ - static inline void store_with_release(volatile T &location, T value) { \ - __asm__ __volatile__("lwsync\n\t" \ - stx " %[val],0(%[ptr])" \ - : "=m"(location) /* redundant with "memory" */ \ - : [ptr]"b"(&location) /* cannot use register 0 here */ \ - , [val]"r"(value) \ - : "memory"/*compiler release fence*/ /*(cr0 not affected)*/); \ - } \ - }; \ - \ - template \ - struct machine_load_store_relaxed { \ - static inline T load (const __TBB_atomic T& location) { \ - T result; \ - __asm__ __volatile__(ldx " %[res],0(%[ptr])" \ - : [res]"=r"(result) \ - : [ptr]"b"(&location) /* cannot use register 0 here */ \ - , "m"(location) \ - ); /*(no compiler fence)*/ /*(cr0 not affected)*/ \ - return result; \ - } \ - static inline void store (__TBB_atomic T &location, T value) { \ - __asm__ __volatile__(stx " %[val],0(%[ptr])" \ - : "=m"(location) \ - : [ptr]"b"(&location) /* cannot use register 0 here */ \ - , [val]"r"(value) \ - ); /*(no compiler fence)*/ /*(cr0 not affected)*/ \ - } \ - }; - -namespace tbb { -namespace internal { - __TBB_MACHINE_DEFINE_LOAD_STORE(1,"lbz","stb","cmpw") - __TBB_MACHINE_DEFINE_LOAD_STORE(2,"lhz","sth","cmpw") - __TBB_MACHINE_DEFINE_LOAD_STORE(4,"lwz","stw","cmpw") - -#if __TBB_WORDSIZE==8 - - __TBB_MACHINE_DEFINE_LOAD_STORE(8,"ld" ,"std","cmpd") - -#elif __TBB_64BIT_ATOMICS /* && __TBB_WORDSIZE==4 */ - - template - struct machine_load_store { - static inline T load_with_acquire(const volatile T& location) { - T result; - T result_register; // dummy variable to allocate a register - __asm__ __volatile__("ld %[res],0(%[ptr])\n\t" - "std %[res],%[resm]\n" - "0:\n\t" - "cmpd %[res],%[res]\n\t" - "bne- 0b\n\t" - "isync" - : [resm]"=m"(result) - , [res]"=&r"(result_register) - : [ptr]"b"(&location) /* cannot use register 0 here */ - , "m"(location) /* redundant with "memory" */ - : "memory" /* compiler acquire fence */ - , "cr0" /* clobbered by cmpd */); - return result; - } - - static inline void store_with_release(volatile T &location, T value) { - T value_register; // dummy variable to allocate a register - __asm__ __volatile__("lwsync\n\t" - "ld %[val],%[valm]\n\t" - "std %[val],0(%[ptr])" - : "=m"(location) /* redundant with "memory" */ - , [val]"=&r"(value_register) - : [ptr]"b"(&location) /* cannot use register 0 here */ - , [valm]"m"(value) - : "memory"/*compiler release fence*/ /*(cr0 not affected)*/); - } - }; - - struct machine_load_store_relaxed { - static inline T load (const volatile T& location) { - T result; - T result_register; // dummy variable to allocate a register - __asm__ __volatile__("ld %[res],0(%[ptr])\n\t" - "std %[res],%[resm]" - : [resm]"=m"(result) - , [res]"=&r"(result_register) - : [ptr]"b"(&location) /* cannot use register 0 here */ - , "m"(location) - ); /*(no compiler fence)*/ /*(cr0 not affected)*/ - return result; - } - - static inline void store (volatile T &location, T value) { - T value_register; // dummy variable to allocate a register - __asm__ __volatile__("ld %[val],%[valm]\n\t" - "std %[val],0(%[ptr])" - : "=m"(location) - , [val]"=&r"(value_register) - : [ptr]"b"(&location) /* cannot use register 0 here */ - , [valm]"m"(value) - ); /*(no compiler fence)*/ /*(cr0 not affected)*/ - } - }; - #define __TBB_machine_load_store_relaxed_8 - -#endif /* __TBB_WORDSIZE==4 && __TBB_64BIT_ATOMICS */ - -}} // namespaces internal, tbb - -#undef __TBB_MACHINE_DEFINE_LOAD_STORE - -#define __TBB_USE_GENERIC_PART_WORD_CAS 1 -#define __TBB_USE_GENERIC_FETCH_ADD 1 -#define __TBB_USE_GENERIC_FETCH_STORE 1 -#define __TBB_USE_GENERIC_SEQUENTIAL_CONSISTENCY_LOAD_STORE 1 - -#define __TBB_control_consistency_helper() __asm__ __volatile__("isync": : :"memory") -#define __TBB_full_memory_fence() __asm__ __volatile__( "sync": : :"memory") - -static inline intptr_t __TBB_machine_lg( uintptr_t x ) { - __TBB_ASSERT(x, "__TBB_Log2(0) undefined"); - // cntlzd/cntlzw starts counting at 2^63/2^31 (ignoring any higher-order bits), and does not affect cr0 -#if __TBB_WORDSIZE==8 - __asm__ __volatile__ ("cntlzd %0,%0" : "+r"(x)); - return 63-static_cast(x); -#else - __asm__ __volatile__ ("cntlzw %0,%0" : "+r"(x)); - return 31-static_cast(x); -#endif -} -#define __TBB_Log2(V) __TBB_machine_lg(V) - -// Assumes implicit alignment for any 32-bit value -typedef uint32_t __TBB_Flag; -#define __TBB_Flag __TBB_Flag - -inline bool __TBB_machine_trylockbyte( __TBB_atomic __TBB_Flag &flag ) { - return __TBB_machine_cmpswp4(&flag,1,0)==0; -} -#define __TBB_TryLockByte(P) __TBB_machine_trylockbyte(P) diff --git a/inst/include/tbb/machine/macos_common.h b/inst/include/tbb/machine/macos_common.h deleted file mode 100644 index 119ad979a..000000000 --- a/inst/include/tbb/machine/macos_common.h +++ /dev/null @@ -1,133 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#if !defined(__TBB_machine_H) || defined(__TBB_machine_macos_common_H) -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#define __TBB_machine_macos_common_H - -#include -#define __TBB_Yield() sched_yield() - -// __TBB_HardwareConcurrency - -#include -#include - -static inline int __TBB_macos_available_cpu() { - int name[2] = {CTL_HW, HW_AVAILCPU}; - int ncpu; - size_t size = sizeof(ncpu); - sysctl( name, 2, &ncpu, &size, NULL, 0 ); - return ncpu; -} - -#define __TBB_HardwareConcurrency() __TBB_macos_available_cpu() - -#ifndef __TBB_full_memory_fence - // TBB has not recognized the architecture (none of the architecture abstraction - // headers was included). - #define __TBB_UnknownArchitecture 1 -#endif - -#if __TBB_UnknownArchitecture -// Implementation of atomic operations based on OS provided primitives -#include - -static inline int64_t __TBB_machine_cmpswp8_OsX(volatile void *ptr, int64_t value, int64_t comparand) -{ - __TBB_ASSERT( tbb::internal::is_aligned(ptr,8), "address not properly aligned for macOS* atomics"); - int64_t* address = (int64_t*)ptr; - while( !OSAtomicCompareAndSwap64Barrier(comparand, value, address) ){ -#if __TBB_WORDSIZE==8 - int64_t snapshot = *address; -#else - int64_t snapshot = OSAtomicAdd64( 0, address ); -#endif - if( snapshot!=comparand ) return snapshot; - } - return comparand; -} - -#define __TBB_machine_cmpswp8 __TBB_machine_cmpswp8_OsX - -#endif /* __TBB_UnknownArchitecture */ - -#if __TBB_UnknownArchitecture - -#ifndef __TBB_WORDSIZE -#define __TBB_WORDSIZE __SIZEOF_POINTER__ -#endif - -#ifdef __TBB_ENDIANNESS - // Already determined based on hardware architecture. -#elif __BIG_ENDIAN__ - #define __TBB_ENDIANNESS __TBB_ENDIAN_BIG -#elif __LITTLE_ENDIAN__ - #define __TBB_ENDIANNESS __TBB_ENDIAN_LITTLE -#else - #define __TBB_ENDIANNESS __TBB_ENDIAN_UNSUPPORTED -#endif - -/** As this generic implementation has absolutely no information about underlying - hardware, its performance most likely will be sub-optimal because of full memory - fence usages where a more lightweight synchronization means (or none at all) - could suffice. Thus if you use this header to enable TBB on a new platform, - consider forking it and relaxing below helpers as appropriate. **/ -#define __TBB_control_consistency_helper() OSMemoryBarrier() -#define __TBB_acquire_consistency_helper() OSMemoryBarrier() -#define __TBB_release_consistency_helper() OSMemoryBarrier() -#define __TBB_full_memory_fence() OSMemoryBarrier() - -static inline int32_t __TBB_machine_cmpswp4(volatile void *ptr, int32_t value, int32_t comparand) -{ - __TBB_ASSERT( tbb::internal::is_aligned(ptr,4), "address not properly aligned for macOS atomics"); - int32_t* address = (int32_t*)ptr; - while( !OSAtomicCompareAndSwap32Barrier(comparand, value, address) ){ - int32_t snapshot = *address; - if( snapshot!=comparand ) return snapshot; - } - return comparand; -} - -static inline int32_t __TBB_machine_fetchadd4(volatile void *ptr, int32_t addend) -{ - __TBB_ASSERT( tbb::internal::is_aligned(ptr,4), "address not properly aligned for macOS atomics"); - return OSAtomicAdd32Barrier(addend, (int32_t*)ptr) - addend; -} - -static inline int64_t __TBB_machine_fetchadd8(volatile void *ptr, int64_t addend) -{ - __TBB_ASSERT( tbb::internal::is_aligned(ptr,8), "address not properly aligned for macOS atomics"); - return OSAtomicAdd64Barrier(addend, (int64_t*)ptr) - addend; -} - -#define __TBB_USE_GENERIC_PART_WORD_CAS 1 -#define __TBB_USE_GENERIC_PART_WORD_FETCH_ADD 1 -#define __TBB_USE_GENERIC_FETCH_STORE 1 -#define __TBB_USE_GENERIC_HALF_FENCED_LOAD_STORE 1 -#define __TBB_USE_GENERIC_RELAXED_LOAD_STORE 1 -#if __TBB_WORDSIZE == 4 - #define __TBB_USE_GENERIC_DWORD_LOAD_STORE 1 -#endif -#define __TBB_USE_GENERIC_SEQUENTIAL_CONSISTENCY_LOAD_STORE 1 - -#endif /* __TBB_UnknownArchitecture */ diff --git a/inst/include/tbb/machine/mic_common.h b/inst/include/tbb/machine/mic_common.h deleted file mode 100644 index 8765d39fb..000000000 --- a/inst/include/tbb/machine/mic_common.h +++ /dev/null @@ -1,57 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_mic_common_H -#define __TBB_mic_common_H - -#ifndef __TBB_machine_H -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#if ! __TBB_DEFINE_MIC - #error mic_common.h should be included only when building for Intel(R) Many Integrated Core Architecture -#endif - -#ifndef __TBB_PREFETCHING -#define __TBB_PREFETCHING 1 -#endif -#if __TBB_PREFETCHING -#include -#define __TBB_cl_prefetch(p) _mm_prefetch((const char*)p, _MM_HINT_T1) -#define __TBB_cl_evict(p) _mm_clevict(p, _MM_HINT_T1) -#endif - -/** Intel(R) Many Integrated Core Architecture does not support mfence and pause instructions **/ -#define __TBB_full_memory_fence() __asm__ __volatile__("lock; addl $0,(%%rsp)":::"memory") -#define __TBB_Pause(x) _mm_delay_32(16*(x)) -#define __TBB_STEALING_PAUSE 1500/16 -#include -#define __TBB_Yield() sched_yield() - -/** Specifics **/ -#define __TBB_STEALING_ABORT_ON_CONTENTION 1 -#define __TBB_YIELD2P 1 -#define __TBB_HOARD_NONLOCAL_TASKS 1 - -#if ! ( __FreeBSD__ || __linux__ ) - #error Intel(R) Many Integrated Core Compiler does not define __FreeBSD__ or __linux__ anymore. Check for the __TBB_XXX_BROKEN defined under __FreeBSD__ or __linux__. -#endif /* ! ( __FreeBSD__ || __linux__ ) */ - -#endif /* __TBB_mic_common_H */ diff --git a/inst/include/tbb/machine/msvc_armv7.h b/inst/include/tbb/machine/msvc_armv7.h deleted file mode 100644 index 40d220207..000000000 --- a/inst/include/tbb/machine/msvc_armv7.h +++ /dev/null @@ -1,171 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#if !defined(__TBB_machine_H) || defined(__TBB_msvc_armv7_H) -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#define __TBB_msvc_armv7_H - -#include -#include - -#define __TBB_WORDSIZE 4 - -#define __TBB_ENDIANNESS __TBB_ENDIAN_UNSUPPORTED - -#if defined(TBB_WIN32_USE_CL_BUILTINS) -// We can test this on _M_IX86 -#pragma intrinsic(_ReadWriteBarrier) -#pragma intrinsic(_mm_mfence) -#define __TBB_compiler_fence() _ReadWriteBarrier() -#define __TBB_full_memory_fence() _mm_mfence() -#define __TBB_control_consistency_helper() __TBB_compiler_fence() -#define __TBB_acquire_consistency_helper() __TBB_compiler_fence() -#define __TBB_release_consistency_helper() __TBB_compiler_fence() -#else -//Now __dmb(_ARM_BARRIER_SY) is used for both compiler and memory fences -//This might be changed later after testing -#define __TBB_compiler_fence() __dmb(_ARM_BARRIER_SY) -#define __TBB_full_memory_fence() __dmb(_ARM_BARRIER_SY) -#define __TBB_control_consistency_helper() __TBB_compiler_fence() -#define __TBB_acquire_consistency_helper() __TBB_full_memory_fence() -#define __TBB_release_consistency_helper() __TBB_full_memory_fence() -#endif - -//-------------------------------------------------- -// Compare and swap -//-------------------------------------------------- - -/** - * Atomic CAS for 32 bit values, if *ptr==comparand, then *ptr=value, returns *ptr - * @param ptr pointer to value in memory to be swapped with value if *ptr==comparand - * @param value value to assign *ptr to if *ptr==comparand - * @param comparand value to compare with *ptr - * @return value originally in memory at ptr, regardless of success -*/ - -#define __TBB_MACHINE_DEFINE_ATOMICS_CMPSWP(S,T,F) \ -inline T __TBB_machine_cmpswp##S( volatile void *ptr, T value, T comparand ) { \ - return _InterlockedCompareExchange##F(reinterpret_cast(ptr),value,comparand); \ -} \ - -#define __TBB_MACHINE_DEFINE_ATOMICS_FETCHADD(S,T,F) \ -inline T __TBB_machine_fetchadd##S( volatile void *ptr, T value ) { \ - return _InterlockedExchangeAdd##F(reinterpret_cast(ptr),value); \ -} \ - -__TBB_MACHINE_DEFINE_ATOMICS_CMPSWP(1,char,8) -__TBB_MACHINE_DEFINE_ATOMICS_CMPSWP(2,short,16) -__TBB_MACHINE_DEFINE_ATOMICS_CMPSWP(4,long,) -__TBB_MACHINE_DEFINE_ATOMICS_CMPSWP(8,__int64,64) -__TBB_MACHINE_DEFINE_ATOMICS_FETCHADD(4,long,) -#if defined(TBB_WIN32_USE_CL_BUILTINS) -// No _InterlockedExchangeAdd64 intrinsic on _M_IX86 -#define __TBB_64BIT_ATOMICS 0 -#else -__TBB_MACHINE_DEFINE_ATOMICS_FETCHADD(8,__int64,64) -#endif - -inline void __TBB_machine_pause (int32_t delay ) -{ - while(delay>0) - { - __TBB_compiler_fence(); - delay--; - } -} - -// API to retrieve/update FPU control setting -#define __TBB_CPU_CTL_ENV_PRESENT 1 - -namespace tbb { -namespace internal { - -template -struct machine_load_store_relaxed { - static inline T load ( const volatile T& location ) { - const T value = location; - - /* - * An extra memory barrier is required for errata #761319 - * Please see http://infocenter.arm.com/help/topic/com.arm.doc.uan0004a - */ - __TBB_acquire_consistency_helper(); - return value; - } - - static inline void store ( volatile T& location, T value ) { - location = value; - } -}; - -class cpu_ctl_env { -private: - unsigned int my_ctl; -public: - bool operator!=( const cpu_ctl_env& ctl ) const { return my_ctl != ctl.my_ctl; } - void get_env() { my_ctl = _control87(0, 0); } - void set_env() const { _control87( my_ctl, ~0U ); } -}; - -} // namespace internal -} // namespaces tbb - -// Machine specific atomic operations -#define __TBB_CompareAndSwap4(P,V,C) __TBB_machine_cmpswp4(P,V,C) -#define __TBB_CompareAndSwap8(P,V,C) __TBB_machine_cmpswp8(P,V,C) -#define __TBB_Pause(V) __TBB_machine_pause(V) - -// Use generics for some things -#define __TBB_USE_FETCHSTORE_AS_FULL_FENCED_STORE 1 -#define __TBB_USE_GENERIC_HALF_FENCED_LOAD_STORE 1 -#define __TBB_USE_GENERIC_PART_WORD_FETCH_ADD 1 -#define __TBB_USE_GENERIC_PART_WORD_FETCH_STORE 1 -#define __TBB_USE_GENERIC_FETCH_STORE 1 -#define __TBB_USE_GENERIC_DWORD_LOAD_STORE 1 -#define __TBB_USE_GENERIC_SEQUENTIAL_CONSISTENCY_LOAD_STORE 1 - -#if defined(TBB_WIN32_USE_CL_BUILTINS) -#if !__TBB_WIN8UI_SUPPORT -extern "C" __declspec(dllimport) int __stdcall SwitchToThread( void ); -#define __TBB_Yield() SwitchToThread() -#else -#include -#define __TBB_Yield() std::this_thread::yield() -#endif -#else -#define __TBB_Yield() __yield() -#endif - -// Machine specific atomic operations -#define __TBB_AtomicOR(P,V) __TBB_machine_OR(P,V) -#define __TBB_AtomicAND(P,V) __TBB_machine_AND(P,V) - -template -inline void __TBB_machine_OR( T1 *operand, T2 addend ) { - _InterlockedOr((long volatile *)operand, (long)addend); -} - -template -inline void __TBB_machine_AND( T1 *operand, T2 addend ) { - _InterlockedAnd((long volatile *)operand, (long)addend); -} - diff --git a/inst/include/tbb/machine/msvc_ia32_common.h b/inst/include/tbb/machine/msvc_ia32_common.h deleted file mode 100644 index 8b4814bcc..000000000 --- a/inst/include/tbb/machine/msvc_ia32_common.h +++ /dev/null @@ -1,280 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#if !defined(__TBB_machine_H) || defined(__TBB_machine_msvc_ia32_common_H) -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#define __TBB_machine_msvc_ia32_common_H - -#include - -//TODO: consider moving this macro to tbb_config.h and using where MSVC asm is used -#if !_M_X64 || __INTEL_COMPILER - #define __TBB_X86_MSVC_INLINE_ASM_AVAILABLE 1 -#else - //MSVC in x64 mode does not accept inline assembler - #define __TBB_X86_MSVC_INLINE_ASM_AVAILABLE 0 - #define __TBB_NO_X86_MSVC_INLINE_ASM_MSG "The compiler being used is not supported (outdated?)" -#endif - -#if _M_X64 - #define __TBB_r(reg_name) r##reg_name - #define __TBB_W(name) name##64 - namespace tbb { namespace internal { namespace msvc_intrinsics { - typedef __int64 word; - }}} -#else - #define __TBB_r(reg_name) e##reg_name - #define __TBB_W(name) name - namespace tbb { namespace internal { namespace msvc_intrinsics { - typedef long word; - }}} -#endif - -#if _MSC_VER>=1600 && (!__INTEL_COMPILER || __INTEL_COMPILER>=1310) - // S is the operand size in bytes, B is the suffix for intrinsics for that size - #define __TBB_MACHINE_DEFINE_ATOMICS(S,B,T,U) \ - __pragma(intrinsic( _InterlockedCompareExchange##B )) \ - static inline T __TBB_machine_cmpswp##S ( volatile void * ptr, U value, U comparand ) { \ - return _InterlockedCompareExchange##B ( (T*)ptr, value, comparand ); \ - } \ - __pragma(intrinsic( _InterlockedExchangeAdd##B )) \ - static inline T __TBB_machine_fetchadd##S ( volatile void * ptr, U addend ) { \ - return _InterlockedExchangeAdd##B ( (T*)ptr, addend ); \ - } \ - __pragma(intrinsic( _InterlockedExchange##B )) \ - static inline T __TBB_machine_fetchstore##S ( volatile void * ptr, U value ) { \ - return _InterlockedExchange##B ( (T*)ptr, value ); \ - } - - // Atomic intrinsics for 1, 2, and 4 bytes are available for x86 & x64 - __TBB_MACHINE_DEFINE_ATOMICS(1,8,char,__int8) - __TBB_MACHINE_DEFINE_ATOMICS(2,16,short,__int16) - __TBB_MACHINE_DEFINE_ATOMICS(4,,long,__int32) - - #if __TBB_WORDSIZE==8 - __TBB_MACHINE_DEFINE_ATOMICS(8,64,__int64,__int64) - #endif - - #undef __TBB_MACHINE_DEFINE_ATOMICS - #define __TBB_ATOMIC_PRIMITIVES_DEFINED -#endif /*_MSC_VER>=1600*/ - -#if _MSC_VER>=1300 || __INTEL_COMPILER>=1100 - #pragma intrinsic(_ReadWriteBarrier) - #pragma intrinsic(_mm_mfence) - #define __TBB_compiler_fence() _ReadWriteBarrier() - #define __TBB_full_memory_fence() _mm_mfence() -#elif __TBB_X86_MSVC_INLINE_ASM_AVAILABLE - #define __TBB_compiler_fence() __asm { __asm nop } - #define __TBB_full_memory_fence() __asm { __asm mfence } -#else - #error Unsupported compiler; define __TBB_{control,acquire,release}_consistency_helper to support it -#endif - -#define __TBB_control_consistency_helper() __TBB_compiler_fence() -#define __TBB_acquire_consistency_helper() __TBB_compiler_fence() -#define __TBB_release_consistency_helper() __TBB_compiler_fence() - -#if (_MSC_VER>=1300) || (__INTEL_COMPILER) - #pragma intrinsic(_mm_pause) - namespace tbb { namespace internal { namespace msvc_intrinsics { - static inline void pause (uintptr_t delay ) { - for (;delay>0; --delay ) - _mm_pause(); - } - }}} - #define __TBB_Pause(V) tbb::internal::msvc_intrinsics::pause(V) - #define __TBB_SINGLE_PAUSE _mm_pause() -#else - #if !__TBB_X86_MSVC_INLINE_ASM_AVAILABLE - #error __TBB_NO_X86_MSVC_INLINE_ASM_MSG - #endif - namespace tbb { namespace internal { namespace msvc_inline_asm - static inline void pause (uintptr_t delay ) { - _asm - { - mov __TBB_r(ax), delay - __TBB_L1: - pause - add __TBB_r(ax), -1 - jne __TBB_L1 - } - return; - } - }}} - #define __TBB_Pause(V) tbb::internal::msvc_inline_asm::pause(V) - #define __TBB_SINGLE_PAUSE __asm pause -#endif - -#if (_MSC_VER>=1400 && !__INTEL_COMPILER) || (__INTEL_COMPILER>=1200) -// MSVC did not have this intrinsic prior to VC8. -// ICL 11.1 fails to compile a TBB example if __TBB_Log2 uses the intrinsic. - #pragma intrinsic(__TBB_W(_BitScanReverse)) - namespace tbb { namespace internal { namespace msvc_intrinsics { - static inline uintptr_t lg_bsr( uintptr_t i ){ - unsigned long j; - __TBB_W(_BitScanReverse)( &j, i ); - return j; - } - }}} - #define __TBB_Log2(V) tbb::internal::msvc_intrinsics::lg_bsr(V) -#else - #if !__TBB_X86_MSVC_INLINE_ASM_AVAILABLE - #error __TBB_NO_X86_MSVC_INLINE_ASM_MSG - #endif - namespace tbb { namespace internal { namespace msvc_inline_asm { - static inline uintptr_t lg_bsr( uintptr_t i ){ - uintptr_t j; - __asm - { - bsr __TBB_r(ax), i - mov j, __TBB_r(ax) - } - return j; - } - }}} - #define __TBB_Log2(V) tbb::internal::msvc_inline_asm::lg_bsr(V) -#endif - -#if _MSC_VER>=1400 - #pragma intrinsic(__TBB_W(_InterlockedOr)) - #pragma intrinsic(__TBB_W(_InterlockedAnd)) - namespace tbb { namespace internal { namespace msvc_intrinsics { - static inline void lock_or( volatile void *operand, intptr_t addend ){ - __TBB_W(_InterlockedOr)((volatile word*)operand, addend); - } - static inline void lock_and( volatile void *operand, intptr_t addend ){ - __TBB_W(_InterlockedAnd)((volatile word*)operand, addend); - } - }}} - #define __TBB_AtomicOR(P,V) tbb::internal::msvc_intrinsics::lock_or(P,V) - #define __TBB_AtomicAND(P,V) tbb::internal::msvc_intrinsics::lock_and(P,V) -#else - #if !__TBB_X86_MSVC_INLINE_ASM_AVAILABLE - #error __TBB_NO_X86_MSVC_INLINE_ASM_MSG - #endif - namespace tbb { namespace internal { namespace msvc_inline_asm { - static inline void lock_or( volatile void *operand, __int32 addend ) { - __asm - { - mov eax, addend - mov edx, [operand] - lock or [edx], eax - } - } - static inline void lock_and( volatile void *operand, __int32 addend ) { - __asm - { - mov eax, addend - mov edx, [operand] - lock and [edx], eax - } - } - }}} - #define __TBB_AtomicOR(P,V) tbb::internal::msvc_inline_asm::lock_or(P,V) - #define __TBB_AtomicAND(P,V) tbb::internal::msvc_inline_asm::lock_and(P,V) -#endif - -#pragma intrinsic(__rdtsc) -namespace tbb { namespace internal { typedef uint64_t machine_tsc_t; } } -static inline tbb::internal::machine_tsc_t __TBB_machine_time_stamp() { - return __rdtsc(); -} -#define __TBB_time_stamp() __TBB_machine_time_stamp() - -// API to retrieve/update FPU control setting -#define __TBB_CPU_CTL_ENV_PRESENT 1 - -namespace tbb { namespace internal { class cpu_ctl_env; } } -#if __TBB_X86_MSVC_INLINE_ASM_AVAILABLE - inline void __TBB_get_cpu_ctl_env ( tbb::internal::cpu_ctl_env* ctl ) { - __asm { - __asm mov __TBB_r(ax), ctl - __asm stmxcsr [__TBB_r(ax)] - __asm fstcw [__TBB_r(ax)+4] - } - } - inline void __TBB_set_cpu_ctl_env ( const tbb::internal::cpu_ctl_env* ctl ) { - __asm { - __asm mov __TBB_r(ax), ctl - __asm ldmxcsr [__TBB_r(ax)] - __asm fldcw [__TBB_r(ax)+4] - } - } -#else - extern "C" { - void __TBB_EXPORTED_FUNC __TBB_get_cpu_ctl_env ( tbb::internal::cpu_ctl_env* ); - void __TBB_EXPORTED_FUNC __TBB_set_cpu_ctl_env ( const tbb::internal::cpu_ctl_env* ); - } -#endif - -namespace tbb { -namespace internal { -class cpu_ctl_env { -private: - int mxcsr; - short x87cw; - static const int MXCSR_CONTROL_MASK = ~0x3f; /* all except last six status bits */ -public: - bool operator!=( const cpu_ctl_env& ctl ) const { return mxcsr != ctl.mxcsr || x87cw != ctl.x87cw; } - void get_env() { - __TBB_get_cpu_ctl_env( this ); - mxcsr &= MXCSR_CONTROL_MASK; - } - void set_env() const { __TBB_set_cpu_ctl_env( this ); } -}; -} // namespace internal -} // namespace tbb - -#if !__TBB_WIN8UI_SUPPORT -extern "C" __declspec(dllimport) int __stdcall SwitchToThread( void ); -#define __TBB_Yield() SwitchToThread() -#else -#include -#define __TBB_Yield() std::this_thread::yield() -#endif - -#undef __TBB_r -#undef __TBB_W -#undef __TBB_word - -extern "C" { - __int8 __TBB_EXPORTED_FUNC __TBB_machine_try_lock_elided (volatile void* ptr); - void __TBB_EXPORTED_FUNC __TBB_machine_unlock_elided (volatile void* ptr); - - // 'pause' instruction aborts HLE/RTM transactions - inline static void __TBB_machine_try_lock_elided_cancel() { __TBB_SINGLE_PAUSE; } - -#if __TBB_TSX_INTRINSICS_PRESENT - #define __TBB_machine_is_in_transaction _xtest - #define __TBB_machine_begin_transaction _xbegin - #define __TBB_machine_end_transaction _xend - // The value (0xFF) below comes from the - // Intel(R) 64 and IA-32 Architectures Optimization Reference Manual 12.4.5 lock not free - #define __TBB_machine_transaction_conflict_abort() _xabort(0xFF) -#else - __int8 __TBB_EXPORTED_FUNC __TBB_machine_is_in_transaction(); - unsigned __int32 __TBB_EXPORTED_FUNC __TBB_machine_begin_transaction(); - void __TBB_EXPORTED_FUNC __TBB_machine_end_transaction(); - void __TBB_EXPORTED_FUNC __TBB_machine_transaction_conflict_abort(); -#endif /* __TBB_TSX_INTRINSICS_PRESENT */ -} diff --git a/inst/include/tbb/machine/sunos_sparc.h b/inst/include/tbb/machine/sunos_sparc.h deleted file mode 100644 index 9119f4028..000000000 --- a/inst/include/tbb/machine/sunos_sparc.h +++ /dev/null @@ -1,203 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - - -#if !defined(__TBB_machine_H) || defined(__TBB_machine_sunos_sparc_H) -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#define __TBB_machine_sunos_sparc_H - -#include -#include - -#define __TBB_WORDSIZE 8 -// Big endian is assumed for SPARC. -// While hardware may support page-specific bi-endianness, only big endian pages may be exposed to TBB -#define __TBB_ENDIANNESS __TBB_ENDIAN_BIG - -/** To those working on SPARC hardware. Consider relaxing acquire and release - consistency helpers to no-op (as this port covers TSO mode only). **/ -#define __TBB_compiler_fence() __asm__ __volatile__ ("": : :"memory") -#define __TBB_control_consistency_helper() __TBB_compiler_fence() -#define __TBB_acquire_consistency_helper() __TBB_compiler_fence() -#define __TBB_release_consistency_helper() __TBB_compiler_fence() -#define __TBB_full_memory_fence() __asm__ __volatile__("membar #LoadLoad|#LoadStore|#StoreStore|#StoreLoad": : : "memory") - -//-------------------------------------------------- -// Compare and swap -//-------------------------------------------------- - -/** - * Atomic CAS for 32 bit values, if *ptr==comparand, then *ptr=value, returns *ptr - * @param ptr pointer to value in memory to be swapped with value if *ptr==comparand - * @param value value to assign *ptr to if *ptr==comparand - * @param comparand value to compare with *ptr - ( @return value originally in memory at ptr, regardless of success -*/ -static inline int32_t __TBB_machine_cmpswp4(volatile void *ptr, int32_t value, int32_t comparand ){ - int32_t result; - __asm__ __volatile__( - "cas\t[%5],%4,%1" - : "=m"(*(int32_t *)ptr), "=r"(result) - : "m"(*(int32_t *)ptr), "1"(value), "r"(comparand), "r"(ptr) - : "memory"); - return result; -} - -/** - * Atomic CAS for 64 bit values, if *ptr==comparand, then *ptr=value, returns *ptr - * @param ptr pointer to value in memory to be swapped with value if *ptr==comparand - * @param value value to assign *ptr to if *ptr==comparand - * @param comparand value to compare with *ptr - ( @return value originally in memory at ptr, regardless of success - */ -static inline int64_t __TBB_machine_cmpswp8(volatile void *ptr, int64_t value, int64_t comparand ){ - int64_t result; - __asm__ __volatile__( - "casx\t[%5],%4,%1" - : "=m"(*(int64_t *)ptr), "=r"(result) - : "m"(*(int64_t *)ptr), "1"(value), "r"(comparand), "r"(ptr) - : "memory"); - return result; -} - -//--------------------------------------------------- -// Fetch and add -//--------------------------------------------------- - -/** - * Atomic fetch and add for 32 bit values, in this case implemented by continuously checking success of atomicity - * @param ptr pointer to value to add addend to - * @param addened value to add to *ptr - * @return value at ptr before addened was added - */ -static inline int32_t __TBB_machine_fetchadd4(volatile void *ptr, int32_t addend){ - int32_t result; - __asm__ __volatile__ ( - "0:\t add\t %3, %4, %0\n" // do addition - "\t cas\t [%2], %3, %0\n" // cas to store result in memory - "\t cmp\t %3, %0\n" // check if value from memory is original - "\t bne,a,pn\t %%icc, 0b\n" // if not try again - "\t mov %0, %3\n" // use branch delay slot to move new value in memory to be added - : "=&r"(result), "=m"(*(int32_t *)ptr) - : "r"(ptr), "r"(*(int32_t *)ptr), "r"(addend), "m"(*(int32_t *)ptr) - : "ccr", "memory"); - return result; -} - -/** - * Atomic fetch and add for 64 bit values, in this case implemented by continuously checking success of atomicity - * @param ptr pointer to value to add addend to - * @param addened value to add to *ptr - * @return value at ptr before addened was added - */ -static inline int64_t __TBB_machine_fetchadd8(volatile void *ptr, int64_t addend){ - int64_t result; - __asm__ __volatile__ ( - "0:\t add\t %3, %4, %0\n" // do addition - "\t casx\t [%2], %3, %0\n" // cas to store result in memory - "\t cmp\t %3, %0\n" // check if value from memory is original - "\t bne,a,pn\t %%xcc, 0b\n" // if not try again - "\t mov %0, %3\n" // use branch delay slot to move new value in memory to be added - : "=&r"(result), "=m"(*(int64_t *)ptr) - : "r"(ptr), "r"(*(int64_t *)ptr), "r"(addend), "m"(*(int64_t *)ptr) - : "ccr", "memory"); - return result; -} - -//-------------------------------------------------------- -// Logarithm (base two, integer) -//-------------------------------------------------------- - -static inline int64_t __TBB_machine_lg( uint64_t x ) { - __TBB_ASSERT(x, "__TBB_Log2(0) undefined"); - uint64_t count; - // one hot encode - x |= (x >> 1); - x |= (x >> 2); - x |= (x >> 4); - x |= (x >> 8); - x |= (x >> 16); - x |= (x >> 32); - // count 1's - __asm__ ("popc %1, %0" : "=r"(count) : "r"(x) ); - return count-1; -} - -//-------------------------------------------------------- - -static inline void __TBB_machine_or( volatile void *ptr, uint64_t value ) { - __asm__ __volatile__ ( - "0:\t or\t %2, %3, %%g1\n" // do operation - "\t casx\t [%1], %2, %%g1\n" // cas to store result in memory - "\t cmp\t %2, %%g1\n" // check if value from memory is original - "\t bne,a,pn\t %%xcc, 0b\n" // if not try again - "\t mov %%g1, %2\n" // use branch delay slot to move new value in memory to be added - : "=m"(*(int64_t *)ptr) - : "r"(ptr), "r"(*(int64_t *)ptr), "r"(value), "m"(*(int64_t *)ptr) - : "ccr", "g1", "memory"); -} - -static inline void __TBB_machine_and( volatile void *ptr, uint64_t value ) { - __asm__ __volatile__ ( - "0:\t and\t %2, %3, %%g1\n" // do operation - "\t casx\t [%1], %2, %%g1\n" // cas to store result in memory - "\t cmp\t %2, %%g1\n" // check if value from memory is original - "\t bne,a,pn\t %%xcc, 0b\n" // if not try again - "\t mov %%g1, %2\n" // use branch delay slot to move new value in memory to be added - : "=m"(*(int64_t *)ptr) - : "r"(ptr), "r"(*(int64_t *)ptr), "r"(value), "m"(*(int64_t *)ptr) - : "ccr", "g1", "memory"); -} - - -static inline void __TBB_machine_pause( int32_t delay ) { - // do nothing, inlined, doesn't matter -} - -// put 0xff in memory location, return memory value, -// generic trylockbyte puts 0x01, however this is fine -// because all that matters is that 0 is unlocked -static inline bool __TBB_machine_trylockbyte(unsigned char &flag){ - unsigned char result; - __asm__ __volatile__ ( - "ldstub\t [%2], %0\n" - : "=r"(result), "=m"(flag) - : "r"(&flag), "m"(flag) - : "memory"); - return result == 0; -} - -#define __TBB_USE_GENERIC_PART_WORD_CAS 1 -#define __TBB_USE_GENERIC_PART_WORD_FETCH_ADD 1 -#define __TBB_USE_GENERIC_FETCH_STORE 1 -#define __TBB_USE_GENERIC_HALF_FENCED_LOAD_STORE 1 -#define __TBB_USE_GENERIC_RELAXED_LOAD_STORE 1 -#define __TBB_USE_GENERIC_SEQUENTIAL_CONSISTENCY_LOAD_STORE 1 - -#define __TBB_AtomicOR(P,V) __TBB_machine_or(P,V) -#define __TBB_AtomicAND(P,V) __TBB_machine_and(P,V) - -// Definition of other functions -#define __TBB_Pause(V) __TBB_machine_pause(V) -#define __TBB_Log2(V) __TBB_machine_lg(V) - -#define __TBB_TryLockByte(P) __TBB_machine_trylockbyte(P) diff --git a/inst/include/tbb/machine/windows_api.h b/inst/include/tbb/machine/windows_api.h deleted file mode 100644 index d362abc5e..000000000 --- a/inst/include/tbb/machine/windows_api.h +++ /dev/null @@ -1,69 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_machine_windows_api_H -#define __TBB_machine_windows_api_H - -#if _WIN32 || _WIN64 - -#include - -#if _WIN32_WINNT < 0x0600 -// The following Windows API function is declared explicitly; -// otherwise it fails to compile by VS2005. -#if !defined(WINBASEAPI) || (_WIN32_WINNT < 0x0501 && _MSC_VER == 1400) -#define __TBB_WINBASEAPI extern "C" -#else -#define __TBB_WINBASEAPI WINBASEAPI -#endif -__TBB_WINBASEAPI BOOL WINAPI TryEnterCriticalSection( LPCRITICAL_SECTION ); -__TBB_WINBASEAPI BOOL WINAPI InitializeCriticalSectionAndSpinCount( LPCRITICAL_SECTION, DWORD ); -// Overloading WINBASEAPI macro and using local functions missing in Windows XP/2003 -#define InitializeCriticalSectionEx inlineInitializeCriticalSectionEx -#define CreateSemaphoreEx inlineCreateSemaphoreEx -#define CreateEventEx inlineCreateEventEx -inline BOOL WINAPI inlineInitializeCriticalSectionEx( LPCRITICAL_SECTION lpCriticalSection, DWORD dwSpinCount, DWORD ) -{ - return InitializeCriticalSectionAndSpinCount( lpCriticalSection, dwSpinCount ); -} -inline HANDLE WINAPI inlineCreateSemaphoreEx( LPSECURITY_ATTRIBUTES lpSemaphoreAttributes, LONG lInitialCount, LONG lMaximumCount, LPCTSTR lpName, DWORD, DWORD ) -{ - return CreateSemaphore( lpSemaphoreAttributes, lInitialCount, lMaximumCount, lpName ); -} -inline HANDLE WINAPI inlineCreateEventEx( LPSECURITY_ATTRIBUTES lpEventAttributes, LPCTSTR lpName, DWORD dwFlags, DWORD ) -{ - BOOL manual_reset = dwFlags&0x00000001 ? TRUE : FALSE; // CREATE_EVENT_MANUAL_RESET - BOOL initial_set = dwFlags&0x00000002 ? TRUE : FALSE; // CREATE_EVENT_INITIAL_SET - return CreateEvent( lpEventAttributes, manual_reset, initial_set, lpName ); -} -#endif - -#if defined(RTL_SRWLOCK_INIT) -#ifndef __TBB_USE_SRWLOCK -// TODO: turn it on when bug 1952 will be fixed -#define __TBB_USE_SRWLOCK 0 -#endif -#endif - -#else -#error tbb/machine/windows_api.h should only be used for Windows based platforms -#endif // _WIN32 || _WIN64 - -#endif // __TBB_machine_windows_api_H diff --git a/inst/include/tbb/machine/windows_ia32.h b/inst/include/tbb/machine/windows_ia32.h deleted file mode 100644 index 8db0d2b82..000000000 --- a/inst/include/tbb/machine/windows_ia32.h +++ /dev/null @@ -1,109 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#if !defined(__TBB_machine_H) || defined(__TBB_machine_windows_ia32_H) -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#define __TBB_machine_windows_ia32_H - -#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) - // Workaround for overzealous compiler warnings in /Wp64 mode - #pragma warning (push) - #pragma warning (disable: 4244 4267) -#endif - -#include "msvc_ia32_common.h" - -#define __TBB_WORDSIZE 4 -#define __TBB_ENDIANNESS __TBB_ENDIAN_LITTLE - -extern "C" { - __int64 __TBB_EXPORTED_FUNC __TBB_machine_cmpswp8 (volatile void *ptr, __int64 value, __int64 comparand ); - __int64 __TBB_EXPORTED_FUNC __TBB_machine_fetchadd8 (volatile void *ptr, __int64 addend ); - __int64 __TBB_EXPORTED_FUNC __TBB_machine_fetchstore8 (volatile void *ptr, __int64 value ); - void __TBB_EXPORTED_FUNC __TBB_machine_store8 (volatile void *ptr, __int64 value ); - __int64 __TBB_EXPORTED_FUNC __TBB_machine_load8 (const volatile void *ptr); -} - -#ifndef __TBB_ATOMIC_PRIMITIVES_DEFINED - -#define __TBB_MACHINE_DEFINE_ATOMICS(S,T,U,A,C) \ -static inline T __TBB_machine_cmpswp##S ( volatile void * ptr, U value, U comparand ) { \ - T result; \ - volatile T *p = (T *)ptr; \ - __asm \ - { \ - __asm mov edx, p \ - __asm mov C , value \ - __asm mov A , comparand \ - __asm lock cmpxchg [edx], C \ - __asm mov result, A \ - } \ - return result; \ -} \ -\ -static inline T __TBB_machine_fetchadd##S ( volatile void * ptr, U addend ) { \ - T result; \ - volatile T *p = (T *)ptr; \ - __asm \ - { \ - __asm mov edx, p \ - __asm mov A, addend \ - __asm lock xadd [edx], A \ - __asm mov result, A \ - } \ - return result; \ -}\ -\ -static inline T __TBB_machine_fetchstore##S ( volatile void * ptr, U value ) { \ - T result; \ - volatile T *p = (T *)ptr; \ - __asm \ - { \ - __asm mov edx, p \ - __asm mov A, value \ - __asm lock xchg [edx], A \ - __asm mov result, A \ - } \ - return result; \ -} - - -__TBB_MACHINE_DEFINE_ATOMICS(1, __int8, __int8, al, cl) -__TBB_MACHINE_DEFINE_ATOMICS(2, __int16, __int16, ax, cx) -__TBB_MACHINE_DEFINE_ATOMICS(4, ptrdiff_t, ptrdiff_t, eax, ecx) - -#undef __TBB_MACHINE_DEFINE_ATOMICS - -#endif /*__TBB_ATOMIC_PRIMITIVES_DEFINED*/ - -//TODO: Check if it possible and profitable for IA-32 architecture on (Linux and Windows) -//to use of 64-bit load/store via floating point registers together with full fence -//for sequentially consistent load/store, instead of CAS. -#define __TBB_USE_FETCHSTORE_AS_FULL_FENCED_STORE 1 -#define __TBB_USE_GENERIC_HALF_FENCED_LOAD_STORE 1 -#define __TBB_USE_GENERIC_RELAXED_LOAD_STORE 1 -#define __TBB_USE_GENERIC_SEQUENTIAL_CONSISTENCY_LOAD_STORE 1 - - -#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) - #pragma warning (pop) -#endif // warnings 4244, 4267 are back diff --git a/inst/include/tbb/machine/windows_intel64.h b/inst/include/tbb/machine/windows_intel64.h deleted file mode 100644 index 86abd6ad1..000000000 --- a/inst/include/tbb/machine/windows_intel64.h +++ /dev/null @@ -1,74 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#if !defined(__TBB_machine_H) || defined(__TBB_machine_windows_intel64_H) -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#define __TBB_machine_windows_intel64_H - -#define __TBB_WORDSIZE 8 -#define __TBB_ENDIANNESS __TBB_ENDIAN_LITTLE - -#include "msvc_ia32_common.h" - -#ifndef __TBB_ATOMIC_PRIMITIVES_DEFINED - -#include -#pragma intrinsic(_InterlockedCompareExchange,_InterlockedExchangeAdd,_InterlockedExchange) -#pragma intrinsic(_InterlockedCompareExchange64,_InterlockedExchangeAdd64,_InterlockedExchange64) - -// ATTENTION: if you ever change argument types in machine-specific primitives, -// please take care of atomic_word<> specializations in tbb/atomic.h -extern "C" { - __int8 __TBB_EXPORTED_FUNC __TBB_machine_cmpswp1 (volatile void *ptr, __int8 value, __int8 comparand ); - __int8 __TBB_EXPORTED_FUNC __TBB_machine_fetchadd1 (volatile void *ptr, __int8 addend ); - __int8 __TBB_EXPORTED_FUNC __TBB_machine_fetchstore1 (volatile void *ptr, __int8 value ); - __int16 __TBB_EXPORTED_FUNC __TBB_machine_cmpswp2 (volatile void *ptr, __int16 value, __int16 comparand ); - __int16 __TBB_EXPORTED_FUNC __TBB_machine_fetchadd2 (volatile void *ptr, __int16 addend ); - __int16 __TBB_EXPORTED_FUNC __TBB_machine_fetchstore2 (volatile void *ptr, __int16 value ); -} - -inline long __TBB_machine_cmpswp4 (volatile void *ptr, __int32 value, __int32 comparand ) { - return _InterlockedCompareExchange( (long*)ptr, value, comparand ); -} -inline long __TBB_machine_fetchadd4 (volatile void *ptr, __int32 addend ) { - return _InterlockedExchangeAdd( (long*)ptr, addend ); -} -inline long __TBB_machine_fetchstore4 (volatile void *ptr, __int32 value ) { - return _InterlockedExchange( (long*)ptr, value ); -} - -inline __int64 __TBB_machine_cmpswp8 (volatile void *ptr, __int64 value, __int64 comparand ) { - return _InterlockedCompareExchange64( (__int64*)ptr, value, comparand ); -} -inline __int64 __TBB_machine_fetchadd8 (volatile void *ptr, __int64 addend ) { - return _InterlockedExchangeAdd64( (__int64*)ptr, addend ); -} -inline __int64 __TBB_machine_fetchstore8 (volatile void *ptr, __int64 value ) { - return _InterlockedExchange64( (__int64*)ptr, value ); -} - -#endif /*__TBB_ATOMIC_PRIMITIVES_DEFINED*/ - -#define __TBB_USE_FETCHSTORE_AS_FULL_FENCED_STORE 1 -#define __TBB_USE_GENERIC_HALF_FENCED_LOAD_STORE 1 -#define __TBB_USE_GENERIC_RELAXED_LOAD_STORE 1 -#define __TBB_USE_GENERIC_SEQUENTIAL_CONSISTENCY_LOAD_STORE 1 diff --git a/inst/include/tbb/machine/xbox360_ppc.h b/inst/include/tbb/machine/xbox360_ppc.h deleted file mode 100644 index 148e5b1d0..000000000 --- a/inst/include/tbb/machine/xbox360_ppc.h +++ /dev/null @@ -1,119 +0,0 @@ -/* - Copyright 2005-2014 Intel Corporation. All Rights Reserved. - - This file is part of Threading Building Blocks. Threading Building Blocks is free software; - you can redistribute it and/or modify it under the terms of the GNU General Public License - version 2 as published by the Free Software Foundation. Threading Building Blocks is - distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the - implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. You should have received a copy of - the GNU General Public License along with Threading Building Blocks; if not, write to the - Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - - As a special exception, you may use this file as part of a free software library without - restriction. Specifically, if other files instantiate templates or use macros or inline - functions from this file, or you compile this file and link it with other files to produce - an executable, this file does not by itself cause the resulting executable to be covered - by the GNU General Public License. This exception does not however invalidate any other - reasons why the executable file might be covered by the GNU General Public License. -*/ - -// TODO: revise by comparing with mac_ppc.h - -#if !defined(__TBB_machine_H) || defined(__TBB_machine_xbox360_ppc_H) -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#define __TBB_machine_xbox360_ppc_H - -#define NONET -#define NOD3D -#include "xtl.h" -#include "ppcintrinsics.h" - -#if _MSC_VER >= 1300 -extern "C" void _MemoryBarrier(); -#pragma intrinsic(_MemoryBarrier) -#define __TBB_control_consistency_helper() __isync() -#define __TBB_acquire_consistency_helper() _MemoryBarrier() -#define __TBB_release_consistency_helper() _MemoryBarrier() -#endif - -#define __TBB_full_memory_fence() __sync() - -#define __TBB_WORDSIZE 4 -#define __TBB_ENDIANNESS __TBB_ENDIAN_BIG - -//todo: define __TBB_USE_FENCED_ATOMICS and define acquire/release primitives to maximize performance - -inline __int32 __TBB_machine_cmpswp4(volatile void *ptr, __int32 value, __int32 comparand ) { - __sync(); - __int32 result = InterlockedCompareExchange((volatile LONG*)ptr, value, comparand); - __isync(); - return result; -} - -inline __int64 __TBB_machine_cmpswp8(volatile void *ptr, __int64 value, __int64 comparand ) -{ - __sync(); - __int64 result = InterlockedCompareExchange64((volatile LONG64*)ptr, value, comparand); - __isync(); - return result; -} - -#define __TBB_USE_GENERIC_PART_WORD_CAS 1 -#define __TBB_USE_GENERIC_FETCH_ADD 1 -#define __TBB_USE_GENERIC_FETCH_STORE 1 -#define __TBB_USE_GENERIC_HALF_FENCED_LOAD_STORE 1 -#define __TBB_USE_GENERIC_RELAXED_LOAD_STORE 1 -#define __TBB_USE_GENERIC_DWORD_LOAD_STORE 1 -#define __TBB_USE_GENERIC_SEQUENTIAL_CONSISTENCY_LOAD_STORE 1 - -#pragma optimize( "", off ) -inline void __TBB_machine_pause (__int32 delay ) -{ - for (__int32 i=0; i> 0) & 1) + - ((__TBB_XBOX360_HARDWARE_THREAD_MASK >> 1) & 1) + - ((__TBB_XBOX360_HARDWARE_THREAD_MASK >> 2) & 1) + - ((__TBB_XBOX360_HARDWARE_THREAD_MASK >> 3) & 1) + - ((__TBB_XBOX360_HARDWARE_THREAD_MASK >> 4) & 1) + - ((__TBB_XBOX360_HARDWARE_THREAD_MASK >> 5) & 1) + 1; // +1 accomodates for the master thread -} - -static inline int __TBB_XBOX360_GetHardwareThreadIndex(int workerThreadIndex) -{ - workerThreadIndex %= __TBB_XBOX360_DetectNumberOfWorkers()-1; - int m = __TBB_XBOX360_HARDWARE_THREAD_MASK; - int index = 0; - int skipcount = workerThreadIndex; - while (true) - { - if ((m & 1)!=0) - { - if (skipcount==0) break; - skipcount--; - } - m >>= 1; - index++; - } - return index; -} - -#define __TBB_HardwareConcurrency() __TBB_XBOX360_DetectNumberOfWorkers() diff --git a/inst/include/tbb/mutex.h b/inst/include/tbb/mutex.h deleted file mode 100644 index e40b4cd02..000000000 --- a/inst/include/tbb/mutex.h +++ /dev/null @@ -1,233 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_mutex_H -#define __TBB_mutex_H - -#if _WIN32||_WIN64 -#include "machine/windows_api.h" -#else -#include -#endif /* _WIN32||_WIN64 */ - -#include -#include "aligned_space.h" -#include "tbb_stddef.h" -#include "tbb_profiling.h" - -namespace tbb { - -//! Wrapper around the platform's native lock. -/** @ingroup synchronization */ -class mutex : internal::mutex_copy_deprecated_and_disabled { -public: - //! Construct unacquired mutex. - mutex() { -#if TBB_USE_ASSERT || TBB_USE_THREADING_TOOLS - internal_construct(); -#else - #if _WIN32||_WIN64 - InitializeCriticalSectionEx(&impl, 4000, 0); - #else - int error_code = pthread_mutex_init(&impl,NULL); - if( error_code ) - tbb::internal::handle_perror(error_code,"mutex: pthread_mutex_init failed"); - #endif /* _WIN32||_WIN64*/ -#endif /* TBB_USE_ASSERT */ - }; - - ~mutex() { -#if TBB_USE_ASSERT - internal_destroy(); -#else - #if _WIN32||_WIN64 - DeleteCriticalSection(&impl); - #else - pthread_mutex_destroy(&impl); - - #endif /* _WIN32||_WIN64 */ -#endif /* TBB_USE_ASSERT */ - }; - - class scoped_lock; - friend class scoped_lock; - - //! The scoped locking pattern - /** It helps to avoid the common problem of forgetting to release lock. - It also nicely provides the "node" for queuing locks. */ - class scoped_lock : internal::no_copy { - public: - //! Construct lock that has not acquired a mutex. - scoped_lock() : my_mutex(NULL) {}; - - //! Acquire lock on given mutex. - scoped_lock( mutex& mutex ) { - acquire( mutex ); - } - - //! Release lock (if lock is held). - ~scoped_lock() { - if( my_mutex ) - release(); - } - - //! Acquire lock on given mutex. - void acquire( mutex& mutex ) { -#if TBB_USE_ASSERT - internal_acquire(mutex); -#else - mutex.lock(); - my_mutex = &mutex; -#endif /* TBB_USE_ASSERT */ - } - - //! Try acquire lock on given mutex. - bool try_acquire( mutex& mutex ) { -#if TBB_USE_ASSERT - return internal_try_acquire (mutex); -#else - bool result = mutex.try_lock(); - if( result ) - my_mutex = &mutex; - return result; -#endif /* TBB_USE_ASSERT */ - } - - //! Release lock - void release() { -#if TBB_USE_ASSERT - internal_release (); -#else - my_mutex->unlock(); - my_mutex = NULL; -#endif /* TBB_USE_ASSERT */ - } - - private: - //! The pointer to the current mutex to work - mutex* my_mutex; - - //! All checks from acquire using mutex.state were moved here - void __TBB_EXPORTED_METHOD internal_acquire( mutex& m ); - - //! All checks from try_acquire using mutex.state were moved here - bool __TBB_EXPORTED_METHOD internal_try_acquire( mutex& m ); - - //! All checks from release using mutex.state were moved here - void __TBB_EXPORTED_METHOD internal_release(); - - friend class mutex; - }; - - // Mutex traits - static const bool is_rw_mutex = false; - static const bool is_recursive_mutex = false; - static const bool is_fair_mutex = false; - - // ISO C++0x compatibility methods - - //! Acquire lock - void lock() { -#if TBB_USE_ASSERT - aligned_space tmp; - new(tmp.begin()) scoped_lock(*this); -#else - #if _WIN32||_WIN64 - EnterCriticalSection(&impl); - #else - int error_code = pthread_mutex_lock(&impl); - if( error_code ) - tbb::internal::handle_perror(error_code,"mutex: pthread_mutex_lock failed"); - #endif /* _WIN32||_WIN64 */ -#endif /* TBB_USE_ASSERT */ - } - - //! Try acquiring lock (non-blocking) - /** Return true if lock acquired; false otherwise. */ - bool try_lock() { -#if TBB_USE_ASSERT - aligned_space tmp; - scoped_lock& s = *tmp.begin(); - s.my_mutex = NULL; - return s.internal_try_acquire(*this); -#else - #if _WIN32||_WIN64 - return TryEnterCriticalSection(&impl)!=0; - #else - return pthread_mutex_trylock(&impl)==0; - #endif /* _WIN32||_WIN64 */ -#endif /* TBB_USE_ASSERT */ - } - - //! Release lock - void unlock() { -#if TBB_USE_ASSERT - aligned_space tmp; - scoped_lock& s = *tmp.begin(); - s.my_mutex = this; - s.internal_release(); -#else - #if _WIN32||_WIN64 - LeaveCriticalSection(&impl); - #else - pthread_mutex_unlock(&impl); - #endif /* _WIN32||_WIN64 */ -#endif /* TBB_USE_ASSERT */ - } - - //! Return native_handle - #if _WIN32||_WIN64 - typedef LPCRITICAL_SECTION native_handle_type; - #else - typedef pthread_mutex_t* native_handle_type; - #endif - native_handle_type native_handle() { return (native_handle_type) &impl; } - - enum state_t { - INITIALIZED=0x1234, - DESTROYED=0x789A, - HELD=0x56CD - }; -private: -#if _WIN32||_WIN64 - CRITICAL_SECTION impl; - enum state_t state; -#else - pthread_mutex_t impl; -#endif /* _WIN32||_WIN64 */ - - //! All checks from mutex constructor using mutex.state were moved here - void __TBB_EXPORTED_METHOD internal_construct(); - - //! All checks from mutex destructor using mutex.state were moved here - void __TBB_EXPORTED_METHOD internal_destroy(); - -#if _WIN32||_WIN64 -public: - //! Set the internal state - void set_state( state_t to ) { state = to; } -#endif -}; - -__TBB_DEFINE_PROFILING_SET_NAME(mutex) - -} // namespace tbb - -#endif /* __TBB_mutex_H */ diff --git a/inst/include/tbb/null_mutex.h b/inst/include/tbb/null_mutex.h deleted file mode 100644 index 85c660e31..000000000 --- a/inst/include/tbb/null_mutex.h +++ /dev/null @@ -1,54 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_null_mutex_H -#define __TBB_null_mutex_H - -#include "tbb_stddef.h" - -namespace tbb { - -//! A mutex which does nothing -/** A null_mutex does no operation and simulates success. - @ingroup synchronization */ -class null_mutex : internal::mutex_copy_deprecated_and_disabled { -public: - //! Represents acquisition of a mutex. - class scoped_lock : internal::no_copy { - public: - scoped_lock() {} - scoped_lock( null_mutex& ) {} - ~scoped_lock() {} - void acquire( null_mutex& ) {} - bool try_acquire( null_mutex& ) { return true; } - void release() {} - }; - - null_mutex() {} - - // Mutex traits - static const bool is_rw_mutex = false; - static const bool is_recursive_mutex = true; - static const bool is_fair_mutex = true; -}; - -} - -#endif /* __TBB_null_mutex_H */ diff --git a/inst/include/tbb/null_rw_mutex.h b/inst/include/tbb/null_rw_mutex.h deleted file mode 100644 index fa0c80359..000000000 --- a/inst/include/tbb/null_rw_mutex.h +++ /dev/null @@ -1,56 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_null_rw_mutex_H -#define __TBB_null_rw_mutex_H - -#include "tbb_stddef.h" - -namespace tbb { - -//! A rw mutex which does nothing -/** A null_rw_mutex is a rw mutex that does nothing and simulates successful operation. - @ingroup synchronization */ -class null_rw_mutex : internal::mutex_copy_deprecated_and_disabled { -public: - //! Represents acquisition of a mutex. - class scoped_lock : internal::no_copy { - public: - scoped_lock() {} - scoped_lock( null_rw_mutex& , bool = true ) {} - ~scoped_lock() {} - void acquire( null_rw_mutex& , bool = true ) {} - bool upgrade_to_writer() { return true; } - bool downgrade_to_reader() { return true; } - bool try_acquire( null_rw_mutex& , bool = true ) { return true; } - void release() {} - }; - - null_rw_mutex() {} - - // Mutex traits - static const bool is_rw_mutex = true; - static const bool is_recursive_mutex = true; - static const bool is_fair_mutex = true; -}; - -} - -#endif /* __TBB_null_rw_mutex_H */ diff --git a/inst/include/tbb/parallel_do.h b/inst/include/tbb/parallel_do.h deleted file mode 100644 index 152756825..000000000 --- a/inst/include/tbb/parallel_do.h +++ /dev/null @@ -1,551 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_parallel_do_H -#define __TBB_parallel_do_H - -#include "internal/_range_iterator.h" -#include "internal/_template_helpers.h" -#include "task.h" -#include "aligned_space.h" -#include - -namespace tbb { -namespace interface9 { -//! @cond INTERNAL -namespace internal { - template class parallel_do_feeder_impl; -} // namespace internal -//! @endcond - -//! Class the user supplied algorithm body uses to add new tasks -/** \param Item Work item type **/ - template - class parallel_do_feeder: ::tbb::internal::no_copy - { - parallel_do_feeder() {} - virtual ~parallel_do_feeder () {} - virtual void internal_add_copy( const Item& item ) = 0; -#if __TBB_CPP11_RVALUE_REF_PRESENT - virtual void internal_add_move( Item&& item ) = 0; -#endif - template friend class internal::parallel_do_feeder_impl; - public: - //! Add a work item to a running parallel_do. - void add( const Item& item ) {internal_add_copy(item);} -#if __TBB_CPP11_RVALUE_REF_PRESENT - void add( Item&& item ) {internal_add_move(std::move(item));} -#endif - }; - -//! @cond INTERNAL -namespace internal { - template class do_group_task; - - //! For internal use only. - /** Selects one of the two possible forms of function call member operator. - @ingroup algorithms **/ - template - class parallel_do_operator_selector - { - typedef parallel_do_feeder Feeder; - template - static void internal_call( const Body& obj, __TBB_FORWARDING_REF(A1) arg1, A2&, void (Body::*)(CvItem) const ) { - obj(tbb::internal::forward(arg1)); - } - template - static void internal_call( const Body& obj, __TBB_FORWARDING_REF(A1) arg1, A2& arg2, void (Body::*)(CvItem, parallel_do_feeder&) const ) { - obj(tbb::internal::forward(arg1), arg2); - } - template - static void internal_call( const Body& obj, __TBB_FORWARDING_REF(A1) arg1, A2&, void (Body::*)(CvItem&) const ) { - obj(arg1); - } - template - static void internal_call( const Body& obj, __TBB_FORWARDING_REF(A1) arg1, A2& arg2, void (Body::*)(CvItem&, parallel_do_feeder&) const ) { - obj(arg1, arg2); - } - public: - template - static void call( const Body& obj, __TBB_FORWARDING_REF(A1) arg1, A2& arg2 ) - { - internal_call( obj, tbb::internal::forward(arg1), arg2, &Body::operator() ); - } - }; - - //! For internal use only. - /** Executes one iteration of a do. - @ingroup algorithms */ - template - class do_iteration_task: public task - { - typedef parallel_do_feeder_impl feeder_type; - - Item my_value; - feeder_type& my_feeder; - - do_iteration_task( const Item& value, feeder_type& feeder ) : - my_value(value), my_feeder(feeder) - {} - -#if __TBB_CPP11_RVALUE_REF_PRESENT - do_iteration_task( Item&& value, feeder_type& feeder ) : - my_value(std::move(value)), my_feeder(feeder) - {} -#endif - - task* execute() __TBB_override - { - parallel_do_operator_selector::call(*my_feeder.my_body, tbb::internal::move(my_value), my_feeder); - return NULL; - } - - template friend class parallel_do_feeder_impl; - }; // class do_iteration_task - - template - class do_iteration_task_iter: public task - { - typedef parallel_do_feeder_impl feeder_type; - - Iterator my_iter; - feeder_type& my_feeder; - - do_iteration_task_iter( const Iterator& iter, feeder_type& feeder ) : - my_iter(iter), my_feeder(feeder) - {} - - task* execute() __TBB_override - { - parallel_do_operator_selector::call(*my_feeder.my_body, *my_iter, my_feeder); - return NULL; - } - - template friend class do_group_task_forward; - template friend class do_group_task_input; - template friend class do_task_iter; - }; // class do_iteration_task_iter - - //! For internal use only. - /** Implements new task adding procedure. - @ingroup algorithms **/ - template - class parallel_do_feeder_impl : public parallel_do_feeder - { -#if __TBB_CPP11_RVALUE_REF_PRESENT - //Avoiding use of copy constructor in a virtual method if the type does not support it - void internal_add_copy_impl(std::true_type, const Item& item) { - typedef do_iteration_task iteration_type; - iteration_type& t = *new (task::allocate_additional_child_of(*my_barrier)) iteration_type(item, *this); - task::spawn(t); - } - void internal_add_copy_impl(std::false_type, const Item&) { - __TBB_ASSERT(false, "Overloading for r-value reference doesn't work or it's not movable and not copyable object"); - } - void internal_add_copy( const Item& item ) __TBB_override - { -#if __TBB_CPP11_IS_COPY_CONSTRUCTIBLE_PRESENT - internal_add_copy_impl(typename std::is_copy_constructible::type(), item); -#else - internal_add_copy_impl(std::true_type(), item); -#endif - } - void internal_add_move( Item&& item ) __TBB_override - { - typedef do_iteration_task iteration_type; - iteration_type& t = *new (task::allocate_additional_child_of(*my_barrier)) iteration_type(std::move(item), *this); - task::spawn(t); - } -#else /* ! __TBB_CPP11_RVALUE_REF_PRESENT */ - void internal_add_copy(const Item& item) __TBB_override { - typedef do_iteration_task iteration_type; - iteration_type& t = *new (task::allocate_additional_child_of(*my_barrier)) iteration_type(item, *this); - task::spawn(t); - } -#endif /* __TBB_CPP11_RVALUE_REF_PRESENT */ - public: - const Body* my_body; - empty_task* my_barrier; - - parallel_do_feeder_impl() - { - my_barrier = new( task::allocate_root() ) empty_task(); - __TBB_ASSERT(my_barrier, "root task allocation failed"); - } - -#if __TBB_TASK_GROUP_CONTEXT - parallel_do_feeder_impl(tbb::task_group_context &context) - { - my_barrier = new( task::allocate_root(context) ) empty_task(); - __TBB_ASSERT(my_barrier, "root task allocation failed"); - } -#endif - - ~parallel_do_feeder_impl() - { - my_barrier->destroy(*my_barrier); - } - }; // class parallel_do_feeder_impl - - - //! For internal use only - /** Unpacks a block of iterations. - @ingroup algorithms */ - - template - class do_group_task_forward: public task - { - static const size_t max_arg_size = 4; - - typedef parallel_do_feeder_impl feeder_type; - - feeder_type& my_feeder; - Iterator my_first; - size_t my_size; - - do_group_task_forward( Iterator first, size_t size, feeder_type& feeder ) - : my_feeder(feeder), my_first(first), my_size(size) - {} - - task* execute() __TBB_override - { - typedef do_iteration_task_iter iteration_type; - __TBB_ASSERT( my_size>0, NULL ); - task_list list; - task* t; - size_t k=0; - for(;;) { - t = new( allocate_child() ) iteration_type( my_first, my_feeder ); - ++my_first; - if( ++k==my_size ) break; - list.push_back(*t); - } - set_ref_count(int(k+1)); - spawn(list); - spawn_and_wait_for_all(*t); - return NULL; - } - - template friend class do_task_iter; - }; // class do_group_task_forward - - template - class do_group_task_input: public task - { - static const size_t max_arg_size = 4; - - typedef parallel_do_feeder_impl feeder_type; - - feeder_type& my_feeder; - size_t my_size; - aligned_space my_arg; - - do_group_task_input( feeder_type& feeder ) - : my_feeder(feeder), my_size(0) - {} - - task* execute() __TBB_override - { -#if __TBB_CPP11_RVALUE_REF_PRESENT - typedef std::move_iterator Item_iterator; -#else - typedef Item* Item_iterator; -#endif - typedef do_iteration_task_iter iteration_type; - __TBB_ASSERT( my_size>0, NULL ); - task_list list; - task* t; - size_t k=0; - for(;;) { - t = new( allocate_child() ) iteration_type( Item_iterator(my_arg.begin() + k), my_feeder ); - if( ++k==my_size ) break; - list.push_back(*t); - } - set_ref_count(int(k+1)); - spawn(list); - spawn_and_wait_for_all(*t); - return NULL; - } - - ~do_group_task_input(){ - for( size_t k=0; k~Item(); - } - - template friend class do_task_iter; - }; // class do_group_task_input - - //! For internal use only. - /** Gets block of iterations and packages them into a do_group_task. - @ingroup algorithms */ - template - class do_task_iter: public task - { - typedef parallel_do_feeder_impl feeder_type; - - public: - do_task_iter( Iterator first, Iterator last , feeder_type& feeder ) : - my_first(first), my_last(last), my_feeder(feeder) - {} - - private: - Iterator my_first; - Iterator my_last; - feeder_type& my_feeder; - - /* Do not merge run(xxx) and run_xxx() methods. They are separated in order - to make sure that compilers will eliminate unused argument of type xxx - (that is will not put it on stack). The sole purpose of this argument - is overload resolution. - - An alternative could be using template functions, but explicit specialization - of member function templates is not supported for non specialized class - templates. Besides template functions would always fall back to the least - efficient variant (the one for input iterators) in case of iterators having - custom tags derived from basic ones. */ - task* execute() __TBB_override - { - typedef typename std::iterator_traits::iterator_category iterator_tag; - return run( (iterator_tag*)NULL ); - } - - /** This is the most restricted variant that operates on input iterators or - iterators with unknown tags (tags not derived from the standard ones). **/ - inline task* run( void* ) { return run_for_input_iterator(); } - - task* run_for_input_iterator() { - typedef do_group_task_input block_type; - - block_type& t = *new( allocate_additional_child_of(*my_feeder.my_barrier) ) block_type(my_feeder); - size_t k=0; - while( !(my_first == my_last) ) { - // Move semantics are automatically used when supported by the iterator - new (t.my_arg.begin() + k) Item(*my_first); - ++my_first; - if( ++k==block_type::max_arg_size ) { - if ( !(my_first == my_last) ) - recycle_to_reexecute(); - break; - } - } - if( k==0 ) { - destroy(t); - return NULL; - } else { - t.my_size = k; - return &t; - } - } - - inline task* run( std::forward_iterator_tag* ) { return run_for_forward_iterator(); } - - task* run_for_forward_iterator() { - typedef do_group_task_forward block_type; - - Iterator first = my_first; - size_t k=0; - while( !(my_first==my_last) ) { - ++my_first; - if( ++k==block_type::max_arg_size ) { - if ( !(my_first==my_last) ) - recycle_to_reexecute(); - break; - } - } - return k==0 ? NULL : new( allocate_additional_child_of(*my_feeder.my_barrier) ) block_type(first, k, my_feeder); - } - - inline task* run( std::random_access_iterator_tag* ) { return run_for_random_access_iterator(); } - - task* run_for_random_access_iterator() { - typedef do_group_task_forward block_type; - typedef do_iteration_task_iter iteration_type; - - size_t k = static_cast(my_last-my_first); - if( k > block_type::max_arg_size ) { - Iterator middle = my_first + k/2; - - empty_task& c = *new( allocate_continuation() ) empty_task; - do_task_iter& b = *new( c.allocate_child() ) do_task_iter(middle, my_last, my_feeder); - recycle_as_child_of(c); - - my_last = middle; - c.set_ref_count(2); - c.spawn(b); - return this; - }else if( k != 0 ) { - task_list list; - task* t; - size_t k1=0; - for(;;) { - t = new( allocate_child() ) iteration_type(my_first, my_feeder); - ++my_first; - if( ++k1==k ) break; - list.push_back(*t); - } - set_ref_count(int(k+1)); - spawn(list); - spawn_and_wait_for_all(*t); - } - return NULL; - } - }; // class do_task_iter - - //! For internal use only. - /** Implements parallel iteration over a range. - @ingroup algorithms */ - template - void run_parallel_do( Iterator first, Iterator last, const Body& body -#if __TBB_TASK_GROUP_CONTEXT - , task_group_context& context -#endif - ) - { - typedef do_task_iter root_iteration_task; -#if __TBB_TASK_GROUP_CONTEXT - parallel_do_feeder_impl feeder(context); -#else - parallel_do_feeder_impl feeder; -#endif - feeder.my_body = &body; - - root_iteration_task &t = *new( feeder.my_barrier->allocate_child() ) root_iteration_task(first, last, feeder); - - feeder.my_barrier->set_ref_count(2); - feeder.my_barrier->spawn_and_wait_for_all(t); - } - - //! For internal use only. - /** Detects types of Body's operator function arguments. - @ingroup algorithms **/ - template - void select_parallel_do( Iterator first, Iterator last, const Body& body, void (Body::*)(Item) const -#if __TBB_TASK_GROUP_CONTEXT - , task_group_context& context -#endif - ) - { - run_parallel_do::type>( first, last, body -#if __TBB_TASK_GROUP_CONTEXT - , context -#endif - ); - } - - //! For internal use only. - /** Detects types of Body's operator function arguments. - @ingroup algorithms **/ - template - void select_parallel_do( Iterator first, Iterator last, const Body& body, void (Body::*)(Item, parallel_do_feeder<_Item>&) const -#if __TBB_TASK_GROUP_CONTEXT - , task_group_context& context -#endif - ) - { - run_parallel_do::type>( first, last, body -#if __TBB_TASK_GROUP_CONTEXT - , context -#endif - ); - } - -} // namespace internal -} // namespace interface9 -//! @endcond - -/** \page parallel_do_body_req Requirements on parallel_do body - Class \c Body implementing the concept of parallel_do body must define: - - \code - B::operator()( - cv_item_type item, - parallel_do_feeder& feeder - ) const - - OR - - B::operator()( cv_item_type& item ) const - \endcode Process item. - May be invoked concurrently for the same \c this but different \c item. - - - \code item_type( const item_type& ) \endcode - Copy a work item. - - \code ~item_type() \endcode Destroy a work item -**/ - -/** \name parallel_do - See also requirements on \ref parallel_do_body_req "parallel_do Body". **/ -//@{ -//! Parallel iteration over a range, with optional addition of more work. -/** @ingroup algorithms */ -template -void parallel_do( Iterator first, Iterator last, const Body& body ) -{ - if ( first == last ) - return; -#if __TBB_TASK_GROUP_CONTEXT - task_group_context context; -#endif - interface9::internal::select_parallel_do( first, last, body, &Body::operator() -#if __TBB_TASK_GROUP_CONTEXT - , context -#endif - ); -} - -template -void parallel_do(Range& rng, const Body& body) { - parallel_do(tbb::internal::first(rng), tbb::internal::last(rng), body); -} - -template -void parallel_do(const Range& rng, const Body& body) { - parallel_do(tbb::internal::first(rng), tbb::internal::last(rng), body); -} - -#if __TBB_TASK_GROUP_CONTEXT -//! Parallel iteration over a range, with optional addition of more work and user-supplied context -/** @ingroup algorithms */ -template -void parallel_do( Iterator first, Iterator last, const Body& body, task_group_context& context ) -{ - if ( first == last ) - return; - interface9::internal::select_parallel_do( first, last, body, &Body::operator(), context ); -} - -template -void parallel_do(Range& rng, const Body& body, task_group_context& context) { - parallel_do(tbb::internal::first(rng), tbb::internal::last(rng), body, context); -} - -template -void parallel_do(const Range& rng, const Body& body, task_group_context& context) { - parallel_do(tbb::internal::first(rng), tbb::internal::last(rng), body, context); -} - -#endif // __TBB_TASK_GROUP_CONTEXT - -//@} - -using interface9::parallel_do_feeder; - -} // namespace - -#endif /* __TBB_parallel_do_H */ diff --git a/inst/include/tbb/parallel_for.h b/inst/include/tbb/parallel_for.h deleted file mode 100644 index 2cab65860..000000000 --- a/inst/include/tbb/parallel_for.h +++ /dev/null @@ -1,407 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_parallel_for_H -#define __TBB_parallel_for_H - -#include -#include "task.h" -#include "partitioner.h" -#include "blocked_range.h" -#include "tbb_exception.h" - -namespace tbb { - -namespace interface9 { -//! @cond INTERNAL -namespace internal { - - //! allocate right task with new parent - void* allocate_sibling(task* start_for_task, size_t bytes); - - //! Task type used in parallel_for - /** @ingroup algorithms */ - template - class start_for: public task { - Range my_range; - const Body my_body; - typename Partitioner::task_partition_type my_partition; - task* execute() __TBB_override; - - //! Update affinity info, if any. - void note_affinity( affinity_id id ) __TBB_override { - my_partition.note_affinity( id ); - } - - public: - //! Constructor for root task. - start_for( const Range& range, const Body& body, Partitioner& partitioner ) : - my_range(range), - my_body(body), - my_partition(partitioner) - { - } - //! Splitting constructor used to generate children. - /** parent_ becomes left child. Newly constructed object is right child. */ - start_for( start_for& parent_, typename Partitioner::split_type& split_obj) : - my_range(parent_.my_range, split_obj), - my_body(parent_.my_body), - my_partition(parent_.my_partition, split_obj) - { - my_partition.set_affinity(*this); - } - //! Construct right child from the given range as response to the demand. - /** parent_ remains left child. Newly constructed object is right child. */ - start_for( start_for& parent_, const Range& r, depth_t d ) : - my_range(r), - my_body(parent_.my_body), - my_partition(parent_.my_partition, split()) - { - my_partition.set_affinity(*this); - my_partition.align_depth( d ); - } - static void run( const Range& range, const Body& body, Partitioner& partitioner ) { - if( !range.empty() ) { -#if !__TBB_TASK_GROUP_CONTEXT || TBB_JOIN_OUTER_TASK_GROUP - start_for& a = *new(task::allocate_root()) start_for(range,body,partitioner); -#else - // Bound context prevents exceptions from body to affect nesting or sibling algorithms, - // and allows users to handle exceptions safely by wrapping parallel_for in the try-block. - task_group_context context; - start_for& a = *new(task::allocate_root(context)) start_for(range,body,partitioner); -#endif /* __TBB_TASK_GROUP_CONTEXT && !TBB_JOIN_OUTER_TASK_GROUP */ - task::spawn_root_and_wait(a); - } - } -#if __TBB_TASK_GROUP_CONTEXT - static void run( const Range& range, const Body& body, Partitioner& partitioner, task_group_context& context ) { - if( !range.empty() ) { - start_for& a = *new(task::allocate_root(context)) start_for(range,body,partitioner); - task::spawn_root_and_wait(a); - } - } -#endif /* __TBB_TASK_GROUP_CONTEXT */ - //! Run body for range, serves as callback for partitioner - void run_body( Range &r ) { my_body( r ); } - - //! spawn right task, serves as callback for partitioner - void offer_work(typename Partitioner::split_type& split_obj) { - spawn( *new( allocate_sibling(static_cast(this), sizeof(start_for)) ) start_for(*this, split_obj) ); - } - //! spawn right task, serves as callback for partitioner - void offer_work(const Range& r, depth_t d = 0) { - spawn( *new( allocate_sibling(static_cast(this), sizeof(start_for)) ) start_for(*this, r, d) ); - } - }; - - //! allocate right task with new parent - // TODO: 'inline' here is to avoid multiple definition error but for sake of code size this should not be inlined - inline void* allocate_sibling(task* start_for_task, size_t bytes) { - task* parent_ptr = new( start_for_task->allocate_continuation() ) flag_task(); - start_for_task->set_parent(parent_ptr); - parent_ptr->set_ref_count(2); - return &parent_ptr->allocate_child().allocate(bytes); - } - - //! execute task for parallel_for - template - task* start_for::execute() { - my_partition.check_being_stolen( *this ); - my_partition.execute(*this, my_range); - return NULL; - } -} // namespace internal -//! @endcond -} // namespace interfaceX - -//! @cond INTERNAL -namespace internal { - using interface9::internal::start_for; - - //! Calls the function with values from range [begin, end) with a step provided - template - class parallel_for_body : internal::no_assign { - const Function &my_func; - const Index my_begin; - const Index my_step; - public: - parallel_for_body( const Function& _func, Index& _begin, Index& _step ) - : my_func(_func), my_begin(_begin), my_step(_step) {} - - void operator()( const tbb::blocked_range& r ) const { - // A set of local variables to help the compiler with vectorization of the following loop. - Index b = r.begin(); - Index e = r.end(); - Index ms = my_step; - Index k = my_begin + b*ms; - -#if __INTEL_COMPILER -#pragma ivdep -#if __TBB_ASSERT_ON_VECTORIZATION_FAILURE -#pragma vector always assert -#endif -#endif - for ( Index i = b; i < e; ++i, k += ms ) { - my_func( k ); - } - } - }; -} // namespace internal -//! @endcond - -// Requirements on Range concept are documented in blocked_range.h - -/** \page parallel_for_body_req Requirements on parallel_for body - Class \c Body implementing the concept of parallel_for body must define: - - \code Body::Body( const Body& ); \endcode Copy constructor - - \code Body::~Body(); \endcode Destructor - - \code void Body::operator()( Range& r ) const; \endcode Function call operator applying the body to range \c r. -**/ - -/** \name parallel_for - See also requirements on \ref range_req "Range" and \ref parallel_for_body_req "parallel_for Body". **/ -//@{ - -//! Parallel iteration over range with default partitioner. -/** @ingroup algorithms **/ -template -void parallel_for( const Range& range, const Body& body ) { - internal::start_for::run(range,body,__TBB_DEFAULT_PARTITIONER()); -} - -//! Parallel iteration over range with simple partitioner. -/** @ingroup algorithms **/ -template -void parallel_for( const Range& range, const Body& body, const simple_partitioner& partitioner ) { - internal::start_for::run(range,body,partitioner); -} - -//! Parallel iteration over range with auto_partitioner. -/** @ingroup algorithms **/ -template -void parallel_for( const Range& range, const Body& body, const auto_partitioner& partitioner ) { - internal::start_for::run(range,body,partitioner); -} - -//! Parallel iteration over range with static_partitioner. -/** @ingroup algorithms **/ -template -void parallel_for( const Range& range, const Body& body, const static_partitioner& partitioner ) { - internal::start_for::run(range,body,partitioner); -} - -//! Parallel iteration over range with affinity_partitioner. -/** @ingroup algorithms **/ -template -void parallel_for( const Range& range, const Body& body, affinity_partitioner& partitioner ) { - internal::start_for::run(range,body,partitioner); -} - -#if __TBB_TASK_GROUP_CONTEXT -//! Parallel iteration over range with default partitioner and user-supplied context. -/** @ingroup algorithms **/ -template -void parallel_for( const Range& range, const Body& body, task_group_context& context ) { - internal::start_for::run(range, body, __TBB_DEFAULT_PARTITIONER(), context); -} - -//! Parallel iteration over range with simple partitioner and user-supplied context. -/** @ingroup algorithms **/ -template -void parallel_for( const Range& range, const Body& body, const simple_partitioner& partitioner, task_group_context& context ) { - internal::start_for::run(range, body, partitioner, context); -} - -//! Parallel iteration over range with auto_partitioner and user-supplied context. -/** @ingroup algorithms **/ -template -void parallel_for( const Range& range, const Body& body, const auto_partitioner& partitioner, task_group_context& context ) { - internal::start_for::run(range, body, partitioner, context); -} - -//! Parallel iteration over range with static_partitioner and user-supplied context. -/** @ingroup algorithms **/ -template -void parallel_for( const Range& range, const Body& body, const static_partitioner& partitioner, task_group_context& context ) { - internal::start_for::run(range, body, partitioner, context); -} - -//! Parallel iteration over range with affinity_partitioner and user-supplied context. -/** @ingroup algorithms **/ -template -void parallel_for( const Range& range, const Body& body, affinity_partitioner& partitioner, task_group_context& context ) { - internal::start_for::run(range,body,partitioner, context); -} -#endif /* __TBB_TASK_GROUP_CONTEXT */ -//@} - -namespace strict_ppl { - -//@{ -//! Implementation of parallel iteration over stepped range of integers with explicit step and partitioner -template -void parallel_for_impl(Index first, Index last, Index step, const Function& f, Partitioner& partitioner) { - if (step <= 0 ) - internal::throw_exception(internal::eid_nonpositive_step); // throws std::invalid_argument - else if (last > first) { - // Above "else" avoids "potential divide by zero" warning on some platforms - Index end = (last - first - Index(1)) / step + Index(1); - tbb::blocked_range range(static_cast(0), end); - internal::parallel_for_body body(f, first, step); - tbb::parallel_for(range, body, partitioner); - } -} - -//! Parallel iteration over a range of integers with a step provided and default partitioner -template -void parallel_for(Index first, Index last, Index step, const Function& f) { - parallel_for_impl(first, last, step, f, auto_partitioner()); -} -//! Parallel iteration over a range of integers with a step provided and simple partitioner -template -void parallel_for(Index first, Index last, Index step, const Function& f, const simple_partitioner& partitioner) { - parallel_for_impl(first, last, step, f, partitioner); -} -//! Parallel iteration over a range of integers with a step provided and auto partitioner -template -void parallel_for(Index first, Index last, Index step, const Function& f, const auto_partitioner& partitioner) { - parallel_for_impl(first, last, step, f, partitioner); -} -//! Parallel iteration over a range of integers with a step provided and static partitioner -template -void parallel_for(Index first, Index last, Index step, const Function& f, const static_partitioner& partitioner) { - parallel_for_impl(first, last, step, f, partitioner); -} -//! Parallel iteration over a range of integers with a step provided and affinity partitioner -template -void parallel_for(Index first, Index last, Index step, const Function& f, affinity_partitioner& partitioner) { - parallel_for_impl(first, last, step, f, partitioner); -} - -//! Parallel iteration over a range of integers with a default step value and default partitioner -template -void parallel_for(Index first, Index last, const Function& f) { - parallel_for_impl(first, last, static_cast(1), f, auto_partitioner()); -} -//! Parallel iteration over a range of integers with a default step value and simple partitioner -template -void parallel_for(Index first, Index last, const Function& f, const simple_partitioner& partitioner) { - parallel_for_impl(first, last, static_cast(1), f, partitioner); -} -//! Parallel iteration over a range of integers with a default step value and auto partitioner -template -void parallel_for(Index first, Index last, const Function& f, const auto_partitioner& partitioner) { - parallel_for_impl(first, last, static_cast(1), f, partitioner); -} -//! Parallel iteration over a range of integers with a default step value and static partitioner -template -void parallel_for(Index first, Index last, const Function& f, const static_partitioner& partitioner) { - parallel_for_impl(first, last, static_cast(1), f, partitioner); -} -//! Parallel iteration over a range of integers with a default step value and affinity partitioner -template -void parallel_for(Index first, Index last, const Function& f, affinity_partitioner& partitioner) { - parallel_for_impl(first, last, static_cast(1), f, partitioner); -} - -#if __TBB_TASK_GROUP_CONTEXT -//! Implementation of parallel iteration over stepped range of integers with explicit step, task group context, and partitioner -template -void parallel_for_impl(Index first, Index last, Index step, const Function& f, Partitioner& partitioner, tbb::task_group_context &context) { - if (step <= 0 ) - internal::throw_exception(internal::eid_nonpositive_step); // throws std::invalid_argument - else if (last > first) { - // Above "else" avoids "potential divide by zero" warning on some platforms - Index end = (last - first - Index(1)) / step + Index(1); - tbb::blocked_range range(static_cast(0), end); - internal::parallel_for_body body(f, first, step); - tbb::parallel_for(range, body, partitioner, context); - } -} - -//! Parallel iteration over a range of integers with explicit step, task group context, and default partitioner -template -void parallel_for(Index first, Index last, Index step, const Function& f, tbb::task_group_context &context) { - parallel_for_impl(first, last, step, f, auto_partitioner(), context); -} -//! Parallel iteration over a range of integers with explicit step, task group context, and simple partitioner - template -void parallel_for(Index first, Index last, Index step, const Function& f, const simple_partitioner& partitioner, tbb::task_group_context &context) { - parallel_for_impl(first, last, step, f, partitioner, context); -} -//! Parallel iteration over a range of integers with explicit step, task group context, and auto partitioner - template -void parallel_for(Index first, Index last, Index step, const Function& f, const auto_partitioner& partitioner, tbb::task_group_context &context) { - parallel_for_impl(first, last, step, f, partitioner, context); -} -//! Parallel iteration over a range of integers with explicit step, task group context, and static partitioner -template -void parallel_for(Index first, Index last, Index step, const Function& f, const static_partitioner& partitioner, tbb::task_group_context &context) { - parallel_for_impl(first, last, step, f, partitioner, context); -} -//! Parallel iteration over a range of integers with explicit step, task group context, and affinity partitioner - template -void parallel_for(Index first, Index last, Index step, const Function& f, affinity_partitioner& partitioner, tbb::task_group_context &context) { - parallel_for_impl(first, last, step, f, partitioner, context); -} - - -//! Parallel iteration over a range of integers with a default step value, explicit task group context, and default partitioner -template -void parallel_for(Index first, Index last, const Function& f, tbb::task_group_context &context) { - parallel_for_impl(first, last, static_cast(1), f, auto_partitioner(), context); -} -//! Parallel iteration over a range of integers with a default step value, explicit task group context, and simple partitioner - template -void parallel_for(Index first, Index last, const Function& f, const simple_partitioner& partitioner, tbb::task_group_context &context) { - parallel_for_impl(first, last, static_cast(1), f, partitioner, context); -} -//! Parallel iteration over a range of integers with a default step value, explicit task group context, and auto partitioner - template -void parallel_for(Index first, Index last, const Function& f, const auto_partitioner& partitioner, tbb::task_group_context &context) { - parallel_for_impl(first, last, static_cast(1), f, partitioner, context); -} -//! Parallel iteration over a range of integers with a default step value, explicit task group context, and static partitioner -template -void parallel_for(Index first, Index last, const Function& f, const static_partitioner& partitioner, tbb::task_group_context &context) { - parallel_for_impl(first, last, static_cast(1), f, partitioner, context); -} -//! Parallel iteration over a range of integers with a default step value, explicit task group context, and affinity_partitioner - template -void parallel_for(Index first, Index last, const Function& f, affinity_partitioner& partitioner, tbb::task_group_context &context) { - parallel_for_impl(first, last, static_cast(1), f, partitioner, context); -} - -#endif /* __TBB_TASK_GROUP_CONTEXT */ -//@} - -} // namespace strict_ppl - -using strict_ppl::parallel_for; - -} // namespace tbb - -#if TBB_PREVIEW_SERIAL_SUBSET -#define __TBB_NORMAL_EXECUTION -#include "../serial/tbb/parallel_for.h" -#undef __TBB_NORMAL_EXECUTION -#endif - -#endif /* __TBB_parallel_for_H */ diff --git a/inst/include/tbb/parallel_for_each.h b/inst/include/tbb/parallel_for_each.h deleted file mode 100644 index 6c2ec9f02..000000000 --- a/inst/include/tbb/parallel_for_each.h +++ /dev/null @@ -1,137 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_parallel_for_each_H -#define __TBB_parallel_for_each_H - -#include "parallel_do.h" -#include "parallel_for.h" - -namespace tbb { - -//! @cond INTERNAL -namespace internal { - // The class calls user function in operator() - template - class parallel_for_each_body_do : internal::no_assign { - const Function &my_func; - public: - parallel_for_each_body_do(const Function &_func) : my_func(_func) {} - - void operator()(typename std::iterator_traits::reference value) const { - my_func(value); - } - }; - - // The class calls user function in operator() - template - class parallel_for_each_body_for : internal::no_assign { - const Function &my_func; - public: - parallel_for_each_body_for(const Function &_func) : my_func(_func) {} - - void operator()(tbb::blocked_range range) const { -#if __INTEL_COMPILER -#pragma ivdep -#endif - for(Iterator it = range.begin(), end = range.end(); it != end; ++it) { - my_func(*it); - } - } - }; - - template - struct parallel_for_each_impl { -#if __TBB_TASK_GROUP_CONTEXT - static void doit(Iterator first, Iterator last, const Function& f, task_group_context &context) { - internal::parallel_for_each_body_do body(f); - tbb::parallel_do(first, last, body, context); - } -#endif - static void doit(Iterator first, Iterator last, const Function& f) { - internal::parallel_for_each_body_do body(f); - tbb::parallel_do(first, last, body); - } - }; - template - struct parallel_for_each_impl { -#if __TBB_TASK_GROUP_CONTEXT - static void doit(Iterator first, Iterator last, const Function& f, task_group_context &context) { - internal::parallel_for_each_body_for body(f); - tbb::parallel_for(tbb::blocked_range(first, last), body, context); - } -#endif - static void doit(Iterator first, Iterator last, const Function& f) { - internal::parallel_for_each_body_for body(f); - tbb::parallel_for(tbb::blocked_range(first, last), body); - } - }; -} // namespace internal -//! @endcond - -/** \name parallel_for_each - **/ -//@{ -//! Calls function f for all items from [first, last) interval using user-supplied context -/** @ingroup algorithms */ -#if __TBB_TASK_GROUP_CONTEXT -template -void parallel_for_each(Iterator first, Iterator last, const Function& f, task_group_context &context) { - internal::parallel_for_each_impl::iterator_category>::doit(first, last, f, context); -} - -//! Calls function f for all items from rng using user-supplied context -/** @ingroup algorithms */ -template -void parallel_for_each(Range& rng, const Function& f, task_group_context& context) { - parallel_for_each(tbb::internal::first(rng), tbb::internal::last(rng), f, context); -} - -//! Calls function f for all items from const rng user-supplied context -/** @ingroup algorithms */ -template -void parallel_for_each(const Range& rng, const Function& f, task_group_context& context) { - parallel_for_each(tbb::internal::first(rng), tbb::internal::last(rng), f, context); -} -#endif /* __TBB_TASK_GROUP_CONTEXT */ - -//! Uses default context -template -void parallel_for_each(Iterator first, Iterator last, const Function& f) { - internal::parallel_for_each_impl::iterator_category>::doit(first, last, f); -} - -//! Uses default context -template -void parallel_for_each(Range& rng, const Function& f) { - parallel_for_each(tbb::internal::first(rng), tbb::internal::last(rng), f); -} - -//! Uses default context -template -void parallel_for_each(const Range& rng, const Function& f) { - parallel_for_each(tbb::internal::first(rng), tbb::internal::last(rng), f); -} - -//@} - -} // namespace - -#endif /* __TBB_parallel_for_each_H */ diff --git a/inst/include/tbb/parallel_invoke.h b/inst/include/tbb/parallel_invoke.h deleted file mode 100644 index 0dd7590e8..000000000 --- a/inst/include/tbb/parallel_invoke.h +++ /dev/null @@ -1,455 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_parallel_invoke_H -#define __TBB_parallel_invoke_H - -#include "task.h" - -#if __TBB_VARIADIC_PARALLEL_INVOKE - #include // std::forward -#endif - -namespace tbb { - -#if !__TBB_TASK_GROUP_CONTEXT - /** Dummy to avoid cluttering the bulk of the header with enormous amount of ifdefs. **/ - struct task_group_context {}; -#endif /* __TBB_TASK_GROUP_CONTEXT */ - -//! @cond INTERNAL -namespace internal { - // Simple task object, executing user method - template - class function_invoker : public task{ - public: - function_invoker(const function& _function) : my_function(_function) {} - private: - const function &my_function; - task* execute() __TBB_override - { - my_function(); - return NULL; - } - }; - - // The class spawns two or three child tasks - template - class spawner : public task { - private: - const function1& my_func1; - const function2& my_func2; - const function3& my_func3; - bool is_recycled; - - task* execute () __TBB_override { - if(is_recycled){ - return NULL; - }else{ - __TBB_ASSERT(N==2 || N==3, "Number of arguments passed to spawner is wrong"); - set_ref_count(N); - recycle_as_safe_continuation(); - internal::function_invoker* invoker2 = new (allocate_child()) internal::function_invoker(my_func2); - __TBB_ASSERT(invoker2, "Child task allocation failed"); - spawn(*invoker2); - size_t n = N; // To prevent compiler warnings - if (n>2) { - internal::function_invoker* invoker3 = new (allocate_child()) internal::function_invoker(my_func3); - __TBB_ASSERT(invoker3, "Child task allocation failed"); - spawn(*invoker3); - } - my_func1(); - is_recycled = true; - return NULL; - } - } // execute - - public: - spawner(const function1& _func1, const function2& _func2, const function3& _func3) : my_func1(_func1), my_func2(_func2), my_func3(_func3), is_recycled(false) {} - }; - - // Creates and spawns child tasks - class parallel_invoke_helper : public empty_task { - public: - // Dummy functor class - class parallel_invoke_noop { - public: - void operator() () const {} - }; - // Creates a helper object with user-defined number of children expected - parallel_invoke_helper(int number_of_children) - { - set_ref_count(number_of_children + 1); - } - -#if __TBB_VARIADIC_PARALLEL_INVOKE - void add_children() {} - void add_children(tbb::task_group_context&) {} - - template - void add_children(function&& _func) - { - internal::function_invoker* invoker = new (allocate_child()) internal::function_invoker(std::forward(_func)); - __TBB_ASSERT(invoker, "Child task allocation failed"); - spawn(*invoker); - } - - template - void add_children(function&& _func, tbb::task_group_context&) - { - add_children(std::forward(_func)); - } - - // Adds child(ren) task(s) and spawns them - template - void add_children(function1&& _func1, function2&& _func2, function&&... _func) - { - // The third argument is dummy, it is ignored actually. - parallel_invoke_noop noop; - typedef internal::spawner<2, function1, function2, parallel_invoke_noop> spawner_type; - spawner_type & sub_root = *new(allocate_child()) spawner_type(std::forward(_func1), std::forward(_func2), noop); - spawn(sub_root); - add_children(std::forward(_func)...); - } -#else - // Adds child task and spawns it - template - void add_children (const function &_func) - { - internal::function_invoker* invoker = new (allocate_child()) internal::function_invoker(_func); - __TBB_ASSERT(invoker, "Child task allocation failed"); - spawn(*invoker); - } - - // Adds a task with multiple child tasks and spawns it - // two arguments - template - void add_children (const function1& _func1, const function2& _func2) - { - // The third argument is dummy, it is ignored actually. - parallel_invoke_noop noop; - internal::spawner<2, function1, function2, parallel_invoke_noop>& sub_root = *new(allocate_child())internal::spawner<2, function1, function2, parallel_invoke_noop>(_func1, _func2, noop); - spawn(sub_root); - } - // three arguments - template - void add_children (const function1& _func1, const function2& _func2, const function3& _func3) - { - internal::spawner<3, function1, function2, function3>& sub_root = *new(allocate_child())internal::spawner<3, function1, function2, function3>(_func1, _func2, _func3); - spawn(sub_root); - } -#endif // __TBB_VARIADIC_PARALLEL_INVOKE - - // Waits for all child tasks - template - void run_and_finish(const F0& f0) - { - internal::function_invoker* invoker = new (allocate_child()) internal::function_invoker(f0); - __TBB_ASSERT(invoker, "Child task allocation failed"); - spawn_and_wait_for_all(*invoker); - } - }; - // The class destroys root if exception occurred as well as in normal case - class parallel_invoke_cleaner: internal::no_copy { - public: -#if __TBB_TASK_GROUP_CONTEXT - parallel_invoke_cleaner(int number_of_children, tbb::task_group_context& context) - : root(*new(task::allocate_root(context)) internal::parallel_invoke_helper(number_of_children)) -#else - parallel_invoke_cleaner(int number_of_children, tbb::task_group_context&) - : root(*new(task::allocate_root()) internal::parallel_invoke_helper(number_of_children)) -#endif /* !__TBB_TASK_GROUP_CONTEXT */ - {} - - ~parallel_invoke_cleaner(){ - root.destroy(root); - } - internal::parallel_invoke_helper& root; - }; - -#if __TBB_VARIADIC_PARALLEL_INVOKE -// Determine whether the last parameter in a pack is task_group_context - template struct impl_selector; // to workaround a GCC bug - - template struct impl_selector { - typedef typename impl_selector::type type; - }; - - template struct impl_selector { - typedef false_type type; - }; - template<> struct impl_selector { - typedef true_type type; - }; - - // Select task_group_context parameter from the back of a pack - inline task_group_context& get_context( task_group_context& tgc ) { return tgc; } - - template - task_group_context& get_context( T1&& /*ignored*/, T&&... t ) - { return get_context( std::forward(t)... ); } - - // task_group_context is known to be at the back of the parameter pack - template - void parallel_invoke_impl(true_type, F0&& f0, F1&& f1, F&&... f) { - __TBB_STATIC_ASSERT(sizeof...(F)>0, "Variadic parallel_invoke implementation broken?"); - // # of child tasks: f0, f1, and a task for each two elements of the pack except the last - const size_t number_of_children = 2 + sizeof...(F)/2; - parallel_invoke_cleaner cleaner(number_of_children, get_context(std::forward(f)...)); - parallel_invoke_helper& root = cleaner.root; - - root.add_children(std::forward(f)...); - root.add_children(std::forward(f1)); - root.run_and_finish(std::forward(f0)); - } - - // task_group_context is not in the pack, needs to be added - template - void parallel_invoke_impl(false_type, F0&& f0, F1&& f1, F&&... f) { - tbb::task_group_context context; - // Add context to the arguments, and redirect to the other overload - parallel_invoke_impl(true_type(), std::forward(f0), std::forward(f1), std::forward(f)..., context); - } -#endif -} // namespace internal -//! @endcond - -/** \name parallel_invoke - **/ -//@{ -//! Executes a list of tasks in parallel and waits for all tasks to complete. -/** @ingroup algorithms */ - -#if __TBB_VARIADIC_PARALLEL_INVOKE - -// parallel_invoke for two or more arguments via variadic templates -// presence of task_group_context is defined automatically -template -void parallel_invoke(F0&& f0, F1&& f1, F&&... f) { - typedef typename internal::impl_selector::type selector_type; - internal::parallel_invoke_impl(selector_type(), std::forward(f0), std::forward(f1), std::forward(f)...); -} - -#else - -// parallel_invoke with user-defined context -// two arguments -template -void parallel_invoke(const F0& f0, const F1& f1, tbb::task_group_context& context) { - internal::parallel_invoke_cleaner cleaner(2, context); - internal::parallel_invoke_helper& root = cleaner.root; - - root.add_children(f1); - - root.run_and_finish(f0); -} - -// three arguments -template -void parallel_invoke(const F0& f0, const F1& f1, const F2& f2, tbb::task_group_context& context) { - internal::parallel_invoke_cleaner cleaner(3, context); - internal::parallel_invoke_helper& root = cleaner.root; - - root.add_children(f2); - root.add_children(f1); - - root.run_and_finish(f0); -} - -// four arguments -template -void parallel_invoke(const F0& f0, const F1& f1, const F2& f2, const F3& f3, - tbb::task_group_context& context) -{ - internal::parallel_invoke_cleaner cleaner(4, context); - internal::parallel_invoke_helper& root = cleaner.root; - - root.add_children(f3); - root.add_children(f2); - root.add_children(f1); - - root.run_and_finish(f0); -} - -// five arguments -template -void parallel_invoke(const F0& f0, const F1& f1, const F2& f2, const F3& f3, const F4& f4, - tbb::task_group_context& context) -{ - internal::parallel_invoke_cleaner cleaner(3, context); - internal::parallel_invoke_helper& root = cleaner.root; - - root.add_children(f4, f3); - root.add_children(f2, f1); - - root.run_and_finish(f0); -} - -// six arguments -template -void parallel_invoke(const F0& f0, const F1& f1, const F2& f2, const F3& f3, const F4& f4, const F5& f5, - tbb::task_group_context& context) -{ - internal::parallel_invoke_cleaner cleaner(3, context); - internal::parallel_invoke_helper& root = cleaner.root; - - root.add_children(f5, f4, f3); - root.add_children(f2, f1); - - root.run_and_finish(f0); -} - -// seven arguments -template -void parallel_invoke(const F0& f0, const F1& f1, const F2& f2, const F3& f3, const F4& f4, - const F5& f5, const F6& f6, - tbb::task_group_context& context) -{ - internal::parallel_invoke_cleaner cleaner(3, context); - internal::parallel_invoke_helper& root = cleaner.root; - - root.add_children(f6, f5, f4); - root.add_children(f3, f2, f1); - - root.run_and_finish(f0); -} - -// eight arguments -template -void parallel_invoke(const F0& f0, const F1& f1, const F2& f2, const F3& f3, const F4& f4, - const F5& f5, const F6& f6, const F7& f7, - tbb::task_group_context& context) -{ - internal::parallel_invoke_cleaner cleaner(4, context); - internal::parallel_invoke_helper& root = cleaner.root; - - root.add_children(f7, f6, f5); - root.add_children(f4, f3); - root.add_children(f2, f1); - - root.run_and_finish(f0); -} - -// nine arguments -template -void parallel_invoke(const F0& f0, const F1& f1, const F2& f2, const F3& f3, const F4& f4, - const F5& f5, const F6& f6, const F7& f7, const F8& f8, - tbb::task_group_context& context) -{ - internal::parallel_invoke_cleaner cleaner(4, context); - internal::parallel_invoke_helper& root = cleaner.root; - - root.add_children(f8, f7, f6); - root.add_children(f5, f4, f3); - root.add_children(f2, f1); - - root.run_and_finish(f0); -} - -// ten arguments -template -void parallel_invoke(const F0& f0, const F1& f1, const F2& f2, const F3& f3, const F4& f4, - const F5& f5, const F6& f6, const F7& f7, const F8& f8, const F9& f9, - tbb::task_group_context& context) -{ - internal::parallel_invoke_cleaner cleaner(4, context); - internal::parallel_invoke_helper& root = cleaner.root; - - root.add_children(f9, f8, f7); - root.add_children(f6, f5, f4); - root.add_children(f3, f2, f1); - - root.run_and_finish(f0); -} - -// two arguments -template -void parallel_invoke(const F0& f0, const F1& f1) { - task_group_context context; - parallel_invoke(f0, f1, context); -} -// three arguments -template -void parallel_invoke(const F0& f0, const F1& f1, const F2& f2) { - task_group_context context; - parallel_invoke(f0, f1, f2, context); -} -// four arguments -template -void parallel_invoke(const F0& f0, const F1& f1, const F2& f2, const F3& f3) { - task_group_context context; - parallel_invoke(f0, f1, f2, f3, context); -} -// five arguments -template -void parallel_invoke(const F0& f0, const F1& f1, const F2& f2, const F3& f3, const F4& f4) { - task_group_context context; - parallel_invoke(f0, f1, f2, f3, f4, context); -} -// six arguments -template -void parallel_invoke(const F0& f0, const F1& f1, const F2& f2, const F3& f3, const F4& f4, const F5& f5) { - task_group_context context; - parallel_invoke(f0, f1, f2, f3, f4, f5, context); -} -// seven arguments -template -void parallel_invoke(const F0& f0, const F1& f1, const F2& f2, const F3& f3, const F4& f4, - const F5& f5, const F6& f6) -{ - task_group_context context; - parallel_invoke(f0, f1, f2, f3, f4, f5, f6, context); -} -// eight arguments -template -void parallel_invoke(const F0& f0, const F1& f1, const F2& f2, const F3& f3, const F4& f4, - const F5& f5, const F6& f6, const F7& f7) -{ - task_group_context context; - parallel_invoke(f0, f1, f2, f3, f4, f5, f6, f7, context); -} -// nine arguments -template -void parallel_invoke(const F0& f0, const F1& f1, const F2& f2, const F3& f3, const F4& f4, - const F5& f5, const F6& f6, const F7& f7, const F8& f8) -{ - task_group_context context; - parallel_invoke(f0, f1, f2, f3, f4, f5, f6, f7, f8, context); -} -// ten arguments -template -void parallel_invoke(const F0& f0, const F1& f1, const F2& f2, const F3& f3, const F4& f4, - const F5& f5, const F6& f6, const F7& f7, const F8& f8, const F9& f9) -{ - task_group_context context; - parallel_invoke(f0, f1, f2, f3, f4, f5, f6, f7, f8, f9, context); -} -#endif // __TBB_VARIADIC_PARALLEL_INVOKE -//@} - -} // namespace - -#endif /* __TBB_parallel_invoke_H */ diff --git a/inst/include/tbb/parallel_reduce.h b/inst/include/tbb/parallel_reduce.h deleted file mode 100644 index 0596ae03e..000000000 --- a/inst/include/tbb/parallel_reduce.h +++ /dev/null @@ -1,569 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_parallel_reduce_H -#define __TBB_parallel_reduce_H - -#include -#include "task.h" -#include "aligned_space.h" -#include "partitioner.h" -#include "tbb_profiling.h" - -namespace tbb { - -namespace interface9 { -//! @cond INTERNAL -namespace internal { - - using namespace tbb::internal; - - /** Values for reduction_context. */ - enum { - root_task, left_child, right_child - }; - - /** Represented as a char, not enum, for compactness. */ - typedef char reduction_context; - - //! Task type used to combine the partial results of parallel_reduce. - /** @ingroup algorithms */ - template - class finish_reduce: public flag_task { - //! Pointer to body, or NULL if the left child has not yet finished. - bool has_right_zombie; - const reduction_context my_context; - Body* my_body; - aligned_space zombie_space; - finish_reduce( reduction_context context_ ) : - has_right_zombie(false), // TODO: substitute by flag_task::child_stolen? - my_context(context_), - my_body(NULL) - { - } - ~finish_reduce() { - if( has_right_zombie ) - zombie_space.begin()->~Body(); - } - task* execute() __TBB_override { - if( has_right_zombie ) { - // Right child was stolen. - Body* s = zombie_space.begin(); - my_body->join( *s ); - // Body::join() won't be called if canceled. Defer destruction to destructor - } - if( my_context==left_child ) - itt_store_word_with_release( static_cast(parent())->my_body, my_body ); - return NULL; - } - template - friend class start_reduce; - }; - - //! allocate right task with new parent - void allocate_sibling(task* start_reduce_task, task *tasks[], size_t start_bytes, size_t finish_bytes); - - //! Task type used to split the work of parallel_reduce. - /** @ingroup algorithms */ - template - class start_reduce: public task { - typedef finish_reduce finish_type; - Body* my_body; - Range my_range; - typename Partitioner::task_partition_type my_partition; - reduction_context my_context; - task* execute() __TBB_override; - //! Update affinity info, if any - void note_affinity( affinity_id id ) __TBB_override { - my_partition.note_affinity( id ); - } - template - friend class finish_reduce; - -public: - //! Constructor used for root task - start_reduce( const Range& range, Body* body, Partitioner& partitioner ) : - my_body(body), - my_range(range), - my_partition(partitioner), - my_context(root_task) - { - } - //! Splitting constructor used to generate children. - /** parent_ becomes left child. Newly constructed object is right child. */ - start_reduce( start_reduce& parent_, typename Partitioner::split_type& split_obj ) : - my_body(parent_.my_body), - my_range(parent_.my_range, split_obj), - my_partition(parent_.my_partition, split_obj), - my_context(right_child) - { - my_partition.set_affinity(*this); - parent_.my_context = left_child; - } - //! Construct right child from the given range as response to the demand. - /** parent_ remains left child. Newly constructed object is right child. */ - start_reduce( start_reduce& parent_, const Range& r, depth_t d ) : - my_body(parent_.my_body), - my_range(r), - my_partition(parent_.my_partition, split()), - my_context(right_child) - { - my_partition.set_affinity(*this); - my_partition.align_depth( d ); // TODO: move into constructor of partitioner - parent_.my_context = left_child; - } - static void run( const Range& range, Body& body, Partitioner& partitioner ) { - if( !range.empty() ) { -#if !__TBB_TASK_GROUP_CONTEXT || TBB_JOIN_OUTER_TASK_GROUP - task::spawn_root_and_wait( *new(task::allocate_root()) start_reduce(range,&body,partitioner) ); -#else - // Bound context prevents exceptions from body to affect nesting or sibling algorithms, - // and allows users to handle exceptions safely by wrapping parallel_for in the try-block. - task_group_context context; - task::spawn_root_and_wait( *new(task::allocate_root(context)) start_reduce(range,&body,partitioner) ); -#endif /* __TBB_TASK_GROUP_CONTEXT && !TBB_JOIN_OUTER_TASK_GROUP */ - } - } -#if __TBB_TASK_GROUP_CONTEXT - static void run( const Range& range, Body& body, Partitioner& partitioner, task_group_context& context ) { - if( !range.empty() ) - task::spawn_root_and_wait( *new(task::allocate_root(context)) start_reduce(range,&body,partitioner) ); - } -#endif /* __TBB_TASK_GROUP_CONTEXT */ - //! Run body for range - void run_body( Range &r ) { (*my_body)( r ); } - - //! spawn right task, serves as callback for partitioner - // TODO: remove code duplication from 'offer_work' methods - void offer_work(typename Partitioner::split_type& split_obj) { - task *tasks[2]; - allocate_sibling(static_cast(this), tasks, sizeof(start_reduce), sizeof(finish_type)); - new((void*)tasks[0]) finish_type(my_context); - new((void*)tasks[1]) start_reduce(*this, split_obj); - spawn(*tasks[1]); - } - //! spawn right task, serves as callback for partitioner - void offer_work(const Range& r, depth_t d = 0) { - task *tasks[2]; - allocate_sibling(static_cast(this), tasks, sizeof(start_reduce), sizeof(finish_type)); - new((void*)tasks[0]) finish_type(my_context); - new((void*)tasks[1]) start_reduce(*this, r, d); - spawn(*tasks[1]); - } - }; - - //! allocate right task with new parent - // TODO: 'inline' here is to avoid multiple definition error but for sake of code size this should not be inlined - inline void allocate_sibling(task* start_reduce_task, task *tasks[], size_t start_bytes, size_t finish_bytes) { - tasks[0] = &start_reduce_task->allocate_continuation().allocate(finish_bytes); - start_reduce_task->set_parent(tasks[0]); - tasks[0]->set_ref_count(2); - tasks[1] = &tasks[0]->allocate_child().allocate(start_bytes); - } - - template - task* start_reduce::execute() { - my_partition.check_being_stolen( *this ); - if( my_context==right_child ) { - finish_type* parent_ptr = static_cast(parent()); - if( !itt_load_word_with_acquire(parent_ptr->my_body) ) { // TODO: replace by is_stolen_task() or by parent_ptr->ref_count() == 2??? - my_body = new( parent_ptr->zombie_space.begin() ) Body(*my_body,split()); - parent_ptr->has_right_zombie = true; - } - } else __TBB_ASSERT(my_context==root_task,NULL);// because left leaf spawns right leafs without recycling - my_partition.execute(*this, my_range); - if( my_context==left_child ) { - finish_type* parent_ptr = static_cast(parent()); - __TBB_ASSERT(my_body!=parent_ptr->zombie_space.begin(),NULL); - itt_store_word_with_release(parent_ptr->my_body, my_body ); - } - return NULL; - } - - //! Task type used to combine the partial results of parallel_deterministic_reduce. - /** @ingroup algorithms */ - template - class finish_deterministic_reduce: public task { - Body &my_left_body; - Body my_right_body; - - finish_deterministic_reduce( Body &body ) : - my_left_body( body ), - my_right_body( body, split() ) - { - } - task* execute() __TBB_override { - my_left_body.join( my_right_body ); - return NULL; - } - template - friend class start_deterministic_reduce; - }; - - //! Task type used to split the work of parallel_deterministic_reduce. - /** @ingroup algorithms */ - template - class start_deterministic_reduce: public task { - typedef finish_deterministic_reduce finish_type; - Body &my_body; - Range my_range; - task* execute() __TBB_override; - - //! Constructor used for root task - start_deterministic_reduce( const Range& range, Body& body ) : - my_body( body ), - my_range( range ) - { - } - //! Splitting constructor used to generate children. - /** parent_ becomes left child. Newly constructed object is right child. */ - start_deterministic_reduce( start_deterministic_reduce& parent_, finish_type& c ) : - my_body( c.my_right_body ), - my_range( parent_.my_range, split() ) - { - } - -public: - static void run( const Range& range, Body& body ) { - if( !range.empty() ) { -#if !__TBB_TASK_GROUP_CONTEXT || TBB_JOIN_OUTER_TASK_GROUP - task::spawn_root_and_wait( *new(task::allocate_root()) start_deterministic_reduce(range,&body) ); -#else - // Bound context prevents exceptions from body to affect nesting or sibling algorithms, - // and allows users to handle exceptions safely by wrapping parallel_for in the try-block. - task_group_context context; - task::spawn_root_and_wait( *new(task::allocate_root(context)) start_deterministic_reduce(range,body) ); -#endif /* __TBB_TASK_GROUP_CONTEXT && !TBB_JOIN_OUTER_TASK_GROUP */ - } - } -#if __TBB_TASK_GROUP_CONTEXT - static void run( const Range& range, Body& body, task_group_context& context ) { - if( !range.empty() ) - task::spawn_root_and_wait( *new(task::allocate_root(context)) start_deterministic_reduce(range,body) ); - } -#endif /* __TBB_TASK_GROUP_CONTEXT */ - }; - - template - task* start_deterministic_reduce::execute() { - if( !my_range.is_divisible() ) { - my_body( my_range ); - return NULL; - } else { - finish_type& c = *new( allocate_continuation() ) finish_type( my_body ); - recycle_as_child_of(c); - c.set_ref_count(2); - start_deterministic_reduce& b = *new( c.allocate_child() ) start_deterministic_reduce( *this, c ); - task::spawn(b); - return this; - } - } -} // namespace internal -//! @endcond -} //namespace interfaceX - -//! @cond INTERNAL -namespace internal { - using interface9::internal::start_reduce; - using interface9::internal::start_deterministic_reduce; - //! Auxiliary class for parallel_reduce; for internal use only. - /** The adaptor class that implements \ref parallel_reduce_body_req "parallel_reduce Body" - using given \ref parallel_reduce_lambda_req "anonymous function objects". - **/ - /** @ingroup algorithms */ - template - class lambda_reduce_body { - -//FIXME: decide if my_real_body, my_reduction, and identity_element should be copied or referenced -// (might require some performance measurements) - - const Value& identity_element; - const RealBody& my_real_body; - const Reduction& my_reduction; - Value my_value; - lambda_reduce_body& operator= ( const lambda_reduce_body& other ); - public: - lambda_reduce_body( const Value& identity, const RealBody& body, const Reduction& reduction ) - : identity_element(identity) - , my_real_body(body) - , my_reduction(reduction) - , my_value(identity) - { } - lambda_reduce_body( const lambda_reduce_body& other ) - : identity_element(other.identity_element) - , my_real_body(other.my_real_body) - , my_reduction(other.my_reduction) - , my_value(other.my_value) - { } - lambda_reduce_body( lambda_reduce_body& other, tbb::split ) - : identity_element(other.identity_element) - , my_real_body(other.my_real_body) - , my_reduction(other.my_reduction) - , my_value(other.identity_element) - { } - void operator()(Range& range) { - my_value = my_real_body(range, const_cast(my_value)); - } - void join( lambda_reduce_body& rhs ) { - my_value = my_reduction(const_cast(my_value), const_cast(rhs.my_value)); - } - Value result() const { - return my_value; - } - }; - -} // namespace internal -//! @endcond - -// Requirements on Range concept are documented in blocked_range.h - -/** \page parallel_reduce_body_req Requirements on parallel_reduce body - Class \c Body implementing the concept of parallel_reduce body must define: - - \code Body::Body( Body&, split ); \endcode Splitting constructor. - Must be able to run concurrently with operator() and method \c join - - \code Body::~Body(); \endcode Destructor - - \code void Body::operator()( Range& r ); \endcode Function call operator applying body to range \c r - and accumulating the result - - \code void Body::join( Body& b ); \endcode Join results. - The result in \c b should be merged into the result of \c this -**/ - -/** \page parallel_reduce_lambda_req Requirements on parallel_reduce anonymous function objects (lambda functions) - TO BE DOCUMENTED -**/ - -/** \name parallel_reduce - See also requirements on \ref range_req "Range" and \ref parallel_reduce_body_req "parallel_reduce Body". **/ -//@{ - -//! Parallel iteration with reduction and default partitioner. -/** @ingroup algorithms **/ -template -void parallel_reduce( const Range& range, Body& body ) { - internal::start_reduce::run( range, body, __TBB_DEFAULT_PARTITIONER() ); -} - -//! Parallel iteration with reduction and simple_partitioner -/** @ingroup algorithms **/ -template -void parallel_reduce( const Range& range, Body& body, const simple_partitioner& partitioner ) { - internal::start_reduce::run( range, body, partitioner ); -} - -//! Parallel iteration with reduction and auto_partitioner -/** @ingroup algorithms **/ -template -void parallel_reduce( const Range& range, Body& body, const auto_partitioner& partitioner ) { - internal::start_reduce::run( range, body, partitioner ); -} - -//! Parallel iteration with reduction and static_partitioner -/** @ingroup algorithms **/ -template -void parallel_reduce( const Range& range, Body& body, const static_partitioner& partitioner ) { - internal::start_reduce::run( range, body, partitioner ); -} - -//! Parallel iteration with reduction and affinity_partitioner -/** @ingroup algorithms **/ -template -void parallel_reduce( const Range& range, Body& body, affinity_partitioner& partitioner ) { - internal::start_reduce::run( range, body, partitioner ); -} - -#if __TBB_TASK_GROUP_CONTEXT -//! Parallel iteration with reduction, simple partitioner and user-supplied context. -/** @ingroup algorithms **/ -template -void parallel_reduce( const Range& range, Body& body, const simple_partitioner& partitioner, task_group_context& context ) { - internal::start_reduce::run( range, body, partitioner, context ); -} - -//! Parallel iteration with reduction, auto_partitioner and user-supplied context -/** @ingroup algorithms **/ -template -void parallel_reduce( const Range& range, Body& body, const auto_partitioner& partitioner, task_group_context& context ) { - internal::start_reduce::run( range, body, partitioner, context ); -} - -//! Parallel iteration with reduction, static_partitioner and user-supplied context -/** @ingroup algorithms **/ -template -void parallel_reduce( const Range& range, Body& body, const static_partitioner& partitioner, task_group_context& context ) { - internal::start_reduce::run( range, body, partitioner, context ); -} - -//! Parallel iteration with reduction, affinity_partitioner and user-supplied context -/** @ingroup algorithms **/ -template -void parallel_reduce( const Range& range, Body& body, affinity_partitioner& partitioner, task_group_context& context ) { - internal::start_reduce::run( range, body, partitioner, context ); -} -#endif /* __TBB_TASK_GROUP_CONTEXT */ - -/** parallel_reduce overloads that work with anonymous function objects - (see also \ref parallel_reduce_lambda_req "requirements on parallel_reduce anonymous function objects"). **/ - -//! Parallel iteration with reduction and default partitioner. -/** @ingroup algorithms **/ -template -Value parallel_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction ) { - internal::lambda_reduce_body body(identity, real_body, reduction); - internal::start_reduce,const __TBB_DEFAULT_PARTITIONER> - ::run(range, body, __TBB_DEFAULT_PARTITIONER() ); - return body.result(); -} - -//! Parallel iteration with reduction and simple_partitioner. -/** @ingroup algorithms **/ -template -Value parallel_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, - const simple_partitioner& partitioner ) { - internal::lambda_reduce_body body(identity, real_body, reduction); - internal::start_reduce,const simple_partitioner> - ::run(range, body, partitioner ); - return body.result(); -} - -//! Parallel iteration with reduction and auto_partitioner -/** @ingroup algorithms **/ -template -Value parallel_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, - const auto_partitioner& partitioner ) { - internal::lambda_reduce_body body(identity, real_body, reduction); - internal::start_reduce,const auto_partitioner> - ::run( range, body, partitioner ); - return body.result(); -} - -//! Parallel iteration with reduction and static_partitioner -/** @ingroup algorithms **/ -template -Value parallel_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, - const static_partitioner& partitioner ) { - internal::lambda_reduce_body body(identity, real_body, reduction); - internal::start_reduce,const static_partitioner> - ::run( range, body, partitioner ); - return body.result(); -} - -//! Parallel iteration with reduction and affinity_partitioner -/** @ingroup algorithms **/ -template -Value parallel_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, - affinity_partitioner& partitioner ) { - internal::lambda_reduce_body body(identity, real_body, reduction); - internal::start_reduce,affinity_partitioner> - ::run( range, body, partitioner ); - return body.result(); -} - -#if __TBB_TASK_GROUP_CONTEXT -//! Parallel iteration with reduction, simple partitioner and user-supplied context. -/** @ingroup algorithms **/ -template -Value parallel_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, - const simple_partitioner& partitioner, task_group_context& context ) { - internal::lambda_reduce_body body(identity, real_body, reduction); - internal::start_reduce,const simple_partitioner> - ::run( range, body, partitioner, context ); - return body.result(); -} - -//! Parallel iteration with reduction, auto_partitioner and user-supplied context -/** @ingroup algorithms **/ -template -Value parallel_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, - const auto_partitioner& partitioner, task_group_context& context ) { - internal::lambda_reduce_body body(identity, real_body, reduction); - internal::start_reduce,const auto_partitioner> - ::run( range, body, partitioner, context ); - return body.result(); -} - -//! Parallel iteration with reduction, static_partitioner and user-supplied context -/** @ingroup algorithms **/ -template -Value parallel_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, - const static_partitioner& partitioner, task_group_context& context ) { - internal::lambda_reduce_body body(identity, real_body, reduction); - internal::start_reduce,const static_partitioner> - ::run( range, body, partitioner, context ); - return body.result(); -} - -//! Parallel iteration with reduction, affinity_partitioner and user-supplied context -/** @ingroup algorithms **/ -template -Value parallel_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, - affinity_partitioner& partitioner, task_group_context& context ) { - internal::lambda_reduce_body body(identity, real_body, reduction); - internal::start_reduce,affinity_partitioner> - ::run( range, body, partitioner, context ); - return body.result(); -} -#endif /* __TBB_TASK_GROUP_CONTEXT */ - -//! Parallel iteration with deterministic reduction and default partitioner. -/** @ingroup algorithms **/ -template -void parallel_deterministic_reduce( const Range& range, Body& body ) { - internal::start_deterministic_reduce::run( range, body ); -} - -#if __TBB_TASK_GROUP_CONTEXT -//! Parallel iteration with deterministic reduction, simple partitioner and user-supplied context. -/** @ingroup algorithms **/ -template -void parallel_deterministic_reduce( const Range& range, Body& body, task_group_context& context ) { - internal::start_deterministic_reduce::run( range, body, context ); -} -#endif /* __TBB_TASK_GROUP_CONTEXT */ - -/** parallel_reduce overloads that work with anonymous function objects - (see also \ref parallel_reduce_lambda_req "requirements on parallel_reduce anonymous function objects"). **/ - -//! Parallel iteration with deterministic reduction and default partitioner. -/** @ingroup algorithms **/ -template -Value parallel_deterministic_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction ) { - internal::lambda_reduce_body body(identity, real_body, reduction); - internal::start_deterministic_reduce > - ::run(range, body); - return body.result(); -} - -#if __TBB_TASK_GROUP_CONTEXT -//! Parallel iteration with deterministic reduction, simple partitioner and user-supplied context. -/** @ingroup algorithms **/ -template -Value parallel_deterministic_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, - task_group_context& context ) { - internal::lambda_reduce_body body(identity, real_body, reduction); - internal::start_deterministic_reduce > - ::run( range, body, context ); - return body.result(); -} -#endif /* __TBB_TASK_GROUP_CONTEXT */ -//@} - -} // namespace tbb - -#endif /* __TBB_parallel_reduce_H */ diff --git a/inst/include/tbb/parallel_scan.h b/inst/include/tbb/parallel_scan.h deleted file mode 100644 index faf6b3164..000000000 --- a/inst/include/tbb/parallel_scan.h +++ /dev/null @@ -1,348 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_parallel_scan_H -#define __TBB_parallel_scan_H - -#include "task.h" -#include "aligned_space.h" -#include -#include "partitioner.h" - -namespace tbb { - -//! Used to indicate that the initial scan is being performed. -/** @ingroup algorithms */ -struct pre_scan_tag { - static bool is_final_scan() {return false;} -}; - -//! Used to indicate that the final scan is being performed. -/** @ingroup algorithms */ -struct final_scan_tag { - static bool is_final_scan() {return true;} -}; - -//! @cond INTERNAL -namespace internal { - - //! Performs final scan for a leaf - /** @ingroup algorithms */ - template - class final_sum: public task { - public: - Body my_body; - private: - aligned_space my_range; - //! Where to put result of last subrange, or NULL if not last subrange. - Body* my_stuff_last; - public: - final_sum( Body& body_ ) : - my_body(body_,split()) - { - poison_pointer(my_stuff_last); - } - ~final_sum() { - my_range.begin()->~Range(); - } - void finish_construction( const Range& range_, Body* stuff_last_ ) { - new( my_range.begin() ) Range(range_); - my_stuff_last = stuff_last_; - } - private: - task* execute() __TBB_override { - my_body( *my_range.begin(), final_scan_tag() ); - if( my_stuff_last ) - my_stuff_last->assign(my_body); - return NULL; - } - }; - - //! Split work to be done in the scan. - /** @ingroup algorithms */ - template - class sum_node: public task { - typedef final_sum final_sum_type; - public: - final_sum_type *my_incoming; - final_sum_type *my_body; - Body *my_stuff_last; - private: - final_sum_type *my_left_sum; - sum_node *my_left; - sum_node *my_right; - bool my_left_is_final; - Range my_range; - sum_node( const Range range_, bool left_is_final_ ) : - my_stuff_last(NULL), - my_left_sum(NULL), - my_left(NULL), - my_right(NULL), - my_left_is_final(left_is_final_), - my_range(range_) - { - // Poison fields that will be set by second pass. - poison_pointer(my_body); - poison_pointer(my_incoming); - } - task* create_child( const Range& range_, final_sum_type& f, sum_node* n, final_sum_type* incoming_, Body* stuff_last_ ) { - if( !n ) { - f.recycle_as_child_of( *this ); - f.finish_construction( range_, stuff_last_ ); - return &f; - } else { - n->my_body = &f; - n->my_incoming = incoming_; - n->my_stuff_last = stuff_last_; - return n; - } - } - task* execute() __TBB_override { - if( my_body ) { - if( my_incoming ) - my_left_sum->my_body.reverse_join( my_incoming->my_body ); - recycle_as_continuation(); - sum_node& c = *this; - task* b = c.create_child(Range(my_range,split()),*my_left_sum,my_right,my_left_sum,my_stuff_last); - task* a = my_left_is_final ? NULL : c.create_child(my_range,*my_body,my_left,my_incoming,NULL); - set_ref_count( (a!=NULL)+(b!=NULL) ); - my_body = NULL; - if( a ) spawn(*b); - else a = b; - return a; - } else { - return NULL; - } - } - template - friend class start_scan; - - template - friend class finish_scan; - }; - - //! Combine partial results - /** @ingroup algorithms */ - template - class finish_scan: public task { - typedef sum_node sum_node_type; - typedef final_sum final_sum_type; - final_sum_type** const my_sum; - sum_node_type*& my_return_slot; - public: - final_sum_type* my_right_zombie; - sum_node_type& my_result; - - task* execute() __TBB_override { - __TBB_ASSERT( my_result.ref_count()==(my_result.my_left!=NULL)+(my_result.my_right!=NULL), NULL ); - if( my_result.my_left ) - my_result.my_left_is_final = false; - if( my_right_zombie && my_sum ) - ((*my_sum)->my_body).reverse_join(my_result.my_left_sum->my_body); - __TBB_ASSERT( !my_return_slot, NULL ); - if( my_right_zombie || my_result.my_right ) { - my_return_slot = &my_result; - } else { - destroy( my_result ); - } - if( my_right_zombie && !my_sum && !my_result.my_right ) { - destroy(*my_right_zombie); - my_right_zombie = NULL; - } - return NULL; - } - - finish_scan( sum_node_type*& return_slot_, final_sum_type** sum_, sum_node_type& result_ ) : - my_sum(sum_), - my_return_slot(return_slot_), - my_right_zombie(NULL), - my_result(result_) - { - __TBB_ASSERT( !my_return_slot, NULL ); - } - }; - - //! Initial task to split the work - /** @ingroup algorithms */ - template - class start_scan: public task { - typedef sum_node sum_node_type; - typedef final_sum final_sum_type; - final_sum_type* my_body; - /** Non-null if caller is requesting total. */ - final_sum_type** my_sum; - sum_node_type** my_return_slot; - /** Null if computing root. */ - sum_node_type* my_parent_sum; - bool my_is_final; - bool my_is_right_child; - Range my_range; - typename Partitioner::partition_type my_partition; - task* execute() __TBB_override ; - public: - start_scan( sum_node_type*& return_slot_, start_scan& parent_, sum_node_type* parent_sum_ ) : - my_body(parent_.my_body), - my_sum(parent_.my_sum), - my_return_slot(&return_slot_), - my_parent_sum(parent_sum_), - my_is_final(parent_.my_is_final), - my_is_right_child(false), - my_range(parent_.my_range,split()), - my_partition(parent_.my_partition,split()) - { - __TBB_ASSERT( !*my_return_slot, NULL ); - } - - start_scan( sum_node_type*& return_slot_, const Range& range_, final_sum_type& body_, const Partitioner& partitioner_) : - my_body(&body_), - my_sum(NULL), - my_return_slot(&return_slot_), - my_parent_sum(NULL), - my_is_final(true), - my_is_right_child(false), - my_range(range_), - my_partition(partitioner_) - { - __TBB_ASSERT( !*my_return_slot, NULL ); - } - - static void run( const Range& range_, Body& body_, const Partitioner& partitioner_ ) { - if( !range_.empty() ) { - typedef internal::start_scan start_pass1_type; - internal::sum_node* root = NULL; - typedef internal::final_sum final_sum_type; - final_sum_type* temp_body = new(task::allocate_root()) final_sum_type( body_ ); - start_pass1_type& pass1 = *new(task::allocate_root()) start_pass1_type( - /*my_return_slot=*/root, - range_, - *temp_body, - partitioner_ ); - temp_body->my_body.reverse_join(body_); - task::spawn_root_and_wait( pass1 ); - if( root ) { - root->my_body = temp_body; - root->my_incoming = NULL; - root->my_stuff_last = &body_; - task::spawn_root_and_wait( *root ); - } else { - body_.assign(temp_body->my_body); - temp_body->finish_construction( range_, NULL ); - temp_body->destroy(*temp_body); - } - } - } - }; - - template - task* start_scan::execute() { - typedef internal::finish_scan finish_pass1_type; - finish_pass1_type* p = my_parent_sum ? static_cast( parent() ) : NULL; - // Inspecting p->result.left_sum would ordinarily be a race condition. - // But we inspect it only if we are not a stolen task, in which case we - // know that task assigning to p->result.left_sum has completed. - bool treat_as_stolen = my_is_right_child && (is_stolen_task() || my_body!=p->my_result.my_left_sum); - if( treat_as_stolen ) { - // Invocation is for right child that has been really stolen or needs to be virtually stolen - p->my_right_zombie = my_body = new( allocate_root() ) final_sum_type(my_body->my_body); - my_is_final = false; - } - task* next_task = NULL; - if( (my_is_right_child && !treat_as_stolen) || !my_range.is_divisible() || my_partition.should_execute_range(*this) ) { - if( my_is_final ) - (my_body->my_body)( my_range, final_scan_tag() ); - else if( my_sum ) - (my_body->my_body)( my_range, pre_scan_tag() ); - if( my_sum ) - *my_sum = my_body; - __TBB_ASSERT( !*my_return_slot, NULL ); - } else { - sum_node_type* result; - if( my_parent_sum ) - result = new(allocate_additional_child_of(*my_parent_sum)) sum_node_type(my_range,/*my_left_is_final=*/my_is_final); - else - result = new(task::allocate_root()) sum_node_type(my_range,/*my_left_is_final=*/my_is_final); - finish_pass1_type& c = *new( allocate_continuation()) finish_pass1_type(*my_return_slot,my_sum,*result); - // Split off right child - start_scan& b = *new( c.allocate_child() ) start_scan( /*my_return_slot=*/result->my_right, *this, result ); - b.my_is_right_child = true; - // Left child is recycling of *this. Must recycle this before spawning b, - // otherwise b might complete and decrement c.ref_count() to zero, which - // would cause c.execute() to run prematurely. - recycle_as_child_of(c); - c.set_ref_count(2); - c.spawn(b); - my_sum = &result->my_left_sum; - my_return_slot = &result->my_left; - my_is_right_child = false; - next_task = this; - my_parent_sum = result; - __TBB_ASSERT( !*my_return_slot, NULL ); - } - return next_task; - } -} // namespace internal -//! @endcond - -// Requirements on Range concept are documented in blocked_range.h - -/** \page parallel_scan_body_req Requirements on parallel_scan body - Class \c Body implementing the concept of parallel_scan body must define: - - \code Body::Body( Body&, split ); \endcode Splitting constructor. - Split \c b so that \c this and \c b can accumulate separately - - \code Body::~Body(); \endcode Destructor - - \code void Body::operator()( const Range& r, pre_scan_tag ); \endcode - Preprocess iterations for range \c r - - \code void Body::operator()( const Range& r, final_scan_tag ); \endcode - Do final processing for iterations of range \c r - - \code void Body::reverse_join( Body& a ); \endcode - Merge preprocessing state of \c a into \c this, where \c a was - created earlier from \c b by b's splitting constructor -**/ - -/** \name parallel_scan - See also requirements on \ref range_req "Range" and \ref parallel_scan_body_req "parallel_scan Body". **/ -//@{ - -//! Parallel prefix with default partitioner -/** @ingroup algorithms **/ -template -void parallel_scan( const Range& range, Body& body ) { - internal::start_scan::run(range,body,__TBB_DEFAULT_PARTITIONER()); -} - -//! Parallel prefix with simple_partitioner -/** @ingroup algorithms **/ -template -void parallel_scan( const Range& range, Body& body, const simple_partitioner& partitioner ) { - internal::start_scan::run(range,body,partitioner); -} - -//! Parallel prefix with auto_partitioner -/** @ingroup algorithms **/ -template -void parallel_scan( const Range& range, Body& body, const auto_partitioner& partitioner ) { - internal::start_scan::run(range,body,partitioner); -} -//@} - -} // namespace tbb - -#endif /* __TBB_parallel_scan_H */ - diff --git a/inst/include/tbb/parallel_sort.h b/inst/include/tbb/parallel_sort.h deleted file mode 100644 index 07d3907b1..000000000 --- a/inst/include/tbb/parallel_sort.h +++ /dev/null @@ -1,266 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_parallel_sort_H -#define __TBB_parallel_sort_H - -#include "parallel_for.h" -#include "blocked_range.h" -#include "internal/_range_iterator.h" -#include -#include -#include - -namespace tbb { - -namespace interface9 { -//! @cond INTERNAL -namespace internal { - -using tbb::internal::no_assign; - -//! Range used in quicksort to split elements into subranges based on a value. -/** The split operation selects a splitter and places all elements less than or equal - to the value in the first range and the remaining elements in the second range. - @ingroup algorithms */ -template -class quick_sort_range: private no_assign { - - inline size_t median_of_three(const RandomAccessIterator &array, size_t l, size_t m, size_t r) const { - return comp(array[l], array[m]) ? ( comp(array[m], array[r]) ? m : ( comp( array[l], array[r]) ? r : l ) ) - : ( comp(array[r], array[m]) ? m : ( comp( array[r], array[l] ) ? r : l ) ); - } - - inline size_t pseudo_median_of_nine( const RandomAccessIterator &array, const quick_sort_range &range ) const { - size_t offset = range.size/8u; - return median_of_three(array, - median_of_three(array, 0, offset, offset*2), - median_of_three(array, offset*3, offset*4, offset*5), - median_of_three(array, offset*6, offset*7, range.size - 1) ); - - } - - size_t split_range( quick_sort_range& range ) { - using std::iter_swap; - RandomAccessIterator array = range.begin; - RandomAccessIterator key0 = range.begin; - size_t m = pseudo_median_of_nine(array, range); - if (m) iter_swap ( array, array+m ); - - size_t i=0; - size_t j=range.size; - // Partition interval [i+1,j-1] with key *key0. - for(;;) { - __TBB_ASSERT( i=grainsize;} - - quick_sort_range( quick_sort_range& range, split ) - : comp(range.comp) - , size(split_range(range)) - // +1 accounts for the pivot element, which is at its correct place - // already and, therefore, is not included into subranges. - , begin(range.begin+range.size+1) {} -}; - -#if __TBB_TASK_GROUP_CONTEXT -//! Body class used to test if elements in a range are presorted -/** @ingroup algorithms */ -template -class quick_sort_pretest_body : no_assign { - const Compare ∁ - -public: - quick_sort_pretest_body(const Compare &_comp) : comp(_comp) {} - - void operator()( const blocked_range& range ) const { - task &my_task = task::self(); - RandomAccessIterator my_end = range.end(); - - int i = 0; - for (RandomAccessIterator k = range.begin(); k != my_end; ++k, ++i) { - if ( i%64 == 0 && my_task.is_cancelled() ) break; - - // The k-1 is never out-of-range because the first chunk starts at begin+serial_cutoff+1 - if ( comp( *(k), *(k-1) ) ) { - my_task.cancel_group_execution(); - break; - } - } - } - -}; -#endif /* __TBB_TASK_GROUP_CONTEXT */ - -//! Body class used to sort elements in a range that is smaller than the grainsize. -/** @ingroup algorithms */ -template -struct quick_sort_body { - void operator()( const quick_sort_range& range ) const { - //SerialQuickSort( range.begin, range.size, range.comp ); - std::sort( range.begin, range.begin + range.size, range.comp ); - } -}; - -//! Wrapper method to initiate the sort by calling parallel_for. -/** @ingroup algorithms */ -template -void parallel_quick_sort( RandomAccessIterator begin, RandomAccessIterator end, const Compare& comp ) { -#if __TBB_TASK_GROUP_CONTEXT - task_group_context my_context; - const int serial_cutoff = 9; - - __TBB_ASSERT( begin + serial_cutoff < end, "min_parallel_size is smaller than serial cutoff?" ); - RandomAccessIterator k = begin; - for ( ; k != begin + serial_cutoff; ++k ) { - if ( comp( *(k+1), *k ) ) { - goto do_parallel_quick_sort; - } - } - - parallel_for( blocked_range(k+1, end), - quick_sort_pretest_body(comp), - auto_partitioner(), - my_context); - - if (my_context.is_group_execution_cancelled()) -do_parallel_quick_sort: -#endif /* __TBB_TASK_GROUP_CONTEXT */ - parallel_for( quick_sort_range(begin, end-begin, comp ), - quick_sort_body(), - auto_partitioner() ); -} - -} // namespace internal -//! @endcond -} // namespace interfaceX - -/** \page parallel_sort_iter_req Requirements on iterators for parallel_sort - Requirements on the iterator type \c It and its value type \c T for \c parallel_sort: - - - \code void iter_swap( It a, It b ) \endcode Swaps the values of the elements the given - iterators \c a and \c b are pointing to. \c It should be a random access iterator. - - - \code bool Compare::operator()( const T& x, const T& y ) \endcode True if x comes before y; -**/ - -/** \name parallel_sort - See also requirements on \ref parallel_sort_iter_req "iterators for parallel_sort". **/ -//@{ - -//! Sorts the data in [begin,end) using the given comparator -/** The compare function object is used for all comparisons between elements during sorting. - The compare object must define a bool operator() function. - @ingroup algorithms **/ -template -void parallel_sort( RandomAccessIterator begin, RandomAccessIterator end, const Compare& comp) { - const int min_parallel_size = 500; - if( end > begin ) { - if (end - begin < min_parallel_size) { - std::sort(begin, end, comp); - } else { - interface9::internal::parallel_quick_sort(begin, end, comp); - } - } -} - -//! Sorts the data in [begin,end) with a default comparator \c std::less -/** @ingroup algorithms **/ -template -inline void parallel_sort( RandomAccessIterator begin, RandomAccessIterator end ) { - parallel_sort( begin, end, std::less< typename std::iterator_traits::value_type >() ); -} - -//! Sorts the data in rng using the given comparator -/** @ingroup algorithms **/ -template -void parallel_sort(Range& rng, const Compare& comp) { - parallel_sort(tbb::internal::first(rng), tbb::internal::last(rng), comp); -} - -//! Sorts the data in const rng using the given comparator -/** @ingroup algorithms **/ -template -void parallel_sort(const Range& rng, const Compare& comp) { - parallel_sort(tbb::internal::first(rng), tbb::internal::last(rng), comp); -} - -//! Sorts the data in rng with a default comparator \c std::less -/** @ingroup algorithms **/ -template -void parallel_sort(Range& rng) { - parallel_sort(tbb::internal::first(rng), tbb::internal::last(rng)); -} - -//! Sorts the data in const rng with a default comparator \c std::less -/** @ingroup algorithms **/ -template -void parallel_sort(const Range& rng) { - parallel_sort(tbb::internal::first(rng), tbb::internal::last(rng)); -} - -//! Sorts the data in the range \c [begin,end) with a default comparator \c std::less -/** @ingroup algorithms **/ -template -inline void parallel_sort( T * begin, T * end ) { - parallel_sort( begin, end, std::less< T >() ); -} -//@} - - -} // namespace tbb - -#endif - diff --git a/inst/include/tbb/parallel_while.h b/inst/include/tbb/parallel_while.h deleted file mode 100644 index 2f37a41a4..000000000 --- a/inst/include/tbb/parallel_while.h +++ /dev/null @@ -1,186 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_parallel_while -#define __TBB_parallel_while - -#include "task.h" -#include - -namespace tbb { - -template -class parallel_while; - -//! @cond INTERNAL -namespace internal { - - template class while_task; - - //! For internal use only. - /** Executes one iteration of a while. - @ingroup algorithms */ - template - class while_iteration_task: public task { - const Body& my_body; - typename Body::argument_type my_value; - task* execute() __TBB_override { - my_body(my_value); - return NULL; - } - while_iteration_task( const typename Body::argument_type& value, const Body& body ) : - my_body(body), my_value(value) - {} - template friend class while_group_task; - friend class tbb::parallel_while; - }; - - //! For internal use only - /** Unpacks a block of iterations. - @ingroup algorithms */ - template - class while_group_task: public task { - static const size_t max_arg_size = 4; - const Body& my_body; - size_t size; - typename Body::argument_type my_arg[max_arg_size]; - while_group_task( const Body& body ) : my_body(body), size(0) {} - task* execute() __TBB_override { - typedef while_iteration_task iteration_type; - __TBB_ASSERT( size>0, NULL ); - task_list list; - task* t; - size_t k=0; - for(;;) { - t = new( allocate_child() ) iteration_type(my_arg[k],my_body); - if( ++k==size ) break; - list.push_back(*t); - } - set_ref_count(int(k+1)); - spawn(list); - spawn_and_wait_for_all(*t); - return NULL; - } - template friend class while_task; - }; - - //! For internal use only. - /** Gets block of iterations from a stream and packages them into a while_group_task. - @ingroup algorithms */ - template - class while_task: public task { - Stream& my_stream; - const Body& my_body; - empty_task& my_barrier; - task* execute() __TBB_override { - typedef while_group_task block_type; - block_type& t = *new( allocate_additional_child_of(my_barrier) ) block_type(my_body); - size_t k=0; - while( my_stream.pop_if_present(t.my_arg[k]) ) { - if( ++k==block_type::max_arg_size ) { - // There might be more iterations. - recycle_to_reexecute(); - break; - } - } - if( k==0 ) { - destroy(t); - return NULL; - } else { - t.size = k; - return &t; - } - } - while_task( Stream& stream, const Body& body, empty_task& barrier ) : - my_stream(stream), - my_body(body), - my_barrier(barrier) - {} - friend class tbb::parallel_while; - }; - -} // namespace internal -//! @endcond - -//! Parallel iteration over a stream, with optional addition of more work. -/** The Body b has the requirement: \n - "b(v)" \n - "b.argument_type" \n - where v is an argument_type - @ingroup algorithms */ -template -class parallel_while: internal::no_copy { -public: - //! Construct empty non-running parallel while. - parallel_while() : my_body(NULL), my_barrier(NULL) {} - - //! Destructor cleans up data members before returning. - ~parallel_while() { - if( my_barrier ) { - my_barrier->destroy(*my_barrier); - my_barrier = NULL; - } - } - - //! Type of items - typedef typename Body::argument_type value_type; - - //! Apply body.apply to each item in the stream. - /** A Stream s has the requirements \n - "S::value_type" \n - "s.pop_if_present(value) is convertible to bool */ - template - void run( Stream& stream, const Body& body ); - - //! Add a work item while running. - /** Should be executed only by body.apply or a thread spawned therefrom. */ - void add( const value_type& item ); - -private: - const Body* my_body; - empty_task* my_barrier; -}; - -template -template -void parallel_while::run( Stream& stream, const Body& body ) { - using namespace internal; - empty_task& barrier = *new( task::allocate_root() ) empty_task(); - my_body = &body; - my_barrier = &barrier; - my_barrier->set_ref_count(2); - while_task& w = *new( my_barrier->allocate_child() ) while_task( stream, body, barrier ); - my_barrier->spawn_and_wait_for_all(w); - my_barrier->destroy(*my_barrier); - my_barrier = NULL; - my_body = NULL; -} - -template -void parallel_while::add( const value_type& item ) { - __TBB_ASSERT(my_barrier,"attempt to add to parallel_while that is not running"); - typedef internal::while_iteration_task iteration_type; - iteration_type& i = *new( task::allocate_additional_child_of(*my_barrier) ) iteration_type(item,*my_body); - task::self().spawn( i ); -} - -} // namespace - -#endif /* __TBB_parallel_while */ diff --git a/inst/include/tbb/pipeline.h b/inst/include/tbb/pipeline.h deleted file mode 100644 index 20a8ec9bb..000000000 --- a/inst/include/tbb/pipeline.h +++ /dev/null @@ -1,665 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_pipeline_H -#define __TBB_pipeline_H - -#include "atomic.h" -#include "task.h" -#include "tbb_allocator.h" -#include - -#if __TBB_CPP11_TYPE_PROPERTIES_PRESENT || __TBB_TR1_TYPE_PROPERTIES_IN_STD_PRESENT -#include -#endif - -namespace tbb { - -class pipeline; -class filter; - -//! @cond INTERNAL -namespace internal { - -// The argument for PIPELINE_VERSION should be an integer between 2 and 9 -#define __TBB_PIPELINE_VERSION(x) ((unsigned char)(x-2)<<1) - -typedef unsigned long Token; -typedef long tokendiff_t; -class stage_task; -class input_buffer; -class pipeline_root_task; -class pipeline_cleaner; - -} // namespace internal - -namespace interface6 { - template class filter_t; - - namespace internal { - class pipeline_proxy; - } -} - -//! @endcond - -//! A stage in a pipeline. -/** @ingroup algorithms */ -class filter: internal::no_copy { -private: - //! Value used to mark "not in pipeline" - static filter* not_in_pipeline() {return reinterpret_cast(intptr_t(-1));} -protected: - //! The lowest bit 0 is for parallel vs. serial - static const unsigned char filter_is_serial = 0x1; - - //! 4th bit distinguishes ordered vs unordered filters. - /** The bit was not set for parallel filters in TBB 2.1 and earlier, - but is_ordered() function always treats parallel filters as out of order. */ - static const unsigned char filter_is_out_of_order = 0x1<<4; - - //! 5th bit distinguishes thread-bound and regular filters. - static const unsigned char filter_is_bound = 0x1<<5; - - //! 6th bit marks input filters emitting small objects - static const unsigned char filter_may_emit_null = 0x1<<6; - - //! 7th bit defines exception propagation mode expected by the application. - static const unsigned char exact_exception_propagation = -#if TBB_USE_CAPTURED_EXCEPTION - 0x0; -#else - 0x1<<7; -#endif /* TBB_USE_CAPTURED_EXCEPTION */ - - static const unsigned char current_version = __TBB_PIPELINE_VERSION(5); - static const unsigned char version_mask = 0x7<<1; // bits 1-3 are for version -public: - enum mode { - //! processes multiple items in parallel and in no particular order - parallel = current_version | filter_is_out_of_order, - //! processes items one at a time; all such filters process items in the same order - serial_in_order = current_version | filter_is_serial, - //! processes items one at a time and in no particular order - serial_out_of_order = current_version | filter_is_serial | filter_is_out_of_order, - //! @deprecated use serial_in_order instead - serial = serial_in_order - }; -protected: - explicit filter( bool is_serial_ ) : - next_filter_in_pipeline(not_in_pipeline()), - my_input_buffer(NULL), - my_filter_mode(static_cast((is_serial_ ? serial : parallel) | exact_exception_propagation)), - prev_filter_in_pipeline(not_in_pipeline()), - my_pipeline(NULL), - next_segment(NULL) - {} - - explicit filter( mode filter_mode ) : - next_filter_in_pipeline(not_in_pipeline()), - my_input_buffer(NULL), - my_filter_mode(static_cast(filter_mode | exact_exception_propagation)), - prev_filter_in_pipeline(not_in_pipeline()), - my_pipeline(NULL), - next_segment(NULL) - {} - - // signal end-of-input for concrete_filters - void __TBB_EXPORTED_METHOD set_end_of_input(); - -public: - //! True if filter is serial. - bool is_serial() const { - return bool( my_filter_mode & filter_is_serial ); - } - - //! True if filter must receive stream in order. - bool is_ordered() const { - return (my_filter_mode & (filter_is_out_of_order|filter_is_serial))==filter_is_serial; - } - - //! True if filter is thread-bound. - bool is_bound() const { - return ( my_filter_mode & filter_is_bound )==filter_is_bound; - } - - //! true if an input filter can emit null - bool object_may_be_null() { - return ( my_filter_mode & filter_may_emit_null ) == filter_may_emit_null; - } - - //! Operate on an item from the input stream, and return item for output stream. - /** Returns NULL if filter is a sink. */ - virtual void* operator()( void* item ) = 0; - - //! Destroy filter. - /** If the filter was added to a pipeline, the pipeline must be destroyed first. */ - virtual __TBB_EXPORTED_METHOD ~filter(); - -#if __TBB_TASK_GROUP_CONTEXT - //! Destroys item if pipeline was cancelled. - /** Required to prevent memory leaks. - Note it can be called concurrently even for serial filters.*/ - virtual void finalize( void* /*item*/ ) {}; -#endif - -private: - //! Pointer to next filter in the pipeline. - filter* next_filter_in_pipeline; - - //! has the filter not yet processed all the tokens it will ever see? - // (pipeline has not yet reached end_of_input or this filter has not yet - // seen the last token produced by input_filter) - bool has_more_work(); - - //! Buffer for incoming tokens, or NULL if not required. - /** The buffer is required if the filter is serial or follows a thread-bound one. */ - internal::input_buffer* my_input_buffer; - - friend class internal::stage_task; - friend class internal::pipeline_root_task; - friend class pipeline; - friend class thread_bound_filter; - - //! Storage for filter mode and dynamically checked implementation version. - const unsigned char my_filter_mode; - - //! Pointer to previous filter in the pipeline. - filter* prev_filter_in_pipeline; - - //! Pointer to the pipeline. - pipeline* my_pipeline; - - //! Pointer to the next "segment" of filters, or NULL if not required. - /** In each segment, the first filter is not thread-bound but follows a thread-bound one. */ - filter* next_segment; -}; - -//! A stage in a pipeline served by a user thread. -/** @ingroup algorithms */ -class thread_bound_filter: public filter { -public: - enum result_type { - // item was processed - success, - // item is currently not available - item_not_available, - // there are no more items to process - end_of_stream - }; -protected: - explicit thread_bound_filter(mode filter_mode): - filter(static_cast(filter_mode | filter::filter_is_bound)) - { - __TBB_ASSERT(filter_mode & filter::filter_is_serial, "thread-bound filters must be serial"); - } -public: - //! If a data item is available, invoke operator() on that item. - /** This interface is non-blocking. - Returns 'success' if an item was processed. - Returns 'item_not_available' if no item can be processed now - but more may arrive in the future, or if token limit is reached. - Returns 'end_of_stream' if there are no more items to process. */ - result_type __TBB_EXPORTED_METHOD try_process_item(); - - //! Wait until a data item becomes available, and invoke operator() on that item. - /** This interface is blocking. - Returns 'success' if an item was processed. - Returns 'end_of_stream' if there are no more items to process. - Never returns 'item_not_available', as it blocks until another return condition applies. */ - result_type __TBB_EXPORTED_METHOD process_item(); - -private: - //! Internal routine for item processing - result_type internal_process_item(bool is_blocking); -}; - -//! A processing pipeline that applies filters to items. -/** @ingroup algorithms */ -class pipeline { -public: - //! Construct empty pipeline. - __TBB_EXPORTED_METHOD pipeline(); - - /** Though the current implementation declares the destructor virtual, do not rely on this - detail. The virtualness is deprecated and may disappear in future versions of TBB. */ - virtual __TBB_EXPORTED_METHOD ~pipeline(); - - //! Add filter to end of pipeline. - void __TBB_EXPORTED_METHOD add_filter( filter& filter_ ); - - //! Run the pipeline to completion. - void __TBB_EXPORTED_METHOD run( size_t max_number_of_live_tokens ); - -#if __TBB_TASK_GROUP_CONTEXT - //! Run the pipeline to completion with user-supplied context. - void __TBB_EXPORTED_METHOD run( size_t max_number_of_live_tokens, tbb::task_group_context& context ); -#endif - - //! Remove all filters from the pipeline. - void __TBB_EXPORTED_METHOD clear(); - -private: - friend class internal::stage_task; - friend class internal::pipeline_root_task; - friend class filter; - friend class thread_bound_filter; - friend class internal::pipeline_cleaner; - friend class tbb::interface6::internal::pipeline_proxy; - - //! Pointer to first filter in the pipeline. - filter* filter_list; - - //! Pointer to location where address of next filter to be added should be stored. - filter* filter_end; - - //! task who's reference count is used to determine when all stages are done. - task* end_counter; - - //! Number of idle tokens waiting for input stage. - atomic input_tokens; - - //! Global counter of tokens - atomic token_counter; - - //! False until fetch_input returns NULL. - bool end_of_input; - - //! True if the pipeline contains a thread-bound filter; false otherwise. - bool has_thread_bound_filters; - - //! Remove filter from pipeline. - void remove_filter( filter& filter_ ); - - //! Not used, but retained to satisfy old export files. - void __TBB_EXPORTED_METHOD inject_token( task& self ); - -#if __TBB_TASK_GROUP_CONTEXT - //! Does clean up if pipeline is cancelled or exception occurred - void clear_filters(); -#endif -}; - -//------------------------------------------------------------------------ -// Support for lambda-friendly parallel_pipeline interface -//------------------------------------------------------------------------ - -namespace interface6 { - -namespace internal { - template class concrete_filter; -} - -//! input_filter control to signal end-of-input for parallel_pipeline -class flow_control { - bool is_pipeline_stopped; - flow_control() { is_pipeline_stopped = false; } - template friend class internal::concrete_filter; -public: - void stop() { is_pipeline_stopped = true; } -}; - -//! @cond INTERNAL -namespace internal { - -template struct tbb_large_object {enum { value = sizeof(T) > sizeof(void *) }; }; - -// Obtain type properties in one or another way -#if __TBB_CPP11_TYPE_PROPERTIES_PRESENT -template struct tbb_trivially_copyable { enum { value = std::is_trivially_copyable::value }; }; -#elif __TBB_TR1_TYPE_PROPERTIES_IN_STD_PRESENT -template struct tbb_trivially_copyable { enum { value = std::has_trivial_copy_constructor::value }; }; -#else -// Explicitly list the types we wish to be placed as-is in the pipeline input_buffers. -template struct tbb_trivially_copyable { enum { value = false }; }; -template struct tbb_trivially_copyable { enum { value = true }; }; -template<> struct tbb_trivially_copyable { enum { value = true }; }; -template<> struct tbb_trivially_copyable { enum { value = true }; }; -template<> struct tbb_trivially_copyable { enum { value = !tbb_large_object::value }; }; -template<> struct tbb_trivially_copyable { enum { value = !tbb_large_object::value }; }; -template<> struct tbb_trivially_copyable { enum { value = !tbb_large_object::value }; }; -template<> struct tbb_trivially_copyable { enum { value = !tbb_large_object::value }; }; -template<> struct tbb_trivially_copyable { enum { value = !tbb_large_object::value }; }; -template<> struct tbb_trivially_copyable { enum { value = !tbb_large_object::value }; }; -#endif // Obtaining type properties - -template struct is_large_object {enum { value = tbb_large_object::value || !tbb_trivially_copyable::value }; }; - -template class token_helper; - -// large object helper (uses tbb_allocator) -template -class token_helper { - public: - typedef typename tbb::tbb_allocator allocator; - typedef T* pointer; - typedef T value_type; - static pointer create_token(const value_type & source) { - pointer output_t = allocator().allocate(1); - return new (output_t) T(source); - } - static value_type & token(pointer & t) { return *t;} - static void * cast_to_void_ptr(pointer ref) { return (void *) ref; } - static pointer cast_from_void_ptr(void * ref) { return (pointer)ref; } - static void destroy_token(pointer token) { - allocator().destroy(token); - allocator().deallocate(token,1); - } -}; - -// pointer specialization -template -class token_helper { - public: - typedef T* pointer; - typedef T* value_type; - static pointer create_token(const value_type & source) { return source; } - static value_type & token(pointer & t) { return t;} - static void * cast_to_void_ptr(pointer ref) { return (void *)ref; } - static pointer cast_from_void_ptr(void * ref) { return (pointer)ref; } - static void destroy_token( pointer /*token*/) {} -}; - -// small object specialization (converts void* to the correct type, passes objects directly.) -template -class token_helper { - typedef union { - T actual_value; - void * void_overlay; - } type_to_void_ptr_map; - public: - typedef T pointer; // not really a pointer in this case. - typedef T value_type; - static pointer create_token(const value_type & source) { - return source; } - static value_type & token(pointer & t) { return t;} - static void * cast_to_void_ptr(pointer ref) { - type_to_void_ptr_map mymap; - mymap.void_overlay = NULL; - mymap.actual_value = ref; - return mymap.void_overlay; - } - static pointer cast_from_void_ptr(void * ref) { - type_to_void_ptr_map mymap; - mymap.void_overlay = ref; - return mymap.actual_value; - } - static void destroy_token( pointer /*token*/) {} -}; - -template -class concrete_filter: public tbb::filter { - const Body& my_body; - typedef token_helper::value > t_helper; - typedef typename t_helper::pointer t_pointer; - typedef token_helper::value > u_helper; - typedef typename u_helper::pointer u_pointer; - - void* operator()(void* input) __TBB_override { - t_pointer temp_input = t_helper::cast_from_void_ptr(input); - u_pointer output_u = u_helper::create_token(my_body(t_helper::token(temp_input))); - t_helper::destroy_token(temp_input); - return u_helper::cast_to_void_ptr(output_u); - } - - void finalize(void * input) __TBB_override { - t_pointer temp_input = t_helper::cast_from_void_ptr(input); - t_helper::destroy_token(temp_input); - } - -public: - concrete_filter(tbb::filter::mode filter_mode, const Body& body) : filter(filter_mode), my_body(body) {} -}; - -// input -template -class concrete_filter: public filter { - const Body& my_body; - typedef token_helper::value > u_helper; - typedef typename u_helper::pointer u_pointer; - - void* operator()(void*) __TBB_override { - flow_control control; - u_pointer output_u = u_helper::create_token(my_body(control)); - if(control.is_pipeline_stopped) { - u_helper::destroy_token(output_u); - set_end_of_input(); - return NULL; - } - return u_helper::cast_to_void_ptr(output_u); - } - -public: - concrete_filter(tbb::filter::mode filter_mode, const Body& body) : - filter(static_cast(filter_mode | filter_may_emit_null)), - my_body(body) - {} -}; - -template -class concrete_filter: public filter { - const Body& my_body; - typedef token_helper::value > t_helper; - typedef typename t_helper::pointer t_pointer; - - void* operator()(void* input) __TBB_override { - t_pointer temp_input = t_helper::cast_from_void_ptr(input); - my_body(t_helper::token(temp_input)); - t_helper::destroy_token(temp_input); - return NULL; - } - void finalize(void* input) __TBB_override { - t_pointer temp_input = t_helper::cast_from_void_ptr(input); - t_helper::destroy_token(temp_input); - } - -public: - concrete_filter(tbb::filter::mode filter_mode, const Body& body) : filter(filter_mode), my_body(body) {} -}; - -template -class concrete_filter: public filter { - const Body& my_body; - - /** Override privately because it is always called virtually */ - void* operator()(void*) __TBB_override { - flow_control control; - my_body(control); - void* output = control.is_pipeline_stopped ? NULL : (void*)(intptr_t)-1; - return output; - } -public: - concrete_filter(filter::mode filter_mode, const Body& body) : filter(filter_mode), my_body(body) {} -}; - -//! The class that represents an object of the pipeline for parallel_pipeline(). -/** It primarily serves as RAII class that deletes heap-allocated filter instances. */ -class pipeline_proxy { - tbb::pipeline my_pipe; -public: - pipeline_proxy( const filter_t& filter_chain ); - ~pipeline_proxy() { - while( filter* f = my_pipe.filter_list ) - delete f; // filter destructor removes it from the pipeline - } - tbb::pipeline* operator->() { return &my_pipe; } -}; - -//! Abstract base class that represents a node in a parse tree underlying a filter_t. -/** These nodes are always heap-allocated and can be shared by filter_t objects. */ -class filter_node: tbb::internal::no_copy { - /** Count must be atomic because it is hidden state for user, but might be shared by threads. */ - tbb::atomic ref_count; -protected: - filter_node() { - ref_count = 0; -#ifdef __TBB_TEST_FILTER_NODE_COUNT - ++(__TBB_TEST_FILTER_NODE_COUNT); -#endif - } -public: - //! Add concrete_filter to pipeline - virtual void add_to( pipeline& ) = 0; - //! Increment reference count - void add_ref() {++ref_count;} - //! Decrement reference count and delete if it becomes zero. - void remove_ref() { - __TBB_ASSERT(ref_count>0,"ref_count underflow"); - if( --ref_count==0 ) - delete this; - } - virtual ~filter_node() { -#ifdef __TBB_TEST_FILTER_NODE_COUNT - --(__TBB_TEST_FILTER_NODE_COUNT); -#endif - } -}; - -//! Node in parse tree representing result of make_filter. -template -class filter_node_leaf: public filter_node { - const tbb::filter::mode mode; - const Body body; - void add_to( pipeline& p ) __TBB_override { - concrete_filter* f = new concrete_filter(mode,body); - p.add_filter( *f ); - } -public: - filter_node_leaf( tbb::filter::mode m, const Body& b ) : mode(m), body(b) {} -}; - -//! Node in parse tree representing join of two filters. -class filter_node_join: public filter_node { - friend class filter_node; // to suppress GCC 3.2 warnings - filter_node& left; - filter_node& right; - ~filter_node_join() { - left.remove_ref(); - right.remove_ref(); - } - void add_to( pipeline& p ) __TBB_override { - left.add_to(p); - right.add_to(p); - } -public: - filter_node_join( filter_node& x, filter_node& y ) : left(x), right(y) { - left.add_ref(); - right.add_ref(); - } -}; - -} // namespace internal -//! @endcond - -//! Create a filter to participate in parallel_pipeline -template -filter_t make_filter(tbb::filter::mode mode, const Body& body) { - return new internal::filter_node_leaf(mode, body); -} - -template -filter_t operator& (const filter_t& left, const filter_t& right) { - __TBB_ASSERT(left.root,"cannot use default-constructed filter_t as left argument of '&'"); - __TBB_ASSERT(right.root,"cannot use default-constructed filter_t as right argument of '&'"); - return new internal::filter_node_join(*left.root,*right.root); -} - -//! Class representing a chain of type-safe pipeline filters -template -class filter_t { - typedef internal::filter_node filter_node; - filter_node* root; - filter_t( filter_node* root_ ) : root(root_) { - root->add_ref(); - } - friend class internal::pipeline_proxy; - template - friend filter_t make_filter(tbb::filter::mode, const Body& ); - template - friend filter_t operator& (const filter_t& , const filter_t& ); -public: - // TODO: add move-constructors, move-assignment, etc. where C++11 is available. - filter_t() : root(NULL) {} - filter_t( const filter_t& rhs ) : root(rhs.root) { - if( root ) root->add_ref(); - } - template - filter_t( tbb::filter::mode mode, const Body& body ) : - root( new internal::filter_node_leaf(mode, body) ) { - root->add_ref(); - } - - void operator=( const filter_t& rhs ) { - // Order of operations below carefully chosen so that reference counts remain correct - // in unlikely event that remove_ref throws exception. - filter_node* old = root; - root = rhs.root; - if( root ) root->add_ref(); - if( old ) old->remove_ref(); - } - ~filter_t() { - if( root ) root->remove_ref(); - } - void clear() { - // Like operator= with filter_t() on right side. - if( root ) { - filter_node* old = root; - root = NULL; - old->remove_ref(); - } - } -}; - -inline internal::pipeline_proxy::pipeline_proxy( const filter_t& filter_chain ) : my_pipe() { - __TBB_ASSERT( filter_chain.root, "cannot apply parallel_pipeline to default-constructed filter_t" ); - filter_chain.root->add_to(my_pipe); -} - -inline void parallel_pipeline(size_t max_number_of_live_tokens, const filter_t& filter_chain -#if __TBB_TASK_GROUP_CONTEXT - , tbb::task_group_context& context -#endif - ) { - internal::pipeline_proxy pipe(filter_chain); - // tbb::pipeline::run() is called via the proxy - pipe->run(max_number_of_live_tokens -#if __TBB_TASK_GROUP_CONTEXT - , context -#endif - ); -} - -#if __TBB_TASK_GROUP_CONTEXT -inline void parallel_pipeline(size_t max_number_of_live_tokens, const filter_t& filter_chain) { - tbb::task_group_context context; - parallel_pipeline(max_number_of_live_tokens, filter_chain, context); -} -#endif // __TBB_TASK_GROUP_CONTEXT - -} // interface6 - -using interface6::flow_control; -using interface6::filter_t; -using interface6::make_filter; -using interface6::parallel_pipeline; - -} // tbb - -#endif /* __TBB_pipeline_H */ diff --git a/inst/include/tbb/queuing_mutex.h b/inst/include/tbb/queuing_mutex.h deleted file mode 100644 index 0fe4b3ea1..000000000 --- a/inst/include/tbb/queuing_mutex.h +++ /dev/null @@ -1,123 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_queuing_mutex_H -#define __TBB_queuing_mutex_H - -#include "tbb_config.h" - -#if !TBB_USE_EXCEPTIONS && _MSC_VER - // Suppress "C++ exception handler used, but unwind semantics are not enabled" warning in STL headers - #pragma warning (push) - #pragma warning (disable: 4530) -#endif - -#include - -#if !TBB_USE_EXCEPTIONS && _MSC_VER - #pragma warning (pop) -#endif - -#include "atomic.h" -#include "tbb_profiling.h" - -namespace tbb { - -//! Queuing mutex with local-only spinning. -/** @ingroup synchronization */ -class queuing_mutex : internal::mutex_copy_deprecated_and_disabled { -public: - //! Construct unacquired mutex. - queuing_mutex() { - q_tail = NULL; -#if TBB_USE_THREADING_TOOLS - internal_construct(); -#endif - } - - //! The scoped locking pattern - /** It helps to avoid the common problem of forgetting to release lock. - It also nicely provides the "node" for queuing locks. */ - class scoped_lock: internal::no_copy { - //! Initialize fields to mean "no lock held". - void initialize() { - mutex = NULL; -#if TBB_USE_ASSERT - internal::poison_pointer(next); -#endif /* TBB_USE_ASSERT */ - } - - public: - //! Construct lock that has not acquired a mutex. - /** Equivalent to zero-initialization of *this. */ - scoped_lock() {initialize();} - - //! Acquire lock on given mutex. - scoped_lock( queuing_mutex& m ) { - initialize(); - acquire(m); - } - - //! Release lock (if lock is held). - ~scoped_lock() { - if( mutex ) release(); - } - - //! Acquire lock on given mutex. - void __TBB_EXPORTED_METHOD acquire( queuing_mutex& m ); - - //! Acquire lock on given mutex if free (i.e. non-blocking) - bool __TBB_EXPORTED_METHOD try_acquire( queuing_mutex& m ); - - //! Release lock. - void __TBB_EXPORTED_METHOD release(); - - private: - //! The pointer to the mutex owned, or NULL if not holding a mutex. - queuing_mutex* mutex; - - //! The pointer to the next competitor for a mutex - scoped_lock *next; - - //! The local spin-wait variable - /** Inverted (0 - blocked, 1 - acquired the mutex) for the sake of - zero-initialization. Defining it as an entire word instead of - a byte seems to help performance slightly. */ - uintptr_t going; - }; - - void __TBB_EXPORTED_METHOD internal_construct(); - - // Mutex traits - static const bool is_rw_mutex = false; - static const bool is_recursive_mutex = false; - static const bool is_fair_mutex = true; - -private: - //! The last competitor requesting the lock - atomic q_tail; - -}; - -__TBB_DEFINE_PROFILING_SET_NAME(queuing_mutex) - -} // namespace tbb - -#endif /* __TBB_queuing_mutex_H */ diff --git a/inst/include/tbb/queuing_rw_mutex.h b/inst/include/tbb/queuing_rw_mutex.h deleted file mode 100644 index e0224ed58..000000000 --- a/inst/include/tbb/queuing_rw_mutex.h +++ /dev/null @@ -1,163 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_queuing_rw_mutex_H -#define __TBB_queuing_rw_mutex_H - -#include "tbb_config.h" - -#if !TBB_USE_EXCEPTIONS && _MSC_VER - // Suppress "C++ exception handler used, but unwind semantics are not enabled" warning in STL headers - #pragma warning (push) - #pragma warning (disable: 4530) -#endif - -#include - -#if !TBB_USE_EXCEPTIONS && _MSC_VER - #pragma warning (pop) -#endif - -#include "atomic.h" -#include "tbb_profiling.h" - -namespace tbb { - -//! Queuing reader-writer mutex with local-only spinning. -/** Adapted from Krieger, Stumm, et al. pseudocode at - http://www.eecg.toronto.edu/parallel/pubs_abs.html#Krieger_etal_ICPP93 - @ingroup synchronization */ -class queuing_rw_mutex : internal::mutex_copy_deprecated_and_disabled { -public: - //! Construct unacquired mutex. - queuing_rw_mutex() { - q_tail = NULL; -#if TBB_USE_THREADING_TOOLS - internal_construct(); -#endif - } - - //! Destructor asserts if the mutex is acquired, i.e. q_tail is non-NULL - ~queuing_rw_mutex() { -#if TBB_USE_ASSERT - __TBB_ASSERT( !q_tail, "destruction of an acquired mutex"); -#endif - } - - //! The scoped locking pattern - /** It helps to avoid the common problem of forgetting to release lock. - It also nicely provides the "node" for queuing locks. */ - class scoped_lock: internal::no_copy { - //! Initialize fields to mean "no lock held". - void initialize() { - my_mutex = NULL; -#if TBB_USE_ASSERT - my_state = 0xFF; // Set to invalid state - internal::poison_pointer(my_next); - internal::poison_pointer(my_prev); -#endif /* TBB_USE_ASSERT */ - } - - public: - //! Construct lock that has not acquired a mutex. - /** Equivalent to zero-initialization of *this. */ - scoped_lock() {initialize();} - - //! Acquire lock on given mutex. - scoped_lock( queuing_rw_mutex& m, bool write=true ) { - initialize(); - acquire(m,write); - } - - //! Release lock (if lock is held). - ~scoped_lock() { - if( my_mutex ) release(); - } - - //! Acquire lock on given mutex. - void acquire( queuing_rw_mutex& m, bool write=true ); - - //! Acquire lock on given mutex if free (i.e. non-blocking) - bool try_acquire( queuing_rw_mutex& m, bool write=true ); - - //! Release lock. - void release(); - - //! Upgrade reader to become a writer. - /** Returns whether the upgrade happened without releasing and re-acquiring the lock */ - bool upgrade_to_writer(); - - //! Downgrade writer to become a reader. - bool downgrade_to_reader(); - - private: - //! The pointer to the mutex owned, or NULL if not holding a mutex. - queuing_rw_mutex* my_mutex; - - //! The pointer to the previous and next competitors for a mutex - scoped_lock *__TBB_atomic my_prev, *__TBB_atomic my_next; - - typedef unsigned char state_t; - - //! State of the request: reader, writer, active reader, other service states - atomic my_state; - - //! The local spin-wait variable - /** Corresponds to "spin" in the pseudocode but inverted for the sake of zero-initialization */ - unsigned char __TBB_atomic my_going; - - //! A tiny internal lock - unsigned char my_internal_lock; - - //! Acquire the internal lock - void acquire_internal_lock(); - - //! Try to acquire the internal lock - /** Returns true if lock was successfully acquired. */ - bool try_acquire_internal_lock(); - - //! Release the internal lock - void release_internal_lock(); - - //! Wait for internal lock to be released - void wait_for_release_of_internal_lock(); - - //! A helper function - void unblock_or_wait_on_internal_lock( uintptr_t ); - }; - - void __TBB_EXPORTED_METHOD internal_construct(); - - // Mutex traits - static const bool is_rw_mutex = true; - static const bool is_recursive_mutex = false; - static const bool is_fair_mutex = true; - -private: - //! The last competitor requesting the lock - atomic q_tail; - -}; - -__TBB_DEFINE_PROFILING_SET_NAME(queuing_rw_mutex) - -} // namespace tbb - -#endif /* __TBB_queuing_rw_mutex_H */ diff --git a/inst/include/tbb/reader_writer_lock.h b/inst/include/tbb/reader_writer_lock.h deleted file mode 100644 index 353beec5d..000000000 --- a/inst/include/tbb/reader_writer_lock.h +++ /dev/null @@ -1,232 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_reader_writer_lock_H -#define __TBB_reader_writer_lock_H - -#include "tbb_thread.h" -#include "tbb_allocator.h" -#include "atomic.h" - -namespace tbb { -namespace interface5 { -//! Writer-preference reader-writer lock with local-only spinning on readers. -/** Loosely adapted from Mellor-Crummey and Scott pseudocode at - http://www.cs.rochester.edu/research/synchronization/pseudocode/rw.html#s_wp - @ingroup synchronization */ - class reader_writer_lock : tbb::internal::no_copy { - public: - friend class scoped_lock; - friend class scoped_lock_read; - //! Status type for nodes associated with lock instances - /** waiting_nonblocking: the wait state for nonblocking lock - instances; for writes, these transition straight to active - states; for reads, these are unused. - - waiting: the start and spin state for all lock instances; these will - transition to active state when appropriate. Non-blocking write locks - transition from this state to waiting_nonblocking immediately. - - active: the active state means that the lock instance holds - the lock; it will transition to invalid state during node deletion - - invalid: the end state for all nodes; this is set in the - destructor so if we encounter this state, we are looking at - memory that has already been freed - - The state diagrams below describe the status transitions. - Single arrows indicate that the thread that owns the node is - responsible for the transition; double arrows indicate that - any thread could make the transition. - - State diagram for scoped_lock status: - - waiting ----------> waiting_nonblocking - | _____________/ | - V V V - active -----------------> invalid - - State diagram for scoped_lock_read status: - - waiting - | - V - active ----------------->invalid - - */ - enum status_t { waiting_nonblocking, waiting, active, invalid }; - - //! Constructs a new reader_writer_lock - reader_writer_lock() { - internal_construct(); - } - - //! Destructs a reader_writer_lock object - ~reader_writer_lock() { - internal_destroy(); - } - - //! The scoped lock pattern for write locks - /** Scoped locks help avoid the common problem of forgetting to release the lock. - This type also serves as the node for queuing locks. */ - class scoped_lock : tbb::internal::no_copy { - public: - friend class reader_writer_lock; - - //! Construct with blocking attempt to acquire write lock on the passed-in lock - scoped_lock(reader_writer_lock& lock) { - internal_construct(lock); - } - - //! Destructor, releases the write lock - ~scoped_lock() { - internal_destroy(); - } - - void* operator new(size_t s) { - return tbb::internal::allocate_via_handler_v3(s); - } - void operator delete(void* p) { - tbb::internal::deallocate_via_handler_v3(p); - } - - private: - //! The pointer to the mutex to lock - reader_writer_lock *mutex; - //! The next queued competitor for the mutex - scoped_lock* next; - //! Status flag of the thread associated with this node - atomic status; - - //! Construct scoped_lock that is not holding lock - scoped_lock(); - - void __TBB_EXPORTED_METHOD internal_construct(reader_writer_lock&); - void __TBB_EXPORTED_METHOD internal_destroy(); - }; - - //! The scoped lock pattern for read locks - class scoped_lock_read : tbb::internal::no_copy { - public: - friend class reader_writer_lock; - - //! Construct with blocking attempt to acquire read lock on the passed-in lock - scoped_lock_read(reader_writer_lock& lock) { - internal_construct(lock); - } - - //! Destructor, releases the read lock - ~scoped_lock_read() { - internal_destroy(); - } - - void* operator new(size_t s) { - return tbb::internal::allocate_via_handler_v3(s); - } - void operator delete(void* p) { - tbb::internal::deallocate_via_handler_v3(p); - } - - private: - //! The pointer to the mutex to lock - reader_writer_lock *mutex; - //! The next queued competitor for the mutex - scoped_lock_read *next; - //! Status flag of the thread associated with this node - atomic status; - - //! Construct scoped_lock_read that is not holding lock - scoped_lock_read(); - - void __TBB_EXPORTED_METHOD internal_construct(reader_writer_lock&); - void __TBB_EXPORTED_METHOD internal_destroy(); - }; - - //! Acquires the reader_writer_lock for write. - /** If the lock is currently held in write mode by another - context, the writer will block by spinning on a local - variable. Exceptions thrown: improper_lock The context tries - to acquire a reader_writer_lock that it already has write - ownership of.*/ - void __TBB_EXPORTED_METHOD lock(); - - //! Tries to acquire the reader_writer_lock for write. - /** This function does not block. Return Value: True or false, - depending on whether the lock is acquired or not. If the lock - is already held by this acquiring context, try_lock() returns - false. */ - bool __TBB_EXPORTED_METHOD try_lock(); - - //! Acquires the reader_writer_lock for read. - /** If the lock is currently held by a writer, this reader will - block and wait until the writers are done. Exceptions thrown: - improper_lock The context tries to acquire a - reader_writer_lock that it already has write ownership of. */ - void __TBB_EXPORTED_METHOD lock_read(); - - //! Tries to acquire the reader_writer_lock for read. - /** This function does not block. Return Value: True or false, - depending on whether the lock is acquired or not. */ - bool __TBB_EXPORTED_METHOD try_lock_read(); - - //! Releases the reader_writer_lock - void __TBB_EXPORTED_METHOD unlock(); - - private: - void __TBB_EXPORTED_METHOD internal_construct(); - void __TBB_EXPORTED_METHOD internal_destroy(); - - //! Attempts to acquire write lock - /** If unavailable, spins in blocking case, returns false in non-blocking case. */ - bool start_write(scoped_lock *); - //! Sets writer_head to w and attempts to unblock - void set_next_writer(scoped_lock *w); - //! Relinquishes write lock to next waiting writer or group of readers - void end_write(scoped_lock *); - //! Checks if current thread holds write lock - bool is_current_writer(); - - //! Attempts to acquire read lock - /** If unavailable, spins in blocking case, returns false in non-blocking case. */ - void start_read(scoped_lock_read *); - //! Unblocks pending readers - void unblock_readers(); - //! Relinquishes read lock by decrementing counter; last reader wakes pending writer - void end_read(); - - //! The list of pending readers - atomic reader_head; - //! The list of pending writers - atomic writer_head; - //! The last node in the list of pending writers - atomic writer_tail; - //! Writer that owns the mutex; tbb_thread::id() otherwise. - tbb_thread::id my_current_writer; - //! Status of mutex - atomic rdr_count_and_flags; // used with __TBB_AtomicOR, which assumes uintptr_t -}; - -} // namespace interface5 - -using interface5::reader_writer_lock; - -} // namespace tbb - -#endif /* __TBB_reader_writer_lock_H */ diff --git a/inst/include/tbb/recursive_mutex.h b/inst/include/tbb/recursive_mutex.h deleted file mode 100644 index 5a23c097c..000000000 --- a/inst/include/tbb/recursive_mutex.h +++ /dev/null @@ -1,234 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_recursive_mutex_H -#define __TBB_recursive_mutex_H - -#if _WIN32||_WIN64 -#include "machine/windows_api.h" -#else -#include -#endif /* _WIN32||_WIN64 */ - -#include -#include "aligned_space.h" -#include "tbb_stddef.h" -#include "tbb_profiling.h" - -namespace tbb { -//! Mutex that allows recursive mutex acquisition. -/** Mutex that allows recursive mutex acquisition. - @ingroup synchronization */ -class recursive_mutex : internal::mutex_copy_deprecated_and_disabled { -public: - //! Construct unacquired recursive_mutex. - recursive_mutex() { -#if TBB_USE_ASSERT || TBB_USE_THREADING_TOOLS - internal_construct(); -#else - #if _WIN32||_WIN64 - InitializeCriticalSectionEx(&impl, 4000, 0); - #else - pthread_mutexattr_t mtx_attr; - int error_code = pthread_mutexattr_init( &mtx_attr ); - if( error_code ) - tbb::internal::handle_perror(error_code,"recursive_mutex: pthread_mutexattr_init failed"); - - pthread_mutexattr_settype( &mtx_attr, PTHREAD_MUTEX_RECURSIVE ); - error_code = pthread_mutex_init( &impl, &mtx_attr ); - if( error_code ) - tbb::internal::handle_perror(error_code,"recursive_mutex: pthread_mutex_init failed"); - - pthread_mutexattr_destroy( &mtx_attr ); - #endif /* _WIN32||_WIN64*/ -#endif /* TBB_USE_ASSERT */ - }; - - ~recursive_mutex() { -#if TBB_USE_ASSERT - internal_destroy(); -#else - #if _WIN32||_WIN64 - DeleteCriticalSection(&impl); - #else - pthread_mutex_destroy(&impl); - - #endif /* _WIN32||_WIN64 */ -#endif /* TBB_USE_ASSERT */ - }; - - class scoped_lock; - friend class scoped_lock; - - //! The scoped locking pattern - /** It helps to avoid the common problem of forgetting to release lock. - It also nicely provides the "node" for queuing locks. */ - class scoped_lock: internal::no_copy { - public: - //! Construct lock that has not acquired a recursive_mutex. - scoped_lock() : my_mutex(NULL) {}; - - //! Acquire lock on given mutex. - scoped_lock( recursive_mutex& mutex ) { -#if TBB_USE_ASSERT - my_mutex = &mutex; -#endif /* TBB_USE_ASSERT */ - acquire( mutex ); - } - - //! Release lock (if lock is held). - ~scoped_lock() { - if( my_mutex ) - release(); - } - - //! Acquire lock on given mutex. - void acquire( recursive_mutex& mutex ) { -#if TBB_USE_ASSERT - internal_acquire( mutex ); -#else - my_mutex = &mutex; - mutex.lock(); -#endif /* TBB_USE_ASSERT */ - } - - //! Try acquire lock on given recursive_mutex. - bool try_acquire( recursive_mutex& mutex ) { -#if TBB_USE_ASSERT - return internal_try_acquire( mutex ); -#else - bool result = mutex.try_lock(); - if( result ) - my_mutex = &mutex; - return result; -#endif /* TBB_USE_ASSERT */ - } - - //! Release lock - void release() { -#if TBB_USE_ASSERT - internal_release(); -#else - my_mutex->unlock(); - my_mutex = NULL; -#endif /* TBB_USE_ASSERT */ - } - - private: - //! The pointer to the current recursive_mutex to work - recursive_mutex* my_mutex; - - //! All checks from acquire using mutex.state were moved here - void __TBB_EXPORTED_METHOD internal_acquire( recursive_mutex& m ); - - //! All checks from try_acquire using mutex.state were moved here - bool __TBB_EXPORTED_METHOD internal_try_acquire( recursive_mutex& m ); - - //! All checks from release using mutex.state were moved here - void __TBB_EXPORTED_METHOD internal_release(); - - friend class recursive_mutex; - }; - - // Mutex traits - static const bool is_rw_mutex = false; - static const bool is_recursive_mutex = true; - static const bool is_fair_mutex = false; - - // C++0x compatibility interface - - //! Acquire lock - void lock() { -#if TBB_USE_ASSERT - aligned_space tmp; - new(tmp.begin()) scoped_lock(*this); -#else - #if _WIN32||_WIN64 - EnterCriticalSection(&impl); - #else - int error_code = pthread_mutex_lock(&impl); - if( error_code ) - tbb::internal::handle_perror(error_code,"recursive_mutex: pthread_mutex_lock failed"); - #endif /* _WIN32||_WIN64 */ -#endif /* TBB_USE_ASSERT */ - } - - //! Try acquiring lock (non-blocking) - /** Return true if lock acquired; false otherwise. */ - bool try_lock() { -#if TBB_USE_ASSERT - aligned_space tmp; - return (new(tmp.begin()) scoped_lock)->internal_try_acquire(*this); -#else - #if _WIN32||_WIN64 - return TryEnterCriticalSection(&impl)!=0; - #else - return pthread_mutex_trylock(&impl)==0; - #endif /* _WIN32||_WIN64 */ -#endif /* TBB_USE_ASSERT */ - } - - //! Release lock - void unlock() { -#if TBB_USE_ASSERT - aligned_space tmp; - scoped_lock& s = *tmp.begin(); - s.my_mutex = this; - s.internal_release(); -#else - #if _WIN32||_WIN64 - LeaveCriticalSection(&impl); - #else - pthread_mutex_unlock(&impl); - #endif /* _WIN32||_WIN64 */ -#endif /* TBB_USE_ASSERT */ - } - - //! Return native_handle - #if _WIN32||_WIN64 - typedef LPCRITICAL_SECTION native_handle_type; - #else - typedef pthread_mutex_t* native_handle_type; - #endif - native_handle_type native_handle() { return (native_handle_type) &impl; } - -private: -#if _WIN32||_WIN64 - CRITICAL_SECTION impl; - enum state_t { - INITIALIZED=0x1234, - DESTROYED=0x789A, - } state; -#else - pthread_mutex_t impl; -#endif /* _WIN32||_WIN64 */ - - //! All checks from mutex constructor using mutex.state were moved here - void __TBB_EXPORTED_METHOD internal_construct(); - - //! All checks from mutex destructor using mutex.state were moved here - void __TBB_EXPORTED_METHOD internal_destroy(); -}; - -__TBB_DEFINE_PROFILING_SET_NAME(recursive_mutex) - -} // namespace tbb - -#endif /* __TBB_recursive_mutex_H */ diff --git a/inst/include/tbb/runtime_loader.h b/inst/include/tbb/runtime_loader.h deleted file mode 100644 index df2846454..000000000 --- a/inst/include/tbb/runtime_loader.h +++ /dev/null @@ -1,180 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_runtime_loader_H -#define __TBB_runtime_loader_H - -#if ! TBB_PREVIEW_RUNTIME_LOADER - #error Set TBB_PREVIEW_RUNTIME_LOADER to include runtime_loader.h -#endif - -#include "tbb_stddef.h" -#include - -#if _MSC_VER - #if ! __TBB_NO_IMPLICIT_LINKAGE - #ifdef _DEBUG - #pragma comment( linker, "/nodefaultlib:tbb_debug.lib" ) - #pragma comment( linker, "/defaultlib:tbbproxy_debug.lib" ) - #else - #pragma comment( linker, "/nodefaultlib:tbb.lib" ) - #pragma comment( linker, "/defaultlib:tbbproxy.lib" ) - #endif - #endif -#endif - -namespace tbb { - -namespace interface6 { - -//! Load TBB at runtime. -/*! - -\b Usage: - -In source code: - -\code -#include "tbb/runtime_loader.h" - -char const * path[] = { "/lib/ia32", NULL }; -tbb::runtime_loader loader( path ); - -// Now use TBB. -\endcode - -Link with \c tbbproxy.lib (or \c libtbbproxy.a) instead of \c tbb.lib (\c libtbb.dylib, -\c libtbb.so). - -TBB library will be loaded at runtime from \c /lib/ia32 directory. - -\b Attention: - -All \c runtime_loader objects (in the same module, i.e. exe or dll) share some global state. -The most noticeable piece of global state is loaded TBB library. -There are some implications: - - - Only one TBB library can be loaded per module. - - - If one object has already loaded TBB library, another object will not load TBB. - If the loaded TBB library is suitable for the second object, both will use TBB - cooperatively, otherwise the second object will report an error. - - - \c runtime_loader objects will not work (correctly) in parallel due to absence of - synchronization. - -*/ - -class runtime_loader : tbb::internal::no_copy { - - public: - - //! Error mode constants. - enum error_mode { - em_status, //!< Save status of operation and continue. - em_throw, //!< Throw an exception of tbb::runtime_loader::error_code type. - em_abort //!< Print message to \c stderr and call \c abort(). - }; // error_mode - - //! Error codes. - enum error_code { - ec_ok, //!< No errors. - ec_bad_call, //!< Invalid function call (e. g. load() called when TBB is already loaded). - ec_bad_arg, //!< Invalid argument passed. - ec_bad_lib, //!< Invalid library found (e. g. \c TBB_runtime_version symbol not found). - ec_bad_ver, //!< TBB found but version is not suitable. - ec_no_lib //!< No suitable TBB library found. - }; // error_code - - //! Initialize object but do not load TBB. - runtime_loader( error_mode mode = em_abort ); - - //! Initialize object and load TBB. - /*! - See load() for details. - - If error mode is \c em_status, call status() to check whether TBB was loaded or not. - */ - runtime_loader( - char const * path[], //!< List of directories to search TBB in. - int min_ver = TBB_INTERFACE_VERSION, //!< Minimal suitable version of TBB. - int max_ver = INT_MAX, //!< Maximal suitable version of TBB. - error_mode mode = em_abort //!< Error mode for this object. - ); - - //! Destroy object. - ~runtime_loader(); - - //! Load TBB. - /*! - The method searches the directories specified in \c path[] array for the TBB library. - When the library is found, it is loaded and its version is checked. If the version is - not suitable, the library is unloaded, and the search continues. - - \b Note: - - For security reasons, avoid using relative directory names. For example, never load - TBB from current (\c "."), parent (\c "..") or any other relative directory (like - \c "lib" ). Use only absolute directory names (e. g. "/usr/local/lib"). - - For the same security reasons, avoid using system default directories (\c "") on - Windows. (See http://www.microsoft.com/technet/security/advisory/2269637.mspx for - details.) - - Neglecting these rules may cause your program to execute 3-rd party malicious code. - - \b Errors: - - \c ec_bad_call - TBB already loaded by this object. - - \c ec_bad_arg - \p min_ver and/or \p max_ver negative or zero, - or \p min_ver > \p max_ver. - - \c ec_bad_ver - TBB of unsuitable version already loaded by another object. - - \c ec_no_lib - No suitable library found. - */ - error_code - load( - char const * path[], //!< List of directories to search TBB in. - int min_ver = TBB_INTERFACE_VERSION, //!< Minimal suitable version of TBB. - int max_ver = INT_MAX //!< Maximal suitable version of TBB. - - ); - - - //! Report status. - /*! - If error mode is \c em_status, the function returns status of the last operation. - */ - error_code status(); - - private: - - error_mode const my_mode; - error_code my_status; - bool my_loaded; - -}; // class runtime_loader - -} // namespace interface6 - -using interface6::runtime_loader; - -} // namespace tbb - -#endif /* __TBB_runtime_loader_H */ - diff --git a/inst/include/tbb/scalable_allocator.h b/inst/include/tbb/scalable_allocator.h deleted file mode 100644 index c2a81493f..000000000 --- a/inst/include/tbb/scalable_allocator.h +++ /dev/null @@ -1,341 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_scalable_allocator_H -#define __TBB_scalable_allocator_H -/** @file */ - -#include /* Need ptrdiff_t and size_t from here. */ -#if !_MSC_VER -#include /* Need intptr_t from here. */ -#endif - -#if !defined(__cplusplus) && __ICC==1100 - #pragma warning (push) - #pragma warning (disable: 991) -#endif - -#ifdef __cplusplus -extern "C" { -#endif /* __cplusplus */ - -#if _MSC_VER >= 1400 -#define __TBB_EXPORTED_FUNC __cdecl -#else -#define __TBB_EXPORTED_FUNC -#endif - -/** The "malloc" analogue to allocate block of memory of size bytes. - * @ingroup memory_allocation */ -void * __TBB_EXPORTED_FUNC scalable_malloc (size_t size); - -/** The "free" analogue to discard a previously allocated piece of memory. - @ingroup memory_allocation */ -void __TBB_EXPORTED_FUNC scalable_free (void* ptr); - -/** The "realloc" analogue complementing scalable_malloc. - @ingroup memory_allocation */ -void * __TBB_EXPORTED_FUNC scalable_realloc (void* ptr, size_t size); - -/** The "calloc" analogue complementing scalable_malloc. - @ingroup memory_allocation */ -void * __TBB_EXPORTED_FUNC scalable_calloc (size_t nobj, size_t size); - -/** The "posix_memalign" analogue. - @ingroup memory_allocation */ -int __TBB_EXPORTED_FUNC scalable_posix_memalign (void** memptr, size_t alignment, size_t size); - -/** The "_aligned_malloc" analogue. - @ingroup memory_allocation */ -void * __TBB_EXPORTED_FUNC scalable_aligned_malloc (size_t size, size_t alignment); - -/** The "_aligned_realloc" analogue. - @ingroup memory_allocation */ -void * __TBB_EXPORTED_FUNC scalable_aligned_realloc (void* ptr, size_t size, size_t alignment); - -/** The "_aligned_free" analogue. - @ingroup memory_allocation */ -void __TBB_EXPORTED_FUNC scalable_aligned_free (void* ptr); - -/** The analogue of _msize/malloc_size/malloc_usable_size. - Returns the usable size of a memory block previously allocated by scalable_*, - or 0 (zero) if ptr does not point to such a block. - @ingroup memory_allocation */ -size_t __TBB_EXPORTED_FUNC scalable_msize (void* ptr); - -/* Results for scalable_allocation_* functions */ -typedef enum { - TBBMALLOC_OK, - TBBMALLOC_INVALID_PARAM, - TBBMALLOC_UNSUPPORTED, - TBBMALLOC_NO_MEMORY, - TBBMALLOC_NO_EFFECT -} ScalableAllocationResult; - -/* Setting TBB_MALLOC_USE_HUGE_PAGES environment variable to 1 enables huge pages. - scalable_allocation_mode call has priority over environment variable. */ -typedef enum { - TBBMALLOC_USE_HUGE_PAGES, /* value turns using huge pages on and off */ - /* deprecated, kept for backward compatibility only */ - USE_HUGE_PAGES = TBBMALLOC_USE_HUGE_PAGES, - /* try to limit memory consumption value Bytes, clean internal buffers - if limit is exceeded, but not prevents from requesting memory from OS */ - TBBMALLOC_SET_SOFT_HEAP_LIMIT -} AllocationModeParam; - -/** Set TBB allocator-specific allocation modes. - @ingroup memory_allocation */ -int __TBB_EXPORTED_FUNC scalable_allocation_mode(int param, intptr_t value); - -typedef enum { - /* Clean internal allocator buffers for all threads. - Returns TBBMALLOC_NO_EFFECT if no buffers cleaned, - TBBMALLOC_OK if some memory released from buffers. */ - TBBMALLOC_CLEAN_ALL_BUFFERS, - /* Clean internal allocator buffer for current thread only. - Return values same as for TBBMALLOC_CLEAN_ALL_BUFFERS. */ - TBBMALLOC_CLEAN_THREAD_BUFFERS -} ScalableAllocationCmd; - -/** Call TBB allocator-specific commands. - @ingroup memory_allocation */ -int __TBB_EXPORTED_FUNC scalable_allocation_command(int cmd, void *param); - -#ifdef __cplusplus -} /* extern "C" */ -#endif /* __cplusplus */ - -#ifdef __cplusplus - -//! The namespace rml contains components of low-level memory pool interface. -namespace rml { -class MemoryPool; - -typedef void *(*rawAllocType)(intptr_t pool_id, size_t &bytes); -// returns non-zero in case of error -typedef int (*rawFreeType)(intptr_t pool_id, void* raw_ptr, size_t raw_bytes); - -/* -MemPoolPolicy extension must be compatible with such structure fields layout - -struct MemPoolPolicy { - rawAllocType pAlloc; - rawFreeType pFree; - size_t granularity; // granularity of pAlloc allocations -}; -*/ - -struct MemPoolPolicy { - enum { - TBBMALLOC_POOL_VERSION = 1 - }; - - rawAllocType pAlloc; - rawFreeType pFree; - // granularity of pAlloc allocations. 0 means default used. - size_t granularity; - int version; - // all memory consumed at 1st pAlloc call and never returned, - // no more pAlloc calls after 1st - unsigned fixedPool : 1, - // memory consumed but returned only at pool termination - keepAllMemory : 1, - reserved : 30; - - MemPoolPolicy(rawAllocType pAlloc_, rawFreeType pFree_, - size_t granularity_ = 0, bool fixedPool_ = false, - bool keepAllMemory_ = false) : - pAlloc(pAlloc_), pFree(pFree_), granularity(granularity_), version(TBBMALLOC_POOL_VERSION), - fixedPool(fixedPool_), keepAllMemory(keepAllMemory_), - reserved(0) {} -}; - -// enums have same values as appropriate enums from ScalableAllocationResult -// TODO: use ScalableAllocationResult in pool_create directly -enum MemPoolError { - // pool created successfully - POOL_OK = TBBMALLOC_OK, - // invalid policy parameters found - INVALID_POLICY = TBBMALLOC_INVALID_PARAM, - // requested pool policy is not supported by allocator library - UNSUPPORTED_POLICY = TBBMALLOC_UNSUPPORTED, - // lack of memory during pool creation - NO_MEMORY = TBBMALLOC_NO_MEMORY, - // action takes no effect - NO_EFFECT = TBBMALLOC_NO_EFFECT -}; - -MemPoolError pool_create_v1(intptr_t pool_id, const MemPoolPolicy *policy, - rml::MemoryPool **pool); - -bool pool_destroy(MemoryPool* memPool); -void *pool_malloc(MemoryPool* memPool, size_t size); -void *pool_realloc(MemoryPool* memPool, void *object, size_t size); -void *pool_aligned_malloc(MemoryPool* mPool, size_t size, size_t alignment); -void *pool_aligned_realloc(MemoryPool* mPool, void *ptr, size_t size, size_t alignment); -bool pool_reset(MemoryPool* memPool); -bool pool_free(MemoryPool *memPool, void *object); -MemoryPool *pool_identify(void *object); -} - -#include /* To use new with the placement argument */ - -/* Ensure that including this header does not cause implicit linkage with TBB */ -#ifndef __TBB_NO_IMPLICIT_LINKAGE - #define __TBB_NO_IMPLICIT_LINKAGE 1 - #include "tbb_stddef.h" - #undef __TBB_NO_IMPLICIT_LINKAGE -#else - #include "tbb_stddef.h" -#endif - -#if __TBB_ALLOCATOR_CONSTRUCT_VARIADIC - #include // std::forward -#endif - -namespace tbb { - -#if _MSC_VER && !defined(__INTEL_COMPILER) - // Workaround for erroneous "unreferenced parameter" warning in method destroy. - #pragma warning (push) - #pragma warning (disable: 4100) -#endif - -//! @cond INTERNAL -namespace internal { - -#if TBB_USE_EXCEPTIONS -// forward declaration is for inlining prevention -template __TBB_NOINLINE( void throw_exception(const E &e) ); -#endif - -// keep throw in a separate function to prevent code bloat -template -void throw_exception(const E &e) { - __TBB_THROW(e); -} - -} // namespace internal -//! @endcond - -//! Meets "allocator" requirements of ISO C++ Standard, Section 20.1.5 -/** The members are ordered the same way they are in section 20.4.1 - of the ISO C++ standard. - @ingroup memory_allocation */ -template -class scalable_allocator { -public: - typedef typename internal::allocator_type::value_type value_type; - typedef value_type* pointer; - typedef const value_type* const_pointer; - typedef value_type& reference; - typedef const value_type& const_reference; - typedef size_t size_type; - typedef ptrdiff_t difference_type; - template struct rebind { - typedef scalable_allocator other; - }; - - scalable_allocator() throw() {} - scalable_allocator( const scalable_allocator& ) throw() {} - template scalable_allocator(const scalable_allocator&) throw() {} - - pointer address(reference x) const {return &x;} - const_pointer address(const_reference x) const {return &x;} - - //! Allocate space for n objects. - pointer allocate( size_type n, const void* /*hint*/ =0 ) { - pointer p = static_cast( scalable_malloc( n * sizeof(value_type) ) ); - if (!p) - internal::throw_exception(std::bad_alloc()); - return p; - } - - //! Free previously allocated block of memory - void deallocate( pointer p, size_type ) { - scalable_free( p ); - } - - //! Largest value for which method allocate might succeed. - size_type max_size() const throw() { - size_type absolutemax = static_cast(-1) / sizeof (value_type); - return (absolutemax > 0 ? absolutemax : 1); - } -#if __TBB_ALLOCATOR_CONSTRUCT_VARIADIC - template - void construct(U *p, Args&&... args) - { ::new((void *)p) U(std::forward(args)...); } -#else /* __TBB_ALLOCATOR_CONSTRUCT_VARIADIC */ -#if __TBB_CPP11_RVALUE_REF_PRESENT - void construct( pointer p, value_type&& value ) { ::new((void*)(p)) value_type( std::move( value ) ); } -#endif - void construct( pointer p, const value_type& value ) {::new((void*)(p)) value_type(value);} -#endif /* __TBB_ALLOCATOR_CONSTRUCT_VARIADIC */ - void destroy( pointer p ) {p->~value_type();} -}; - -#if _MSC_VER && !defined(__INTEL_COMPILER) - #pragma warning (pop) -#endif /* warning 4100 is back */ - -//! Analogous to std::allocator, as defined in ISO C++ Standard, Section 20.4.1 -/** @ingroup memory_allocation */ -template<> -class scalable_allocator { -public: - typedef void* pointer; - typedef const void* const_pointer; - typedef void value_type; - template struct rebind { - typedef scalable_allocator other; - }; -}; - -template -inline bool operator==( const scalable_allocator&, const scalable_allocator& ) {return true;} - -template -inline bool operator!=( const scalable_allocator&, const scalable_allocator& ) {return false;} - -} // namespace tbb - -#if _MSC_VER - #if (__TBB_BUILD || __TBBMALLOC_BUILD) && !defined(__TBBMALLOC_NO_IMPLICIT_LINKAGE) - #define __TBBMALLOC_NO_IMPLICIT_LINKAGE 1 - #endif - - #if !__TBBMALLOC_NO_IMPLICIT_LINKAGE - #ifdef _DEBUG - #pragma comment(lib, "tbbmalloc_debug.lib") - #else - #pragma comment(lib, "tbbmalloc.lib") - #endif - #endif - - -#endif - -#endif /* __cplusplus */ - -#if !defined(__cplusplus) && __ICC==1100 - #pragma warning (pop) -#endif /* ICC 11.0 warning 991 is back */ - -#endif /* __TBB_scalable_allocator_H */ diff --git a/inst/include/tbb/spin_mutex.h b/inst/include/tbb/spin_mutex.h deleted file mode 100644 index 99ef15c22..000000000 --- a/inst/include/tbb/spin_mutex.h +++ /dev/null @@ -1,212 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_spin_mutex_H -#define __TBB_spin_mutex_H - -#include -#include -#include "aligned_space.h" -#include "tbb_stddef.h" -#include "tbb_machine.h" -#include "tbb_profiling.h" -#include "internal/_mutex_padding.h" - -namespace tbb { - -//! A lock that occupies a single byte. -/** A spin_mutex is a spin mutex that fits in a single byte. - It should be used only for locking short critical sections - (typically less than 20 instructions) when fairness is not an issue. - If zero-initialized, the mutex is considered unheld. - @ingroup synchronization */ -class spin_mutex : internal::mutex_copy_deprecated_and_disabled { - //! 0 if lock is released, 1 if lock is acquired. - __TBB_atomic_flag flag; - -public: - //! Construct unacquired lock. - /** Equivalent to zero-initialization of *this. */ - spin_mutex() : flag(0) { -#if TBB_USE_THREADING_TOOLS - internal_construct(); -#endif - } - - //! Represents acquisition of a mutex. - class scoped_lock : internal::no_copy { - private: - //! Points to currently held mutex, or NULL if no lock is held. - spin_mutex* my_mutex; - - //! Value to store into spin_mutex::flag to unlock the mutex. - /** This variable is no longer used. Instead, 0 and 1 are used to - represent that the lock is free and acquired, respectively. - We keep the member variable here to ensure backward compatibility */ - __TBB_Flag my_unlock_value; - - //! Like acquire, but with ITT instrumentation. - void __TBB_EXPORTED_METHOD internal_acquire( spin_mutex& m ); - - //! Like try_acquire, but with ITT instrumentation. - bool __TBB_EXPORTED_METHOD internal_try_acquire( spin_mutex& m ); - - //! Like release, but with ITT instrumentation. - void __TBB_EXPORTED_METHOD internal_release(); - - friend class spin_mutex; - - public: - //! Construct without acquiring a mutex. - scoped_lock() : my_mutex(NULL), my_unlock_value(0) {} - - //! Construct and acquire lock on a mutex. - scoped_lock( spin_mutex& m ) : my_unlock_value(0) { - internal::suppress_unused_warning(my_unlock_value); -#if TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT - my_mutex=NULL; - internal_acquire(m); -#else - my_mutex=&m; - __TBB_LockByte(m.flag); -#endif /* TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT*/ - } - - //! Acquire lock. - void acquire( spin_mutex& m ) { -#if TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT - internal_acquire(m); -#else - my_mutex = &m; - __TBB_LockByte(m.flag); -#endif /* TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT*/ - } - - //! Try acquiring lock (non-blocking) - /** Return true if lock acquired; false otherwise. */ - bool try_acquire( spin_mutex& m ) { -#if TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT - return internal_try_acquire(m); -#else - bool result = __TBB_TryLockByte(m.flag); - if( result ) - my_mutex = &m; - return result; -#endif /* TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT*/ - } - - //! Release lock - void release() { -#if TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT - internal_release(); -#else - __TBB_UnlockByte(my_mutex->flag); - my_mutex = NULL; -#endif /* TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT */ - } - - //! Destroy lock. If holding a lock, releases the lock first. - ~scoped_lock() { - if( my_mutex ) { -#if TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT - internal_release(); -#else - __TBB_UnlockByte(my_mutex->flag); -#endif /* TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT */ - } - } - }; - - //! Internal constructor with ITT instrumentation. - void __TBB_EXPORTED_METHOD internal_construct(); - - // Mutex traits - static const bool is_rw_mutex = false; - static const bool is_recursive_mutex = false; - static const bool is_fair_mutex = false; - - // ISO C++0x compatibility methods - - //! Acquire lock - void lock() { -#if TBB_USE_THREADING_TOOLS - aligned_space tmp; - new(tmp.begin()) scoped_lock(*this); -#else - __TBB_LockByte(flag); -#endif /* TBB_USE_THREADING_TOOLS*/ - } - - //! Try acquiring lock (non-blocking) - /** Return true if lock acquired; false otherwise. */ - bool try_lock() { -#if TBB_USE_THREADING_TOOLS - aligned_space tmp; - return (new(tmp.begin()) scoped_lock)->internal_try_acquire(*this); -#else - return __TBB_TryLockByte(flag); -#endif /* TBB_USE_THREADING_TOOLS*/ - } - - //! Release lock - void unlock() { -#if TBB_USE_THREADING_TOOLS - aligned_space tmp; - scoped_lock& s = *tmp.begin(); - s.my_mutex = this; - s.internal_release(); -#else - __TBB_UnlockByte(flag); -#endif /* TBB_USE_THREADING_TOOLS */ - } - - friend class scoped_lock; -}; // end of spin_mutex - -__TBB_DEFINE_PROFILING_SET_NAME(spin_mutex) - -} // namespace tbb - -#if ( __TBB_x86_32 || __TBB_x86_64 ) -#include "internal/_x86_eliding_mutex_impl.h" -#endif - -namespace tbb { -//! A cross-platform spin mutex with speculative lock acquisition. -/** On platforms with proper HW support, this lock may speculatively execute - its critical sections, using HW mechanisms to detect real data races and - ensure atomicity of the critical sections. In particular, it uses - Intel(R) Transactional Synchronization Extensions (Intel(R) TSX). - Without such HW support, it behaves like a spin_mutex. - It should be used for locking short critical sections where the lock is - contended but the data it protects are not. If zero-initialized, the - mutex is considered unheld. - @ingroup synchronization */ - -#if ( __TBB_x86_32 || __TBB_x86_64 ) -typedef interface7::internal::padded_mutex speculative_spin_mutex; -#else -typedef interface7::internal::padded_mutex speculative_spin_mutex; -#endif -__TBB_DEFINE_PROFILING_SET_NAME(speculative_spin_mutex) - -} // namespace tbb - -#endif /* __TBB_spin_mutex_H */ diff --git a/inst/include/tbb/spin_rw_mutex.h b/inst/include/tbb/spin_rw_mutex.h deleted file mode 100644 index b20f4ebda..000000000 --- a/inst/include/tbb/spin_rw_mutex.h +++ /dev/null @@ -1,259 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_spin_rw_mutex_H -#define __TBB_spin_rw_mutex_H - -#include "tbb_stddef.h" -#include "tbb_machine.h" -#include "tbb_profiling.h" -#include "internal/_mutex_padding.h" - -namespace tbb { - -#if __TBB_TSX_AVAILABLE -namespace interface8 { namespace internal { - class x86_rtm_rw_mutex; -}} -#endif - -class spin_rw_mutex_v3; -typedef spin_rw_mutex_v3 spin_rw_mutex; - -//! Fast, unfair, spinning reader-writer lock with backoff and writer-preference -/** @ingroup synchronization */ -class spin_rw_mutex_v3 : internal::mutex_copy_deprecated_and_disabled { - //! @cond INTERNAL - - //! Internal acquire write lock. - bool __TBB_EXPORTED_METHOD internal_acquire_writer(); - - //! Out of line code for releasing a write lock. - /** This code has debug checking and instrumentation for Intel(R) Thread Checker and Intel(R) Thread Profiler. */ - void __TBB_EXPORTED_METHOD internal_release_writer(); - - //! Internal acquire read lock. - void __TBB_EXPORTED_METHOD internal_acquire_reader(); - - //! Internal upgrade reader to become a writer. - bool __TBB_EXPORTED_METHOD internal_upgrade(); - - //! Out of line code for downgrading a writer to a reader. - /** This code has debug checking and instrumentation for Intel(R) Thread Checker and Intel(R) Thread Profiler. */ - void __TBB_EXPORTED_METHOD internal_downgrade(); - - //! Internal release read lock. - void __TBB_EXPORTED_METHOD internal_release_reader(); - - //! Internal try_acquire write lock. - bool __TBB_EXPORTED_METHOD internal_try_acquire_writer(); - - //! Internal try_acquire read lock. - bool __TBB_EXPORTED_METHOD internal_try_acquire_reader(); - - //! @endcond -public: - //! Construct unacquired mutex. - spin_rw_mutex_v3() : state(0) { -#if TBB_USE_THREADING_TOOLS - internal_construct(); -#endif - } - -#if TBB_USE_ASSERT - //! Destructor asserts if the mutex is acquired, i.e. state is zero. - ~spin_rw_mutex_v3() { - __TBB_ASSERT( !state, "destruction of an acquired mutex"); - }; -#endif /* TBB_USE_ASSERT */ - - //! The scoped locking pattern - /** It helps to avoid the common problem of forgetting to release lock. - It also nicely provides the "node" for queuing locks. */ - class scoped_lock : internal::no_copy { -#if __TBB_TSX_AVAILABLE - friend class tbb::interface8::internal::x86_rtm_rw_mutex; - // helper methods for x86_rtm_rw_mutex - spin_rw_mutex *internal_get_mutex() const { return mutex; } - void internal_set_mutex(spin_rw_mutex* m) { mutex = m; } -#endif - public: - //! Construct lock that has not acquired a mutex. - /** Equivalent to zero-initialization of *this. */ - scoped_lock() : mutex(NULL), is_writer(false) {} - - //! Acquire lock on given mutex. - scoped_lock( spin_rw_mutex& m, bool write = true ) : mutex(NULL) { - acquire(m, write); - } - - //! Release lock (if lock is held). - ~scoped_lock() { - if( mutex ) release(); - } - - //! Acquire lock on given mutex. - void acquire( spin_rw_mutex& m, bool write = true ) { - __TBB_ASSERT( !mutex, "holding mutex already" ); - is_writer = write; - mutex = &m; - if( write ) mutex->internal_acquire_writer(); - else mutex->internal_acquire_reader(); - } - - //! Upgrade reader to become a writer. - /** Returns whether the upgrade happened without releasing and re-acquiring the lock */ - bool upgrade_to_writer() { - __TBB_ASSERT( mutex, "lock is not acquired" ); - __TBB_ASSERT( !is_writer, "not a reader" ); - is_writer = true; - return mutex->internal_upgrade(); - } - - //! Release lock. - void release() { - __TBB_ASSERT( mutex, "lock is not acquired" ); - spin_rw_mutex *m = mutex; - mutex = NULL; -#if TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT - if( is_writer ) m->internal_release_writer(); - else m->internal_release_reader(); -#else - if( is_writer ) __TBB_AtomicAND( &m->state, READERS ); - else __TBB_FetchAndAddWrelease( &m->state, -(intptr_t)ONE_READER); -#endif /* TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT */ - } - - //! Downgrade writer to become a reader. - bool downgrade_to_reader() { - __TBB_ASSERT( mutex, "lock is not acquired" ); - __TBB_ASSERT( is_writer, "not a writer" ); -#if TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT - mutex->internal_downgrade(); -#else - __TBB_FetchAndAddW( &mutex->state, ((intptr_t)ONE_READER-WRITER)); -#endif /* TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT */ - is_writer = false; - return true; - } - - //! Try acquire lock on given mutex. - bool try_acquire( spin_rw_mutex& m, bool write = true ) { - __TBB_ASSERT( !mutex, "holding mutex already" ); - bool result; - is_writer = write; - result = write? m.internal_try_acquire_writer() - : m.internal_try_acquire_reader(); - if( result ) - mutex = &m; - return result; - } - - protected: - - //! The pointer to the current mutex that is held, or NULL if no mutex is held. - spin_rw_mutex* mutex; - - //! If mutex!=NULL, then is_writer is true if holding a writer lock, false if holding a reader lock. - /** Not defined if not holding a lock. */ - bool is_writer; - }; - - // Mutex traits - static const bool is_rw_mutex = true; - static const bool is_recursive_mutex = false; - static const bool is_fair_mutex = false; - - // ISO C++0x compatibility methods - - //! Acquire writer lock - void lock() {internal_acquire_writer();} - - //! Try acquiring writer lock (non-blocking) - /** Return true if lock acquired; false otherwise. */ - bool try_lock() {return internal_try_acquire_writer();} - - //! Release lock - void unlock() { -#if TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT - if( state&WRITER ) internal_release_writer(); - else internal_release_reader(); -#else - if( state&WRITER ) __TBB_AtomicAND( &state, READERS ); - else __TBB_FetchAndAddWrelease( &state, -(intptr_t)ONE_READER); -#endif /* TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT */ - } - - // Methods for reader locks that resemble ISO C++0x compatibility methods. - - //! Acquire reader lock - void lock_read() {internal_acquire_reader();} - - //! Try acquiring reader lock (non-blocking) - /** Return true if reader lock acquired; false otherwise. */ - bool try_lock_read() {return internal_try_acquire_reader();} - -protected: - typedef intptr_t state_t; - static const state_t WRITER = 1; - static const state_t WRITER_PENDING = 2; - static const state_t READERS = ~(WRITER | WRITER_PENDING); - static const state_t ONE_READER = 4; - static const state_t BUSY = WRITER | READERS; - //! State of lock - /** Bit 0 = writer is holding lock - Bit 1 = request by a writer to acquire lock (hint to readers to wait) - Bit 2..N = number of readers holding lock */ - state_t state; - -private: - void __TBB_EXPORTED_METHOD internal_construct(); -}; - -__TBB_DEFINE_PROFILING_SET_NAME(spin_rw_mutex) - -} // namespace tbb - -#if __TBB_TSX_AVAILABLE -#include "internal/_x86_rtm_rw_mutex_impl.h" -#endif - -namespace tbb { -namespace interface8 { -//! A cross-platform spin reader/writer mutex with speculative lock acquisition. -/** On platforms with proper HW support, this lock may speculatively execute - its critical sections, using HW mechanisms to detect real data races and - ensure atomicity of the critical sections. In particular, it uses - Intel(R) Transactional Synchronization Extensions (Intel(R) TSX). - Without such HW support, it behaves like a spin_rw_mutex. - It should be used for locking short critical sections where the lock is - contended but the data it protects are not. - @ingroup synchronization */ -#if __TBB_TSX_AVAILABLE -typedef interface7::internal::padded_mutex speculative_spin_rw_mutex; -#else -typedef interface7::internal::padded_mutex speculative_spin_rw_mutex; -#endif -} // namespace interface8 - -using interface8::speculative_spin_rw_mutex; -__TBB_DEFINE_PROFILING_SET_NAME(speculative_spin_rw_mutex) -} // namespace tbb -#endif /* __TBB_spin_rw_mutex_H */ diff --git a/inst/include/tbb/task.h b/inst/include/tbb/task.h deleted file mode 100644 index 246684ab6..000000000 --- a/inst/include/tbb/task.h +++ /dev/null @@ -1,1050 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_task_H -#define __TBB_task_H - -#include "tbb_stddef.h" -#include "tbb_machine.h" -#include "tbb_profiling.h" -#include - -typedef struct ___itt_caller *__itt_caller; - -namespace tbb { - -class task; -class task_list; -class task_group_context; - -// MSVC does not allow taking the address of a member that was defined -// privately in task_base and made public in class task via a using declaration. -#if _MSC_VER || (__GNUC__==3 && __GNUC_MINOR__<3) -#define __TBB_TASK_BASE_ACCESS public -#else -#define __TBB_TASK_BASE_ACCESS private -#endif - -namespace internal { //< @cond INTERNAL - - class allocate_additional_child_of_proxy: no_assign { - //! No longer used, but retained for binary layout compatibility. Always NULL. - task* self; - task& parent; - public: - explicit allocate_additional_child_of_proxy( task& parent_ ) : self(NULL), parent(parent_) {} - task& __TBB_EXPORTED_METHOD allocate( size_t size ) const; - void __TBB_EXPORTED_METHOD free( task& ) const; - }; - - struct cpu_ctl_env_space { int space[sizeof(internal::uint64_t)/sizeof(int)]; }; -} //< namespace internal @endcond - -namespace interface5 { - namespace internal { - //! Base class for methods that became static in TBB 3.0. - /** TBB's evolution caused the "this" argument for several methods to become obsolete. - However, for backwards binary compatibility, the new methods need distinct names, - otherwise the One Definition Rule would be broken. Hence the new methods are - defined in this private base class, and then exposed in class task via - using declarations. */ - class task_base: tbb::internal::no_copy { - __TBB_TASK_BASE_ACCESS: - friend class tbb::task; - - //! Schedule task for execution when a worker becomes available. - static void spawn( task& t ); - - //! Spawn multiple tasks and clear list. - static void spawn( task_list& list ); - - //! Like allocate_child, except that task's parent becomes "t", not this. - /** Typically used in conjunction with schedule_to_reexecute to implement while loops. - Atomically increments the reference count of t.parent() */ - static tbb::internal::allocate_additional_child_of_proxy allocate_additional_child_of( task& t ) { - return tbb::internal::allocate_additional_child_of_proxy(t); - } - - //! Destroy a task. - /** Usually, calling this method is unnecessary, because a task is - implicitly deleted after its execute() method runs. However, - sometimes a task needs to be explicitly deallocated, such as - when a root task is used as the parent in spawn_and_wait_for_all. */ - static void __TBB_EXPORTED_FUNC destroy( task& victim ); - }; - } // internal -} // interface5 - -//! @cond INTERNAL -namespace internal { - - class scheduler: no_copy { - public: - //! For internal use only - virtual void spawn( task& first, task*& next ) = 0; - - //! For internal use only - virtual void wait_for_all( task& parent, task* child ) = 0; - - //! For internal use only - virtual void spawn_root_and_wait( task& first, task*& next ) = 0; - - //! Pure virtual destructor; - // Have to have it just to shut up overzealous compilation warnings - virtual ~scheduler() = 0; - - //! For internal use only - virtual void enqueue( task& t, void* reserved ) = 0; - }; - - //! A reference count - /** Should always be non-negative. A signed type is used so that underflow can be detected. */ - typedef intptr_t reference_count; - - //! An id as used for specifying affinity. - typedef unsigned short affinity_id; - -#if __TBB_TASK_ISOLATION - //! A tag for task isolation. - typedef intptr_t isolation_tag; - const isolation_tag no_isolation = 0; -#endif /* __TBB_TASK_ISOLATION */ - -#if __TBB_TASK_GROUP_CONTEXT - class generic_scheduler; - - struct context_list_node_t { - context_list_node_t *my_prev, - *my_next; - }; - - class allocate_root_with_context_proxy: no_assign { - task_group_context& my_context; - public: - allocate_root_with_context_proxy ( task_group_context& ctx ) : my_context(ctx) {} - task& __TBB_EXPORTED_METHOD allocate( size_t size ) const; - void __TBB_EXPORTED_METHOD free( task& ) const; - }; -#endif /* __TBB_TASK_GROUP_CONTEXT */ - - class allocate_root_proxy: no_assign { - public: - static task& __TBB_EXPORTED_FUNC allocate( size_t size ); - static void __TBB_EXPORTED_FUNC free( task& ); - }; - - class allocate_continuation_proxy: no_assign { - public: - task& __TBB_EXPORTED_METHOD allocate( size_t size ) const; - void __TBB_EXPORTED_METHOD free( task& ) const; - }; - - class allocate_child_proxy: no_assign { - public: - task& __TBB_EXPORTED_METHOD allocate( size_t size ) const; - void __TBB_EXPORTED_METHOD free( task& ) const; - }; - - //! Memory prefix to a task object. - /** This class is internal to the library. - Do not reference it directly, except within the library itself. - Fields are ordered in way that preserves backwards compatibility and yields - good packing on typical 32-bit and 64-bit platforms. New fields should be - added at the beginning for backward compatibility with accesses to the task - prefix inlined into application code. - - In case task prefix size exceeds 32 or 64 bytes on IA32 and Intel64 - architectures correspondingly, consider dynamic setting of task_alignment - and task_prefix_reservation_size based on the maximal operand size supported - by the current CPU. - - @ingroup task_scheduling */ - class task_prefix { - private: - friend class tbb::task; - friend class tbb::interface5::internal::task_base; - friend class tbb::task_list; - friend class internal::scheduler; - friend class internal::allocate_root_proxy; - friend class internal::allocate_child_proxy; - friend class internal::allocate_continuation_proxy; - friend class internal::allocate_additional_child_of_proxy; - -#if __TBB_TASK_ISOLATION - //! The tag used for task isolation. - isolation_tag isolation; -#endif /* __TBB_TASK_ISOLATION */ - -#if __TBB_TASK_GROUP_CONTEXT - //! Shared context that is used to communicate asynchronous state changes - /** Currently it is used to broadcast cancellation requests generated both - by users and as the result of unhandled exceptions in the task::execute() - methods. */ - task_group_context *context; -#endif /* __TBB_TASK_GROUP_CONTEXT */ - - //! The scheduler that allocated the task, or NULL if the task is big. - /** Small tasks are pooled by the scheduler that allocated the task. - If a scheduler needs to free a small task allocated by another scheduler, - it returns the task to that other scheduler. This policy avoids - memory space blowup issues for memory allocators that allocate from - thread-specific pools. */ - scheduler* origin; - -#if __TBB_TASK_PRIORITY - union { -#endif /* __TBB_TASK_PRIORITY */ - //! Obsolete. The scheduler that owns the task. - /** Retained only for the sake of backward binary compatibility. - Still used by inline methods in the task.h header. **/ - scheduler* owner; - -#if __TBB_TASK_PRIORITY - //! Pointer to the next offloaded lower priority task. - /** Used to maintain a list of offloaded tasks inside the scheduler. **/ - task* next_offloaded; - }; -#endif /* __TBB_TASK_PRIORITY */ - - //! The task whose reference count includes me. - /** In the "blocking style" of programming, this field points to the parent task. - In the "continuation-passing style" of programming, this field points to the - continuation of the parent. */ - tbb::task* parent; - - //! Reference count used for synchronization. - /** In the "continuation-passing style" of programming, this field is - the difference of the number of allocated children minus the - number of children that have completed. - In the "blocking style" of programming, this field is one more than the difference. */ - __TBB_atomic reference_count ref_count; - - //! Obsolete. Used to be scheduling depth before TBB 2.2 - /** Retained only for the sake of backward binary compatibility. - Not used by TBB anymore. **/ - int depth; - - //! A task::state_type, stored as a byte for compactness. - /** This state is exposed to users via method task::state(). */ - unsigned char state; - - //! Miscellaneous state that is not directly visible to users, stored as a byte for compactness. - /** 0x0 -> version 1.0 task - 0x1 -> version >=2.1 task - 0x10 -> task was enqueued - 0x20 -> task_proxy - 0x40 -> task has live ref_count - 0x80 -> a stolen task */ - unsigned char extra_state; - - affinity_id affinity; - - //! "next" field for list of task - tbb::task* next; - - //! The task corresponding to this task_prefix. - tbb::task& task() {return *reinterpret_cast(this+1);} - }; - -} // namespace internal -//! @endcond - -#if __TBB_TASK_GROUP_CONTEXT - -#if __TBB_TASK_PRIORITY -namespace internal { - static const int priority_stride_v4 = INT_MAX / 4; -} - -enum priority_t { - priority_normal = internal::priority_stride_v4 * 2, - priority_low = priority_normal - internal::priority_stride_v4, - priority_high = priority_normal + internal::priority_stride_v4 -}; - -#endif /* __TBB_TASK_PRIORITY */ - -#if TBB_USE_CAPTURED_EXCEPTION - class tbb_exception; -#else - namespace internal { - class tbb_exception_ptr; - } -#endif /* !TBB_USE_CAPTURED_EXCEPTION */ - -class task_scheduler_init; -namespace interface7 { class task_arena; } - -//! Used to form groups of tasks -/** @ingroup task_scheduling - The context services explicit cancellation requests from user code, and unhandled - exceptions intercepted during tasks execution. Intercepting an exception results - in generating internal cancellation requests (which is processed in exactly the - same way as external ones). - - The context is associated with one or more root tasks and defines the cancellation - group that includes all the descendants of the corresponding root task(s). Association - is established when a context object is passed as an argument to the task::allocate_root() - method. See task_group_context::task_group_context for more details. - - The context can be bound to another one, and other contexts can be bound to it, - forming a tree-like structure: parent -> this -> children. Arrows here designate - cancellation propagation direction. If a task in a cancellation group is cancelled - all the other tasks in this group and groups bound to it (as children) get cancelled too. - - IMPLEMENTATION NOTE: - When adding new members to task_group_context or changing types of existing ones, - update the size of both padding buffers (_leading_padding and _trailing_padding) - appropriately. See also VERSIONING NOTE at the constructor definition below. **/ -class task_group_context : internal::no_copy { -private: - friend class internal::generic_scheduler; - friend class task_scheduler_init; - friend class interface7::task_arena; - -#if TBB_USE_CAPTURED_EXCEPTION - typedef tbb_exception exception_container_type; -#else - typedef internal::tbb_exception_ptr exception_container_type; -#endif - - enum version_traits_word_layout { - traits_offset = 16, - version_mask = 0xFFFF, - traits_mask = 0xFFFFul << traits_offset - }; - -public: - enum kind_type { - isolated, - bound - }; - - enum traits_type { - exact_exception = 0x0001ul << traits_offset, -#if __TBB_FP_CONTEXT - fp_settings = 0x0002ul << traits_offset, -#endif - concurrent_wait = 0x0004ul << traits_offset, -#if TBB_USE_CAPTURED_EXCEPTION - default_traits = 0 -#else - default_traits = exact_exception -#endif /* !TBB_USE_CAPTURED_EXCEPTION */ - }; - -private: - enum state { - may_have_children = 1, - // the following enumerations must be the last, new 2^x values must go above - next_state_value, low_unused_state_bit = (next_state_value-1)*2 - }; - - union { - //! Flavor of this context: bound or isolated. - // TODO: describe asynchronous use, and whether any memory semantics are needed - __TBB_atomic kind_type my_kind; - uintptr_t _my_kind_aligner; - }; - - //! Pointer to the context of the parent cancellation group. NULL for isolated contexts. - task_group_context *my_parent; - - //! Used to form the thread specific list of contexts without additional memory allocation. - /** A context is included into the list of the current thread when its binding to - its parent happens. Any context can be present in the list of one thread only. **/ - internal::context_list_node_t my_node; - - //! Used to set and maintain stack stitching point for Intel Performance Tools. - __itt_caller itt_caller; - - //! Leading padding protecting accesses to frequently used members from false sharing. - /** Read accesses to the field my_cancellation_requested are on the hot path inside - the scheduler. This padding ensures that this field never shares the same cache - line with a local variable that is frequently written to. **/ - char _leading_padding[internal::NFS_MaxLineSize - - 2 * sizeof(uintptr_t)- sizeof(void*) - sizeof(internal::context_list_node_t) - - sizeof(__itt_caller) -#if __TBB_FP_CONTEXT - - sizeof(internal::cpu_ctl_env_space) -#endif - ]; - -#if __TBB_FP_CONTEXT - //! Space for platform-specific FPU settings. - /** Must only be accessed inside TBB binaries, and never directly in user - code or inline methods. */ - internal::cpu_ctl_env_space my_cpu_ctl_env; -#endif - - //! Specifies whether cancellation was requested for this task group. - uintptr_t my_cancellation_requested; - - //! Version for run-time checks and behavioral traits of the context. - /** Version occupies low 16 bits, and traits (zero or more ORed enumerators - from the traits_type enumerations) take the next 16 bits. - Original (zeroth) version of the context did not support any traits. **/ - uintptr_t my_version_and_traits; - - //! Pointer to the container storing exception being propagated across this task group. - exception_container_type *my_exception; - - //! Scheduler instance that registered this context in its thread specific list. - internal::generic_scheduler *my_owner; - - //! Internal state (combination of state flags, currently only may_have_children). - uintptr_t my_state; - -#if __TBB_TASK_PRIORITY - //! Priority level of the task group (in normalized representation) - intptr_t my_priority; -#endif /* __TBB_TASK_PRIORITY */ - - //! Trailing padding protecting accesses to frequently used members from false sharing - /** \sa _leading_padding **/ - char _trailing_padding[internal::NFS_MaxLineSize - 2 * sizeof(uintptr_t) - 2 * sizeof(void*) -#if __TBB_TASK_PRIORITY - - sizeof(intptr_t) -#endif /* __TBB_TASK_PRIORITY */ - ]; - -public: - //! Default & binding constructor. - /** By default a bound context is created. That is this context will be bound - (as child) to the context of the task calling task::allocate_root(this_context) - method. Cancellation requests passed to the parent context are propagated - to all the contexts bound to it. Similarly priority change is propagated - from the parent context to its children. - - If task_group_context::isolated is used as the argument, then the tasks associated - with this context will never be affected by events in any other context. - - Creating isolated contexts involve much less overhead, but they have limited - utility. Normally when an exception occurs in an algorithm that has nested - ones running, it is desirably to have all the nested algorithms cancelled - as well. Such a behavior requires nested algorithms to use bound contexts. - - There is one good place where using isolated algorithms is beneficial. It is - a master thread. That is if a particular algorithm is invoked directly from - the master thread (not from a TBB task), supplying it with explicitly - created isolated context will result in a faster algorithm startup. - - VERSIONING NOTE: - Implementation(s) of task_group_context constructor(s) cannot be made - entirely out-of-line because the run-time version must be set by the user - code. This will become critically important for binary compatibility, if - we ever have to change the size of the context object. - - Boosting the runtime version will also be necessary if new data fields are - introduced in the currently unused padding areas and these fields are updated - by inline methods. **/ - task_group_context ( kind_type relation_with_parent = bound, - uintptr_t t = default_traits ) - : my_kind(relation_with_parent) - , my_version_and_traits(2 | t) - { - init(); - } - - // Do not introduce standalone unbind method since it will break state propagation assumptions - __TBB_EXPORTED_METHOD ~task_group_context (); - - //! Forcefully reinitializes the context after the task tree it was associated with is completed. - /** Because the method assumes that all the tasks that used to be associated with - this context have already finished, calling it while the context is still - in use somewhere in the task hierarchy leads to undefined behavior. - - IMPORTANT: This method is not thread safe! - - The method does not change the context's parent if it is set. **/ - void __TBB_EXPORTED_METHOD reset (); - - //! Initiates cancellation of all tasks in this cancellation group and its subordinate groups. - /** \return false if cancellation has already been requested, true otherwise. - - Note that canceling never fails. When false is returned, it just means that - another thread (or this one) has already sent cancellation request to this - context or to one of its ancestors (if this context is bound). It is guaranteed - that when this method is concurrently called on the same not yet cancelled - context, true will be returned by one and only one invocation. **/ - bool __TBB_EXPORTED_METHOD cancel_group_execution (); - - //! Returns true if the context received cancellation request. - bool __TBB_EXPORTED_METHOD is_group_execution_cancelled () const; - - //! Records the pending exception, and cancels the task group. - /** May be called only from inside a catch-block. If the context is already - cancelled, does nothing. - The method brings the task group associated with this context exactly into - the state it would be in, if one of its tasks threw the currently pending - exception during its execution. In other words, it emulates the actions - of the scheduler's dispatch loop exception handler. **/ - void __TBB_EXPORTED_METHOD register_pending_exception (); - -#if __TBB_FP_CONTEXT - //! Captures the current FPU control settings to the context. - /** Because the method assumes that all the tasks that used to be associated with - this context have already finished, calling it while the context is still - in use somewhere in the task hierarchy leads to undefined behavior. - - IMPORTANT: This method is not thread safe! - - The method does not change the FPU control settings of the context's parent. **/ - void __TBB_EXPORTED_METHOD capture_fp_settings (); -#endif - -#if __TBB_TASK_PRIORITY - //! Changes priority of the task group - void set_priority ( priority_t ); - - //! Retrieves current priority of the current task group - priority_t priority () const; -#endif /* __TBB_TASK_PRIORITY */ - - //! Returns the context's trait - uintptr_t traits() const { return my_version_and_traits & traits_mask; } - -protected: - //! Out-of-line part of the constructor. - /** Singled out to ensure backward binary compatibility of the future versions. **/ - void __TBB_EXPORTED_METHOD init (); - -private: - friend class task; - friend class internal::allocate_root_with_context_proxy; - - static const kind_type binding_required = bound; - static const kind_type binding_completed = kind_type(bound+1); - static const kind_type detached = kind_type(binding_completed+1); - static const kind_type dying = kind_type(detached+1); - - //! Propagates any state change detected to *this, and as an optimisation possibly also upward along the heritage line. - template - void propagate_task_group_state ( T task_group_context::*mptr_state, task_group_context& src, T new_state ); - - //! Registers this context with the local scheduler and binds it to its parent context - void bind_to ( internal::generic_scheduler *local_sched ); - - //! Registers this context with the local scheduler - void register_with ( internal::generic_scheduler *local_sched ); - -#if __TBB_FP_CONTEXT - //! Copies FPU control setting from another context - // TODO: Consider adding #else stub in order to omit #if sections in other code - void copy_fp_settings( const task_group_context &src ); -#endif /* __TBB_FP_CONTEXT */ -}; // class task_group_context - -#endif /* __TBB_TASK_GROUP_CONTEXT */ - -//! Base class for user-defined tasks. -/** @ingroup task_scheduling */ -class task: __TBB_TASK_BASE_ACCESS interface5::internal::task_base { - - //! Set reference count - void __TBB_EXPORTED_METHOD internal_set_ref_count( int count ); - - //! Decrement reference count and return its new value. - internal::reference_count __TBB_EXPORTED_METHOD internal_decrement_ref_count(); - -protected: - //! Default constructor. - task() {prefix().extra_state=1;} - -public: - //! Destructor. - virtual ~task() {} - - //! Should be overridden by derived classes. - virtual task* execute() = 0; - - //! Enumeration of task states that the scheduler considers. - enum state_type { - //! task is running, and will be destroyed after method execute() completes. - executing, - //! task to be rescheduled. - reexecute, - //! task is in ready pool, or is going to be put there, or was just taken off. - ready, - //! task object is freshly allocated or recycled. - allocated, - //! task object is on free list, or is going to be put there, or was just taken off. - freed, - //! task to be recycled as continuation - recycle -#if __TBB_RECYCLE_TO_ENQUEUE - //! task to be scheduled for starvation-resistant execution - ,to_enqueue -#endif - }; - - //------------------------------------------------------------------------ - // Allocating tasks - //------------------------------------------------------------------------ - - //! Returns proxy for overloaded new that allocates a root task. - static internal::allocate_root_proxy allocate_root() { - return internal::allocate_root_proxy(); - } - -#if __TBB_TASK_GROUP_CONTEXT - //! Returns proxy for overloaded new that allocates a root task associated with user supplied context. - static internal::allocate_root_with_context_proxy allocate_root( task_group_context& ctx ) { - return internal::allocate_root_with_context_proxy(ctx); - } -#endif /* __TBB_TASK_GROUP_CONTEXT */ - - //! Returns proxy for overloaded new that allocates a continuation task of *this. - /** The continuation's parent becomes the parent of *this. */ - internal::allocate_continuation_proxy& allocate_continuation() { - return *reinterpret_cast(this); - } - - //! Returns proxy for overloaded new that allocates a child task of *this. - internal::allocate_child_proxy& allocate_child() { - return *reinterpret_cast(this); - } - - //! Define recommended static form via import from base class. - using task_base::allocate_additional_child_of; - -#if __TBB_DEPRECATED_TASK_INTERFACE - //! Destroy a task. - /** Usually, calling this method is unnecessary, because a task is - implicitly deleted after its execute() method runs. However, - sometimes a task needs to be explicitly deallocated, such as - when a root task is used as the parent in spawn_and_wait_for_all. */ - void __TBB_EXPORTED_METHOD destroy( task& t ); -#else /* !__TBB_DEPRECATED_TASK_INTERFACE */ - //! Define recommended static form via import from base class. - using task_base::destroy; -#endif /* !__TBB_DEPRECATED_TASK_INTERFACE */ - - //------------------------------------------------------------------------ - // Recycling of tasks - //------------------------------------------------------------------------ - - //! Change this to be a continuation of its former self. - /** The caller must guarantee that the task's refcount does not become zero until - after the method execute() returns. Typically, this is done by having - method execute() return a pointer to a child of the task. If the guarantee - cannot be made, use method recycle_as_safe_continuation instead. - - Because of the hazard, this method may be deprecated in the future. */ - void recycle_as_continuation() { - __TBB_ASSERT( prefix().state==executing, "execute not running?" ); - prefix().state = allocated; - } - - //! Recommended to use, safe variant of recycle_as_continuation - /** For safety, it requires additional increment of ref_count. - With no descendants and ref_count of 1, it has the semantics of recycle_to_reexecute. */ - void recycle_as_safe_continuation() { - __TBB_ASSERT( prefix().state==executing, "execute not running?" ); - prefix().state = recycle; - } - - //! Change this to be a child of new_parent. - void recycle_as_child_of( task& new_parent ) { - internal::task_prefix& p = prefix(); - __TBB_ASSERT( prefix().state==executing||prefix().state==allocated, "execute not running, or already recycled" ); - __TBB_ASSERT( prefix().ref_count==0, "no child tasks allowed when recycled as a child" ); - __TBB_ASSERT( p.parent==NULL, "parent must be null" ); - __TBB_ASSERT( new_parent.prefix().state<=recycle, "corrupt parent's state" ); - __TBB_ASSERT( new_parent.prefix().state!=freed, "parent already freed" ); - p.state = allocated; - p.parent = &new_parent; -#if __TBB_TASK_GROUP_CONTEXT - p.context = new_parent.prefix().context; -#endif /* __TBB_TASK_GROUP_CONTEXT */ - } - - //! Schedule this for reexecution after current execute() returns. - /** Made obsolete by recycle_as_safe_continuation; may become deprecated. */ - void recycle_to_reexecute() { - __TBB_ASSERT( prefix().state==executing, "execute not running, or already recycled" ); - __TBB_ASSERT( prefix().ref_count==0, "no child tasks allowed when recycled for reexecution" ); - prefix().state = reexecute; - } - -#if __TBB_RECYCLE_TO_ENQUEUE - //! Schedule this to enqueue after descendant tasks complete. - /** Save enqueue/spawn difference, it has the semantics of recycle_as_safe_continuation. */ - void recycle_to_enqueue() { - __TBB_ASSERT( prefix().state==executing, "execute not running, or already recycled" ); - prefix().state = to_enqueue; - } -#endif /* __TBB_RECYCLE_TO_ENQUEUE */ - - //------------------------------------------------------------------------ - // Spawning and blocking - //------------------------------------------------------------------------ - - //! Set reference count - void set_ref_count( int count ) { -#if TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT - internal_set_ref_count(count); -#else - prefix().ref_count = count; -#endif /* TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT */ - } - - //! Atomically increment reference count. - /** Has acquire semantics */ - void increment_ref_count() { - __TBB_FetchAndIncrementWacquire( &prefix().ref_count ); - } - - //! Atomically adds to reference count and returns its new value. - /** Has release-acquire semantics */ - int add_ref_count( int count ) { - internal::call_itt_notify( internal::releasing, &prefix().ref_count ); - internal::reference_count k = count+__TBB_FetchAndAddW( &prefix().ref_count, count ); - __TBB_ASSERT( k>=0, "task's reference count underflowed" ); - if( k==0 ) - internal::call_itt_notify( internal::acquired, &prefix().ref_count ); - return int(k); - } - - //! Atomically decrement reference count and returns its new value. - /** Has release semantics. */ - int decrement_ref_count() { -#if TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT - return int(internal_decrement_ref_count()); -#else - return int(__TBB_FetchAndDecrementWrelease( &prefix().ref_count ))-1; -#endif /* TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT */ - } - - //! Define recommended static forms via import from base class. - using task_base::spawn; - - //! Similar to spawn followed by wait_for_all, but more efficient. - void spawn_and_wait_for_all( task& child ) { - prefix().owner->wait_for_all( *this, &child ); - } - - //! Similar to spawn followed by wait_for_all, but more efficient. - void __TBB_EXPORTED_METHOD spawn_and_wait_for_all( task_list& list ); - - //! Spawn task allocated by allocate_root, wait for it to complete, and deallocate it. - static void spawn_root_and_wait( task& root ) { - root.prefix().owner->spawn_root_and_wait( root, root.prefix().next ); - } - - //! Spawn root tasks on list and wait for all of them to finish. - /** If there are more tasks than worker threads, the tasks are spawned in - order of front to back. */ - static void spawn_root_and_wait( task_list& root_list ); - - //! Wait for reference count to become one, and set reference count to zero. - /** Works on tasks while waiting. */ - void wait_for_all() { - prefix().owner->wait_for_all( *this, NULL ); - } - - //! Enqueue task for starvation-resistant execution. -#if __TBB_TASK_PRIORITY - /** The task will be enqueued on the normal priority level disregarding the - priority of its task group. - - The rationale of such semantics is that priority of an enqueued task is - statically fixed at the moment of its enqueuing, while task group priority - is dynamic. Thus automatic priority inheritance would be generally a subject - to the race, which may result in unexpected behavior. - - Use enqueue() overload with explicit priority value and task::group_priority() - method to implement such priority inheritance when it is really necessary. **/ -#endif /* __TBB_TASK_PRIORITY */ - static void enqueue( task& t ) { - t.prefix().owner->enqueue( t, NULL ); - } - -#if __TBB_TASK_PRIORITY - //! Enqueue task for starvation-resistant execution on the specified priority level. - static void enqueue( task& t, priority_t p ) { - __TBB_ASSERT( p == priority_low || p == priority_normal || p == priority_high, "Invalid priority level value" ); - t.prefix().owner->enqueue( t, (void*)p ); - } -#endif /* __TBB_TASK_PRIORITY */ - - //! The innermost task being executed or destroyed by the current thread at the moment. - static task& __TBB_EXPORTED_FUNC self(); - - //! task on whose behalf this task is working, or NULL if this is a root. - task* parent() const {return prefix().parent;} - - //! sets parent task pointer to specified value - void set_parent(task* p) { -#if __TBB_TASK_GROUP_CONTEXT - __TBB_ASSERT(!p || prefix().context == p->prefix().context, "The tasks must be in the same context"); -#endif - prefix().parent = p; - } - -#if __TBB_TASK_GROUP_CONTEXT - //! This method is deprecated and will be removed in the future. - /** Use method group() instead. **/ - task_group_context* context() {return prefix().context;} - - //! Pointer to the task group descriptor. - task_group_context* group () { return prefix().context; } -#endif /* __TBB_TASK_GROUP_CONTEXT */ - - //! True if task was stolen from the task pool of another thread. - bool is_stolen_task() const { - return (prefix().extra_state & 0x80)!=0; - } - - //------------------------------------------------------------------------ - // Debugging - //------------------------------------------------------------------------ - - //! Current execution state - state_type state() const {return state_type(prefix().state);} - - //! The internal reference count. - int ref_count() const { -#if TBB_USE_ASSERT - internal::reference_count ref_count_ = prefix().ref_count; - __TBB_ASSERT( ref_count_==int(ref_count_), "integer overflow error"); -#endif - return int(prefix().ref_count); - } - - //! Obsolete, and only retained for the sake of backward compatibility. Always returns true. - bool __TBB_EXPORTED_METHOD is_owned_by_current_thread() const; - - //------------------------------------------------------------------------ - // Affinity - //------------------------------------------------------------------------ - - //! An id as used for specifying affinity. - /** Guaranteed to be integral type. Value of 0 means no affinity. */ - typedef internal::affinity_id affinity_id; - - //! Set affinity for this task. - void set_affinity( affinity_id id ) {prefix().affinity = id;} - - //! Current affinity of this task - affinity_id affinity() const {return prefix().affinity;} - - //! Invoked by scheduler to notify task that it ran on unexpected thread. - /** Invoked before method execute() runs, if task is stolen, or task has - affinity but will be executed on another thread. - - The default action does nothing. */ - virtual void __TBB_EXPORTED_METHOD note_affinity( affinity_id id ); - -#if __TBB_TASK_GROUP_CONTEXT - //! Moves this task from its current group into another one. - /** Argument ctx specifies the new group. - - The primary purpose of this method is to associate unique task group context - with a task allocated for subsequent enqueuing. In contrast to spawned tasks - enqueued ones normally outlive the scope where they were created. This makes - traditional usage model where task group context are allocated locally on - the stack inapplicable. Dynamic allocation of context objects is performance - inefficient. Method change_group() allows to make task group context object - a member of the task class, and then associate it with its containing task - object in the latter's constructor. **/ - void __TBB_EXPORTED_METHOD change_group ( task_group_context& ctx ); - - //! Initiates cancellation of all tasks in this cancellation group and its subordinate groups. - /** \return false if cancellation has already been requested, true otherwise. **/ - bool cancel_group_execution () { return prefix().context->cancel_group_execution(); } - - //! Returns true if the context has received cancellation request. - bool is_cancelled () const { return prefix().context->is_group_execution_cancelled(); } -#else - bool is_cancelled () const { return false; } -#endif /* __TBB_TASK_GROUP_CONTEXT */ - -#if __TBB_TASK_PRIORITY - //! Changes priority of the task group this task belongs to. - void set_group_priority ( priority_t p ) { prefix().context->set_priority(p); } - - //! Retrieves current priority of the task group this task belongs to. - priority_t group_priority () const { return prefix().context->priority(); } - -#endif /* __TBB_TASK_PRIORITY */ - -private: - friend class interface5::internal::task_base; - friend class task_list; - friend class internal::scheduler; - friend class internal::allocate_root_proxy; -#if __TBB_TASK_GROUP_CONTEXT - friend class internal::allocate_root_with_context_proxy; -#endif /* __TBB_TASK_GROUP_CONTEXT */ - friend class internal::allocate_continuation_proxy; - friend class internal::allocate_child_proxy; - friend class internal::allocate_additional_child_of_proxy; - - //! Get reference to corresponding task_prefix. - /** Version tag prevents loader on Linux from using the wrong symbol in debug builds. **/ - internal::task_prefix& prefix( internal::version_tag* = NULL ) const { - return reinterpret_cast(const_cast(this))[-1]; - } -}; // class task - -//! task that does nothing. Useful for synchronization. -/** @ingroup task_scheduling */ -class empty_task: public task { - task* execute() __TBB_override { - return NULL; - } -}; - -//! @cond INTERNAL -namespace internal { - template - class function_task : public task { -#if __TBB_ALLOW_MUTABLE_FUNCTORS - F my_func; -#else - const F my_func; -#endif - task* execute() __TBB_override { - my_func(); - return NULL; - } - public: - function_task( const F& f ) : my_func(f) {} - }; -} // namespace internal -//! @endcond - -//! A list of children. -/** Used for method task::spawn_children - @ingroup task_scheduling */ -class task_list: internal::no_copy { -private: - task* first; - task** next_ptr; - friend class task; - friend class interface5::internal::task_base; -public: - //! Construct empty list - task_list() : first(NULL), next_ptr(&first) {} - - //! Destroys the list, but does not destroy the task objects. - ~task_list() {} - - //! True if list if empty; false otherwise. - bool empty() const {return !first;} - - //! Push task onto back of list. - void push_back( task& task ) { - task.prefix().next = NULL; - *next_ptr = &task; - next_ptr = &task.prefix().next; - } -#if __TBB_TODO - // TODO: add this method and implement&document the local execution ordering. See more in generic_scheduler::local_spawn - //! Push task onto front of list (FIFO local execution, like individual spawning in the same order). - void push_front( task& task ) { - if( empty() ) { - push_back(task); - } else { - task.prefix().next = first; - first = &task; - } - } -#endif - //! Pop the front task from the list. - task& pop_front() { - __TBB_ASSERT( !empty(), "attempt to pop item from empty task_list" ); - task* result = first; - first = result->prefix().next; - if( !first ) next_ptr = &first; - return *result; - } - - //! Clear the list - void clear() { - first=NULL; - next_ptr=&first; - } -}; - -inline void interface5::internal::task_base::spawn( task& t ) { - t.prefix().owner->spawn( t, t.prefix().next ); -} - -inline void interface5::internal::task_base::spawn( task_list& list ) { - if( task* t = list.first ) { - t->prefix().owner->spawn( *t, *list.next_ptr ); - list.clear(); - } -} - -inline void task::spawn_root_and_wait( task_list& root_list ) { - if( task* t = root_list.first ) { - t->prefix().owner->spawn_root_and_wait( *t, *root_list.next_ptr ); - root_list.clear(); - } -} - -} // namespace tbb - -inline void *operator new( size_t bytes, const tbb::internal::allocate_root_proxy& ) { - return &tbb::internal::allocate_root_proxy::allocate(bytes); -} - -inline void operator delete( void* task, const tbb::internal::allocate_root_proxy& ) { - tbb::internal::allocate_root_proxy::free( *static_cast(task) ); -} - -#if __TBB_TASK_GROUP_CONTEXT -inline void *operator new( size_t bytes, const tbb::internal::allocate_root_with_context_proxy& p ) { - return &p.allocate(bytes); -} - -inline void operator delete( void* task, const tbb::internal::allocate_root_with_context_proxy& p ) { - p.free( *static_cast(task) ); -} -#endif /* __TBB_TASK_GROUP_CONTEXT */ - -inline void *operator new( size_t bytes, const tbb::internal::allocate_continuation_proxy& p ) { - return &p.allocate(bytes); -} - -inline void operator delete( void* task, const tbb::internal::allocate_continuation_proxy& p ) { - p.free( *static_cast(task) ); -} - -inline void *operator new( size_t bytes, const tbb::internal::allocate_child_proxy& p ) { - return &p.allocate(bytes); -} - -inline void operator delete( void* task, const tbb::internal::allocate_child_proxy& p ) { - p.free( *static_cast(task) ); -} - -inline void *operator new( size_t bytes, const tbb::internal::allocate_additional_child_of_proxy& p ) { - return &p.allocate(bytes); -} - -inline void operator delete( void* task, const tbb::internal::allocate_additional_child_of_proxy& p ) { - p.free( *static_cast(task) ); -} - -#endif /* __TBB_task_H */ diff --git a/inst/include/tbb/task_arena.h b/inst/include/tbb/task_arena.h deleted file mode 100644 index f33135b6a..000000000 --- a/inst/include/tbb/task_arena.h +++ /dev/null @@ -1,333 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_task_arena_H -#define __TBB_task_arena_H - -#include "task.h" -#include "tbb_exception.h" -#if TBB_USE_THREADING_TOOLS -#include "atomic.h" // for as_atomic -#endif - -namespace tbb { - -namespace this_task_arena { - int max_concurrency(); -} // namespace this_task_arena - -//! @cond INTERNAL -namespace internal { - //! Internal to library. Should not be used by clients. - /** @ingroup task_scheduling */ - class arena; - class task_scheduler_observer_v3; -} // namespace internal -//! @endcond - -namespace interface7 { -class task_arena; - -//! @cond INTERNAL -namespace internal { -using namespace tbb::internal; //e.g. function_task from task.h - -class delegate_base : no_assign { -public: - virtual void operator()() const = 0; - virtual ~delegate_base() {} -}; - -template -class delegated_function : public delegate_base { - F &my_func; - void operator()() const __TBB_override { - my_func(); - } -public: - delegated_function ( F& f ) : my_func(f) {} -}; - -class task_arena_base { -protected: - //! NULL if not currently initialized. - internal::arena* my_arena; - -#if __TBB_TASK_GROUP_CONTEXT - //! default context of the arena - task_group_context *my_context; -#endif - - //! Concurrency level for deferred initialization - int my_max_concurrency; - - //! Reserved master slots - unsigned my_master_slots; - - //! Special settings - intptr_t my_version_and_traits; - - enum { - default_flags = 0 -#if __TBB_TASK_GROUP_CONTEXT - | (task_group_context::default_traits & task_group_context::exact_exception) // 0 or 1 << 16 - , exact_exception_flag = task_group_context::exact_exception // used to specify flag for context directly -#endif - }; - - task_arena_base(int max_concurrency, unsigned reserved_for_masters) - : my_arena(0) -#if __TBB_TASK_GROUP_CONTEXT - , my_context(0) -#endif - , my_max_concurrency(max_concurrency) - , my_master_slots(reserved_for_masters) - , my_version_and_traits(default_flags) - {} - - void __TBB_EXPORTED_METHOD internal_initialize(); - void __TBB_EXPORTED_METHOD internal_terminate(); - void __TBB_EXPORTED_METHOD internal_attach(); - void __TBB_EXPORTED_METHOD internal_enqueue( task&, intptr_t ) const; - void __TBB_EXPORTED_METHOD internal_execute( delegate_base& ) const; - void __TBB_EXPORTED_METHOD internal_wait() const; - static int __TBB_EXPORTED_FUNC internal_current_slot(); - static int __TBB_EXPORTED_FUNC internal_max_concurrency( const task_arena * ); -public: - //! Typedef for number of threads that is automatic. - static const int automatic = -1; - static const int not_initialized = -2; - -}; - -#if __TBB_TASK_ISOLATION -void __TBB_EXPORTED_FUNC isolate_within_arena( delegate_base& d, intptr_t reserved = 0 ); -#endif /* __TBB_TASK_ISOLATION */ -} // namespace internal -//! @endcond - -/** 1-to-1 proxy representation class of scheduler's arena - * Constructors set up settings only, real construction is deferred till the first method invocation - * Destructor only removes one of the references to the inner arena representation. - * Final destruction happens when all the references (and the work) are gone. - */ -class task_arena : public internal::task_arena_base { - friend class tbb::internal::task_scheduler_observer_v3; - friend int tbb::this_task_arena::max_concurrency(); - bool my_initialized; - void mark_initialized() { - __TBB_ASSERT( my_arena, "task_arena initialization is incomplete" ); -#if __TBB_TASK_GROUP_CONTEXT - __TBB_ASSERT( my_context, "task_arena initialization is incomplete" ); -#endif -#if TBB_USE_THREADING_TOOLS - // Actual synchronization happens in internal_initialize & internal_attach. - // The race on setting my_initialized is benign, but should be hidden from Intel(R) Inspector - internal::as_atomic(my_initialized).fetch_and_store(true); -#else - my_initialized = true; -#endif - } - -public: - //! Creates task_arena with certain concurrency limits - /** Sets up settings only, real construction is deferred till the first method invocation - * @arg max_concurrency specifies total number of slots in arena where threads work - * @arg reserved_for_masters specifies number of slots to be used by master threads only. - * Value of 1 is default and reflects behavior of implicit arenas. - **/ - task_arena(int max_concurrency_ = automatic, unsigned reserved_for_masters = 1) - : task_arena_base(max_concurrency_, reserved_for_masters) - , my_initialized(false) - {} - - //! Copies settings from another task_arena - task_arena(const task_arena &s) // copy settings but not the reference or instance - : task_arena_base(s.my_max_concurrency, s.my_master_slots) - , my_initialized(false) - {} - - //! Tag class used to indicate the "attaching" constructor - struct attach {}; - - //! Creates an instance of task_arena attached to the current arena of the thread - explicit task_arena( attach ) - : task_arena_base(automatic, 1) // use default settings if attach fails - , my_initialized(false) - { - internal_attach(); - if( my_arena ) my_initialized = true; - } - - //! Forces allocation of the resources for the task_arena as specified in constructor arguments - inline void initialize() { - if( !my_initialized ) { - internal_initialize(); - mark_initialized(); - } - } - - //! Overrides concurrency level and forces initialization of internal representation - inline void initialize(int max_concurrency_, unsigned reserved_for_masters = 1) { - // TODO: decide if this call must be thread-safe - __TBB_ASSERT( !my_arena, "Impossible to modify settings of an already initialized task_arena"); - if( !my_initialized ) { - my_max_concurrency = max_concurrency_; - my_master_slots = reserved_for_masters; - initialize(); - } - } - - //! Attaches this instance to the current arena of the thread - inline void initialize(attach) { - // TODO: decide if this call must be thread-safe - __TBB_ASSERT( !my_arena, "Impossible to modify settings of an already initialized task_arena"); - if( !my_initialized ) { - internal_attach(); - if( !my_arena ) internal_initialize(); - mark_initialized(); - } - } - - //! Removes the reference to the internal arena representation. - //! Not thread safe wrt concurrent invocations of other methods. - inline void terminate() { - if( my_initialized ) { - internal_terminate(); - my_initialized = false; - } - } - - //! Removes the reference to the internal arena representation, and destroys the external object. - //! Not thread safe wrt concurrent invocations of other methods. - ~task_arena() { - terminate(); - } - - //! Returns true if the arena is active (initialized); false otherwise. - //! The name was chosen to match a task_scheduler_init method with the same semantics. - bool is_active() const { return my_initialized; } - - //! Enqueues a task into the arena to process a functor, and immediately returns. - //! Does not require the calling thread to join the arena - template - void enqueue( const F& f ) { - initialize(); -#if __TBB_TASK_GROUP_CONTEXT - internal_enqueue( *new( task::allocate_root(*my_context) ) internal::function_task(f), 0 ); -#else - internal_enqueue( *new( task::allocate_root() ) internal::function_task(f), 0 ); -#endif - } - -#if __TBB_TASK_PRIORITY - //! Enqueues a task with priority p into the arena to process a functor f, and immediately returns. - //! Does not require the calling thread to join the arena - template - void enqueue( const F& f, priority_t p ) { - __TBB_ASSERT( p == priority_low || p == priority_normal || p == priority_high, "Invalid priority level value" ); - initialize(); -#if __TBB_TASK_GROUP_CONTEXT - internal_enqueue( *new( task::allocate_root(*my_context) ) internal::function_task(f), (intptr_t)p ); -#else - internal_enqueue( *new( task::allocate_root() ) internal::function_task(f), (intptr_t)p ); -#endif - } -#endif// __TBB_TASK_PRIORITY - - //! Joins the arena and executes a functor, then returns - //! If not possible to join, wraps the functor into a task, enqueues it and waits for task completion - //! Can decrement the arena demand for workers, causing a worker to leave and free a slot to the calling thread - template - void execute(F& f) { - initialize(); - internal::delegated_function d(f); - internal_execute( d ); - } - - //! Joins the arena and executes a functor, then returns - //! If not possible to join, wraps the functor into a task, enqueues it and waits for task completion - //! Can decrement the arena demand for workers, causing a worker to leave and free a slot to the calling thread - template - void execute(const F& f) { - initialize(); - internal::delegated_function d(f); - internal_execute( d ); - } - -#if __TBB_EXTRA_DEBUG - //! Wait for all work in the arena to be completed - //! Even submitted by other application threads - //! Joins arena if/when possible (in the same way as execute()) - void debug_wait_until_empty() { - initialize(); - internal_wait(); - } -#endif //__TBB_EXTRA_DEBUG - - //! Returns the index, aka slot number, of the calling thread in its current arena - //! This method is deprecated and replaced with this_task_arena::current_thread_index() - inline static int current_thread_index() { - return internal_current_slot(); - } - - //! Returns the maximal number of threads that can work inside the arena - inline int max_concurrency() const { - // Handle special cases inside the library - return (my_max_concurrency>1) ? my_max_concurrency : internal_max_concurrency(this); - } -}; - -#if __TBB_TASK_ISOLATION -namespace this_task_arena { - template - void isolate( const F& f ) { - internal::delegated_function d(f); - internal::isolate_within_arena( d ); - } -} -#endif /* __TBB_TASK_ISOLATION */ - -} // namespace interfaceX - -using interface7::task_arena; -#if __TBB_TASK_ISOLATION -namespace this_task_arena { - using namespace interface7::this_task_arena; -} -#endif /* __TBB_TASK_ISOLATION */ - -namespace this_task_arena { - //! Returns the index, aka slot number, of the calling thread in its current arena - inline int current_thread_index() { - int idx = tbb::task_arena::current_thread_index(); - return idx == -1 ? tbb::task_arena::not_initialized : idx; - } - - //! Returns the maximal number of threads that can work inside the arena - inline int max_concurrency() { - return tbb::task_arena::internal_max_concurrency(NULL); - } - -} // namespace this_task_arena - -} // namespace tbb - -#endif /* __TBB_task_arena_H */ diff --git a/inst/include/tbb/task_group.h b/inst/include/tbb/task_group.h deleted file mode 100644 index bf6922b9b..000000000 --- a/inst/include/tbb/task_group.h +++ /dev/null @@ -1,223 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_task_group_H -#define __TBB_task_group_H - -#include "task.h" -#include "tbb_exception.h" - -#if __TBB_TASK_GROUP_CONTEXT - -namespace tbb { - -namespace internal { - template class task_handle_task; -} - -class task_group; -class structured_task_group; - -template -class task_handle : internal::no_assign { - template friend class internal::task_handle_task; - friend class task_group; - friend class structured_task_group; - - static const intptr_t scheduled = 0x1; - - F my_func; - intptr_t my_state; - - void mark_scheduled () { - // The check here is intentionally lax to avoid the impact of interlocked operation - if ( my_state & scheduled ) - internal::throw_exception( internal::eid_invalid_multiple_scheduling ); - my_state |= scheduled; - } -public: - task_handle( const F& f ) : my_func(f), my_state(0) {} - - void operator() () const { my_func(); } -}; - -enum task_group_status { - not_complete, - complete, - canceled -}; - -namespace internal { - -template -class task_handle_task : public task { - task_handle& my_handle; - task* execute() __TBB_override { - my_handle(); - return NULL; - } -public: - task_handle_task( task_handle& h ) : my_handle(h) { h.mark_scheduled(); } -}; - -class task_group_base : internal::no_copy { -protected: - empty_task* my_root; - task_group_context my_context; - - task& owner () { return *my_root; } - - template - task_group_status internal_run_and_wait( F& f ) { - __TBB_TRY { - if ( !my_context.is_group_execution_cancelled() ) - f(); - } __TBB_CATCH( ... ) { - my_context.register_pending_exception(); - } - return wait(); - } - - template - void internal_run( F& f ) { - owner().spawn( *new( owner().allocate_additional_child_of(*my_root) ) Task(f) ); - } - -public: - task_group_base( uintptr_t traits = 0 ) - : my_context(task_group_context::bound, task_group_context::default_traits | traits) - { - my_root = new( task::allocate_root(my_context) ) empty_task; - my_root->set_ref_count(1); - } - - ~task_group_base() __TBB_NOEXCEPT(false) { - if( my_root->ref_count() > 1 ) { - bool stack_unwinding_in_progress = std::uncaught_exception(); - // Always attempt to do proper cleanup to avoid inevitable memory corruption - // in case of missing wait (for the sake of better testability & debuggability) - if ( !is_canceling() ) - cancel(); - __TBB_TRY { - my_root->wait_for_all(); - } __TBB_CATCH (...) { - task::destroy(*my_root); - __TBB_RETHROW(); - } - task::destroy(*my_root); - if ( !stack_unwinding_in_progress ) - internal::throw_exception( internal::eid_missing_wait ); - } - else { - task::destroy(*my_root); - } - } - - template - void run( task_handle& h ) { - internal_run< task_handle, internal::task_handle_task >( h ); - } - - task_group_status wait() { - __TBB_TRY { - my_root->wait_for_all(); - } __TBB_CATCH( ... ) { - my_context.reset(); - __TBB_RETHROW(); - } - if ( my_context.is_group_execution_cancelled() ) { - // TODO: the reset method is not thread-safe. Ensure the correct behavior. - my_context.reset(); - return canceled; - } - return complete; - } - - bool is_canceling() { - return my_context.is_group_execution_cancelled(); - } - - void cancel() { - my_context.cancel_group_execution(); - } -}; // class task_group_base - -} // namespace internal - -class task_group : public internal::task_group_base { -public: - task_group () : task_group_base( task_group_context::concurrent_wait ) {} - -#if __SUNPRO_CC - template - void run( task_handle& h ) { - internal_run< task_handle, internal::task_handle_task >( h ); - } -#else - using task_group_base::run; -#endif - - template - void run( const F& f ) { - internal_run< const F, internal::function_task >( f ); - } - - template - task_group_status run_and_wait( const F& f ) { - return internal_run_and_wait( f ); - } - - template - task_group_status run_and_wait( task_handle& h ) { - h.mark_scheduled(); - return internal_run_and_wait< task_handle >( h ); - } -}; // class task_group - -class structured_task_group : public internal::task_group_base { -public: - template - task_group_status run_and_wait ( task_handle& h ) { - h.mark_scheduled(); - return internal_run_and_wait< task_handle >( h ); - } - - task_group_status wait() { - task_group_status res = task_group_base::wait(); - my_root->set_ref_count(1); - return res; - } -}; // class structured_task_group - -inline -bool is_current_task_group_canceling() { - return task::self().is_cancelled(); -} - -template -task_handle make_task( const F& f ) { - return task_handle( f ); -} - -} // namespace tbb - -#endif /* __TBB_TASK_GROUP_CONTEXT */ - -#endif /* __TBB_task_group_H */ diff --git a/inst/include/tbb/task_scheduler_init.h b/inst/include/tbb/task_scheduler_init.h deleted file mode 100644 index 928e7a4e5..000000000 --- a/inst/include/tbb/task_scheduler_init.h +++ /dev/null @@ -1,161 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_task_scheduler_init_H -#define __TBB_task_scheduler_init_H - -#include "tbb_stddef.h" -#include "limits.h" -#if __TBB_SUPPORTS_WORKERS_WAITING_IN_TERMINATE -#include // nothrow_t -#endif - -namespace tbb { - -typedef std::size_t stack_size_type; - -//! @cond INTERNAL -namespace internal { - //! Internal to library. Should not be used by clients. - /** @ingroup task_scheduling */ - class scheduler; -} // namespace internal -//! @endcond - -//! Class delimiting the scope of task scheduler activity. -/** A thread can construct a task_scheduler_init object and keep it alive - while it uses TBB's tasking subsystem (including parallel algorithms). - - This class allows to customize properties of the TBB task pool to some extent. - For example it can limit concurrency level of parallel work initiated by the - given thread. It also can be used to specify stack size of the TBB worker threads, - though this setting is not effective if the thread pool has already been created. - - If a parallel construct is used without task_scheduler_init object previously - created, the scheduler will be initialized automatically with default settings, - and will persist until this thread exits. Default concurrency level is defined - as described in task_scheduler_init::initialize(). - @ingroup task_scheduling */ -class task_scheduler_init: internal::no_copy { - enum ExceptionPropagationMode { - propagation_mode_exact = 1u, - propagation_mode_captured = 2u, - propagation_mode_mask = propagation_mode_exact | propagation_mode_captured - }; - - /** NULL if not currently initialized. */ - internal::scheduler* my_scheduler; - - bool internal_terminate( bool blocking ); -#if __TBB_SUPPORTS_WORKERS_WAITING_IN_TERMINATE - bool __TBB_EXPORTED_METHOD internal_blocking_terminate( bool throwing ); -#endif -public: - - //! Typedef for number of threads that is automatic. - static const int automatic = -1; - - //! Argument to initialize() or constructor that causes initialization to be deferred. - static const int deferred = -2; - - //! Ensure that scheduler exists for this thread - /** A value of -1 lets TBB decide on the number of threads, which is usually - maximal hardware concurrency for this process, that is the number of logical - CPUs on the machine (possibly limited by the processor affinity mask of this - process (Windows) or of this thread (Linux, FreeBSD). It is preferable option - for production code because it helps to avoid nasty surprises when several - TBB based components run side-by-side or in a nested fashion inside the same - process. - - The number_of_threads is ignored if any other task_scheduler_inits - currently exist. A thread may construct multiple task_scheduler_inits. - Doing so does no harm because the underlying scheduler is reference counted. */ - void __TBB_EXPORTED_METHOD initialize( int number_of_threads=automatic ); - - //! The overloaded method with stack size parameter - /** Overloading is necessary to preserve ABI compatibility */ - void __TBB_EXPORTED_METHOD initialize( int number_of_threads, stack_size_type thread_stack_size ); - - //! Inverse of method initialize. - void __TBB_EXPORTED_METHOD terminate(); - -#if __TBB_SUPPORTS_WORKERS_WAITING_IN_TERMINATE -#if TBB_USE_EXCEPTIONS - //! terminate() that waits for worker threads termination. Throws exception on error. - void blocking_terminate() { - internal_blocking_terminate( /*throwing=*/true ); - } -#endif - //! terminate() that waits for worker threads termination. Returns false on error. - bool blocking_terminate(const std::nothrow_t&) __TBB_NOEXCEPT(true) { - return internal_blocking_terminate( /*throwing=*/false ); - } -#endif // __TBB_SUPPORTS_WORKERS_WAITING_IN_TERMINATE - - //! Shorthand for default constructor followed by call to initialize(number_of_threads). - task_scheduler_init( int number_of_threads=automatic, stack_size_type thread_stack_size=0 ) : my_scheduler(NULL) - { - // Two lowest order bits of the stack size argument may be taken to communicate - // default exception propagation mode of the client to be used when the - // client manually creates tasks in the master thread and does not use - // explicit task group context object. This is necessary because newer - // TBB binaries with exact propagation enabled by default may be used - // by older clients that expect tbb::captured_exception wrapper. - // All zeros mean old client - no preference. - __TBB_ASSERT( !(thread_stack_size & propagation_mode_mask), "Requested stack size is not aligned" ); -#if TBB_USE_EXCEPTIONS - thread_stack_size |= TBB_USE_CAPTURED_EXCEPTION ? propagation_mode_captured : propagation_mode_exact; -#endif /* TBB_USE_EXCEPTIONS */ - initialize( number_of_threads, thread_stack_size ); - } - - //! Destroy scheduler for this thread if thread has no other live task_scheduler_inits. - ~task_scheduler_init() { - if( my_scheduler ) - terminate(); - internal::poison_pointer( my_scheduler ); - } - //! Returns the number of threads TBB scheduler would create if initialized by default. - /** Result returned by this method does not depend on whether the scheduler - has already been initialized. - - Because tbb 2.0 does not support blocking tasks yet, you may use this method - to boost the number of threads in the tbb's internal pool, if your tasks are - doing I/O operations. The optimal number of additional threads depends on how - much time your tasks spend in the blocked state. - - Before TBB 3.0 U4 this method returned the number of logical CPU in the - system. Currently on Windows, Linux and FreeBSD it returns the number of - logical CPUs available to the current process in accordance with its affinity - mask. - - NOTE: The return value of this method never changes after its first invocation. - This means that changes in the process affinity mask that took place after - this method was first invoked will not affect the number of worker threads - in the TBB worker threads pool. */ - static int __TBB_EXPORTED_FUNC default_num_threads (); - - //! Returns true if scheduler is active (initialized); false otherwise - bool is_active() const { return my_scheduler != NULL; } -}; - -} // namespace tbb - -#endif /* __TBB_task_scheduler_init_H */ diff --git a/inst/include/tbb/tbb.h b/inst/include/tbb/tbb.h deleted file mode 100644 index 5e385ea2f..000000000 --- a/inst/include/tbb/tbb.h +++ /dev/null @@ -1,85 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_tbb_H -#define __TBB_tbb_H - -/** - This header bulk-includes declarations or definitions of all the functionality - provided by TBB (save for malloc dependent headers). - - If you use only a few TBB constructs, consider including specific headers only. - Any header listed below can be included independently of others. -**/ - -#if TBB_PREVIEW_AGGREGATOR -#include "aggregator.h" -#endif -#include "aligned_space.h" -#include "atomic.h" -#include "blocked_range.h" -#include "blocked_range2d.h" -#include "blocked_range3d.h" -#include "cache_aligned_allocator.h" -#include "combinable.h" -#include "concurrent_hash_map.h" -#if TBB_PREVIEW_CONCURRENT_LRU_CACHE -#include "concurrent_lru_cache.h" -#endif -#include "concurrent_priority_queue.h" -#include "concurrent_queue.h" -#include "concurrent_unordered_map.h" -#include "concurrent_unordered_set.h" -#include "concurrent_vector.h" -#include "critical_section.h" -#include "enumerable_thread_specific.h" -#include "flow_graph.h" -#if TBB_PREVIEW_GLOBAL_CONTROL -#include "global_control.h" -#endif -#include "mutex.h" -#include "null_mutex.h" -#include "null_rw_mutex.h" -#include "parallel_do.h" -#include "parallel_for.h" -#include "parallel_for_each.h" -#include "parallel_invoke.h" -#include "parallel_reduce.h" -#include "parallel_scan.h" -#include "parallel_sort.h" -#include "partitioner.h" -#include "pipeline.h" -#include "queuing_mutex.h" -#include "queuing_rw_mutex.h" -#include "reader_writer_lock.h" -#include "recursive_mutex.h" -#include "spin_mutex.h" -#include "spin_rw_mutex.h" -#include "task.h" -#include "task_arena.h" -#include "task_group.h" -#include "task_scheduler_init.h" -#include "task_scheduler_observer.h" -#include "tbb_allocator.h" -#include "tbb_exception.h" -#include "tbb_thread.h" -#include "tick_count.h" - -#endif /* __TBB_tbb_H */ diff --git a/inst/include/tbb/tbb_allocator.h b/inst/include/tbb/tbb_allocator.h deleted file mode 100644 index a3f1ef0c1..000000000 --- a/inst/include/tbb/tbb_allocator.h +++ /dev/null @@ -1,218 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_tbb_allocator_H -#define __TBB_tbb_allocator_H - -#include "tbb_stddef.h" -#include -#if __TBB_ALLOCATOR_CONSTRUCT_VARIADIC - #include // std::forward -#endif - -#if !TBB_USE_EXCEPTIONS && _MSC_VER - // Suppress "C++ exception handler used, but unwind semantics are not enabled" warning in STL headers - #pragma warning (push) - #pragma warning (disable: 4530) -#endif - -#include - -#if !TBB_USE_EXCEPTIONS && _MSC_VER - #pragma warning (pop) -#endif - -namespace tbb { - -//! @cond INTERNAL -namespace internal { - - //! Deallocates memory using FreeHandler - /** The function uses scalable_free if scalable allocator is available and free if not*/ - void __TBB_EXPORTED_FUNC deallocate_via_handler_v3( void *p ); - - //! Allocates memory using MallocHandler - /** The function uses scalable_malloc if scalable allocator is available and malloc if not*/ - void* __TBB_EXPORTED_FUNC allocate_via_handler_v3( size_t n ); - - //! Returns true if standard malloc/free are used to work with memory. - bool __TBB_EXPORTED_FUNC is_malloc_used_v3(); -} -//! @endcond - -#if _MSC_VER && !defined(__INTEL_COMPILER) - // Workaround for erroneous "unreferenced parameter" warning in method destroy. - #pragma warning (push) - #pragma warning (disable: 4100) -#endif - -//! Meets "allocator" requirements of ISO C++ Standard, Section 20.1.5 -/** The class selects the best memory allocation mechanism available - from scalable_malloc and standard malloc. - The members are ordered the same way they are in section 20.4.1 - of the ISO C++ standard. - @ingroup memory_allocation */ -template -class tbb_allocator { -public: - typedef typename internal::allocator_type::value_type value_type; - typedef value_type* pointer; - typedef const value_type* const_pointer; - typedef value_type& reference; - typedef const value_type& const_reference; - typedef size_t size_type; - typedef ptrdiff_t difference_type; - template struct rebind { - typedef tbb_allocator other; - }; - - //! Specifies current allocator - enum malloc_type { - scalable, - standard - }; - - tbb_allocator() throw() {} - tbb_allocator( const tbb_allocator& ) throw() {} - template tbb_allocator(const tbb_allocator&) throw() {} - - pointer address(reference x) const {return &x;} - const_pointer address(const_reference x) const {return &x;} - - //! Allocate space for n objects. - pointer allocate( size_type n, const void* /*hint*/ = 0) { - return pointer(internal::allocate_via_handler_v3( n * sizeof(value_type) )); - } - - //! Free previously allocated block of memory. - void deallocate( pointer p, size_type ) { - internal::deallocate_via_handler_v3(p); - } - - //! Largest value for which method allocate might succeed. - size_type max_size() const throw() { - size_type max = static_cast(-1) / sizeof (value_type); - return (max > 0 ? max : 1); - } - - //! Copy-construct value at location pointed to by p. -#if __TBB_ALLOCATOR_CONSTRUCT_VARIADIC - template - void construct(U *p, Args&&... args) - { ::new((void *)p) U(std::forward(args)...); } -#else // __TBB_ALLOCATOR_CONSTRUCT_VARIADIC -#if __TBB_CPP11_RVALUE_REF_PRESENT - void construct( pointer p, value_type&& value ) {::new((void*)(p)) value_type(std::move(value));} -#endif - void construct( pointer p, const value_type& value ) {::new((void*)(p)) value_type(value);} -#endif // __TBB_ALLOCATOR_CONSTRUCT_VARIADIC - - //! Destroy value at location pointed to by p. - void destroy( pointer p ) {p->~value_type();} - - //! Returns current allocator - static malloc_type allocator_type() { - return internal::is_malloc_used_v3() ? standard : scalable; - } -}; - -#if _MSC_VER && !defined(__INTEL_COMPILER) - #pragma warning (pop) -#endif // warning 4100 is back - -//! Analogous to std::allocator, as defined in ISO C++ Standard, Section 20.4.1 -/** @ingroup memory_allocation */ -template<> -class tbb_allocator { -public: - typedef void* pointer; - typedef const void* const_pointer; - typedef void value_type; - template struct rebind { - typedef tbb_allocator other; - }; -}; - -template -inline bool operator==( const tbb_allocator&, const tbb_allocator& ) {return true;} - -template -inline bool operator!=( const tbb_allocator&, const tbb_allocator& ) {return false;} - -//! Meets "allocator" requirements of ISO C++ Standard, Section 20.1.5 -/** The class is an adapter over an actual allocator that fills the allocation - using memset function with template argument C as the value. - The members are ordered the same way they are in section 20.4.1 - of the ISO C++ standard. - @ingroup memory_allocation */ -template class Allocator = tbb_allocator> -class zero_allocator : public Allocator -{ -public: - typedef Allocator base_allocator_type; - typedef typename base_allocator_type::value_type value_type; - typedef typename base_allocator_type::pointer pointer; - typedef typename base_allocator_type::const_pointer const_pointer; - typedef typename base_allocator_type::reference reference; - typedef typename base_allocator_type::const_reference const_reference; - typedef typename base_allocator_type::size_type size_type; - typedef typename base_allocator_type::difference_type difference_type; - template struct rebind { - typedef zero_allocator other; - }; - - zero_allocator() throw() { } - zero_allocator(const zero_allocator &a) throw() : base_allocator_type( a ) { } - template - zero_allocator(const zero_allocator &a) throw() : base_allocator_type( Allocator( a ) ) { } - - pointer allocate(const size_type n, const void *hint = 0 ) { - pointer ptr = base_allocator_type::allocate( n, hint ); - std::memset( (void*) ptr, 0, n * sizeof(value_type) ); - return ptr; - } -}; - -//! Analogous to std::allocator, as defined in ISO C++ Standard, Section 20.4.1 -/** @ingroup memory_allocation */ -template class Allocator> -class zero_allocator : public Allocator { -public: - typedef Allocator base_allocator_type; - typedef typename base_allocator_type::value_type value_type; - typedef typename base_allocator_type::pointer pointer; - typedef typename base_allocator_type::const_pointer const_pointer; - template struct rebind { - typedef zero_allocator other; - }; -}; - -template class B1, typename T2, template class B2> -inline bool operator==( const zero_allocator &a, const zero_allocator &b) { - return static_cast< B1 >(a) == static_cast< B2 >(b); -} -template class B1, typename T2, template class B2> -inline bool operator!=( const zero_allocator &a, const zero_allocator &b) { - return static_cast< B1 >(a) != static_cast< B2 >(b); -} - -} // namespace tbb - -#endif /* __TBB_tbb_allocator_H */ diff --git a/inst/include/tbb/tbb_config.h b/inst/include/tbb/tbb_config.h deleted file mode 100644 index bdc8dcb9a..000000000 --- a/inst/include/tbb/tbb_config.h +++ /dev/null @@ -1,779 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_tbb_config_H -#define __TBB_tbb_config_H - -/** This header is supposed to contain macro definitions and C style comments only. - The macros defined here are intended to control such aspects of TBB build as - - presence of compiler features - - compilation modes - - feature sets - - known compiler/platform issues -**/ - -/* This macro marks incomplete code or comments describing ideas which are considered for the future. - * See also for plain comment with TODO and FIXME marks for small improvement opportunities. - */ -#define __TBB_TODO 0 - -/*Check which standard library we use on macOS*.*/ -/*__TBB_SYMBOL is defined only while processing exported symbols list where C++ is not allowed.*/ -#if !defined(__TBB_SYMBOL) && (__APPLE__ || __ANDROID__) - #include -#endif - -// note that when ICC or Clang is in use, __TBB_GCC_VERSION might not fully match -// the actual GCC version on the system. -#define __TBB_GCC_VERSION (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) - -// Since GNU libstdc++ does not have a convenient macro for its version, -// we rely on the version of GCC or the user-specified macro below. -// The format of TBB_USE_GLIBCXX_VERSION should match the __TBB_GCC_VERSION above, -// e.g. it should be set to 40902 for libstdc++ coming with GCC 4.9.2. -#ifdef TBB_USE_GLIBCXX_VERSION -#define __TBB_GLIBCXX_VERSION TBB_USE_GLIBCXX_VERSION -#else -#define __TBB_GLIBCXX_VERSION __TBB_GCC_VERSION -//TODO: analyze __GLIBCXX__ instead of __TBB_GCC_VERSION ? -#endif - -#if __clang__ - /**according to clang documentation version can be vendor specific **/ - #define __TBB_CLANG_VERSION (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) -#endif - -/** Target OS is either iOS* or iOS* simulator **/ -#if __ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__ - #define __TBB_IOS 1 -#endif - -/** Preprocessor symbols to determine HW architecture **/ - -#if _WIN32||_WIN64 -# if defined(_M_X64)||defined(__x86_64__) // the latter for MinGW support -# define __TBB_x86_64 1 -# elif defined(_M_IA64) -# define __TBB_ipf 1 -# elif defined(_M_IX86)||defined(__i386__) // the latter for MinGW support -# define __TBB_x86_32 1 -# else -# define __TBB_generic_arch 1 -# endif -#else /* Assume generic Unix */ -# if !__linux__ && !__APPLE__ -# define __TBB_generic_os 1 -# endif -# if __TBB_IOS -# define __TBB_generic_arch 1 -# elif __x86_64__ -# define __TBB_x86_64 1 -# elif __ia64__ -# define __TBB_ipf 1 -# elif __i386__||__i386 // __i386 is for Sun OS -# define __TBB_x86_32 1 -# else -# define __TBB_generic_arch 1 -# endif -#endif - -#if __MIC__ || __MIC2__ -#define __TBB_DEFINE_MIC 1 -#endif - -#define __TBB_TSX_AVAILABLE ((__TBB_x86_32 || __TBB_x86_64) && !__TBB_DEFINE_MIC) - -/** Presence of compiler features **/ - -#if __INTEL_COMPILER == 9999 && __INTEL_COMPILER_BUILD_DATE == 20110811 -/* Intel(R) Composer XE 2011 Update 6 incorrectly sets __INTEL_COMPILER. Fix it. */ - #undef __INTEL_COMPILER - #define __INTEL_COMPILER 1210 -#endif - -#if __TBB_GCC_VERSION >= 40400 && !defined(__INTEL_COMPILER) - /** warning suppression pragmas available in GCC since 4.4 **/ - #define __TBB_GCC_WARNING_SUPPRESSION_PRESENT 1 -#endif - -/* Select particular features of C++11 based on compiler version. - ICC 12.1 (Linux*), GCC 4.3 and higher, clang 2.9 and higher - set __GXX_EXPERIMENTAL_CXX0X__ in c++11 mode. - - Compilers that mimics other compilers (ICC, clang) must be processed before - compilers they mimic (GCC, MSVC). - - TODO: The following conditions should be extended when new compilers/runtimes - support added. - */ - -/** C++11 mode detection macros for Intel(R) C++ compiler (enabled by -std=c++XY option): - __INTEL_CXX11_MODE__ for version >=13.0 (not available for ICC 15.0 if -std=c++14 is used), - __STDC_HOSTED__ for version >=12.0 (useful only on Windows), - __GXX_EXPERIMENTAL_CXX0X__ for version >=12.0 on Linux and macOS. **/ -#if __INTEL_COMPILER && !__INTEL_CXX11_MODE__ - // __INTEL_CXX11_MODE__ is not set, try to deduce it - #define __INTEL_CXX11_MODE__ (__GXX_EXPERIMENTAL_CXX0X__ || (_MSC_VER && __STDC_HOSTED__)) -#endif - -// Intel(R) C++ Compiler offloading API to the Intel(R) Graphics Technology presence macro -// TODO: add support for ICC 15.00 _GFX_enqueue API and then decrease Intel compiler supported version -// TODO: add linux support and restict it with (__linux__ && __TBB_x86_64 && !__ANDROID__) macro -#if __INTEL_COMPILER >= 1600 && _WIN32 -#define __TBB_GFX_PRESENT 1 -#endif - -#if __INTEL_COMPILER && (!_MSC_VER || __INTEL_CXX11_MODE__) - // On Windows, C++11 features supported by Visual Studio 2010 and higher are enabled by default, - // so in absence of /Qstd= use MSVC branch for __TBB_CPP11_* detection. - // On other platforms, no -std= means C++03. - - #define __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT (__INTEL_CXX11_MODE__ && __VARIADIC_TEMPLATES) - // Both r-value reference support in compiler and std::move/std::forward - // presence in C++ standard library is checked. - #define __TBB_CPP11_RVALUE_REF_PRESENT ((_MSC_VER >= 1700 || __GXX_EXPERIMENTAL_CXX0X__ && (__TBB_GLIBCXX_VERSION >= 40500 || _LIBCPP_VERSION)) && __INTEL_COMPILER >= 1400) - #define __TBB_IMPLICIT_MOVE_PRESENT (__INTEL_CXX11_MODE__ && __INTEL_COMPILER >= 1400 && (_MSC_VER >= 1900 || __TBB_GCC_VERSION >= 40600 || __clang__)) - #if _MSC_VER >= 1600 - #define __TBB_EXCEPTION_PTR_PRESENT ( __INTEL_COMPILER > 1300 \ - /*ICC 12.1 Upd 10 and 13 beta Upd 2 fixed exception_ptr linking issue*/ \ - || (__INTEL_COMPILER == 1300 && __INTEL_COMPILER_BUILD_DATE >= 20120530) \ - || (__INTEL_COMPILER == 1210 && __INTEL_COMPILER_BUILD_DATE >= 20120410) ) - /** libstdc++ that comes with GCC 4.6 use C++11 features not supported by ICC 12.1. - * Because of that ICC 12.1 does not support C++11 mode with gcc 4.6 (or higher), - * and therefore does not define __GXX_EXPERIMENTAL_CXX0X__ macro **/ - #elif __TBB_GLIBCXX_VERSION >= 40404 && __TBB_GLIBCXX_VERSION < 40600 - #define __TBB_EXCEPTION_PTR_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __INTEL_COMPILER >= 1200) - #elif __TBB_GLIBCXX_VERSION >= 40600 - #define __TBB_EXCEPTION_PTR_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __INTEL_COMPILER >= 1300) - #elif _LIBCPP_VERSION - #define __TBB_EXCEPTION_PTR_PRESENT __GXX_EXPERIMENTAL_CXX0X__ - #else - #define __TBB_EXCEPTION_PTR_PRESENT 0 - #endif - #define __TBB_STATIC_ASSERT_PRESENT (__INTEL_CXX11_MODE__ || _MSC_VER >= 1600) - #define __TBB_CPP11_TUPLE_PRESENT (_MSC_VER >= 1600 || __GXX_EXPERIMENTAL_CXX0X__ && (__TBB_GLIBCXX_VERSION >= 40300 || _LIBCPP_VERSION)) - #if (__clang__ && __INTEL_COMPILER > 1400) - /* Older versions of Intel Compiler do not have __has_include */ - #if (__has_feature(__cxx_generalized_initializers__) && __has_include()) - #define __TBB_INITIALIZER_LISTS_PRESENT 1 - #endif - #else - #define __TBB_INITIALIZER_LISTS_PRESENT (__INTEL_CXX11_MODE__ && __INTEL_COMPILER >= 1400 && (_MSC_VER >= 1800 || __TBB_GLIBCXX_VERSION >= 40400 || _LIBCPP_VERSION)) - #endif - #define __TBB_CONSTEXPR_PRESENT (__INTEL_CXX11_MODE__ && __INTEL_COMPILER >= 1400) - #define __TBB_DEFAULTED_AND_DELETED_FUNC_PRESENT (__INTEL_CXX11_MODE__ && __INTEL_COMPILER >= 1200) - /** ICC seems to disable support of noexcept event in c++11 when compiling in compatibility mode for gcc <4.6 **/ - #define __TBB_NOEXCEPT_PRESENT (__INTEL_CXX11_MODE__ && __INTEL_COMPILER >= 1300 && (__TBB_GLIBCXX_VERSION >= 40600 || _LIBCPP_VERSION || _MSC_VER)) - #define __TBB_CPP11_STD_BEGIN_END_PRESENT (_MSC_VER >= 1700 || __GXX_EXPERIMENTAL_CXX0X__ && __INTEL_COMPILER >= 1310 && (__TBB_GLIBCXX_VERSION >= 40600 || _LIBCPP_VERSION)) - #define __TBB_CPP11_AUTO_PRESENT (_MSC_VER >= 1600 || __GXX_EXPERIMENTAL_CXX0X__ && __INTEL_COMPILER >= 1210) - #define __TBB_CPP11_DECLTYPE_PRESENT (_MSC_VER >= 1600 || __GXX_EXPERIMENTAL_CXX0X__ && __INTEL_COMPILER >= 1210) - #define __TBB_CPP11_LAMBDAS_PRESENT (__INTEL_CXX11_MODE__ && __INTEL_COMPILER >= 1200) - #define __TBB_CPP11_DEFAULT_FUNC_TEMPLATE_ARGS_PRESENT (_MSC_VER >= 1800 || __GXX_EXPERIMENTAL_CXX0X__ && __INTEL_COMPILER >= 1210) - #define __TBB_OVERRIDE_PRESENT (__INTEL_CXX11_MODE__ && __INTEL_COMPILER >= 1400) - #define __TBB_ALIGNAS_PRESENT (__INTEL_CXX11_MODE__ && __INTEL_COMPILER >= 1500) - #define __TBB_CPP11_TEMPLATE_ALIASES_PRESENT (__INTEL_CXX11_MODE__ && __INTEL_COMPILER >= 1210) -#elif __clang__ -/** TODO: these options need to be rechecked **/ -/** on macOS the only way to get C++11 is to use clang. For library features (e.g. exception_ptr) libc++ is also - * required. So there is no need to check GCC version for clang**/ - #define __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT __has_feature(__cxx_variadic_templates__) - #define __TBB_CPP11_RVALUE_REF_PRESENT (__has_feature(__cxx_rvalue_references__) && (_LIBCPP_VERSION || __TBB_GLIBCXX_VERSION >= 40500)) - #define __TBB_IMPLICIT_MOVE_PRESENT __has_feature(cxx_implicit_moves) -/** TODO: extend exception_ptr related conditions to cover libstdc++ **/ - #define __TBB_EXCEPTION_PTR_PRESENT (__cplusplus >= 201103L && (_LIBCPP_VERSION || __TBB_GLIBCXX_VERSION >= 40600)) - #define __TBB_STATIC_ASSERT_PRESENT __has_feature(__cxx_static_assert__) - /**Clang (preprocessor) has problems with dealing with expression having __has_include in #ifs - * used inside C++ code. (At least version that comes with OS X 10.8 : Apple LLVM version 4.2 (clang-425.0.28) (based on LLVM 3.2svn)) **/ - #if (__GXX_EXPERIMENTAL_CXX0X__ && __has_include()) - #define __TBB_CPP11_TUPLE_PRESENT 1 - #endif - #if (__has_feature(__cxx_generalized_initializers__) && __has_include()) - #define __TBB_INITIALIZER_LISTS_PRESENT 1 - #endif - #define __TBB_CONSTEXPR_PRESENT __has_feature(__cxx_constexpr__) - #define __TBB_DEFAULTED_AND_DELETED_FUNC_PRESENT (__has_feature(__cxx_defaulted_functions__) && __has_feature(__cxx_deleted_functions__)) - /**For some unknown reason __has_feature(__cxx_noexcept) does not yield true for all cases. Compiler bug ? **/ - #define __TBB_NOEXCEPT_PRESENT (__cplusplus >= 201103L) - #define __TBB_CPP11_STD_BEGIN_END_PRESENT (__has_feature(__cxx_range_for__) && (_LIBCPP_VERSION || __TBB_GLIBCXX_VERSION >= 40600)) - #define __TBB_CPP11_AUTO_PRESENT __has_feature(__cxx_auto_type__) - #define __TBB_CPP11_DECLTYPE_PRESENT __has_feature(__cxx_decltype__) - #define __TBB_CPP11_LAMBDAS_PRESENT __has_feature(cxx_lambdas) - #define __TBB_CPP11_DEFAULT_FUNC_TEMPLATE_ARGS_PRESENT __has_feature(cxx_default_function_template_args) - #define __TBB_OVERRIDE_PRESENT __has_feature(cxx_override_control) - #define __TBB_ALIGNAS_PRESENT __has_feature(cxx_alignas) - #define __TBB_CPP11_TEMPLATE_ALIASES_PRESENT __has_feature(cxx_alias_templates) -#elif __GNUC__ - #define __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT __GXX_EXPERIMENTAL_CXX0X__ - #define __TBB_CPP11_VARIADIC_FIXED_LENGTH_EXP_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GCC_VERSION >= 40700) - #define __TBB_CPP11_RVALUE_REF_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GCC_VERSION >= 40500) - #define __TBB_IMPLICIT_MOVE_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GCC_VERSION >= 40600) - /** __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4 here is a substitution for _GLIBCXX_ATOMIC_BUILTINS_4, which is a prerequisite - for exception_ptr but cannot be used in this file because it is defined in a header, not by the compiler. - If the compiler has no atomic intrinsics, the C++ library should not expect those as well. **/ - #define __TBB_EXCEPTION_PTR_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GCC_VERSION >= 40404 && __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4) - #define __TBB_STATIC_ASSERT_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GCC_VERSION >= 40300) - #define __TBB_CPP11_TUPLE_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GCC_VERSION >= 40300) - #define __TBB_INITIALIZER_LISTS_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GCC_VERSION >= 40400) - /** gcc seems have to support constexpr from 4.4 but tests in (test_atomic) seeming reasonable fail to compile prior 4.6**/ - #define __TBB_CONSTEXPR_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GCC_VERSION >= 40400) - #define __TBB_DEFAULTED_AND_DELETED_FUNC_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GCC_VERSION >= 40400) - #define __TBB_NOEXCEPT_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GCC_VERSION >= 40600) - #define __TBB_CPP11_STD_BEGIN_END_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GCC_VERSION >= 40600) - #define __TBB_CPP11_AUTO_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GCC_VERSION >= 40400) - #define __TBB_CPP11_DECLTYPE_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GCC_VERSION >= 40400) - #define __TBB_CPP11_LAMBDAS_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GCC_VERSION >= 40500) - #define __TBB_CPP11_DEFAULT_FUNC_TEMPLATE_ARGS_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GCC_VERSION >= 40300) - #define __TBB_OVERRIDE_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GCC_VERSION >= 40700) - #define __TBB_ALIGNAS_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GCC_VERSION >= 40800) - #define __TBB_CPP11_TEMPLATE_ALIASES_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GCC_VERSION >= 40700) -#elif _MSC_VER - // These definitions are also used with Intel Compiler in "default" mode; see a comment above. - - #define __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT (_MSC_VER >= 1800) - // Contains a workaround for ICC 13 - #define __TBB_CPP11_RVALUE_REF_PRESENT (_MSC_VER >= 1700 && (!__INTEL_COMPILER || __INTEL_COMPILER >= 1400)) - #define __TBB_IMPLICIT_MOVE_PRESENT (_MSC_VER >= 1900) - #define __TBB_EXCEPTION_PTR_PRESENT (_MSC_VER >= 1600) - #define __TBB_STATIC_ASSERT_PRESENT (_MSC_VER >= 1600) - #define __TBB_CPP11_TUPLE_PRESENT (_MSC_VER >= 1600) - #define __TBB_INITIALIZER_LISTS_PRESENT (_MSC_VER >= 1800) - #define __TBB_CONSTEXPR_PRESENT (_MSC_VER >= 1900) - #define __TBB_DEFAULTED_AND_DELETED_FUNC_PRESENT (_MSC_VER >= 1800) - #define __TBB_NOEXCEPT_PRESENT (_MSC_VER >= 1900) - #define __TBB_CPP11_STD_BEGIN_END_PRESENT (_MSC_VER >= 1700) - #define __TBB_CPP11_AUTO_PRESENT (_MSC_VER >= 1600) - #define __TBB_CPP11_DECLTYPE_PRESENT (_MSC_VER >= 1600) - #define __TBB_CPP11_LAMBDAS_PRESENT (_MSC_VER >= 1600) - #define __TBB_CPP11_DEFAULT_FUNC_TEMPLATE_ARGS_PRESENT (_MSC_VER >= 1800) - #define __TBB_OVERRIDE_PRESENT (_MSC_VER >= 1700) - #define __TBB_ALIGNAS_PRESENT (_MSC_VER >= 1900) - #define __TBB_CPP11_TEMPLATE_ALIASES_PRESENT (_MSC_VER >= 1800) -#else - #define __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT 0 - #define __TBB_CPP11_RVALUE_REF_PRESENT 0 - #define __TBB_IMPLICIT_MOVE_PRESENT 0 - #define __TBB_EXCEPTION_PTR_PRESENT 0 - #define __TBB_STATIC_ASSERT_PRESENT 0 - #define __TBB_CPP11_TUPLE_PRESENT 0 - #define __TBB_INITIALIZER_LISTS_PRESENT 0 - #define __TBB_CONSTEXPR_PRESENT 0 - #define __TBB_DEFAULTED_AND_DELETED_FUNC_PRESENT 0 - #define __TBB_NOEXCEPT_PRESENT 0 - #define __TBB_CPP11_STD_BEGIN_END_PRESENT 0 - #define __TBB_CPP11_AUTO_PRESENT 0 - #define __TBB_CPP11_DECLTYPE_PRESENT 0 - #define __TBB_CPP11_LAMBDAS_PRESENT 0 - #define __TBB_CPP11_DEFAULT_FUNC_TEMPLATE_ARGS_PRESENT 0 - #define __TBB_OVERRIDE_PRESENT 0 - #define __TBB_ALIGNAS_PRESENT 0 - #define __TBB_CPP11_TEMPLATE_ALIASES_PRESENT 0 -#endif - -// C++11 standard library features - -#ifndef __TBB_CPP11_VARIADIC_FIXED_LENGTH_EXP_PRESENT -#define __TBB_CPP11_VARIADIC_FIXED_LENGTH_EXP_PRESENT __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT -#endif -#define __TBB_CPP11_VARIADIC_TUPLE_PRESENT (!_MSC_VER || _MSC_VER >=1800) - -#define __TBB_CPP11_TYPE_PROPERTIES_PRESENT (_LIBCPP_VERSION || _MSC_VER >= 1700 || (__TBB_GLIBCXX_VERSION >= 50000 && (__GXX_EXPERIMENTAL_CXX0X__ || __cplusplus >= 201103L))) -#define __TBB_TR1_TYPE_PROPERTIES_IN_STD_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GLIBCXX_VERSION >= 40300 || _MSC_VER >= 1600) -// GCC supported some of type properties since 4.7 -#define __TBB_CPP11_IS_COPY_CONSTRUCTIBLE_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GLIBCXX_VERSION >= 40700 || __TBB_CPP11_TYPE_PROPERTIES_PRESENT) - -// In GCC, std::move_if_noexcept appeared later than noexcept -#define __TBB_MOVE_IF_NOEXCEPT_PRESENT (__TBB_NOEXCEPT_PRESENT && (__TBB_GLIBCXX_VERSION >= 40700 || _MSC_VER >= 1900 || _LIBCPP_VERSION)) -#define __TBB_ALLOCATOR_TRAITS_PRESENT (__cplusplus >= 201103L && _LIBCPP_VERSION || _MSC_VER >= 1700 || \ - __GXX_EXPERIMENTAL_CXX0X__ && __TBB_GLIBCXX_VERSION >= 40700 && !(__TBB_GLIBCXX_VERSION == 40700 && __TBB_DEFINE_MIC)) -#define __TBB_MAKE_EXCEPTION_PTR_PRESENT (__TBB_EXCEPTION_PTR_PRESENT && (_MSC_VER >= 1700 || __TBB_GLIBCXX_VERSION >= 40600 || _LIBCPP_VERSION)) - -#define __TBB_CPP11_FUTURE_PRESENT (_MSC_VER >= 1700 || __TBB_GLIBCXX_VERSION >= 40600 && _GXX_EXPERIMENTAL_CXX0X__ || _LIBCPP_VERSION) - -// std::swap is in only since C++11, though MSVC had it at least since VS2005 -#if _MSC_VER>=1400 || _LIBCPP_VERSION || __GXX_EXPERIMENTAL_CXX0X__ -#define __TBB_STD_SWAP_HEADER -#else -#define __TBB_STD_SWAP_HEADER -#endif - -//TODO: not clear how exactly this macro affects exception_ptr - investigate -// On linux ICC fails to find existing std::exception_ptr in libstdc++ without this define -#if __INTEL_COMPILER && __GNUC__ && __TBB_EXCEPTION_PTR_PRESENT && !defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_4) - #define __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4 1 -#endif - -// Work around a bug in MinGW32 -#if __MINGW32__ && __TBB_EXCEPTION_PTR_PRESENT && !defined(_GLIBCXX_ATOMIC_BUILTINS_4) - #define _GLIBCXX_ATOMIC_BUILTINS_4 -#endif - -#if __GNUC__ || __SUNPRO_CC || __IBMCPP__ - /* ICC defines __GNUC__ and so is covered */ - #define __TBB_ATTRIBUTE_ALIGNED_PRESENT 1 -#elif _MSC_VER && (_MSC_VER >= 1300 || __INTEL_COMPILER) - #define __TBB_DECLSPEC_ALIGN_PRESENT 1 -#endif - -/* Actually ICC supports gcc __sync_* intrinsics starting 11.1, - * but 64 bit support for 32 bit target comes in later ones*/ -/* TODO: change the version back to 4.1.2 once macro __TBB_WORD_SIZE become optional */ -/* Assumed that all clang versions have these gcc compatible intrinsics. */ -#if __TBB_GCC_VERSION >= 40306 || __INTEL_COMPILER >= 1200 || __clang__ - /** built-in atomics available in GCC since 4.1.2 **/ - #define __TBB_GCC_BUILTIN_ATOMICS_PRESENT 1 -#endif - -#if __INTEL_COMPILER >= 1200 - /** built-in C++11 style atomics available in ICC since 12.0 **/ - #define __TBB_ICC_BUILTIN_ATOMICS_PRESENT 1 -#endif - -#define __TBB_TSX_INTRINSICS_PRESENT ((__RTM__ || _MSC_VER>=1700 || __INTEL_COMPILER>=1300) && !__TBB_DEFINE_MIC && !__ANDROID__) - -/** Macro helpers **/ -#define __TBB_CONCAT_AUX(A,B) A##B -// The additional level of indirection is needed to expand macros A and B (not to get the AB macro). -// See [cpp.subst] and [cpp.concat] for more details. -#define __TBB_CONCAT(A,B) __TBB_CONCAT_AUX(A,B) -// The IGNORED argument and comma are needed to always have 2 arguments (even when A is empty). -#define __TBB_IS_MACRO_EMPTY(A,IGNORED) __TBB_CONCAT_AUX(__TBB_MACRO_EMPTY,A) -#define __TBB_MACRO_EMPTY 1 - -/** User controlled TBB features & modes **/ -#ifndef TBB_USE_DEBUG -/* -There are four cases that are supported: - 1. "_DEBUG is undefined" means "no debug"; - 2. "_DEBUG defined to something that is evaluated to 0 (the "garbage" is also evaluated to 0 [cpp.cond])" means "no debug"; - 3. "_DEBUG defined to something that is evaluated to non-zero value" means "debug"; - 4. "_DEBUG defined to nothing (empty)" means "debug". -*/ -#ifdef _DEBUG -// Check if _DEBUG is empty. -#define __TBB_IS__DEBUG_EMPTY (__TBB_IS_MACRO_EMPTY(_DEBUG,IGNORED)==__TBB_MACRO_EMPTY) -#if __TBB_IS__DEBUG_EMPTY -#define TBB_USE_DEBUG 1 -#else -#define TBB_USE_DEBUG _DEBUG -#endif /* __TBB_IS__DEBUG_EMPTY */ -#else -#define TBB_USE_DEBUG 0 -#endif -#endif /* TBB_USE_DEBUG */ - -#ifndef TBB_USE_ASSERT -#define TBB_USE_ASSERT TBB_USE_DEBUG -#endif /* TBB_USE_ASSERT */ - -#ifndef TBB_USE_THREADING_TOOLS -#define TBB_USE_THREADING_TOOLS TBB_USE_DEBUG -#endif /* TBB_USE_THREADING_TOOLS */ - -#ifndef TBB_USE_PERFORMANCE_WARNINGS -#ifdef TBB_PERFORMANCE_WARNINGS -#define TBB_USE_PERFORMANCE_WARNINGS TBB_PERFORMANCE_WARNINGS -#else -#define TBB_USE_PERFORMANCE_WARNINGS TBB_USE_DEBUG -#endif /* TBB_PEFORMANCE_WARNINGS */ -#endif /* TBB_USE_PERFORMANCE_WARNINGS */ - -#if __TBB_DEFINE_MIC - #if TBB_USE_EXCEPTIONS - #error The platform does not properly support exception handling. Please do not set TBB_USE_EXCEPTIONS macro or set it to 0. - #elif !defined(TBB_USE_EXCEPTIONS) - #define TBB_USE_EXCEPTIONS 0 - #endif -#elif !(__EXCEPTIONS || defined(_CPPUNWIND) || __SUNPRO_CC) - #if TBB_USE_EXCEPTIONS - #error Compilation settings do not support exception handling. Please do not set TBB_USE_EXCEPTIONS macro or set it to 0. - #elif !defined(TBB_USE_EXCEPTIONS) - #define TBB_USE_EXCEPTIONS 0 - #endif -#elif !defined(TBB_USE_EXCEPTIONS) - #define TBB_USE_EXCEPTIONS 1 -#endif - -#if __clang__ && !__INTEL_COMPILER -#define __TBB_USE_OPTIONAL_RTTI __has_feature(cxx_rtti) -#elif defined(_CPPRTTI) -#define __TBB_USE_OPTIONAL_RTTI 1 -#else -#define __TBB_USE_OPTIONAL_RTTI (__GXX_RTTI || __RTTI || __INTEL_RTTI__) -#endif - -#ifndef TBB_IMPLEMENT_CPP0X -/** By default, use C++11 classes if available **/ - #if __clang__ - /* Old versions of Intel Compiler do not have __has_include or cannot use it in #define */ - #if (__INTEL_COMPILER && (__INTEL_COMPILER < 1500 || __INTEL_COMPILER == 1500 && __INTEL_COMPILER_UPDATE <= 1)) - #define TBB_IMPLEMENT_CPP0X (__cplusplus < 201103L || !_LIBCPP_VERSION) - #else - #define TBB_IMPLEMENT_CPP0X (__cplusplus < 201103L || (!__has_include() && !__has_include())) - #endif - #elif __GNUC__ - #define TBB_IMPLEMENT_CPP0X (__TBB_GCC_VERSION < 40400 || !__GXX_EXPERIMENTAL_CXX0X__) - #elif _MSC_VER - #define TBB_IMPLEMENT_CPP0X (_MSC_VER < 1700) - #else - // TODO: Reconsider general approach to be more reliable, e.g. (!(__cplusplus >= 201103L && __ STDC_HOSTED__)) - #define TBB_IMPLEMENT_CPP0X (!__STDCPP_THREADS__) - #endif -#endif /* TBB_IMPLEMENT_CPP0X */ - -/* TBB_USE_CAPTURED_EXCEPTION should be explicitly set to either 0 or 1, as it is used as C++ const */ -#ifndef TBB_USE_CAPTURED_EXCEPTION - /** IA-64 architecture pre-built TBB binaries do not support exception_ptr. **/ - #if __TBB_EXCEPTION_PTR_PRESENT && !defined(__ia64__) - #define TBB_USE_CAPTURED_EXCEPTION 0 - #else - #define TBB_USE_CAPTURED_EXCEPTION 1 - #endif -#else /* defined TBB_USE_CAPTURED_EXCEPTION */ - #if !TBB_USE_CAPTURED_EXCEPTION && !__TBB_EXCEPTION_PTR_PRESENT - #error Current runtime does not support std::exception_ptr. Set TBB_USE_CAPTURED_EXCEPTION and make sure that your code is ready to catch tbb::captured_exception. - #endif -#endif /* defined TBB_USE_CAPTURED_EXCEPTION */ - -/** Check whether the request to use GCC atomics can be satisfied **/ -#if TBB_USE_GCC_BUILTINS && !__TBB_GCC_BUILTIN_ATOMICS_PRESENT - #error "GCC atomic built-ins are not supported." -#endif - -/** Internal TBB features & modes **/ - -/** __TBB_WEAK_SYMBOLS_PRESENT denotes that the system supports the weak symbol mechanism **/ -#ifndef __TBB_WEAK_SYMBOLS_PRESENT -#define __TBB_WEAK_SYMBOLS_PRESENT ( !_WIN32 && !__APPLE__ && !__sun && (__TBB_GCC_VERSION >= 40000 || __INTEL_COMPILER ) ) -#endif - -/** __TBB_DYNAMIC_LOAD_ENABLED describes the system possibility to load shared libraries at run time **/ -#ifndef __TBB_DYNAMIC_LOAD_ENABLED - #define __TBB_DYNAMIC_LOAD_ENABLED 1 -#endif - -/** __TBB_SOURCE_DIRECTLY_INCLUDED is a mode used in whitebox testing when - it's necessary to test internal functions not exported from TBB DLLs -**/ -#if (_WIN32||_WIN64) && (__TBB_SOURCE_DIRECTLY_INCLUDED || TBB_USE_PREVIEW_BINARY) - #define __TBB_NO_IMPLICIT_LINKAGE 1 - #define __TBBMALLOC_NO_IMPLICIT_LINKAGE 1 -#endif - -#ifndef __TBB_COUNT_TASK_NODES - #define __TBB_COUNT_TASK_NODES TBB_USE_ASSERT -#endif - -#ifndef __TBB_TASK_GROUP_CONTEXT - #define __TBB_TASK_GROUP_CONTEXT 1 -#endif /* __TBB_TASK_GROUP_CONTEXT */ - -#ifndef __TBB_SCHEDULER_OBSERVER - #define __TBB_SCHEDULER_OBSERVER 1 -#endif /* __TBB_SCHEDULER_OBSERVER */ - -#ifndef __TBB_FP_CONTEXT - #define __TBB_FP_CONTEXT __TBB_TASK_GROUP_CONTEXT -#endif /* __TBB_FP_CONTEXT */ - -#if __TBB_FP_CONTEXT && !__TBB_TASK_GROUP_CONTEXT - #error __TBB_FP_CONTEXT requires __TBB_TASK_GROUP_CONTEXT to be enabled -#endif - -#define __TBB_RECYCLE_TO_ENQUEUE __TBB_BUILD // keep non-official - -#ifndef __TBB_ARENA_OBSERVER - #define __TBB_ARENA_OBSERVER ((__TBB_BUILD||TBB_PREVIEW_LOCAL_OBSERVER)&& __TBB_SCHEDULER_OBSERVER) -#endif /* __TBB_ARENA_OBSERVER */ - -#ifndef __TBB_SLEEP_PERMISSION - #define __TBB_SLEEP_PERMISSION ((__TBB_CPF_BUILD||TBB_PREVIEW_LOCAL_OBSERVER)&& __TBB_SCHEDULER_OBSERVER) -#endif /* __TBB_SLEEP_PERMISSION */ - -#ifndef __TBB_TASK_ISOLATION - #define __TBB_TASK_ISOLATION (__TBB_CPF_BUILD||TBB_PREVIEW_TASK_ISOLATION) -#endif /* __TBB_TASK_ISOLATION */ - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE -// Users of flow-graph trace need to explicitly link against the preview library. This -// prevents the linker from implicitly linking an application with a preview version of -// TBB and unexpectedly bringing in other community preview features, which might change -// the behavior of the application. -#define __TBB_NO_IMPLICIT_LINKAGE 1 -#endif /* TBB_PREVIEW_FLOW_GRAPH_TRACE */ - -#ifndef __TBB_ITT_STRUCTURE_API -#define __TBB_ITT_STRUCTURE_API ( !__TBB_DEFINE_MIC && (__TBB_CPF_BUILD || TBB_PREVIEW_FLOW_GRAPH_TRACE) ) -#endif - -#if TBB_USE_EXCEPTIONS && !__TBB_TASK_GROUP_CONTEXT - #error TBB_USE_EXCEPTIONS requires __TBB_TASK_GROUP_CONTEXT to be enabled -#endif - -#ifndef __TBB_TASK_PRIORITY - #define __TBB_TASK_PRIORITY (__TBB_TASK_GROUP_CONTEXT) -#endif /* __TBB_TASK_PRIORITY */ - -#if __TBB_TASK_PRIORITY && !__TBB_TASK_GROUP_CONTEXT - #error __TBB_TASK_PRIORITY requires __TBB_TASK_GROUP_CONTEXT to be enabled -#endif - -#if TBB_PREVIEW_WAITING_FOR_WORKERS || __TBB_BUILD - #define __TBB_SUPPORTS_WORKERS_WAITING_IN_TERMINATE 1 -#endif - -#ifndef __TBB_ENQUEUE_ENFORCED_CONCURRENCY - #define __TBB_ENQUEUE_ENFORCED_CONCURRENCY 1 -#endif - -#if !defined(__TBB_SURVIVE_THREAD_SWITCH) && \ - (_WIN32 || _WIN64 || __APPLE__ || (__linux__ && !__ANDROID__)) - #define __TBB_SURVIVE_THREAD_SWITCH 1 -#endif /* __TBB_SURVIVE_THREAD_SWITCH */ - -#ifndef __TBB_DEFAULT_PARTITIONER -#if TBB_DEPRECATED -/** Default partitioner for parallel loop templates in TBB 1.0-2.1 */ -#define __TBB_DEFAULT_PARTITIONER tbb::simple_partitioner -#else -/** Default partitioner for parallel loop templates since TBB 2.2 */ -#define __TBB_DEFAULT_PARTITIONER tbb::auto_partitioner -#endif /* TBB_DEPRECATED */ -#endif /* !defined(__TBB_DEFAULT_PARTITIONER */ - -#ifndef __TBB_USE_PROPORTIONAL_SPLIT_IN_BLOCKED_RANGES -#define __TBB_USE_PROPORTIONAL_SPLIT_IN_BLOCKED_RANGES 1 -#endif - -#ifndef __TBB_ENABLE_RANGE_FEEDBACK -#define __TBB_ENABLE_RANGE_FEEDBACK 0 -#endif - -#ifdef _VARIADIC_MAX - #define __TBB_VARIADIC_MAX _VARIADIC_MAX -#else - #if _MSC_VER == 1700 - #define __TBB_VARIADIC_MAX 5 // VS11 setting, issue resolved in VS12 - #elif _MSC_VER == 1600 - #define __TBB_VARIADIC_MAX 10 // VS10 setting - #else - #define __TBB_VARIADIC_MAX 15 - #endif -#endif - -/** __TBB_WIN8UI_SUPPORT enables support of New Windows*8 Store Apps and limit a possibility to load - shared libraries at run time only from application container **/ -#if defined(WINAPI_FAMILY) && WINAPI_FAMILY == WINAPI_FAMILY_APP - #define __TBB_WIN8UI_SUPPORT 1 -#else - #define __TBB_WIN8UI_SUPPORT 0 -#endif - -/** Macros of the form __TBB_XXX_BROKEN denote known issues that are caused by - the bugs in compilers, standard or OS specific libraries. They should be - removed as soon as the corresponding bugs are fixed or the buggy OS/compiler - versions go out of the support list. -**/ - -#if __SIZEOF_POINTER__ < 8 && __ANDROID__ && __TBB_GCC_VERSION <= 40403 && !__GCC_HAVE_SYNC_COMPARE_AND_SWAP_8 - /** Necessary because on Android 8-byte CAS and F&A are not available for some processor architectures, - but no mandatory warning message appears from GCC 4.4.3. Instead, only a linkage error occurs when - these atomic operations are used (such as in unit test test_atomic.exe). **/ - #define __TBB_GCC_64BIT_ATOMIC_BUILTINS_BROKEN 1 -#elif __TBB_x86_32 && __TBB_GCC_VERSION == 40102 && ! __GNUC_RH_RELEASE__ - /** GCC 4.1.2 erroneously emit call to external function for 64 bit sync_ intrinsics. - However these functions are not defined anywhere. It seems that this problem was fixed later on - and RHEL got an updated version of gcc 4.1.2. **/ - #define __TBB_GCC_64BIT_ATOMIC_BUILTINS_BROKEN 1 -#endif - -#if __GNUC__ && __TBB_x86_64 && __INTEL_COMPILER == 1200 - #define __TBB_ICC_12_0_INL_ASM_FSTCW_BROKEN 1 -#endif - -#if _MSC_VER && __INTEL_COMPILER && (__INTEL_COMPILER<1110 || __INTEL_COMPILER==1110 && __INTEL_COMPILER_BUILD_DATE < 20091012) - /** Necessary to avoid ICL error (or warning in non-strict mode): - "exception specification for implicitly declared virtual destructor is - incompatible with that of overridden one". **/ - #define __TBB_DEFAULT_DTOR_THROW_SPEC_BROKEN 1 -#endif - -#if !__INTEL_COMPILER && (_MSC_VER && _MSC_VER < 1500 || __GNUC__ && __TBB_GCC_VERSION < 40102) - /** gcc 3.4.6 (and earlier) and VS2005 (and earlier) do not allow declaring template class as a friend - of classes defined in other namespaces. **/ - #define __TBB_TEMPLATE_FRIENDS_BROKEN 1 -#endif - -#if __GLIBC__==2 && __GLIBC_MINOR__==3 || (__APPLE__ && ( __INTEL_COMPILER==1200 && !TBB_USE_DEBUG)) - /** Macro controlling EH usages in TBB tests. - Some older versions of glibc crash when exception handling happens concurrently. **/ - #define __TBB_THROW_ACROSS_MODULE_BOUNDARY_BROKEN 1 -#endif - -#if (_WIN32||_WIN64) && __INTEL_COMPILER == 1110 - /** That's a bug in Intel(R) C++ Compiler 11.1.044/IA-32 architecture/Windows* OS, that leads to a worker thread crash on the thread's startup. **/ - #define __TBB_ICL_11_1_CODE_GEN_BROKEN 1 -#endif - -#if __clang__ || (__GNUC__==3 && __GNUC_MINOR__==3 && !defined(__INTEL_COMPILER)) - /** Bugs with access to nested classes declared in protected area */ - #define __TBB_PROTECTED_NESTED_CLASS_BROKEN 1 -#endif - -#if __MINGW32__ && __TBB_GCC_VERSION < 40200 - /** MinGW has a bug with stack alignment for routines invoked from MS RTLs. - Since GCC 4.2, the bug can be worked around via a special attribute. **/ - #define __TBB_SSE_STACK_ALIGNMENT_BROKEN 1 -#endif - -#if __TBB_GCC_VERSION==40300 && !__INTEL_COMPILER && !__clang__ - /* GCC of this version may rashly ignore control dependencies */ - #define __TBB_GCC_OPTIMIZER_ORDERING_BROKEN 1 -#endif - -#if __FreeBSD__ - /** A bug in FreeBSD 8.0 results in kernel panic when there is contention - on a mutex created with this attribute. **/ - #define __TBB_PRIO_INHERIT_BROKEN 1 - - /** A bug in FreeBSD 8.0 results in test hanging when an exception occurs - during (concurrent?) object construction by means of placement new operator. **/ - #define __TBB_PLACEMENT_NEW_EXCEPTION_SAFETY_BROKEN 1 -#endif /* __FreeBSD__ */ - -#if (__linux__ || __APPLE__) && __i386__ && defined(__INTEL_COMPILER) - /** The Intel(R) C++ Compiler for IA-32 architecture (Linux* OS|macOS) crashes or generates - incorrect code when __asm__ arguments have a cast to volatile. **/ - #define __TBB_ICC_ASM_VOLATILE_BROKEN 1 -#endif - -#if !__INTEL_COMPILER && (_MSC_VER || __GNUC__==3 && __GNUC_MINOR__<=2) - /** Bug in GCC 3.2 and MSVC compilers that sometimes return 0 for __alignof(T) - when T has not yet been instantiated. **/ - #define __TBB_ALIGNOF_NOT_INSTANTIATED_TYPES_BROKEN 1 -#endif - -#if __TBB_DEFINE_MIC - /** Main thread and user's thread have different default thread affinity masks. **/ - #define __TBB_MAIN_THREAD_AFFINITY_BROKEN 1 -#endif - -#if __GXX_EXPERIMENTAL_CXX0X__ && !defined(__EXCEPTIONS) && \ - ((!__INTEL_COMPILER && !__clang__ && (__TBB_GCC_VERSION>=40400 && __TBB_GCC_VERSION<40600)) || \ - (__INTEL_COMPILER<=1400 && (__TBB_GLIBCXX_VERSION>=40400 && __TBB_GLIBCXX_VERSION<=40801))) -/* There is an issue for specific GCC toolchain when C++11 is enabled - and exceptions are disabled: - exceprion_ptr.h/nested_exception.h use throw unconditionally. - GCC can ignore 'throw' since 4.6; but with ICC the issue still exists. - */ - #define __TBB_LIBSTDCPP_EXCEPTION_HEADERS_BROKEN 1 -#endif - -#if __INTEL_COMPILER==1300 && __TBB_GLIBCXX_VERSION>=40700 && defined(__GXX_EXPERIMENTAL_CXX0X__) -/* Some C++11 features used inside libstdc++ are not supported by Intel compiler. - * Checking version of gcc instead of libstdc++ because - * - they are directly connected, - * - for now it is not possible to check version of any standard library in this file - */ - #define __TBB_ICC_13_0_CPP11_STDLIB_SUPPORT_BROKEN 1 -#endif - -#if (__GNUC__==4 && __GNUC_MINOR__==4 ) && !defined(__INTEL_COMPILER) && !defined(__clang__) - /** excessive warnings related to strict aliasing rules in GCC 4.4 **/ - #define __TBB_GCC_STRICT_ALIASING_BROKEN 1 - /* topical remedy: #pragma GCC diagnostic ignored "-Wstrict-aliasing" */ - #if !__TBB_GCC_WARNING_SUPPRESSION_PRESENT - #error Warning suppression is not supported, while should. - #endif -#endif - -/*In a PIC mode some versions of GCC 4.1.2 generate incorrect inlined code for 8 byte __sync_val_compare_and_swap intrinsic */ -#if __TBB_GCC_VERSION == 40102 && __PIC__ && !defined(__INTEL_COMPILER) && !defined(__clang__) - #define __TBB_GCC_CAS8_BUILTIN_INLINING_BROKEN 1 -#endif - -#if __TBB_x86_32 && ( __INTEL_COMPILER || (__GNUC__==5 && __GNUC_MINOR__>=2 && __GXX_EXPERIMENTAL_CXX0X__) \ - || (__GNUC__==3 && __GNUC_MINOR__==3) || (__MINGW32__ && __GNUC__==4 && __GNUC_MINOR__==5) || __SUNPRO_CC ) - // Some compilers for IA-32 architecture fail to provide 8-byte alignment of objects on the stack, - // even if the object specifies 8-byte alignment. On such platforms, the implementation - // of 64 bit atomics for IA-32 architecture (e.g. atomic) use different tactics - // depending upon whether the object is properly aligned or not. - #define __TBB_FORCE_64BIT_ALIGNMENT_BROKEN 1 -#else - // Define to 0 explicitly because the macro is used in a compiled code of test_atomic - #define __TBB_FORCE_64BIT_ALIGNMENT_BROKEN 0 -#endif - -#if __GNUC__ && !__INTEL_COMPILER && !__clang__ && __TBB_DEFAULTED_AND_DELETED_FUNC_PRESENT && __TBB_GCC_VERSION < 40700 - #define __TBB_ZERO_INIT_WITH_DEFAULTED_CTOR_BROKEN 1 -#endif - -#if _MSC_VER && _MSC_VER <= 1800 && !__INTEL_COMPILER - // With MSVC, when an array is passed by const reference to a template function, - // constness from the function parameter may get propagated to the template parameter. - #define __TBB_CONST_REF_TO_ARRAY_TEMPLATE_PARAM_BROKEN 1 -#endif - -// A compiler bug: a disabled copy constructor prevents use of the moving constructor -#define __TBB_IF_NO_COPY_CTOR_MOVE_SEMANTICS_BROKEN (_MSC_VER && (__INTEL_COMPILER >= 1300 && __INTEL_COMPILER <= 1310) && !__INTEL_CXX11_MODE__) - -#define __TBB_CPP11_DECLVAL_BROKEN (_MSC_VER == 1600 || (__GNUC__ && __TBB_GCC_VERSION < 40500) ) - -// Intel C++ compiler has difficulties with copying std::pair with VC11 std::reference_wrapper being a const member -#define __TBB_COPY_FROM_NON_CONST_REF_BROKEN (_MSC_VER == 1700 && __INTEL_COMPILER && __INTEL_COMPILER < 1600) - -// The implicit upcasting of the tuple of a reference of a derived class to a base class fails on icc 13.X if the system's gcc environment is 4.8 -// Also in gcc 4.4 standard library the implementation of the tuple<&> conversion (tuple a = tuple, B is inherited from A) is broken. -#if __GXX_EXPERIMENTAL_CXX0X__ && ((__INTEL_COMPILER >=1300 && __INTEL_COMPILER <=1310 && __TBB_GLIBCXX_VERSION>=40700) || (__TBB_GLIBCXX_VERSION < 40500)) -#define __TBB_UPCAST_OF_TUPLE_OF_REF_BROKEN 1 -#endif - -/** End of __TBB_XXX_BROKEN macro section **/ - -#if defined(_MSC_VER) && _MSC_VER>=1500 && !defined(__INTEL_COMPILER) - // A macro to suppress erroneous or benign "unreachable code" MSVC warning (4702) - #define __TBB_MSVC_UNREACHABLE_CODE_IGNORED 1 -#endif - -#define __TBB_ATOMIC_CTORS (__TBB_CONSTEXPR_PRESENT && __TBB_DEFAULTED_AND_DELETED_FUNC_PRESENT && (!__TBB_ZERO_INIT_WITH_DEFAULTED_CTOR_BROKEN)) - -// Many OS versions (Android 4.0.[0-3] for example) need workaround for dlopen to avoid non-recursive loader lock hang -// Setting the workaround for all compile targets ($APP_PLATFORM) below Android 4.4 (android-19) -#if __ANDROID__ -#include -#define __TBB_USE_DLOPEN_REENTRANCY_WORKAROUND (__ANDROID_API__ < 19) -#endif - -#define __TBB_ALLOCATOR_CONSTRUCT_VARIADIC (__TBB_CPP11_VARIADIC_TEMPLATES_PRESENT && __TBB_CPP11_RVALUE_REF_PRESENT) - -#define __TBB_VARIADIC_PARALLEL_INVOKE (TBB_PREVIEW_VARIADIC_PARALLEL_INVOKE && __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT && __TBB_CPP11_RVALUE_REF_PRESENT) -#define __TBB_FLOW_GRAPH_CPP11_FEATURES (__TBB_CPP11_VARIADIC_TEMPLATES_PRESENT \ - && __TBB_CPP11_RVALUE_REF_PRESENT && __TBB_CPP11_AUTO_PRESENT) \ - && __TBB_CPP11_VARIADIC_TUPLE_PRESENT && __TBB_CPP11_DEFAULT_FUNC_TEMPLATE_ARGS_PRESENT \ - && !__TBB_UPCAST_OF_TUPLE_OF_REF_BROKEN -#define __TBB_PREVIEW_STREAMING_NODE (__TBB_CPP11_VARIADIC_FIXED_LENGTH_EXP_PRESENT && __TBB_FLOW_GRAPH_CPP11_FEATURES \ - && TBB_PREVIEW_FLOW_GRAPH_NODES && !TBB_IMPLEMENT_CPP0X && !__TBB_UPCAST_OF_TUPLE_OF_REF_BROKEN) -#define __TBB_PREVIEW_OPENCL_NODE (__TBB_PREVIEW_STREAMING_NODE && __TBB_CPP11_TEMPLATE_ALIASES_PRESENT) -#define __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING (TBB_PREVIEW_FLOW_GRAPH_FEATURES || __TBB_PREVIEW_OPENCL_NODE) -#define __TBB_PREVIEW_ASYNC_MSG (TBB_PREVIEW_FLOW_GRAPH_FEATURES && __TBB_FLOW_GRAPH_CPP11_FEATURES) - -#define __TBB_PREVIEW_GFX_FACTORY (__TBB_GFX_PRESENT && TBB_PREVIEW_FLOW_GRAPH_FEATURES && !__TBB_MIC_OFFLOAD \ - && __TBB_FLOW_GRAPH_CPP11_FEATURES && __TBB_CPP11_TEMPLATE_ALIASES_PRESENT \ - && __TBB_CPP11_FUTURE_PRESENT) -#endif /* __TBB_tbb_config_H */ diff --git a/inst/include/tbb/tbb_disable_exceptions.h b/inst/include/tbb/tbb_disable_exceptions.h deleted file mode 100644 index a1d28a60b..000000000 --- a/inst/include/tbb/tbb_disable_exceptions.h +++ /dev/null @@ -1,35 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -//! To disable use of exceptions, include this header before any other header file from the library. - -//! The macro that prevents use of exceptions in the library files -#undef TBB_USE_EXCEPTIONS -#define TBB_USE_EXCEPTIONS 0 - -//! Prevent compilers from issuing exception related warnings. -/** Note that the warnings are suppressed for all the code after this header is included. */ -#if _MSC_VER -#if __INTEL_COMPILER - #pragma warning (disable: 583) -#else - #pragma warning (disable: 4530 4577) -#endif -#endif diff --git a/inst/include/tbb/tbb_exception.h b/inst/include/tbb/tbb_exception.h deleted file mode 100644 index 1c8433096..000000000 --- a/inst/include/tbb/tbb_exception.h +++ /dev/null @@ -1,371 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_exception_H -#define __TBB_exception_H - -#include "tbb_stddef.h" - -#if !TBB_USE_EXCEPTIONS && _MSC_VER - // Suppress "C++ exception handler used, but unwind semantics are not enabled" warning in STL headers - #pragma warning (push) - #pragma warning (disable: 4530) -#endif - -#include -#include //required for bad_alloc definition, operators new -#include // required to construct std exception classes - -#if !TBB_USE_EXCEPTIONS && _MSC_VER - #pragma warning (pop) -#endif - -namespace tbb { - -//! Exception for concurrent containers -class bad_last_alloc : public std::bad_alloc { -public: - const char* what() const throw() __TBB_override; -#if __TBB_DEFAULT_DTOR_THROW_SPEC_BROKEN - ~bad_last_alloc() throw() __TBB_override {} -#endif -}; - -//! Exception for PPL locks -class improper_lock : public std::exception { -public: - const char* what() const throw() __TBB_override; -}; - -//! Exception for user-initiated abort -class user_abort : public std::exception { -public: - const char* what() const throw() __TBB_override; -}; - -//! Exception for missing wait on structured_task_group -class missing_wait : public std::exception { -public: - const char* what() const throw() __TBB_override; -}; - -//! Exception for repeated scheduling of the same task_handle -class invalid_multiple_scheduling : public std::exception { -public: - const char* what() const throw() __TBB_override; -}; - -namespace internal { -//! Obsolete -void __TBB_EXPORTED_FUNC throw_bad_last_alloc_exception_v4(); - -enum exception_id { - eid_bad_alloc = 1, - eid_bad_last_alloc, - eid_nonpositive_step, - eid_out_of_range, - eid_segment_range_error, - eid_index_range_error, - eid_missing_wait, - eid_invalid_multiple_scheduling, - eid_improper_lock, - eid_possible_deadlock, - eid_operation_not_permitted, - eid_condvar_wait_failed, - eid_invalid_load_factor, - eid_reserved, // free slot for backward compatibility, can be reused. - eid_invalid_swap, - eid_reservation_length_error, - eid_invalid_key, - eid_user_abort, - eid_reserved1, -#if __TBB_SUPPORTS_WORKERS_WAITING_IN_TERMINATE - // This id is used only from inside the library and only for support of CPF functionality. - // So, if we drop the functionality, eid_reserved1 can be safely renamed and reused. - eid_blocking_thread_join_impossible = eid_reserved1, -#endif - eid_bad_tagged_msg_cast, - //! The last enumerator tracks the number of defined IDs. It must remain the last one. - /** When adding new IDs, place them immediately _before_ this comment (that is - _after_ all the existing IDs. NEVER insert new IDs between the existing ones. **/ - eid_max -}; - -//! Gathers all throw operators in one place. -/** Its purpose is to minimize code bloat that can be caused by throw operators - scattered in multiple places, especially in templates. **/ -void __TBB_EXPORTED_FUNC throw_exception_v4 ( exception_id ); - -//! Versionless convenience wrapper for throw_exception_v4() -inline void throw_exception ( exception_id eid ) { throw_exception_v4(eid); } - -} // namespace internal -} // namespace tbb - -#if __TBB_TASK_GROUP_CONTEXT -#include "tbb_allocator.h" -#include //for typeid - -namespace tbb { - -//! Interface to be implemented by all exceptions TBB recognizes and propagates across the threads. -/** If an unhandled exception of the type derived from tbb::tbb_exception is intercepted - by the TBB scheduler in one of the worker threads, it is delivered to and re-thrown in - the root thread. The root thread is the thread that has started the outermost algorithm - or root task sharing the same task_group_context with the guilty algorithm/task (the one - that threw the exception first). - - Note: when documentation mentions workers with respect to exception handling, - masters are implied as well, because they are completely equivalent in this context. - Consequently a root thread can be master or worker thread. - - NOTE: In case of nested algorithms or complex task hierarchies when the nested - levels share (explicitly or by means of implicit inheritance) the task group - context of the outermost level, the exception may be (re-)thrown multiple times - (ultimately - in each worker on each nesting level) before reaching the root - thread at the outermost level. IMPORTANT: if you intercept an exception derived - from this class on a nested level, you must re-throw it in the catch block by means - of the "throw;" operator. - - TBB provides two implementations of this interface: tbb::captured_exception and - template class tbb::movable_exception. See their declarations for more info. **/ -class tbb_exception : public std::exception -{ - /** No operator new is provided because the TBB usage model assumes dynamic - creation of the TBB exception objects only by means of applying move() - operation on an exception thrown out of TBB scheduler. **/ - void* operator new ( size_t ); - -public: -#if __clang__ - // At -O3 or even -O2 optimization level, Clang may fully throw away an empty destructor - // of tbb_exception from destructors of derived classes. As a result, it does not create - // vtable for tbb_exception, which is a required part of TBB binary interface. - // Making the destructor non-empty (with just a semicolon) prevents that optimization. - ~tbb_exception() throw() { /* keep the semicolon! */ ; } -#endif - - //! Creates and returns pointer to the deep copy of this exception object. - /** Move semantics is allowed. **/ - virtual tbb_exception* move () throw() = 0; - - //! Destroys objects created by the move() method. - /** Frees memory and calls destructor for this exception object. - Can and must be used only on objects created by the move method. **/ - virtual void destroy () throw() = 0; - - //! Throws this exception object. - /** Make sure that if you have several levels of derivation from this interface - you implement or override this method on the most derived level. The implementation - is as simple as "throw *this;". Failure to do this will result in exception - of a base class type being thrown. **/ - virtual void throw_self () = 0; - - //! Returns RTTI name of the originally intercepted exception - virtual const char* name() const throw() = 0; - - //! Returns the result of originally intercepted exception's what() method. - virtual const char* what() const throw() __TBB_override = 0; - - /** Operator delete is provided only to allow using existing smart pointers - with TBB exception objects obtained as the result of applying move() - operation on an exception thrown out of TBB scheduler. - - When overriding method move() make sure to override operator delete as well - if memory is allocated not by TBB's scalable allocator. **/ - void operator delete ( void* p ) { - internal::deallocate_via_handler_v3(p); - } -}; - -//! This class is used by TBB to propagate information about unhandled exceptions into the root thread. -/** Exception of this type is thrown by TBB in the root thread (thread that started a parallel - algorithm ) if an unhandled exception was intercepted during the algorithm execution in one - of the workers. - \sa tbb::tbb_exception **/ -class captured_exception : public tbb_exception -{ -public: - captured_exception ( const captured_exception& src ) - : tbb_exception(src), my_dynamic(false) - { - set(src.my_exception_name, src.my_exception_info); - } - - captured_exception ( const char* name_, const char* info ) - : my_dynamic(false) - { - set(name_, info); - } - - __TBB_EXPORTED_METHOD ~captured_exception () throw(); - - captured_exception& operator= ( const captured_exception& src ) { - if ( this != &src ) { - clear(); - set(src.my_exception_name, src.my_exception_info); - } - return *this; - } - - captured_exception* __TBB_EXPORTED_METHOD move () throw() __TBB_override; - - void __TBB_EXPORTED_METHOD destroy () throw() __TBB_override; - - void throw_self () __TBB_override { __TBB_THROW(*this); } - - const char* __TBB_EXPORTED_METHOD name() const throw() __TBB_override; - - const char* __TBB_EXPORTED_METHOD what() const throw() __TBB_override; - - void __TBB_EXPORTED_METHOD set ( const char* name, const char* info ) throw(); - void __TBB_EXPORTED_METHOD clear () throw(); - -private: - //! Used only by method clone(). - captured_exception() {} - - //! Functionally equivalent to {captured_exception e(name,info); return e.clone();} - static captured_exception* allocate ( const char* name, const char* info ); - - bool my_dynamic; - const char* my_exception_name; - const char* my_exception_info; -}; - -//! Template that can be used to implement exception that transfers arbitrary ExceptionData to the root thread -/** Code using TBB can instantiate this template with an arbitrary ExceptionData type - and throw this exception object. Such exceptions are intercepted by the TBB scheduler - and delivered to the root thread (). - \sa tbb::tbb_exception **/ -template -class movable_exception : public tbb_exception -{ - typedef movable_exception self_type; - -public: - movable_exception ( const ExceptionData& data_ ) - : my_exception_data(data_) - , my_dynamic(false) - , my_exception_name( -#if TBB_USE_EXCEPTIONS - typeid(self_type).name() -#else /* !TBB_USE_EXCEPTIONS */ - "movable_exception" -#endif /* !TBB_USE_EXCEPTIONS */ - ) - {} - - movable_exception ( const movable_exception& src ) throw () - : tbb_exception(src) - , my_exception_data(src.my_exception_data) - , my_dynamic(false) - , my_exception_name(src.my_exception_name) - {} - - ~movable_exception () throw() {} - - const movable_exception& operator= ( const movable_exception& src ) { - if ( this != &src ) { - my_exception_data = src.my_exception_data; - my_exception_name = src.my_exception_name; - } - return *this; - } - - ExceptionData& data () throw() { return my_exception_data; } - - const ExceptionData& data () const throw() { return my_exception_data; } - - const char* name () const throw() __TBB_override { return my_exception_name; } - - const char* what () const throw() __TBB_override { return "tbb::movable_exception"; } - - movable_exception* move () throw() __TBB_override { - void* e = internal::allocate_via_handler_v3(sizeof(movable_exception)); - if ( e ) { - ::new (e) movable_exception(*this); - ((movable_exception*)e)->my_dynamic = true; - } - return (movable_exception*)e; - } - void destroy () throw() __TBB_override { - __TBB_ASSERT ( my_dynamic, "Method destroy can be called only on dynamically allocated movable_exceptions" ); - if ( my_dynamic ) { - this->~movable_exception(); - internal::deallocate_via_handler_v3(this); - } - } - void throw_self () __TBB_override { __TBB_THROW( *this ); } - -protected: - //! User data - ExceptionData my_exception_data; - -private: - //! Flag specifying whether this object has been dynamically allocated (by the move method) - bool my_dynamic; - - //! RTTI name of this class - /** We rely on the fact that RTTI names are static string constants. **/ - const char* my_exception_name; -}; - -#if !TBB_USE_CAPTURED_EXCEPTION -namespace internal { - -//! Exception container that preserves the exact copy of the original exception -/** This class can be used only when the appropriate runtime support (mandated - by C++0x) is present **/ -class tbb_exception_ptr { - std::exception_ptr my_ptr; - -public: - static tbb_exception_ptr* allocate (); - static tbb_exception_ptr* allocate ( const tbb_exception& tag ); - //! This overload uses move semantics (i.e. it empties src) - static tbb_exception_ptr* allocate ( captured_exception& src ); - - //! Destroys this objects - /** Note that objects of this type can be created only by the allocate() method. **/ - void destroy () throw(); - - //! Throws the contained exception . - void throw_self () { std::rethrow_exception(my_ptr); } - -private: - tbb_exception_ptr ( const std::exception_ptr& src ) : my_ptr(src) {} - tbb_exception_ptr ( const captured_exception& src ) : - #if __TBB_MAKE_EXCEPTION_PTR_PRESENT - my_ptr(std::make_exception_ptr(src)) // the final function name in C++11 - #else - my_ptr(std::copy_exception(src)) // early C++0x drafts name - #endif - {} -}; // class tbb::internal::tbb_exception_ptr - -} // namespace internal -#endif /* !TBB_USE_CAPTURED_EXCEPTION */ - -} // namespace tbb - -#endif /* __TBB_TASK_GROUP_CONTEXT */ - -#endif /* __TBB_exception_H */ diff --git a/inst/include/tbb/tbb_machine.h b/inst/include/tbb/tbb_machine.h deleted file mode 100644 index 68d1d5d00..000000000 --- a/inst/include/tbb/tbb_machine.h +++ /dev/null @@ -1,974 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_machine_H -#define __TBB_machine_H - -/** This header provides basic platform abstraction layer by hooking up appropriate - architecture/OS/compiler specific headers from the /include/tbb/machine directory. - If a plug-in header does not implement all the required APIs, it must specify - the missing ones by setting one or more of the following macros: - - __TBB_USE_GENERIC_PART_WORD_CAS - __TBB_USE_GENERIC_PART_WORD_FETCH_ADD - __TBB_USE_GENERIC_PART_WORD_FETCH_STORE - __TBB_USE_GENERIC_FETCH_ADD - __TBB_USE_GENERIC_FETCH_STORE - __TBB_USE_GENERIC_DWORD_FETCH_ADD - __TBB_USE_GENERIC_DWORD_FETCH_STORE - __TBB_USE_GENERIC_HALF_FENCED_LOAD_STORE - __TBB_USE_GENERIC_SEQUENTIAL_CONSISTENCY_LOAD_STORE - __TBB_USE_GENERIC_RELAXED_LOAD_STORE - __TBB_USE_FETCHSTORE_AS_FULL_FENCED_STORE - - In this case tbb_machine.h will add missing functionality based on a minimal set - of APIs that are required to be implemented by all plug-n headers as described - further. - Note that these generic implementations may be sub-optimal for a particular - architecture, and thus should be relied upon only after careful evaluation - or as the last resort. - - Additionally __TBB_64BIT_ATOMICS can be set to 0 on a 32-bit architecture to - indicate that the port is not going to support double word atomics. It may also - be set to 1 explicitly, though normally this is not necessary as tbb_machine.h - will set it automatically. - - __TBB_ENDIANNESS macro can be defined by the implementation as well. - It is used only if __TBB_USE_GENERIC_PART_WORD_CAS is set (or for testing), - and must specify the layout of aligned 16-bit and 32-bit data anywhere within a process - (while the details of unaligned 16-bit or 32-bit data or of 64-bit data are irrelevant). - The layout must be the same at all relevant memory locations within the current process; - in case of page-specific endianness, one endianness must be kept "out of sight". - Possible settings, reflecting hardware and possibly O.S. convention, are: - - __TBB_ENDIAN_BIG for big-endian data, - - __TBB_ENDIAN_LITTLE for little-endian data, - - __TBB_ENDIAN_DETECT for run-time detection iff exactly one of the above, - - __TBB_ENDIAN_UNSUPPORTED to prevent undefined behavior if none of the above. - - Prerequisites for each architecture port - ---------------------------------------- - The following functions and macros have no generic implementation. Therefore they must be - implemented in each machine architecture specific header either as a conventional - function or as a functional macro. - - __TBB_WORDSIZE - This is the size of machine word in bytes, i.e. for 32 bit systems it - should be defined to 4. - - __TBB_Yield() - Signals OS that the current thread is willing to relinquish the remainder - of its time quantum. - - __TBB_full_memory_fence() - Must prevent all memory operations from being reordered across it (both - by hardware and compiler). All such fences must be totally ordered (or - sequentially consistent). - - __TBB_machine_cmpswp4( volatile void *ptr, int32_t value, int32_t comparand ) - Must be provided if __TBB_USE_FENCED_ATOMICS is not set. - - __TBB_machine_cmpswp8( volatile void *ptr, int32_t value, int64_t comparand ) - Must be provided for 64-bit architectures if __TBB_USE_FENCED_ATOMICS is not set, - and for 32-bit architectures if __TBB_64BIT_ATOMICS is set - - __TBB_machine_(...), where - = {cmpswp, fetchadd, fetchstore} - = {1, 2, 4, 8} - = {full_fence, acquire, release, relaxed} - Must be provided if __TBB_USE_FENCED_ATOMICS is set. - - __TBB_control_consistency_helper() - Bridges the memory-semantics gap between architectures providing only - implicit C++0x "consume" semantics (like Power Architecture) and those - also implicitly obeying control dependencies (like IA-64 architecture). - It must be used only in conditional code where the condition is itself - data-dependent, and will then make subsequent code behave as if the - original data dependency were acquired. - It needs only a compiler fence where implied by the architecture - either specifically (like IA-64 architecture) or because generally stronger - "acquire" semantics are enforced (like x86). - It is always valid, though potentially suboptimal, to replace - control with acquire on the load and then remove the helper. - - __TBB_acquire_consistency_helper(), __TBB_release_consistency_helper() - Must be provided if __TBB_USE_GENERIC_HALF_FENCED_LOAD_STORE is set. - Enforce acquire and release semantics in generic implementations of fenced - store and load operations. Depending on the particular architecture/compiler - combination they may be a hardware fence, a compiler fence, both or nothing. - **/ - -#include "tbb_stddef.h" - -namespace tbb { -namespace internal { //< @cond INTERNAL - -//////////////////////////////////////////////////////////////////////////////// -// Overridable helpers declarations -// -// A machine/*.h file may choose to define these templates, otherwise it must -// request default implementation by setting appropriate __TBB_USE_GENERIC_XXX macro(s). -// -template -struct machine_load_store; - -template -struct machine_load_store_relaxed; - -template -struct machine_load_store_seq_cst; -// -// End of overridable helpers declarations -//////////////////////////////////////////////////////////////////////////////// - -template struct atomic_selector; - -template<> struct atomic_selector<1> { - typedef int8_t word; - inline static word fetch_store ( volatile void* location, word value ); -}; - -template<> struct atomic_selector<2> { - typedef int16_t word; - inline static word fetch_store ( volatile void* location, word value ); -}; - -template<> struct atomic_selector<4> { -#if _MSC_VER && !_WIN64 - // Work-around that avoids spurious /Wp64 warnings - typedef intptr_t word; -#else - typedef int32_t word; -#endif - inline static word fetch_store ( volatile void* location, word value ); -}; - -template<> struct atomic_selector<8> { - typedef int64_t word; - inline static word fetch_store ( volatile void* location, word value ); -}; - -}} //< namespaces internal @endcond, tbb - -#define __TBB_MACHINE_DEFINE_STORE8_GENERIC_FENCED(M) \ - inline void __TBB_machine_generic_store8##M(volatile void *ptr, int64_t value) { \ - for(;;) { \ - int64_t result = *(volatile int64_t *)ptr; \ - if( __TBB_machine_cmpswp8##M(ptr,value,result)==result ) break; \ - } \ - } \ - -#define __TBB_MACHINE_DEFINE_LOAD8_GENERIC_FENCED(M) \ - inline int64_t __TBB_machine_generic_load8##M(const volatile void *ptr) { \ - /* Comparand and new value may be anything, they only must be equal, and */ \ - /* the value should have a low probability to be actually found in 'location'.*/ \ - const int64_t anyvalue = 2305843009213693951LL; \ - return __TBB_machine_cmpswp8##M(const_cast(ptr),anyvalue,anyvalue); \ - } \ - -// The set of allowed values for __TBB_ENDIANNESS (see above for details) -#define __TBB_ENDIAN_UNSUPPORTED -1 -#define __TBB_ENDIAN_LITTLE 0 -#define __TBB_ENDIAN_BIG 1 -#define __TBB_ENDIAN_DETECT 2 - -#if _WIN32||_WIN64 - -#ifdef _MANAGED -#pragma managed(push, off) -#endif - - #if __MINGW64__ || __MINGW32__ - extern "C" __declspec(dllimport) int __stdcall SwitchToThread( void ); - #define __TBB_Yield() SwitchToThread() - #if (TBB_USE_GCC_BUILTINS && __TBB_GCC_BUILTIN_ATOMICS_PRESENT) - #include "machine/gcc_generic.h" - #elif __MINGW64__ - #include "machine/linux_intel64.h" - #elif __MINGW32__ - #include "machine/linux_ia32.h" - #endif - #elif (TBB_USE_ICC_BUILTINS && __TBB_ICC_BUILTIN_ATOMICS_PRESENT) - #include "machine/icc_generic.h" - #elif defined(_M_IX86) && !defined(__TBB_WIN32_USE_CL_BUILTINS) - #include "machine/windows_ia32.h" - #elif defined(_M_X64) - #include "machine/windows_intel64.h" - #elif defined(_M_ARM) || defined(__TBB_WIN32_USE_CL_BUILTINS) - #include "machine/msvc_armv7.h" - #endif - -#ifdef _MANAGED -#pragma managed(pop) -#endif - -#elif __TBB_DEFINE_MIC - - #include "machine/mic_common.h" - #if (TBB_USE_ICC_BUILTINS && __TBB_ICC_BUILTIN_ATOMICS_PRESENT) - #include "machine/icc_generic.h" - #else - #include "machine/linux_intel64.h" - #endif - -#elif __linux__ || __FreeBSD__ || __NetBSD__ - - #if (TBB_USE_GCC_BUILTINS && __TBB_GCC_BUILTIN_ATOMICS_PRESENT) - #include "machine/gcc_generic.h" - #elif (TBB_USE_ICC_BUILTINS && __TBB_ICC_BUILTIN_ATOMICS_PRESENT) - #include "machine/icc_generic.h" - #elif __i386__ - #include "machine/linux_ia32.h" - #elif __x86_64__ - #include "machine/linux_intel64.h" - #elif __ia64__ - #include "machine/linux_ia64.h" - #elif __powerpc__ - #include "machine/mac_ppc.h" - #elif __ARM_ARCH_7A__ - #include "machine/gcc_armv7.h" - #elif __TBB_GCC_BUILTIN_ATOMICS_PRESENT - #include "machine/gcc_generic.h" - #endif - #include "machine/linux_common.h" - -#elif __APPLE__ - //TODO: TBB_USE_GCC_BUILTINS is not used for Mac, Sun, Aix - #if (TBB_USE_ICC_BUILTINS && __TBB_ICC_BUILTIN_ATOMICS_PRESENT) - #include "machine/icc_generic.h" - #elif __TBB_x86_32 - #include "machine/linux_ia32.h" - #elif __TBB_x86_64 - #include "machine/linux_intel64.h" - #elif __POWERPC__ - #include "machine/mac_ppc.h" - #endif - #include "machine/macos_common.h" - -#elif _AIX - - #include "machine/ibm_aix51.h" - -#elif __sun || __SUNPRO_CC - - #define __asm__ asm - #define __volatile__ volatile - - #if __i386 || __i386__ - #include "machine/linux_ia32.h" - #elif __x86_64__ - #include "machine/linux_intel64.h" - #elif __sparc - #include "machine/sunos_sparc.h" - #endif - #include - - #define __TBB_Yield() sched_yield() - -#endif /* OS selection */ - -#ifndef __TBB_64BIT_ATOMICS - #define __TBB_64BIT_ATOMICS 1 -#endif - -//TODO: replace usage of these functions with usage of tbb::atomic, and then remove them -//TODO: map functions with W suffix to use cast to tbb::atomic and according op, i.e. as_atomic().op() -// Special atomic functions -#if __TBB_USE_FENCED_ATOMICS - #define __TBB_machine_cmpswp1 __TBB_machine_cmpswp1full_fence - #define __TBB_machine_cmpswp2 __TBB_machine_cmpswp2full_fence - #define __TBB_machine_cmpswp4 __TBB_machine_cmpswp4full_fence - #define __TBB_machine_cmpswp8 __TBB_machine_cmpswp8full_fence - - #if __TBB_WORDSIZE==8 - #define __TBB_machine_fetchadd8 __TBB_machine_fetchadd8full_fence - #define __TBB_machine_fetchstore8 __TBB_machine_fetchstore8full_fence - #define __TBB_FetchAndAddWrelease(P,V) __TBB_machine_fetchadd8release(P,V) - #define __TBB_FetchAndIncrementWacquire(P) __TBB_machine_fetchadd8acquire(P,1) - #define __TBB_FetchAndDecrementWrelease(P) __TBB_machine_fetchadd8release(P,(-1)) - #else - #define __TBB_machine_fetchadd4 __TBB_machine_fetchadd4full_fence - #define __TBB_machine_fetchstore4 __TBB_machine_fetchstore4full_fence - #define __TBB_FetchAndAddWrelease(P,V) __TBB_machine_fetchadd4release(P,V) - #define __TBB_FetchAndIncrementWacquire(P) __TBB_machine_fetchadd4acquire(P,1) - #define __TBB_FetchAndDecrementWrelease(P) __TBB_machine_fetchadd4release(P,(-1)) - #endif /* __TBB_WORDSIZE==4 */ -#else /* !__TBB_USE_FENCED_ATOMICS */ - #define __TBB_FetchAndAddWrelease(P,V) __TBB_FetchAndAddW(P,V) - #define __TBB_FetchAndIncrementWacquire(P) __TBB_FetchAndAddW(P,1) - #define __TBB_FetchAndDecrementWrelease(P) __TBB_FetchAndAddW(P,(-1)) -#endif /* !__TBB_USE_FENCED_ATOMICS */ - -#if __TBB_WORDSIZE==4 - #define __TBB_CompareAndSwapW(P,V,C) __TBB_machine_cmpswp4(P,V,C) - #define __TBB_FetchAndAddW(P,V) __TBB_machine_fetchadd4(P,V) - #define __TBB_FetchAndStoreW(P,V) __TBB_machine_fetchstore4(P,V) -#elif __TBB_WORDSIZE==8 - #if __TBB_USE_GENERIC_DWORD_LOAD_STORE || __TBB_USE_GENERIC_DWORD_FETCH_ADD || __TBB_USE_GENERIC_DWORD_FETCH_STORE - #error These macros should only be used on 32-bit platforms. - #endif - - #define __TBB_CompareAndSwapW(P,V,C) __TBB_machine_cmpswp8(P,V,C) - #define __TBB_FetchAndAddW(P,V) __TBB_machine_fetchadd8(P,V) - #define __TBB_FetchAndStoreW(P,V) __TBB_machine_fetchstore8(P,V) -#else /* __TBB_WORDSIZE != 8 */ - #error Unsupported machine word size. -#endif /* __TBB_WORDSIZE */ - -#ifndef __TBB_Pause - inline void __TBB_Pause(int32_t) { - __TBB_Yield(); - } -#endif - -namespace tbb { - -//! Sequentially consistent full memory fence. -inline void atomic_fence () { __TBB_full_memory_fence(); } - -namespace internal { //< @cond INTERNAL - -//! Class that implements exponential backoff. -/** See implementation of spin_wait_while_eq for an example. */ -class atomic_backoff : no_copy { - //! Time delay, in units of "pause" instructions. - /** Should be equal to approximately the number of "pause" instructions - that take the same time as an context switch. Must be a power of two.*/ - static const int32_t LOOPS_BEFORE_YIELD = 16; - int32_t count; -public: - // In many cases, an object of this type is initialized eagerly on hot path, - // as in for(atomic_backoff b; ; b.pause()) { /*loop body*/ } - // For this reason, the construction cost must be very small! - atomic_backoff() : count(1) {} - // This constructor pauses immediately; do not use on hot paths! - atomic_backoff( bool ) : count(1) { pause(); } - - //! Pause for a while. - void pause() { - if( count<=LOOPS_BEFORE_YIELD ) { - __TBB_Pause(count); - // Pause twice as long the next time. - count*=2; - } else { - // Pause is so long that we might as well yield CPU to scheduler. - __TBB_Yield(); - } - } - - //! Pause for a few times and return false if saturated. - bool bounded_pause() { - __TBB_Pause(count); - if( count -void spin_wait_while_eq( const volatile T& location, U value ) { - atomic_backoff backoff; - while( location==value ) backoff.pause(); -} - -//! Spin UNTIL the value of the variable is equal to a given value -/** T and U should be comparable types. */ -template -void spin_wait_until_eq( const volatile T& location, const U value ) { - atomic_backoff backoff; - while( location!=value ) backoff.pause(); -} - -template -void spin_wait_while(predicate_type condition){ - atomic_backoff backoff; - while( condition() ) backoff.pause(); -} - -//////////////////////////////////////////////////////////////////////////////// -// Generic compare-and-swap applied to only a part of a machine word. -// -#ifndef __TBB_ENDIANNESS -#define __TBB_ENDIANNESS __TBB_ENDIAN_DETECT -#endif - -#if __TBB_USE_GENERIC_PART_WORD_CAS && __TBB_ENDIANNESS==__TBB_ENDIAN_UNSUPPORTED -#error Generic implementation of part-word CAS may not be used with __TBB_ENDIAN_UNSUPPORTED -#endif - -#if __TBB_ENDIANNESS!=__TBB_ENDIAN_UNSUPPORTED -// -// This function is the only use of __TBB_ENDIANNESS. -// The following restrictions/limitations apply for this operation: -// - T must be an integer type of at most 4 bytes for the casts and calculations to work -// - T must also be less than 4 bytes to avoid compiler warnings when computing mask -// (and for the operation to be useful at all, so no workaround is applied) -// - the architecture must consistently use either little-endian or big-endian (same for all locations) -// -// TODO: static_assert for the type requirements stated above -template -inline T __TBB_MaskedCompareAndSwap (volatile T * const ptr, const T value, const T comparand ) { - struct endianness{ static bool is_big_endian(){ - #if __TBB_ENDIANNESS==__TBB_ENDIAN_DETECT - const uint32_t probe = 0x03020100; - return (((const char*)(&probe))[0]==0x03); - #elif __TBB_ENDIANNESS==__TBB_ENDIAN_BIG || __TBB_ENDIANNESS==__TBB_ENDIAN_LITTLE - return __TBB_ENDIANNESS==__TBB_ENDIAN_BIG; - #else - #error Unexpected value of __TBB_ENDIANNESS - #endif - }}; - - const uint32_t byte_offset = (uint32_t) ((uintptr_t)ptr & 0x3); - volatile uint32_t * const aligned_ptr = (uint32_t*)((uintptr_t)ptr - byte_offset ); - - // location of T within uint32_t for a C++ shift operation - const uint32_t bits_to_shift = 8*(endianness::is_big_endian() ? (4 - sizeof(T) - (byte_offset)) : byte_offset); - const uint32_t mask = (((uint32_t)1<<(sizeof(T)*8)) - 1 )<> bits_to_shift); - } - else continue; // CAS failed but the bits of interest were not changed - } -} -#endif // __TBB_ENDIANNESS!=__TBB_ENDIAN_UNSUPPORTED -//////////////////////////////////////////////////////////////////////////////// - -template -inline T __TBB_CompareAndSwapGeneric (volatile void *ptr, T value, T comparand ); - -template<> -inline int8_t __TBB_CompareAndSwapGeneric <1,int8_t> (volatile void *ptr, int8_t value, int8_t comparand ) { -#if __TBB_USE_GENERIC_PART_WORD_CAS - return __TBB_MaskedCompareAndSwap((volatile int8_t *)ptr,value,comparand); -#else - return __TBB_machine_cmpswp1(ptr,value,comparand); -#endif -} - -template<> -inline int16_t __TBB_CompareAndSwapGeneric <2,int16_t> (volatile void *ptr, int16_t value, int16_t comparand ) { -#if __TBB_USE_GENERIC_PART_WORD_CAS - return __TBB_MaskedCompareAndSwap((volatile int16_t *)ptr,value,comparand); -#else - return __TBB_machine_cmpswp2(ptr,value,comparand); -#endif -} - -template<> -inline int32_t __TBB_CompareAndSwapGeneric <4,int32_t> (volatile void *ptr, int32_t value, int32_t comparand ) { - // Cast shuts up /Wp64 warning - return (int32_t)__TBB_machine_cmpswp4(ptr,value,comparand); -} - -#if __TBB_64BIT_ATOMICS -template<> -inline int64_t __TBB_CompareAndSwapGeneric <8,int64_t> (volatile void *ptr, int64_t value, int64_t comparand ) { - return __TBB_machine_cmpswp8(ptr,value,comparand); -} -#endif - -template -inline T __TBB_FetchAndAddGeneric (volatile void *ptr, T addend) { - T result; - for( atomic_backoff b;;b.pause() ) { - result = *reinterpret_cast(ptr); - // __TBB_CompareAndSwapGeneric presumed to have full fence. - if( __TBB_CompareAndSwapGeneric ( ptr, result+addend, result )==result ) - break; - } - return result; -} - -template -inline T __TBB_FetchAndStoreGeneric (volatile void *ptr, T value) { - T result; - for( atomic_backoff b;;b.pause() ) { - result = *reinterpret_cast(ptr); - // __TBB_CompareAndSwapGeneric presumed to have full fence. - if( __TBB_CompareAndSwapGeneric ( ptr, value, result )==result ) - break; - } - return result; -} - -#if __TBB_USE_GENERIC_PART_WORD_CAS -#define __TBB_machine_cmpswp1 tbb::internal::__TBB_CompareAndSwapGeneric<1,int8_t> -#define __TBB_machine_cmpswp2 tbb::internal::__TBB_CompareAndSwapGeneric<2,int16_t> -#endif - -#if __TBB_USE_GENERIC_FETCH_ADD || __TBB_USE_GENERIC_PART_WORD_FETCH_ADD -#define __TBB_machine_fetchadd1 tbb::internal::__TBB_FetchAndAddGeneric<1,int8_t> -#define __TBB_machine_fetchadd2 tbb::internal::__TBB_FetchAndAddGeneric<2,int16_t> -#endif - -#if __TBB_USE_GENERIC_FETCH_ADD -#define __TBB_machine_fetchadd4 tbb::internal::__TBB_FetchAndAddGeneric<4,int32_t> -#endif - -#if __TBB_USE_GENERIC_FETCH_ADD || __TBB_USE_GENERIC_DWORD_FETCH_ADD -#define __TBB_machine_fetchadd8 tbb::internal::__TBB_FetchAndAddGeneric<8,int64_t> -#endif - -#if __TBB_USE_GENERIC_FETCH_STORE || __TBB_USE_GENERIC_PART_WORD_FETCH_STORE -#define __TBB_machine_fetchstore1 tbb::internal::__TBB_FetchAndStoreGeneric<1,int8_t> -#define __TBB_machine_fetchstore2 tbb::internal::__TBB_FetchAndStoreGeneric<2,int16_t> -#endif - -#if __TBB_USE_GENERIC_FETCH_STORE -#define __TBB_machine_fetchstore4 tbb::internal::__TBB_FetchAndStoreGeneric<4,int32_t> -#endif - -#if __TBB_USE_GENERIC_FETCH_STORE || __TBB_USE_GENERIC_DWORD_FETCH_STORE -#define __TBB_machine_fetchstore8 tbb::internal::__TBB_FetchAndStoreGeneric<8,int64_t> -#endif - -#if __TBB_USE_FETCHSTORE_AS_FULL_FENCED_STORE -#define __TBB_MACHINE_DEFINE_ATOMIC_SELECTOR_FETCH_STORE(S) \ - atomic_selector::word atomic_selector::fetch_store ( volatile void* location, word value ) { \ - return __TBB_machine_fetchstore##S( location, value ); \ - } - -__TBB_MACHINE_DEFINE_ATOMIC_SELECTOR_FETCH_STORE(1) -__TBB_MACHINE_DEFINE_ATOMIC_SELECTOR_FETCH_STORE(2) -__TBB_MACHINE_DEFINE_ATOMIC_SELECTOR_FETCH_STORE(4) -__TBB_MACHINE_DEFINE_ATOMIC_SELECTOR_FETCH_STORE(8) - -#undef __TBB_MACHINE_DEFINE_ATOMIC_SELECTOR_FETCH_STORE -#endif /* __TBB_USE_FETCHSTORE_AS_FULL_FENCED_STORE */ - -#if __TBB_USE_GENERIC_DWORD_LOAD_STORE -/*TODO: find a more elegant way to handle function names difference*/ -#if ! __TBB_USE_FENCED_ATOMICS - /* This name forwarding is needed for generic implementation of - * load8/store8 defined below (via macro) to pick the right CAS function*/ - #define __TBB_machine_cmpswp8full_fence __TBB_machine_cmpswp8 -#endif -__TBB_MACHINE_DEFINE_LOAD8_GENERIC_FENCED(full_fence) -__TBB_MACHINE_DEFINE_STORE8_GENERIC_FENCED(full_fence) - -#if ! __TBB_USE_FENCED_ATOMICS - #undef __TBB_machine_cmpswp8full_fence -#endif - -#define __TBB_machine_store8 tbb::internal::__TBB_machine_generic_store8full_fence -#define __TBB_machine_load8 tbb::internal::__TBB_machine_generic_load8full_fence -#endif /* __TBB_USE_GENERIC_DWORD_LOAD_STORE */ - -#if __TBB_USE_GENERIC_HALF_FENCED_LOAD_STORE -/** Fenced operations use volatile qualifier to prevent compiler from optimizing - them out, and on architectures with weak memory ordering to induce compiler - to generate code with appropriate acquire/release semantics. - On architectures like IA32, Intel64 (and likely Sparc TSO) volatile has - no effect on code gen, and consistency helpers serve as a compiler fence (the - latter being true for IA64/gcc as well to fix a bug in some gcc versions). - This code assumes that the generated instructions will operate atomically, - which typically requires a type that can be moved in a single instruction, - cooperation from the compiler for effective use of such an instruction, - and appropriate alignment of the data. **/ -template -struct machine_load_store { - static T load_with_acquire ( const volatile T& location ) { - T to_return = location; - __TBB_acquire_consistency_helper(); - return to_return; - } - static void store_with_release ( volatile T &location, T value ) { - __TBB_release_consistency_helper(); - location = value; - } -}; - -//in general, plain load and store of 32bit compiler is not atomic for 64bit types -#if __TBB_WORDSIZE==4 && __TBB_64BIT_ATOMICS -template -struct machine_load_store { - static T load_with_acquire ( const volatile T& location ) { - return (T)__TBB_machine_load8( (const volatile void*)&location ); - } - static void store_with_release ( volatile T& location, T value ) { - __TBB_machine_store8( (volatile void*)&location, (int64_t)value ); - } -}; -#endif /* __TBB_WORDSIZE==4 && __TBB_64BIT_ATOMICS */ -#endif /* __TBB_USE_GENERIC_HALF_FENCED_LOAD_STORE */ - -#if __TBB_USE_GENERIC_SEQUENTIAL_CONSISTENCY_LOAD_STORE -template -struct machine_load_store_seq_cst { - static T load ( const volatile T& location ) { - __TBB_full_memory_fence(); - return machine_load_store::load_with_acquire( location ); - } -#if __TBB_USE_FETCHSTORE_AS_FULL_FENCED_STORE - static void store ( volatile T &location, T value ) { - atomic_selector::fetch_store( (volatile void*)&location, (typename atomic_selector::word)value ); - } -#else /* !__TBB_USE_FETCHSTORE_AS_FULL_FENCED_STORE */ - static void store ( volatile T &location, T value ) { - machine_load_store::store_with_release( location, value ); - __TBB_full_memory_fence(); - } -#endif /* !__TBB_USE_FETCHSTORE_AS_FULL_FENCED_STORE */ -}; - -#if __TBB_WORDSIZE==4 && __TBB_64BIT_ATOMICS -/** The implementation does not use functions __TBB_machine_load8/store8 as they - are not required to be sequentially consistent. **/ -template -struct machine_load_store_seq_cst { - static T load ( const volatile T& location ) { - // Comparand and new value may be anything, they only must be equal, and - // the value should have a low probability to be actually found in 'location'. - const int64_t anyvalue = 2305843009213693951LL; - return __TBB_machine_cmpswp8( (volatile void*)const_cast(&location), anyvalue, anyvalue ); - } - static void store ( volatile T &location, T value ) { - int64_t result = (volatile int64_t&)location; - while ( __TBB_machine_cmpswp8((volatile void*)&location, (int64_t)value, result) != result ) - result = (volatile int64_t&)location; - } -}; -#endif /* __TBB_WORDSIZE==4 && __TBB_64BIT_ATOMICS */ -#endif /*__TBB_USE_GENERIC_SEQUENTIAL_CONSISTENCY_LOAD_STORE */ - -#if __TBB_USE_GENERIC_RELAXED_LOAD_STORE -// Relaxed operations add volatile qualifier to prevent compiler from optimizing them out. -/** Volatile should not incur any additional cost on IA32, Intel64, and Sparc TSO - architectures. However on architectures with weak memory ordering compiler may - generate code with acquire/release semantics for operations on volatile data. **/ -template -struct machine_load_store_relaxed { - static inline T load ( const volatile T& location ) { - return location; - } - static inline void store ( volatile T& location, T value ) { - location = value; - } -}; - -#if __TBB_WORDSIZE==4 && __TBB_64BIT_ATOMICS -template -struct machine_load_store_relaxed { - static inline T load ( const volatile T& location ) { - return (T)__TBB_machine_load8( (const volatile void*)&location ); - } - static inline void store ( volatile T& location, T value ) { - __TBB_machine_store8( (volatile void*)&location, (int64_t)value ); - } -}; -#endif /* __TBB_WORDSIZE==4 && __TBB_64BIT_ATOMICS */ -#endif /* __TBB_USE_GENERIC_RELAXED_LOAD_STORE */ - -#undef __TBB_WORDSIZE //this macro is forbidden to use outside of atomic machinery - -template -inline T __TBB_load_with_acquire(const volatile T &location) { - return machine_load_store::load_with_acquire( location ); -} -template -inline void __TBB_store_with_release(volatile T& location, V value) { - machine_load_store::store_with_release( location, T(value) ); -} -//! Overload that exists solely to avoid /Wp64 warnings. -inline void __TBB_store_with_release(volatile size_t& location, size_t value) { - machine_load_store::store_with_release( location, value ); -} - -template -inline T __TBB_load_full_fence(const volatile T &location) { - return machine_load_store_seq_cst::load( location ); -} -template -inline void __TBB_store_full_fence(volatile T& location, V value) { - machine_load_store_seq_cst::store( location, T(value) ); -} -//! Overload that exists solely to avoid /Wp64 warnings. -inline void __TBB_store_full_fence(volatile size_t& location, size_t value) { - machine_load_store_seq_cst::store( location, value ); -} - -template -inline T __TBB_load_relaxed (const volatile T& location) { - return machine_load_store_relaxed::load( const_cast(location) ); -} -template -inline void __TBB_store_relaxed ( volatile T& location, V value ) { - machine_load_store_relaxed::store( const_cast(location), T(value) ); -} -//! Overload that exists solely to avoid /Wp64 warnings. -inline void __TBB_store_relaxed ( volatile size_t& location, size_t value ) { - machine_load_store_relaxed::store( const_cast(location), value ); -} - -// Macro __TBB_TypeWithAlignmentAtLeastAsStrict(T) should be a type with alignment at least as -// strict as type T. The type should have a trivial default constructor and destructor, so that -// arrays of that type can be declared without initializers. -// It is correct (but perhaps a waste of space) if __TBB_TypeWithAlignmentAtLeastAsStrict(T) expands -// to a type bigger than T. -// The default definition here works on machines where integers are naturally aligned and the -// strictest alignment is 64. -#ifndef __TBB_TypeWithAlignmentAtLeastAsStrict - -#if __TBB_ALIGNAS_PRESENT - -// Use C++11 keywords alignas and alignof -#define __TBB_DefineTypeWithAlignment(PowerOf2) \ -struct alignas(PowerOf2) __TBB_machine_type_with_alignment_##PowerOf2 { \ - uint32_t member[PowerOf2/sizeof(uint32_t)]; \ -}; -#define __TBB_alignof(T) alignof(T) - -#elif __TBB_ATTRIBUTE_ALIGNED_PRESENT - -#define __TBB_DefineTypeWithAlignment(PowerOf2) \ -struct __TBB_machine_type_with_alignment_##PowerOf2 { \ - uint32_t member[PowerOf2/sizeof(uint32_t)]; \ -} __attribute__((aligned(PowerOf2))); -#define __TBB_alignof(T) __alignof__(T) - -#elif __TBB_DECLSPEC_ALIGN_PRESENT - -#define __TBB_DefineTypeWithAlignment(PowerOf2) \ -__declspec(align(PowerOf2)) \ -struct __TBB_machine_type_with_alignment_##PowerOf2 { \ - uint32_t member[PowerOf2/sizeof(uint32_t)]; \ -}; -#define __TBB_alignof(T) __alignof(T) - -#else /* A compiler with unknown syntax for data alignment */ -#error Must define __TBB_TypeWithAlignmentAtLeastAsStrict(T) -#endif - -/* Now declare types aligned to useful powers of two */ -// TODO: Is __TBB_DefineTypeWithAlignment(8) needed on 32 bit platforms? -__TBB_DefineTypeWithAlignment(16) -__TBB_DefineTypeWithAlignment(32) -__TBB_DefineTypeWithAlignment(64) - -typedef __TBB_machine_type_with_alignment_64 __TBB_machine_type_with_strictest_alignment; - -// Primary template is a declaration of incomplete type so that it fails with unknown alignments -template struct type_with_alignment; - -// Specializations for allowed alignments -template<> struct type_with_alignment<1> { char member; }; -template<> struct type_with_alignment<2> { uint16_t member; }; -template<> struct type_with_alignment<4> { uint32_t member; }; -template<> struct type_with_alignment<8> { uint64_t member; }; -template<> struct type_with_alignment<16> {__TBB_machine_type_with_alignment_16 member; }; -template<> struct type_with_alignment<32> {__TBB_machine_type_with_alignment_32 member; }; -template<> struct type_with_alignment<64> {__TBB_machine_type_with_alignment_64 member; }; - -#if __TBB_ALIGNOF_NOT_INSTANTIATED_TYPES_BROKEN -//! Work around for bug in GNU 3.2 and MSVC compilers. -/** Bug is that compiler sometimes returns 0 for __alignof(T) when T has not yet been instantiated. - The work-around forces instantiation by forcing computation of sizeof(T) before __alignof(T). */ -template -struct work_around_alignment_bug { - static const size_t alignment = __TBB_alignof(T); -}; -#define __TBB_TypeWithAlignmentAtLeastAsStrict(T) tbb::internal::type_with_alignment::alignment> -#else -#define __TBB_TypeWithAlignmentAtLeastAsStrict(T) tbb::internal::type_with_alignment<__TBB_alignof(T)> -#endif /* __TBB_ALIGNOF_NOT_INSTANTIATED_TYPES_BROKEN */ - -#endif /* __TBB_TypeWithAlignmentAtLeastAsStrict */ - -// Template class here is to avoid instantiation of the static data for modules that don't use it -template -struct reverse { - static const T byte_table[256]; -}; -// An efficient implementation of the reverse function utilizes a 2^8 lookup table holding the bit-reversed -// values of [0..2^8 - 1]. Those values can also be computed on the fly at a slightly higher cost. -template -const T reverse::byte_table[256] = { - 0x00, 0x80, 0x40, 0xC0, 0x20, 0xA0, 0x60, 0xE0, 0x10, 0x90, 0x50, 0xD0, 0x30, 0xB0, 0x70, 0xF0, - 0x08, 0x88, 0x48, 0xC8, 0x28, 0xA8, 0x68, 0xE8, 0x18, 0x98, 0x58, 0xD8, 0x38, 0xB8, 0x78, 0xF8, - 0x04, 0x84, 0x44, 0xC4, 0x24, 0xA4, 0x64, 0xE4, 0x14, 0x94, 0x54, 0xD4, 0x34, 0xB4, 0x74, 0xF4, - 0x0C, 0x8C, 0x4C, 0xCC, 0x2C, 0xAC, 0x6C, 0xEC, 0x1C, 0x9C, 0x5C, 0xDC, 0x3C, 0xBC, 0x7C, 0xFC, - 0x02, 0x82, 0x42, 0xC2, 0x22, 0xA2, 0x62, 0xE2, 0x12, 0x92, 0x52, 0xD2, 0x32, 0xB2, 0x72, 0xF2, - 0x0A, 0x8A, 0x4A, 0xCA, 0x2A, 0xAA, 0x6A, 0xEA, 0x1A, 0x9A, 0x5A, 0xDA, 0x3A, 0xBA, 0x7A, 0xFA, - 0x06, 0x86, 0x46, 0xC6, 0x26, 0xA6, 0x66, 0xE6, 0x16, 0x96, 0x56, 0xD6, 0x36, 0xB6, 0x76, 0xF6, - 0x0E, 0x8E, 0x4E, 0xCE, 0x2E, 0xAE, 0x6E, 0xEE, 0x1E, 0x9E, 0x5E, 0xDE, 0x3E, 0xBE, 0x7E, 0xFE, - 0x01, 0x81, 0x41, 0xC1, 0x21, 0xA1, 0x61, 0xE1, 0x11, 0x91, 0x51, 0xD1, 0x31, 0xB1, 0x71, 0xF1, - 0x09, 0x89, 0x49, 0xC9, 0x29, 0xA9, 0x69, 0xE9, 0x19, 0x99, 0x59, 0xD9, 0x39, 0xB9, 0x79, 0xF9, - 0x05, 0x85, 0x45, 0xC5, 0x25, 0xA5, 0x65, 0xE5, 0x15, 0x95, 0x55, 0xD5, 0x35, 0xB5, 0x75, 0xF5, - 0x0D, 0x8D, 0x4D, 0xCD, 0x2D, 0xAD, 0x6D, 0xED, 0x1D, 0x9D, 0x5D, 0xDD, 0x3D, 0xBD, 0x7D, 0xFD, - 0x03, 0x83, 0x43, 0xC3, 0x23, 0xA3, 0x63, 0xE3, 0x13, 0x93, 0x53, 0xD3, 0x33, 0xB3, 0x73, 0xF3, - 0x0B, 0x8B, 0x4B, 0xCB, 0x2B, 0xAB, 0x6B, 0xEB, 0x1B, 0x9B, 0x5B, 0xDB, 0x3B, 0xBB, 0x7B, 0xFB, - 0x07, 0x87, 0x47, 0xC7, 0x27, 0xA7, 0x67, 0xE7, 0x17, 0x97, 0x57, 0xD7, 0x37, 0xB7, 0x77, 0xF7, - 0x0F, 0x8F, 0x4F, 0xCF, 0x2F, 0xAF, 0x6F, 0xEF, 0x1F, 0x9F, 0x5F, 0xDF, 0x3F, 0xBF, 0x7F, 0xFF -}; - -} // namespace internal @endcond -} // namespace tbb - -// Preserving access to legacy APIs -using tbb::internal::__TBB_load_with_acquire; -using tbb::internal::__TBB_store_with_release; - -// Mapping historically used names to the ones expected by atomic_load_store_traits -#define __TBB_load_acquire __TBB_load_with_acquire -#define __TBB_store_release __TBB_store_with_release - -#ifndef __TBB_Log2 -inline intptr_t __TBB_Log2( uintptr_t x ) { - if( x==0 ) return -1; - intptr_t result = 0; - -#if !defined(_M_ARM) - uintptr_t tmp_; - if( sizeof(x)>4 && (tmp_ = ((uint64_t)x)>>32) ) { x=tmp_; result += 32; } -#endif - if( uintptr_t tmp = x>>16 ) { x=tmp; result += 16; } - if( uintptr_t tmp = x>>8 ) { x=tmp; result += 8; } - if( uintptr_t tmp = x>>4 ) { x=tmp; result += 4; } - if( uintptr_t tmp = x>>2 ) { x=tmp; result += 2; } - - return (x&2)? result+1: result; -} -#endif - -#ifndef __TBB_AtomicOR -inline void __TBB_AtomicOR( volatile void *operand, uintptr_t addend ) { - for( tbb::internal::atomic_backoff b;;b.pause() ) { - uintptr_t tmp = *(volatile uintptr_t *)operand; - uintptr_t result = __TBB_CompareAndSwapW(operand, tmp|addend, tmp); - if( result==tmp ) break; - } -} -#endif - -#ifndef __TBB_AtomicAND -inline void __TBB_AtomicAND( volatile void *operand, uintptr_t addend ) { - for( tbb::internal::atomic_backoff b;;b.pause() ) { - uintptr_t tmp = *(volatile uintptr_t *)operand; - uintptr_t result = __TBB_CompareAndSwapW(operand, tmp&addend, tmp); - if( result==tmp ) break; - } -} -#endif - -#if __TBB_PREFETCHING -#ifndef __TBB_cl_prefetch -#error This platform does not define cache management primitives required for __TBB_PREFETCHING -#endif - -#ifndef __TBB_cl_evict -#define __TBB_cl_evict(p) -#endif -#endif - -#ifndef __TBB_Flag -typedef unsigned char __TBB_Flag; -#endif -typedef __TBB_atomic __TBB_Flag __TBB_atomic_flag; - -#ifndef __TBB_TryLockByte -inline bool __TBB_TryLockByte( __TBB_atomic_flag &flag ) { - return __TBB_machine_cmpswp1(&flag,1,0)==0; -} -#endif - -#ifndef __TBB_LockByte -inline __TBB_Flag __TBB_LockByte( __TBB_atomic_flag& flag ) { - tbb::internal::atomic_backoff backoff; - while( !__TBB_TryLockByte(flag) ) backoff.pause(); - return 0; -} -#endif - -#ifndef __TBB_UnlockByte -#define __TBB_UnlockByte(addr) __TBB_store_with_release((addr),0) -#endif - -// lock primitives with Intel(R) Transactional Synchronization Extensions (Intel(R) TSX) -#if ( __TBB_x86_32 || __TBB_x86_64 ) /* only on ia32/intel64 */ -inline void __TBB_TryLockByteElidedCancel() { __TBB_machine_try_lock_elided_cancel(); } - -inline bool __TBB_TryLockByteElided( __TBB_atomic_flag& flag ) { - bool res = __TBB_machine_try_lock_elided( &flag )!=0; - // to avoid the "lemming" effect, we need to abort the transaction - // if __TBB_machine_try_lock_elided returns false (i.e., someone else - // has acquired the mutex non-speculatively). - if( !res ) __TBB_TryLockByteElidedCancel(); - return res; -} - -inline void __TBB_LockByteElided( __TBB_atomic_flag& flag ) -{ - for(;;) { - tbb::internal::spin_wait_while_eq( flag, 1 ); - if( __TBB_machine_try_lock_elided( &flag ) ) - return; - // Another thread acquired the lock "for real". - // To avoid the "lemming" effect, we abort the transaction. - __TBB_TryLockByteElidedCancel(); - } -} - -inline void __TBB_UnlockByteElided( __TBB_atomic_flag& flag ) { - __TBB_machine_unlock_elided( &flag ); -} -#endif - -#ifndef __TBB_ReverseByte -inline unsigned char __TBB_ReverseByte(unsigned char src) { - return tbb::internal::reverse::byte_table[src]; -} -#endif - -template -T __TBB_ReverseBits(T src) { - T dst; - unsigned char *original = (unsigned char *) &src; - unsigned char *reversed = (unsigned char *) &dst; - - for( int i = sizeof(T)-1; i >= 0; i-- ) - reversed[i] = __TBB_ReverseByte( original[sizeof(T)-i-1] ); - - return dst; -} - -#endif /* __TBB_machine_H */ diff --git a/inst/include/tbb/tbb_profiling.h b/inst/include/tbb/tbb_profiling.h deleted file mode 100644 index ffaf98b91..000000000 --- a/inst/include/tbb/tbb_profiling.h +++ /dev/null @@ -1,284 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_profiling_H -#define __TBB_profiling_H - -namespace tbb { - namespace internal { - - // - // This is not under __TBB_ITT_STRUCTURE_API because these values are used directly in flow_graph.h. - // - - // include list of index names - #define TBB_STRING_RESOURCE(index_name,str) index_name, - enum string_index { - #include "internal/_tbb_strings.h" - NUM_STRINGS - }; - #undef TBB_STRING_RESOURCE - - enum itt_relation - { - __itt_relation_is_unknown = 0, - __itt_relation_is_dependent_on, /**< "A is dependent on B" means that A cannot start until B completes */ - __itt_relation_is_sibling_of, /**< "A is sibling of B" means that A and B were created as a group */ - __itt_relation_is_parent_of, /**< "A is parent of B" means that A created B */ - __itt_relation_is_continuation_of, /**< "A is continuation of B" means that A assumes the dependencies of B */ - __itt_relation_is_child_of, /**< "A is child of B" means that A was created by B (inverse of is_parent_of) */ - __itt_relation_is_continued_by, /**< "A is continued by B" means that B assumes the dependencies of A (inverse of is_continuation_of) */ - __itt_relation_is_predecessor_to /**< "A is predecessor to B" means that B cannot start until A completes (inverse of is_dependent_on) */ - }; - - } -} - -// Check if the tools support is enabled -#if (_WIN32||_WIN64||__linux__) && !__MINGW32__ && TBB_USE_THREADING_TOOLS - -#if _WIN32||_WIN64 -#include /* mbstowcs_s */ -#endif -#include "tbb_stddef.h" - -namespace tbb { - namespace internal { - -#if _WIN32||_WIN64 - void __TBB_EXPORTED_FUNC itt_set_sync_name_v3( void *obj, const wchar_t* name ); - inline size_t multibyte_to_widechar( wchar_t* wcs, const char* mbs, size_t bufsize) { -#if _MSC_VER>=1400 - size_t len; - mbstowcs_s( &len, wcs, bufsize, mbs, _TRUNCATE ); - return len; // mbstowcs_s counts null terminator -#else - size_t len = mbstowcs( wcs, mbs, bufsize ); - if(wcs && len!=size_t(-1) ) - wcs[len - inline void itt_store_word_with_release(tbb::atomic& dst, U src) { -#if TBB_USE_THREADING_TOOLS - // This assertion should be replaced with static_assert - __TBB_ASSERT(sizeof(T) == sizeof(void *), "Type must be word-sized."); - itt_store_pointer_with_release_v3(&dst, (void *)uintptr_t(src)); -#else - dst = src; -#endif // TBB_USE_THREADING_TOOLS - } - - template - inline T itt_load_word_with_acquire(const tbb::atomic& src) { -#if TBB_USE_THREADING_TOOLS - // This assertion should be replaced with static_assert - __TBB_ASSERT(sizeof(T) == sizeof(void *), "Type must be word-sized."); -#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) - // Workaround for overzealous compiler warnings - #pragma warning (push) - #pragma warning (disable: 4311) -#endif - T result = (T)itt_load_pointer_with_acquire_v3(&src); -#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) - #pragma warning (pop) -#endif - return result; -#else - return src; -#endif // TBB_USE_THREADING_TOOLS - } - - template - inline void itt_store_word_with_release(T& dst, T src) { -#if TBB_USE_THREADING_TOOLS - // This assertion should be replaced with static_assert - __TBB_ASSERT(sizeof(T) == sizeof(void *), "Type must be word-sized."); - itt_store_pointer_with_release_v3(&dst, (void *)src); -#else - __TBB_store_with_release(dst, src); -#endif // TBB_USE_THREADING_TOOLS - } - - template - inline T itt_load_word_with_acquire(const T& src) { -#if TBB_USE_THREADING_TOOLS - // This assertion should be replaced with static_assert - __TBB_ASSERT(sizeof(T) == sizeof(void *), "Type must be word-sized"); - return (T)itt_load_pointer_with_acquire_v3(&src); -#else - return __TBB_load_with_acquire(src); -#endif // TBB_USE_THREADING_TOOLS - } - - template - inline void itt_hide_store_word(T& dst, T src) { -#if TBB_USE_THREADING_TOOLS - //TODO: This assertion should be replaced with static_assert - __TBB_ASSERT(sizeof(T) == sizeof(void *), "Type must be word-sized"); - itt_store_pointer_with_release_v3(&dst, (void *)src); -#else - dst = src; -#endif - } - - //TODO: rename to itt_hide_load_word_relaxed - template - inline T itt_hide_load_word(const T& src) { -#if TBB_USE_THREADING_TOOLS - //TODO: This assertion should be replaced with static_assert - __TBB_ASSERT(sizeof(T) == sizeof(void *), "Type must be word-sized."); - return (T)itt_load_pointer_v3(&src); -#else - return src; -#endif - } - -#if TBB_USE_THREADING_TOOLS - inline void call_itt_notify(notify_type t, void *ptr) { - call_itt_notify_v5((int)t, ptr); - } - -#else - inline void call_itt_notify(notify_type /*t*/, void * /*ptr*/) {} - -#endif // TBB_USE_THREADING_TOOLS - -#if __TBB_ITT_STRUCTURE_API - inline void itt_make_task_group( itt_domain_enum domain, void *group, unsigned long long group_extra, - void *parent, unsigned long long parent_extra, string_index name_index ) { - itt_make_task_group_v7( domain, group, group_extra, parent, parent_extra, name_index ); - } - - inline void itt_metadata_str_add( itt_domain_enum domain, void *addr, unsigned long long addr_extra, - string_index key, const char *value ) { - itt_metadata_str_add_v7( domain, addr, addr_extra, key, value ); - } - - inline void itt_relation_add( itt_domain_enum domain, void *addr0, unsigned long long addr0_extra, - itt_relation relation, void *addr1, unsigned long long addr1_extra ) { - itt_relation_add_v7( domain, addr0, addr0_extra, relation, addr1, addr1_extra ); - } - - inline void itt_task_begin( itt_domain_enum domain, void *task, unsigned long long task_extra, - void *parent, unsigned long long parent_extra, string_index name_index ) { - itt_task_begin_v7( domain, task, task_extra, parent, parent_extra, name_index ); - } - - inline void itt_task_end( itt_domain_enum domain ) { - itt_task_end_v7( domain ); - } - - inline void itt_region_begin( itt_domain_enum domain, void *region, unsigned long long region_extra, - void *parent, unsigned long long parent_extra, string_index name_index ) { - itt_region_begin_v9( domain, region, region_extra, parent, parent_extra, name_index ); - } - - inline void itt_region_end( itt_domain_enum domain, void *region, unsigned long long region_extra ) { - itt_region_end_v9( domain, region, region_extra ); - } -#endif // __TBB_ITT_STRUCTURE_API - - } // namespace internal -} // namespace tbb - -#endif /* __TBB_profiling_H */ diff --git a/inst/include/tbb/tbb_stddef.h b/inst/include/tbb/tbb_stddef.h deleted file mode 100644 index 236f3d830..000000000 --- a/inst/include/tbb/tbb_stddef.h +++ /dev/null @@ -1,542 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_tbb_stddef_H -#define __TBB_tbb_stddef_H - -// Marketing-driven product version -#define TBB_VERSION_MAJOR 2017 -#define TBB_VERSION_MINOR 0 - -// Engineering-focused interface version -#define TBB_INTERFACE_VERSION 9107 -#define TBB_INTERFACE_VERSION_MAJOR TBB_INTERFACE_VERSION/1000 - -// The oldest major interface version still supported -// To be used in SONAME, manifests, etc. -#define TBB_COMPATIBLE_INTERFACE_VERSION 2 - -#define __TBB_STRING_AUX(x) #x -#define __TBB_STRING(x) __TBB_STRING_AUX(x) - -// We do not need defines below for resource processing on windows -#if !defined RC_INVOKED - -// Define groups for Doxygen documentation -/** - * @defgroup algorithms Algorithms - * @defgroup containers Containers - * @defgroup memory_allocation Memory Allocation - * @defgroup synchronization Synchronization - * @defgroup timing Timing - * @defgroup task_scheduling Task Scheduling - */ - -// Simple text that is displayed on the main page of Doxygen documentation. -/** - * \mainpage Main Page - * - * Click the tabs above for information about the - * - Modules (groups of functionality) implemented by the library - * - Classes provided by the library - * - Files constituting the library. - * . - * Please note that significant part of TBB functionality is implemented in the form of - * template functions, descriptions of which are not accessible on the Classes - * tab. Use Modules or Namespace/Namespace Members - * tabs to find them. - * - * Additional pieces of information can be found here - * - \subpage concepts - * . - */ - -/** \page concepts TBB concepts - - A concept is a set of requirements to a type, which are necessary and sufficient - for the type to model a particular behavior or a set of behaviors. Some concepts - are specific to a particular algorithm (e.g. algorithm body), while other ones - are common to several algorithms (e.g. range concept). - - All TBB algorithms make use of different classes implementing various concepts. - Implementation classes are supplied by the user as type arguments of template - parameters and/or as objects passed as function call arguments. The library - provides predefined implementations of some concepts (e.g. several kinds of - \ref range_req "ranges"), while other ones must always be implemented by the user. - - TBB defines a set of minimal requirements each concept must conform to. Here is - the list of different concepts hyperlinked to the corresponding requirements specifications: - - \subpage range_req - - \subpage parallel_do_body_req - - \subpage parallel_for_body_req - - \subpage parallel_reduce_body_req - - \subpage parallel_scan_body_req - - \subpage parallel_sort_iter_req -**/ - -// tbb_config.h should be included the first since it contains macro definitions used in other headers -#include "tbb_config.h" - -#if _MSC_VER >=1400 - #define __TBB_EXPORTED_FUNC __cdecl - #define __TBB_EXPORTED_METHOD __thiscall -#else - #define __TBB_EXPORTED_FUNC - #define __TBB_EXPORTED_METHOD -#endif - -#if __INTEL_COMPILER || _MSC_VER -#define __TBB_NOINLINE(decl) __declspec(noinline) decl -#elif __GNUC__ -#define __TBB_NOINLINE(decl) decl __attribute__ ((noinline)) -#else -#define __TBB_NOINLINE(decl) decl -#endif - -#if __TBB_NOEXCEPT_PRESENT -#define __TBB_NOEXCEPT(expression) noexcept(expression) -#else -#define __TBB_NOEXCEPT(expression) -#endif - -#include /* Need size_t and ptrdiff_t */ - -#if _MSC_VER - #define __TBB_tbb_windef_H - #include "internal/_tbb_windef.h" - #undef __TBB_tbb_windef_H -#endif -#if !defined(_MSC_VER) || _MSC_VER>=1600 - #include -#endif - -//! Type for an assertion handler -typedef void(*assertion_handler_type)( const char* filename, int line, const char* expression, const char * comment ); - -#if __TBBMALLOC_BUILD -namespace rml { namespace internal { - #define __TBB_ASSERT_RELEASE(predicate,message) ((predicate)?((void)0) : rml::internal::assertion_failure(__FILE__,__LINE__,#predicate,message)) -#else -namespace tbb { - #define __TBB_ASSERT_RELEASE(predicate,message) ((predicate)?((void)0) : tbb::assertion_failure(__FILE__,__LINE__,#predicate,message)) -#endif - - //! Set assertion handler and return previous value of it. - assertion_handler_type __TBB_EXPORTED_FUNC set_assertion_handler( assertion_handler_type new_handler ); - - //! Process an assertion failure. - /** Normally called from __TBB_ASSERT macro. - If assertion handler is null, print message for assertion failure and abort. - Otherwise call the assertion handler. */ - void __TBB_EXPORTED_FUNC assertion_failure( const char* filename, int line, const char* expression, const char* comment ); - -#if __TBBMALLOC_BUILD -}} // namespace rml::internal -#else -} // namespace tbb -#endif - -#if TBB_USE_ASSERT - - //! Assert that x is true. - /** If x is false, print assertion failure message. - If the comment argument is not NULL, it is printed as part of the failure message. - The comment argument has no other effect. */ - #define __TBB_ASSERT(predicate,message) __TBB_ASSERT_RELEASE(predicate,message) - - #define __TBB_ASSERT_EX __TBB_ASSERT - -#else /* !TBB_USE_ASSERT */ - - //! No-op version of __TBB_ASSERT. - #define __TBB_ASSERT(predicate,comment) ((void)0) - //! "Extended" version is useful to suppress warnings if a variable is only used with an assert - #define __TBB_ASSERT_EX(predicate,comment) ((void)(1 && (predicate))) - -#endif /* !TBB_USE_ASSERT */ - -//! The namespace tbb contains all components of the library. -namespace tbb { - - namespace internal { -#if _MSC_VER && _MSC_VER<1600 - typedef __int8 int8_t; - typedef __int16 int16_t; - typedef __int32 int32_t; - typedef __int64 int64_t; - typedef unsigned __int8 uint8_t; - typedef unsigned __int16 uint16_t; - typedef unsigned __int32 uint32_t; - typedef unsigned __int64 uint64_t; -#else /* Posix */ - using ::int8_t; - using ::int16_t; - using ::int32_t; - using ::int64_t; - using ::uint8_t; - using ::uint16_t; - using ::uint32_t; - using ::uint64_t; -#endif /* Posix */ - } // namespace internal - - using std::size_t; - using std::ptrdiff_t; - -//! The function returns the interface version of the TBB shared library being used. -/** - * The version it returns is determined at runtime, not at compile/link time. - * So it can be different than the value of TBB_INTERFACE_VERSION obtained at compile time. - */ -extern "C" int __TBB_EXPORTED_FUNC TBB_runtime_interface_version(); - -/** - * @cond INTERNAL - * @brief Identifiers declared inside namespace internal should never be used directly by client code. - */ -namespace internal { - -//! Compile-time constant that is upper bound on cache line/sector size. -/** It should be used only in situations where having a compile-time upper - bound is more useful than a run-time exact answer. - @ingroup memory_allocation */ -const size_t NFS_MaxLineSize = 128; - -/** Label for data that may be accessed from different threads, and that may eventually become wrapped - in a formal atomic type. - - Note that no problems have yet been observed relating to the definition currently being empty, - even if at least "volatile" would seem to be in order to avoid data sometimes temporarily hiding - in a register (although "volatile" as a "poor man's atomic" lacks several other features of a proper - atomic, some of which are now provided instead through specialized functions). - - Note that usage is intentionally compatible with a definition as qualifier "volatile", - both as a way to have the compiler help enforce use of the label and to quickly rule out - one potential issue. - - Note however that, with some architecture/compiler combinations, e.g. on IA-64 architecture, "volatile" - also has non-portable memory semantics that are needlessly expensive for "relaxed" operations. - - Note that this must only be applied to data that will not change bit patterns when cast to/from - an integral type of the same length; tbb::atomic must be used instead for, e.g., floating-point types. - - TODO: apply wherever relevant **/ -#define __TBB_atomic // intentionally empty, see above - -#if __TBB_OVERRIDE_PRESENT -#define __TBB_override override -#else -#define __TBB_override // formal comment only -#endif - -template -struct padded_base : T { - char pad[S - R]; -}; -template struct padded_base : T {}; - -//! Pads type T to fill out to a multiple of cache line size. -template -struct padded : padded_base {}; - -//! Extended variant of the standard offsetof macro -/** The standard offsetof macro is not sufficient for TBB as it can be used for - POD-types only. The constant 0x1000 (not NULL) is necessary to appease GCC. **/ -#define __TBB_offsetof(class_name, member_name) \ - ((ptrdiff_t)&(reinterpret_cast(0x1000)->member_name) - 0x1000) - -//! Returns address of the object containing a member with the given name and address -#define __TBB_get_object_ref(class_name, member_name, member_addr) \ - (*reinterpret_cast((char*)member_addr - __TBB_offsetof(class_name, member_name))) - -//! Throws std::runtime_error with what() returning error_code description prefixed with aux_info -void __TBB_EXPORTED_FUNC handle_perror( int error_code, const char* aux_info ); - -#if TBB_USE_EXCEPTIONS - #define __TBB_TRY try - #define __TBB_CATCH(e) catch(e) - #define __TBB_THROW(e) throw e - #define __TBB_RETHROW() throw -#else /* !TBB_USE_EXCEPTIONS */ - inline bool __TBB_false() { return false; } - #define __TBB_TRY - #define __TBB_CATCH(e) if ( tbb::internal::__TBB_false() ) - #define __TBB_THROW(e) ((void)0) - #define __TBB_RETHROW() ((void)0) -#endif /* !TBB_USE_EXCEPTIONS */ - -//! Report a runtime warning. -void __TBB_EXPORTED_FUNC runtime_warning( const char* format, ... ); - -#if TBB_USE_ASSERT -static void* const poisoned_ptr = reinterpret_cast(-1); - -//! Set p to invalid pointer value. -// Also works for regular (non-__TBB_atomic) pointers. -template -inline void poison_pointer( T* __TBB_atomic & p ) { p = reinterpret_cast(poisoned_ptr); } - -/** Expected to be used in assertions only, thus no empty form is defined. **/ -template -inline bool is_poisoned( T* p ) { return p == reinterpret_cast(poisoned_ptr); } -#else -template -inline void poison_pointer( T* __TBB_atomic & ) {/*do nothing*/} -#endif /* !TBB_USE_ASSERT */ - -//! Cast between unrelated pointer types. -/** This method should be used sparingly as a last resort for dealing with - situations that inherently break strict ISO C++ aliasing rules. */ -// T is a pointer type because it will be explicitly provided by the programmer as a template argument; -// U is a referent type to enable the compiler to check that "ptr" is a pointer, deducing U in the process. -template -inline T punned_cast( U* ptr ) { - uintptr_t x = reinterpret_cast(ptr); - return reinterpret_cast(x); -} - -//! Base class for types that should not be assigned. -class no_assign { - // Deny assignment - void operator=( const no_assign& ); -public: -#if __GNUC__ - //! Explicitly define default construction, because otherwise gcc issues gratuitous warning. - no_assign() {} -#endif /* __GNUC__ */ -}; - -//! Base class for types that should not be copied or assigned. -class no_copy: no_assign { - //! Deny copy construction - no_copy( const no_copy& ); -public: - //! Allow default construction - no_copy() {} -}; - -#if TBB_DEPRECATED_MUTEX_COPYING -class mutex_copy_deprecated_and_disabled {}; -#else -// By default various implementations of mutexes are not copy constructible -// and not copy assignable. -class mutex_copy_deprecated_and_disabled : no_copy {}; -#endif - -//! A function to check if passed in pointer is aligned on a specific border -template -inline bool is_aligned(T* pointer, uintptr_t alignment) { - return 0==((uintptr_t)pointer & (alignment-1)); -} - -//! A function to check if passed integer is a power of 2 -template -inline bool is_power_of_two(integer_type arg) { - return arg && (0 == (arg & (arg - 1))); -} - -//! A function to compute arg modulo divisor where divisor is a power of 2. -template -inline argument_integer_type modulo_power_of_two(argument_integer_type arg, divisor_integer_type divisor) { - __TBB_ASSERT( is_power_of_two(divisor), "Divisor should be a power of two" ); - return (arg & (divisor - 1)); -} - - -//! A function to determine if arg is a power of 2 at least as big as another power of 2. -// i.e. for strictly positive i and j, with j being a power of 2, -// determines whether i==j< -inline bool is_power_of_two_at_least(argument_integer_type arg, power2_integer_type power2) { - __TBB_ASSERT( is_power_of_two(power2), "Divisor should be a power of two" ); - return 0 == (arg & (arg - power2)); -} - -//! Utility template function to prevent "unused" warnings by various compilers. -template void suppress_unused_warning( const T1& ) {} -template void suppress_unused_warning( const T1&, const T2& ) {} -template void suppress_unused_warning( const T1&, const T2&, const T3& ) {} - -// Struct to be used as a version tag for inline functions. -/** Version tag can be necessary to prevent loader on Linux from using the wrong - symbol in debug builds (when inline functions are compiled as out-of-line). **/ -struct version_tag_v3 {}; - -typedef version_tag_v3 version_tag; - -} // internal - -//! Dummy type that distinguishes splitting constructor from copy constructor. -/** - * See description of parallel_for and parallel_reduce for example usages. - * @ingroup algorithms - */ -class split { -}; - -//! Type enables transmission of splitting proportion from partitioners to range objects -/** - * In order to make use of such facility Range objects must implement - * splitting constructor with this type passed and initialize static - * constant boolean field 'is_splittable_in_proportion' with the value - * of 'true' - */ -class proportional_split: internal::no_assign { -public: - proportional_split(size_t _left = 1, size_t _right = 1) : my_left(_left), my_right(_right) { } - - size_t left() const { return my_left; } - size_t right() const { return my_right; } - - // used when range does not support proportional split - operator split() const { return split(); } - -#if __TBB_ENABLE_RANGE_FEEDBACK - void set_proportion(size_t _left, size_t _right) { - my_left = _left; - my_right = _right; - } -#endif -private: - size_t my_left, my_right; -}; - -} // tbb - -// Following is a set of classes and functions typically used in compile-time "metaprogramming". -// TODO: move all that to a separate header - -#if __TBB_ALLOCATOR_TRAITS_PRESENT -#include //for allocator_traits -#endif - -#if __TBB_CPP11_RVALUE_REF_PRESENT || _LIBCPP_VERSION -#include // for std::move -#endif - -namespace tbb { -namespace internal { - -//! Class for determining type of std::allocator::value_type. -template -struct allocator_type { - typedef T value_type; -}; - -#if _MSC_VER -//! Microsoft std::allocator has non-standard extension that strips const from a type. -template -struct allocator_type { - typedef T value_type; -}; -#endif - -// Ad-hoc implementation of true_type & false_type -// Intended strictly for internal use! For public APIs (traits etc), use C++11 analogues. -template -struct bool_constant { - static /*constexpr*/ const bool value = v; -}; -typedef bool_constant true_type; -typedef bool_constant false_type; - -#if __TBB_ALLOCATOR_TRAITS_PRESENT -using std::allocator_traits; -#else -template -struct allocator_traits{ - typedef tbb::internal::false_type propagate_on_container_move_assignment; -}; -#endif - -//! A template to select either 32-bit or 64-bit constant as compile time, depending on machine word size. -template -struct select_size_t_constant { - //Explicit cast is needed to avoid compiler warnings about possible truncation. - //The value of the right size, which is selected by ?:, is anyway not truncated or promoted. - static const size_t value = (size_t)((sizeof(size_t)==sizeof(u)) ? u : ull); -}; - -#if __TBB_CPP11_RVALUE_REF_PRESENT -using std::move; -using std::forward; -#elif defined(_LIBCPP_NAMESPACE) -// libc++ defines "pre-C++11 move and forward" similarly to ours; use it to avoid name conflicts in some cases. -using std::_LIBCPP_NAMESPACE::move; -using std::_LIBCPP_NAMESPACE::forward; -#else -// It is assumed that cv qualifiers, if any, are part of the deduced type. -template -T& move( T& x ) { return x; } -template -T& forward( T& x ) { return x; } -#endif /* __TBB_CPP11_RVALUE_REF_PRESENT */ - -// Helper macros to simplify writing templates working with both C++03 and C++11. -#if __TBB_CPP11_RVALUE_REF_PRESENT -#define __TBB_FORWARDING_REF(A) A&& -#else -// It is assumed that cv qualifiers, if any, are part of a deduced type. -// Thus this macro should not be used in public interfaces. -#define __TBB_FORWARDING_REF(A) A& -#endif -#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT -#define __TBB_PARAMETER_PACK ... -#define __TBB_PACK_EXPANSION(A) A... -#else -#define __TBB_PARAMETER_PACK -#define __TBB_PACK_EXPANSION(A) A -#endif /* __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT */ - -#if __TBB_CPP11_DECLTYPE_PRESENT -#if __TBB_CPP11_DECLVAL_BROKEN -// Ad-hoc implementation of std::declval -template __TBB_FORWARDING_REF(T) declval() /*noexcept*/; -#else -using std::declval; -#endif -#endif - -template -struct STATIC_ASSERTION_FAILED; - -template <> -struct STATIC_ASSERTION_FAILED { enum {value=1};}; - -template<> -struct STATIC_ASSERTION_FAILED; //intentionally left undefined to cause compile time error - -//! @endcond -}} // namespace tbb::internal - -#if __TBB_STATIC_ASSERT_PRESENT -#define __TBB_STATIC_ASSERT(condition,msg) static_assert(condition,msg) -#else -//please note condition is intentionally inverted to get a bit more understandable error msg -#define __TBB_STATIC_ASSERT_IMPL1(condition,msg,line) \ - enum {static_assert_on_line_##line = tbb::internal::STATIC_ASSERTION_FAILED::value} - -#define __TBB_STATIC_ASSERT_IMPL(condition,msg,line) __TBB_STATIC_ASSERT_IMPL1(condition,msg,line) -//! Verify condition, at compile time -#define __TBB_STATIC_ASSERT(condition,msg) __TBB_STATIC_ASSERT_IMPL(condition,msg,__LINE__) -#endif - -#endif /* RC_INVOKED */ -#endif /* __TBB_tbb_stddef_H */ diff --git a/inst/include/tbb/tbb_thread.h b/inst/include/tbb/tbb_thread.h deleted file mode 100644 index d1cafd650..000000000 --- a/inst/include/tbb/tbb_thread.h +++ /dev/null @@ -1,342 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_tbb_thread_H -#define __TBB_tbb_thread_H - -#include "tbb_stddef.h" - -#if _WIN32||_WIN64 -#include "machine/windows_api.h" -#define __TBB_NATIVE_THREAD_ROUTINE unsigned WINAPI -#define __TBB_NATIVE_THREAD_ROUTINE_PTR(r) unsigned (WINAPI* r)( void* ) -namespace tbb { namespace internal { -#if __TBB_WIN8UI_SUPPORT - typedef size_t thread_id_type; -#else // __TBB_WIN8UI_SUPPORT - typedef DWORD thread_id_type; -#endif // __TBB_WIN8UI_SUPPORT -}} //namespace tbb::internal -#else -#define __TBB_NATIVE_THREAD_ROUTINE void* -#define __TBB_NATIVE_THREAD_ROUTINE_PTR(r) void* (*r)( void* ) -#include -namespace tbb { namespace internal { - typedef pthread_t thread_id_type; -}} //namespace tbb::internal -#endif // _WIN32||_WIN64 - -#include "atomic.h" -#include "internal/_tbb_hash_compare_impl.h" -#include "tick_count.h" - -#if !TBB_USE_EXCEPTIONS && _MSC_VER - // Suppress "C++ exception handler used, but unwind semantics are not enabled" warning in STL headers - #pragma warning (push) - #pragma warning (disable: 4530) -#endif - -#include __TBB_STD_SWAP_HEADER -#include - -#if !TBB_USE_EXCEPTIONS && _MSC_VER - #pragma warning (pop) -#endif - -namespace tbb { - -namespace internal { - class tbb_thread_v3; -} - -inline void swap( internal::tbb_thread_v3& t1, internal::tbb_thread_v3& t2 ) __TBB_NOEXCEPT(true); - -namespace internal { - - //! Allocate a closure - void* __TBB_EXPORTED_FUNC allocate_closure_v3( size_t size ); - //! Free a closure allocated by allocate_closure_v3 - void __TBB_EXPORTED_FUNC free_closure_v3( void* ); - - struct thread_closure_base { - void* operator new( size_t size ) {return allocate_closure_v3(size);} - void operator delete( void* ptr ) {free_closure_v3(ptr);} - }; - - template struct thread_closure_0: thread_closure_base { - F function; - - static __TBB_NATIVE_THREAD_ROUTINE start_routine( void* c ) { - thread_closure_0 *self = static_cast(c); - self->function(); - delete self; - return 0; - } - thread_closure_0( const F& f ) : function(f) {} - }; - //! Structure used to pass user function with 1 argument to thread. - template struct thread_closure_1: thread_closure_base { - F function; - X arg1; - //! Routine passed to Windows's _beginthreadex by thread::internal_start() inside tbb.dll - static __TBB_NATIVE_THREAD_ROUTINE start_routine( void* c ) { - thread_closure_1 *self = static_cast(c); - self->function(self->arg1); - delete self; - return 0; - } - thread_closure_1( const F& f, const X& x ) : function(f), arg1(x) {} - }; - template struct thread_closure_2: thread_closure_base { - F function; - X arg1; - Y arg2; - //! Routine passed to Windows's _beginthreadex by thread::internal_start() inside tbb.dll - static __TBB_NATIVE_THREAD_ROUTINE start_routine( void* c ) { - thread_closure_2 *self = static_cast(c); - self->function(self->arg1, self->arg2); - delete self; - return 0; - } - thread_closure_2( const F& f, const X& x, const Y& y ) : function(f), arg1(x), arg2(y) {} - }; - - //! Versioned thread class. - class tbb_thread_v3 { -#if __TBB_IF_NO_COPY_CTOR_MOVE_SEMANTICS_BROKEN - // Workaround for a compiler bug: declaring the copy constructor as public - // enables use of the moving constructor. - // The definition is not provided in order to prohibit copying. - public: -#endif - tbb_thread_v3(const tbb_thread_v3&); // = delete; // Deny access - public: -#if _WIN32||_WIN64 - typedef HANDLE native_handle_type; -#else - typedef pthread_t native_handle_type; -#endif // _WIN32||_WIN64 - - class id; - //! Constructs a thread object that does not represent a thread of execution. - tbb_thread_v3() __TBB_NOEXCEPT(true) : my_handle(0) -#if _WIN32||_WIN64 - , my_thread_id(0) -#endif // _WIN32||_WIN64 - {} - - //! Constructs an object and executes f() in a new thread - template explicit tbb_thread_v3(F f) { - typedef internal::thread_closure_0 closure_type; - internal_start(closure_type::start_routine, new closure_type(f)); - } - //! Constructs an object and executes f(x) in a new thread - template tbb_thread_v3(F f, X x) { - typedef internal::thread_closure_1 closure_type; - internal_start(closure_type::start_routine, new closure_type(f,x)); - } - //! Constructs an object and executes f(x,y) in a new thread - template tbb_thread_v3(F f, X x, Y y) { - typedef internal::thread_closure_2 closure_type; - internal_start(closure_type::start_routine, new closure_type(f,x,y)); - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - tbb_thread_v3(tbb_thread_v3&& x) __TBB_NOEXCEPT(true) - : my_handle(x.my_handle) -#if _WIN32||_WIN64 - , my_thread_id(x.my_thread_id) -#endif - { - x.internal_wipe(); - } - tbb_thread_v3& operator=(tbb_thread_v3&& x) __TBB_NOEXCEPT(true) { - internal_move(x); - return *this; - } - private: - tbb_thread_v3& operator=(const tbb_thread_v3& x); // = delete; - public: -#else // __TBB_CPP11_RVALUE_REF_PRESENT - tbb_thread_v3& operator=(tbb_thread_v3& x) { - internal_move(x); - return *this; - } -#endif // __TBB_CPP11_RVALUE_REF_PRESENT - - void swap( tbb_thread_v3& t ) __TBB_NOEXCEPT(true) {tbb::swap( *this, t );} - bool joinable() const __TBB_NOEXCEPT(true) {return my_handle!=0; } - //! The completion of the thread represented by *this happens before join() returns. - void __TBB_EXPORTED_METHOD join(); - //! When detach() returns, *this no longer represents the possibly continuing thread of execution. - void __TBB_EXPORTED_METHOD detach(); - ~tbb_thread_v3() {if( joinable() ) detach();} - inline id get_id() const __TBB_NOEXCEPT(true); - native_handle_type native_handle() { return my_handle; } - - //! The number of hardware thread contexts. - /** Before TBB 3.0 U4 this methods returned the number of logical CPU in - the system. Currently on Windows, Linux and FreeBSD it returns the - number of logical CPUs available to the current process in accordance - with its affinity mask. - - NOTE: The return value of this method never changes after its first - invocation. This means that changes in the process affinity mask that - took place after this method was first invoked will not affect the - number of worker threads in the TBB worker threads pool. **/ - static unsigned __TBB_EXPORTED_FUNC hardware_concurrency() __TBB_NOEXCEPT(true); - private: - native_handle_type my_handle; -#if _WIN32||_WIN64 - thread_id_type my_thread_id; -#endif // _WIN32||_WIN64 - - void internal_wipe() __TBB_NOEXCEPT(true) { - my_handle = 0; -#if _WIN32||_WIN64 - my_thread_id = 0; -#endif - } - void internal_move(tbb_thread_v3& x) __TBB_NOEXCEPT(true) { - if (joinable()) detach(); - my_handle = x.my_handle; -#if _WIN32||_WIN64 - my_thread_id = x.my_thread_id; -#endif // _WIN32||_WIN64 - x.internal_wipe(); - } - - /** Runs start_routine(closure) on another thread and sets my_handle to the handle of the created thread. */ - void __TBB_EXPORTED_METHOD internal_start( __TBB_NATIVE_THREAD_ROUTINE_PTR(start_routine), - void* closure ); - friend void __TBB_EXPORTED_FUNC move_v3( tbb_thread_v3& t1, tbb_thread_v3& t2 ); - friend void tbb::swap( tbb_thread_v3& t1, tbb_thread_v3& t2 ) __TBB_NOEXCEPT(true); - }; - - class tbb_thread_v3::id { - thread_id_type my_id; - id( thread_id_type id_ ) : my_id(id_) {} - - friend class tbb_thread_v3; - public: - id() __TBB_NOEXCEPT(true) : my_id(0) {} - - friend bool operator==( tbb_thread_v3::id x, tbb_thread_v3::id y ) __TBB_NOEXCEPT(true); - friend bool operator!=( tbb_thread_v3::id x, tbb_thread_v3::id y ) __TBB_NOEXCEPT(true); - friend bool operator<( tbb_thread_v3::id x, tbb_thread_v3::id y ) __TBB_NOEXCEPT(true); - friend bool operator<=( tbb_thread_v3::id x, tbb_thread_v3::id y ) __TBB_NOEXCEPT(true); - friend bool operator>( tbb_thread_v3::id x, tbb_thread_v3::id y ) __TBB_NOEXCEPT(true); - friend bool operator>=( tbb_thread_v3::id x, tbb_thread_v3::id y ) __TBB_NOEXCEPT(true); - - template - friend std::basic_ostream& - operator<< (std::basic_ostream &out, - tbb_thread_v3::id id) - { - out << id.my_id; - return out; - } - friend tbb_thread_v3::id __TBB_EXPORTED_FUNC thread_get_id_v3(); - - friend inline size_t tbb_hasher( const tbb_thread_v3::id& id ) { - __TBB_STATIC_ASSERT(sizeof(id.my_id) <= sizeof(size_t), "Implementaion assumes that thread_id_type fits into machine word"); - return tbb::tbb_hasher(id.my_id); - } - - // A workaround for lack of tbb::atomic (which would require id to be POD in C++03). - friend id atomic_compare_and_swap(id& location, const id& value, const id& comparand){ - return as_atomic(location.my_id).compare_and_swap(value.my_id, comparand.my_id); - } - }; // tbb_thread_v3::id - - tbb_thread_v3::id tbb_thread_v3::get_id() const __TBB_NOEXCEPT(true) { -#if _WIN32||_WIN64 - return id(my_thread_id); -#else - return id(my_handle); -#endif // _WIN32||_WIN64 - } - - void __TBB_EXPORTED_FUNC move_v3( tbb_thread_v3& t1, tbb_thread_v3& t2 ); - tbb_thread_v3::id __TBB_EXPORTED_FUNC thread_get_id_v3(); - void __TBB_EXPORTED_FUNC thread_yield_v3(); - void __TBB_EXPORTED_FUNC thread_sleep_v3(const tick_count::interval_t &i); - - inline bool operator==(tbb_thread_v3::id x, tbb_thread_v3::id y) __TBB_NOEXCEPT(true) - { - return x.my_id == y.my_id; - } - inline bool operator!=(tbb_thread_v3::id x, tbb_thread_v3::id y) __TBB_NOEXCEPT(true) - { - return x.my_id != y.my_id; - } - inline bool operator<(tbb_thread_v3::id x, tbb_thread_v3::id y) __TBB_NOEXCEPT(true) - { - return x.my_id < y.my_id; - } - inline bool operator<=(tbb_thread_v3::id x, tbb_thread_v3::id y) __TBB_NOEXCEPT(true) - { - return x.my_id <= y.my_id; - } - inline bool operator>(tbb_thread_v3::id x, tbb_thread_v3::id y) __TBB_NOEXCEPT(true) - { - return x.my_id > y.my_id; - } - inline bool operator>=(tbb_thread_v3::id x, tbb_thread_v3::id y) __TBB_NOEXCEPT(true) - { - return x.my_id >= y.my_id; - } - -} // namespace internal; - -//! Users reference thread class by name tbb_thread -typedef internal::tbb_thread_v3 tbb_thread; - -using internal::operator==; -using internal::operator!=; -using internal::operator<; -using internal::operator>; -using internal::operator<=; -using internal::operator>=; - -inline void move( tbb_thread& t1, tbb_thread& t2 ) { - internal::move_v3(t1, t2); -} - -inline void swap( internal::tbb_thread_v3& t1, internal::tbb_thread_v3& t2 ) __TBB_NOEXCEPT(true) { - std::swap(t1.my_handle, t2.my_handle); -#if _WIN32||_WIN64 - std::swap(t1.my_thread_id, t2.my_thread_id); -#endif /* _WIN32||_WIN64 */ -} - -namespace this_tbb_thread { - inline tbb_thread::id get_id() { return internal::thread_get_id_v3(); } - //! Offers the operating system the opportunity to schedule another thread. - inline void yield() { internal::thread_yield_v3(); } - //! The current thread blocks at least until the time specified. - inline void sleep(const tick_count::interval_t &i) { - internal::thread_sleep_v3(i); - } -} // namespace this_tbb_thread - -} // namespace tbb - -#endif /* __TBB_tbb_thread_H */ diff --git a/inst/include/tbb/tick_count.h b/inst/include/tbb/tick_count.h deleted file mode 100644 index a7f4e0f37..000000000 --- a/inst/include/tbb/tick_count.h +++ /dev/null @@ -1,140 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_tick_count_H -#define __TBB_tick_count_H - -#include "tbb_stddef.h" - -#if _WIN32||_WIN64 -#include "machine/windows_api.h" -#elif __linux__ -#include -#else /* generic Unix */ -#include -#endif /* (choice of OS) */ - -namespace tbb { - -//! Absolute timestamp -/** @ingroup timing */ -class tick_count { -public: - //! Relative time interval. - class interval_t { - long long value; - explicit interval_t( long long value_ ) : value(value_) {} - public: - //! Construct a time interval representing zero time duration - interval_t() : value(0) {}; - - //! Construct a time interval representing sec seconds time duration - explicit interval_t( double sec ); - - //! Return the length of a time interval in seconds - double seconds() const; - - friend class tbb::tick_count; - - //! Extract the intervals from the tick_counts and subtract them. - friend interval_t operator-( const tick_count& t1, const tick_count& t0 ); - - //! Add two intervals. - friend interval_t operator+( const interval_t& i, const interval_t& j ) { - return interval_t(i.value+j.value); - } - - //! Subtract two intervals. - friend interval_t operator-( const interval_t& i, const interval_t& j ) { - return interval_t(i.value-j.value); - } - - //! Accumulation operator - interval_t& operator+=( const interval_t& i ) {value += i.value; return *this;} - - //! Subtraction operator - interval_t& operator-=( const interval_t& i ) {value -= i.value; return *this;} - private: - static long long ticks_per_second(){ -#if _WIN32||_WIN64 - LARGE_INTEGER qpfreq; - int rval = QueryPerformanceFrequency(&qpfreq); - __TBB_ASSERT_EX(rval, "QueryPerformanceFrequency returned zero"); - return static_cast(qpfreq.QuadPart); -#elif __linux__ - return static_cast(1E9); -#else /* generic Unix */ - return static_cast(1E6); -#endif /* (choice of OS) */ - } - }; - - //! Construct an absolute timestamp initialized to zero. - tick_count() : my_count(0) {}; - - //! Return current time. - static tick_count now(); - - //! Subtract two timestamps to get the time interval between - friend interval_t operator-( const tick_count& t1, const tick_count& t0 ); - - //! Return the resolution of the clock in seconds per tick. - static double resolution() { return 1.0 / interval_t::ticks_per_second(); } - -private: - long long my_count; -}; - -inline tick_count tick_count::now() { - tick_count result; -#if _WIN32||_WIN64 - LARGE_INTEGER qpcnt; - int rval = QueryPerformanceCounter(&qpcnt); - __TBB_ASSERT_EX(rval, "QueryPerformanceCounter failed"); - result.my_count = qpcnt.QuadPart; -#elif __linux__ - struct timespec ts; - int status = clock_gettime( CLOCK_REALTIME, &ts ); - __TBB_ASSERT_EX( status==0, "CLOCK_REALTIME not supported" ); - result.my_count = static_cast(1000000000UL)*static_cast(ts.tv_sec) + static_cast(ts.tv_nsec); -#else /* generic Unix */ - struct timeval tv; - int status = gettimeofday(&tv, NULL); - __TBB_ASSERT_EX( status==0, "gettimeofday failed" ); - result.my_count = static_cast(1000000)*static_cast(tv.tv_sec) + static_cast(tv.tv_usec); -#endif /*(choice of OS) */ - return result; -} - -inline tick_count::interval_t::interval_t( double sec ) { - value = static_cast(sec*interval_t::ticks_per_second()); -} - -inline tick_count::interval_t operator-( const tick_count& t1, const tick_count& t0 ) { - return tick_count::interval_t( t1.my_count-t0.my_count ); -} - -inline double tick_count::interval_t::seconds() const { - return value*tick_count::resolution(); -} - -} // namespace tbb - -#endif /* __TBB_tick_count_H */ diff --git a/inst/include/tthread/tinythread.h b/inst/include/tthread/tinythread.h index 230ac3cf2..290dba434 100644 --- a/inst/include/tthread/tinythread.h +++ b/inst/include/tthread/tinythread.h @@ -130,8 +130,8 @@ freely, subject to the following restrictions: /// destructors) to be declared with the @c thread_local keyword, most pre-C++11 /// compilers only allow for trivial types (e.g. @c int). So, to guarantee /// portable code, only use trivial types for thread local storage. -/// @note This directive is currently not supported on Mac OS X (it will give -/// a compiler error), since compile-time TLS is not supported in the Mac OS X +/// @note This directive is currently not supported on Mac macOS (it will give +/// a compiler error), since compile-time TLS is not supported in the Mac macOS /// executable format. Also, some older versions of MinGW (before GCC 4.x) do /// not support this directive. /// @hideinitializer @@ -492,7 +492,7 @@ class thread { /// Default constructor. /// Construct a @c thread object without an associated thread of execution /// (i.e. non-joinable). - thread() : mHandle(0), mNotAThread(true) + thread() : mHandle(0), mJoinable(false) #if defined(_TTHREAD_WIN32_) , mWin32ThreadID(0) #endif @@ -554,7 +554,7 @@ class thread { private: native_handle_type mHandle; ///< Thread handle. mutable mutex mDataMutex; ///< Serializer for access to the thread private data. - bool mNotAThread; ///< True if this object is not a thread of execution. + bool mJoinable; ///< Is the thread joinable? #if defined(_TTHREAD_WIN32_) unsigned int mWin32ThreadID; ///< Unique thread ID (filled out by _beginthreadex). #endif @@ -871,7 +871,12 @@ inline void * thread::wrapper_function(void * aArg) // The thread is no longer executing lock_guard guard(ti->mThread->mDataMutex); - ti->mThread->mNotAThread = true; + + // On POSIX, we allow the thread to be joined even after execution has finished. + // This is necessary to ensure that thread-local memory can be reclaimed. +#if defined(_TTHREAD_WIN32_) + ti->mThread->mJoinable = false; +#endif // The thread is responsible for freeing the startup information delete ti; @@ -891,8 +896,8 @@ inline thread::thread(void (*aFunction)(void *), void * aArg) ti->mArg = aArg; ti->mThread = this; - // The thread is now alive - mNotAThread = false; + // Mark thread as joinable + mJoinable = true; // Create the thread #if defined(_TTHREAD_WIN32_) @@ -905,9 +910,10 @@ inline thread::thread(void (*aFunction)(void *), void * aArg) // Did we fail to create the thread? if(!mHandle) { - mNotAThread = true; + mJoinable = false; delete ti; } + } inline thread::~thread() @@ -926,28 +932,37 @@ inline void thread::join() #elif defined(_TTHREAD_POSIX_) pthread_join(mHandle, NULL); #endif + + // https://linux.die.net/man/3/pthread_join states: + // + // Joining with a thread that has previously been joined results in undefined behavior. + // + // We just allow a thread to be joined once. + mJoinable = false; } } inline bool thread::joinable() const { mDataMutex.lock(); - bool result = !mNotAThread; + bool result = mJoinable; mDataMutex.unlock(); return result; } inline void thread::detach() { + // TODO: Attempting to detach a non-joinable thread should throw. + // https://en.cppreference.com/w/cpp/thread/thread/detach mDataMutex.lock(); - if(!mNotAThread) + if(mJoinable) { #if defined(_TTHREAD_WIN32_) CloseHandle(mHandle); #elif defined(_TTHREAD_POSIX_) pthread_detach(mHandle); #endif - mNotAThread = true; + mJoinable = false; } mDataMutex.unlock(); } diff --git a/inst/skeleton/vector-sum.cpp b/inst/skeleton/vector-sum.cpp index 30622e519..57b541659 100644 --- a/inst/skeleton/vector-sum.cpp +++ b/inst/skeleton/vector-sum.cpp @@ -19,37 +19,37 @@ using namespace Rcpp; using namespace RcppParallel; struct Sum : public Worker -{ +{ // source vector const RVector input; - + // accumulated value double value; - + // constructors Sum(const NumericVector input) : input(input), value(0) {} Sum(const Sum& sum, Split) : input(sum.input), value(0) {} - + // accumulate just the element of the range I've been asked to void operator()(std::size_t begin, std::size_t end) { value += std::accumulate(input.begin() + begin, input.begin() + end, 0.0); } - + // join my value with that of another Sum - void join(const Sum& rhs) { - value += rhs.value; + void join(const Sum& rhs) { + value += rhs.value; } }; // [[Rcpp::export]] double parallelVectorSum(NumericVector x) { - - // declare the SumBody instance + + // declare the SumBody instance Sum sum(x); - + // call parallel_reduce to start the work parallelReduce(0, x.length(), sum); - + // return the computed sum return sum.value; } diff --git a/inst/tests/cpp/innerproduct.cpp b/inst/tests/cpp/innerproduct.cpp index 7a1957205..fee3e41ce 100644 --- a/inst/tests/cpp/innerproduct.cpp +++ b/inst/tests/cpp/innerproduct.cpp @@ -19,43 +19,43 @@ double innerProduct(NumericVector x, NumericVector y) { using namespace RcppParallel; struct InnerProduct : public Worker -{ +{ // source vectors const RVector x; const RVector y; - + // product that I have accumulated double product; - + // constructors - InnerProduct(const NumericVector x, const NumericVector y) + InnerProduct(const NumericVector x, const NumericVector y) : x(x), y(y), product(0) {} - InnerProduct(const InnerProduct& innerProduct, Split) + InnerProduct(const InnerProduct& innerProduct, Split) : x(innerProduct.x), y(innerProduct.y), product(0) {} - + // process just the elements of the range I have been asked to void operator()(std::size_t begin, std::size_t end) { - product += std::inner_product(x.begin() + begin, - x.begin() + end, - y.begin() + begin, + product += std::inner_product(x.begin() + begin, + x.begin() + end, + y.begin() + begin, 0.0); } - + // join my value with that of another InnerProduct - void join(const InnerProduct& rhs) { - product += rhs.product; + void join(const InnerProduct& rhs) { + product += rhs.product; } }; // [[Rcpp::export]] double parallelInnerProduct(NumericVector x, NumericVector y) { - + // declare the InnerProduct instance that takes a pointer to the vector data InnerProduct innerProduct(x, y); - + // call paralleReduce to start the work parallelReduce(0, x.length(), innerProduct); - + // return the computed product return innerProduct.product; } diff --git a/inst/tests/cpp/malloc.cpp b/inst/tests/cpp/malloc.cpp deleted file mode 100644 index f98667020..000000000 --- a/inst/tests/cpp/malloc.cpp +++ /dev/null @@ -1,26 +0,0 @@ - -#include -using namespace Rcpp; - -// [[Rcpp::depends(RcppParallel)]] -#include -using namespace RcppParallel; - -// [[Rcpp::export]] -void tbbMalloc() { - -#if RCPP_PARALLEL_USE_TBB - - std::vector > vec; - vec.push_back(42); - - std::vector > scalableVec; - scalableVec.push_back(42); - - std::vector > cacheAlignedVec; - cacheAlignedVec.push_back(42); - -#endif - -} - diff --git a/inst/tests/cpp/sum.cpp b/inst/tests/cpp/sum.cpp index aec4895f9..db47c699e 100644 --- a/inst/tests/cpp/sum.cpp +++ b/inst/tests/cpp/sum.cpp @@ -3,7 +3,7 @@ * @author JJ Allaire * @license GPL (>= 2) */ - + #include #include @@ -12,37 +12,37 @@ using namespace RcppParallel; using namespace Rcpp; struct Sum : public Worker -{ +{ // source vector const RVector input; - + // accumulated value double value; - + // constructors Sum(const NumericVector input) : input(input), value(0) {} Sum(const Sum& sum, Split) : input(sum.input), value(0) {} - + // accumulate just the element of the range I have been asked to void operator()(std::size_t begin, std::size_t end) { value += std::accumulate(input.begin() + begin, input.begin() + end, 0.0); } - + // join my value with that of another Sum - void join(const Sum& rhs) { - value += rhs.value; - } + void join(const Sum& rhs) { + value += rhs.value; + } }; // [[Rcpp::export]] double parallelVectorSum(NumericVector x) { - - // declare the SumBody instance + + // declare the SumBody instance Sum sum(x); - + // call parallel_reduce to start the work parallelReduce(0, x.length(), sum); - + // return the computed sum return sum.value; } diff --git a/inst/tests/cpp/truefalse_macros.cpp b/inst/tests/cpp/truefalse_macros.cpp new file mode 100644 index 000000000..412c88399 --- /dev/null +++ b/inst/tests/cpp/truefalse_macros.cpp @@ -0,0 +1,31 @@ +/** + * @title Test for TRUE and FALSE macros + * @author Travers Ching + * @license GPL (>= 2) + */ + +// TRUE and FALSE macros that may come with system headers on some systems +// But conflict with R.h (R_ext/Boolean.h) +// TRUE and FALSE macros should be undef in RcppParallel.h + +#include +#include + +// [[Rcpp::depends(RcppParallel)]] + +#ifndef TRUE +static_assert(true, "Macro TRUE does not exist"); +#else +static_assert(false, "Macro TRUE exists"); +#endif + +#ifndef FALSE +static_assert(true, "Macro FALSE does not exist"); +#else +static_assert(false, "Macro FALSE exists"); +#endif + +// [[Rcpp::export]] +int hush_no_export_warning() { + return 1; +} \ No newline at end of file diff --git a/inst/tests/runit.distance.R b/inst/tests/runit.distance.R index 6e384d481..f85a58020 100644 --- a/inst/tests/runit.distance.R +++ b/inst/tests/runit.distance.R @@ -1,14 +1,18 @@ +library(Rcpp) +library(RUnit) + sourceCpp(system.file("tests/cpp/distance.cpp", package = "RcppParallel")) test.distance <- function() { - n = 1000 - m = matrix(runif(n*10), ncol = 10) - m = m/rowSums(m) + n <- 1000 + m <- matrix(runif(n*10), ncol = 10) + m <- m/rowSums(m) checkEquals( rcpp_js_distance(m), rcpp_parallel_js_distance(m) ) + } diff --git a/inst/tests/runit.innerproduct.R b/inst/tests/runit.innerproduct.R index a28fc5d3e..2f9260cb2 100644 --- a/inst/tests/runit.innerproduct.R +++ b/inst/tests/runit.innerproduct.R @@ -1,4 +1,7 @@ +library(Rcpp) +library(RUnit) + sourceCpp(system.file("tests/cpp/innerproduct.cpp", package = "RcppParallel")) test.innerproduct <- function() { @@ -10,4 +13,5 @@ test.innerproduct <- function() { innerProduct(x, y), parallelInnerProduct(x, y) ) + } diff --git a/inst/tests/runit.malloc.R b/inst/tests/runit.malloc.R deleted file mode 100644 index 65b64f664..000000000 --- a/inst/tests/runit.malloc.R +++ /dev/null @@ -1,6 +0,0 @@ -sourceCpp(system.file("tests/cpp/malloc.cpp", package = "RcppParallel")) - -test.malloc <- function() { - tbbMalloc() -} - diff --git a/inst/tests/runit.sum.R b/inst/tests/runit.sum.R index 286592e35..4792b1cd7 100644 --- a/inst/tests/runit.sum.R +++ b/inst/tests/runit.sum.R @@ -1,4 +1,7 @@ +library(Rcpp) +library(RUnit) + sourceCpp(system.file("tests/cpp/sum.cpp", package = "RcppParallel")) test.sum <- function() { diff --git a/inst/tests/runit.transform.R b/inst/tests/runit.transform.R index 68e13a12b..d056dbdb4 100644 --- a/inst/tests/runit.transform.R +++ b/inst/tests/runit.transform.R @@ -1,4 +1,7 @@ +library(Rcpp) +library(RUnit) + sourceCpp(system.file("tests/cpp/transform.cpp", package = "RcppParallel")) test.transform <- function() { diff --git a/inst/tests/runit.truefalse_macros.R b/inst/tests/runit.truefalse_macros.R new file mode 100644 index 000000000..8e403e364 --- /dev/null +++ b/inst/tests/runit.truefalse_macros.R @@ -0,0 +1,6 @@ + +library(Rcpp) +library(RUnit) + +sourceCpp(system.file("tests/cpp/truefalse_macros.cpp", package = "RcppParallel")) + diff --git a/man/RcppParallel-package.Rd b/man/RcppParallel-package.Rd index 7ebea9992..76121353f 100644 --- a/man/RcppParallel-package.Rd +++ b/man/RcppParallel-package.Rd @@ -1,28 +1,26 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/RcppParallel-package.R \docType{package} \name{RcppParallel-package} -\alias{RcppParallel} \alias{RcppParallel-package} +\alias{RcppParallel} \title{Parallel programming tools for Rcpp} \description{ - High level functions for doing parallel programming with Rcpp. - For example, the \code{parallelFor} function can be used to convert the work of - a standard serial "for" loop into a parallel one and the \code{parallelReduce} - function can be used for accumulating aggregate or other values. - - The high level interface enables safe and robust parallel programming without - direct manipulation of operating system threads. On Windows, OS X, and Linux - systems the underlying implementation is based on Intel TBB (Threading Building - Blocks). On other platforms a less-performant fallback implementation based on the - TinyThread library is used. - - For additional documentation see the package website at: \href{http://rcppcore.github.io/RcppParallel}{http://rcppcore.github.io/RcppParallel}. -} - -\author{ - JJ Allaire, Romain Francois, Gregory Vandenbrouck, Marcus Geelnard, Intel Inc. +High level functions for doing parallel programming with Rcpp. For example, +the \code{parallelFor()} function can be used to convert the work of a +standard serial "for" loop into a parallel one, and the \code{parallelReduce()} +function can be used for accumulating aggregate or other values. } +\details{ +The high level interface enables safe and robust parallel programming +without direct manipulation of operating system threads. On Windows, macOS, +and Linux systems the underlying implementation is based on Intel TBB +(Threading Building Blocks). On other platforms, a less-performant fallback +implementation based on the TinyThread library is used. +For additional documentation, see the package website at: +\url{https://rcppcore.github.io/RcppParallel/} +} \keyword{package} \keyword{parallel} - diff --git a/man/RcppParallel.package.skeleton.Rd b/man/RcppParallel.package.skeleton.Rd index 06b6c22c5..ad58178e6 100644 --- a/man/RcppParallel.package.skeleton.Rd +++ b/man/RcppParallel.package.skeleton.Rd @@ -1,65 +1,67 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/skeleton.R \name{RcppParallel.package.skeleton} \alias{RcppParallel.package.skeleton} -\title{ -Create a skeleton for a new package depending on RcppParallel -} -\description{ - \code{RcppParallel.package.skeleton} automates the creation of - a new source package that intends to use features of RcppParallel. - - It is based on the \link[utils]{package.skeleton} function - which it executes first. -} +\title{Create a skeleton for a new package depending on RcppParallel} \usage{ -RcppParallel.package.skeleton( - name = "anRpackage", - example_code = TRUE, - ... -) +RcppParallel.package.skeleton(name = "anRpackage", example_code = TRUE, ...) } \arguments{ - \item{name}{The name of your R package.} - \item{example_code}{If \code{TRUE}, example C++ code using RcppParallel is added to the package.} - \item{...}{Optional arguments passed to \link[Rcpp]{Rcpp.package.skeleton}.} -} -\details{ - In addition to \link[Rcpp]{Rcpp.package.skeleton} : - - The \samp{DESCRIPTION} file gains an Imports line requesting that - the package depends on RcppParallel and a LinkingTo line so that the package - finds RcppParallel header files. - - The \samp{NAMESPACE} gains a \code{useDynLib} directive as well - as an \code{importFrom(RcppParallel, evalCpp} to ensure instantiation of RcppParallel. - - The \samp{src} directory is created if it does not exists and - a \samp{Makevars} file is added setting the environment variables - \samp{PKG_LIBS} to accomodate the necessary flags to link with the RcppParallel library. - - If the \code{example_code} argument is set to \code{TRUE}, - example files \samp{vector-sum.cpp} is created in the \samp{src} directory. - \code{Rcpp::compileAttributes()} is then called to generate \code{src/RcppExports.cpp} and - \code{R/RcppExports.R}. These files are given as an example and should - eventually by removed from the generated package. +\item{name}{The name of your R package.} + +\item{example_code}{If \code{TRUE}, example C++ code using RcppParallel is +added to the package.} + +\item{...}{Optional arguments passed to \link[Rcpp]{Rcpp.package.skeleton}.} } \value{ Nothing, used for its side effects } -\seealso{ -\link[utils]{package.skeleton} +\description{ +\code{RcppParallel.package.skeleton} automates the creation of a new source +package that intends to use features of RcppParallel. } -\references{ -Read the \emph{Writing R Extensions} manual for more details. +\details{ +It is based on the \link[utils]{package.skeleton} function which it executes +first. -Once you have created a \emph{source} package you need to install it: -see the \emph{R Installation and Administration} manual, -\code{\link{INSTALL}} and \code{\link{install.packages}}. +In addition to \link[Rcpp]{Rcpp.package.skeleton} : + +The \samp{DESCRIPTION} file gains an Imports line requesting that the +package depends on RcppParallel and a LinkingTo line so that the package +finds RcppParallel header files. + +The \samp{NAMESPACE} gains a \code{useDynLib} directive as well as an +\code{importFrom(RcppParallel, evalCpp} to ensure instantiation of +RcppParallel. + +The \samp{src} directory is created if it does not exists and a +\samp{Makevars} file is added setting the environment variables +\samp{PKG_LIBS} to accomodate the necessary flags to link with the +RcppParallel library. + +If the \code{example_code} argument is set to \code{TRUE}, example files +\samp{vector-sum.cpp} is created in the \samp{src} directory. +\code{Rcpp::compileAttributes()} is then called to generate +\code{src/RcppExports.cpp} and \code{R/RcppExports.R}. These files are given +as an example and should eventually by removed from the generated package. } \examples{ + \dontrun{ # simple package -RcppParallel.package.skeleton( "foobar" ) +RcppParallel.package.skeleton("foobar") } + } -\keyword{ programming } +\references{ +Read the \emph{Writing R Extensions} manual for more details. +Once you have created a \emph{source} package you need to install it: see +the \emph{R Installation and Administration} manual, \code{\link{INSTALL}} +and \code{\link{install.packages}}. +} +\seealso{ +\link[utils]{package.skeleton} +} +\keyword{programming} diff --git a/man/RcppParallelFlags.Rd b/man/RcppParallelFlags.Rd deleted file mode 100644 index 1dc163df8..000000000 --- a/man/RcppParallelFlags.Rd +++ /dev/null @@ -1,28 +0,0 @@ -\name{RcppParallelFlags} -\alias{RcppParallelLibs} -\alias{LdFlags} -\alias{CxxFlags} -\title{ -Compilation flags for RcppParallel -} -\description{ -Output the compiler or linker flags required to build against RcppParallel. -} -\usage{ -CxxFlags() -LdFlags() -RcppParallelLibs() -} - -\details{ -These functions are typically called from \code{Makevars} as follows: - -\code{PKG_LIBS += $(shell "${R_HOME}/bin/Rscript" -e "RcppParallel::LdFlags()")} - -} - -\value{ - Returns \code{NULL} invisibly. The function is not called for it's - return value rather for the side effect of outputting the flags. -} - diff --git a/man/flags.Rd b/man/flags.Rd new file mode 100644 index 000000000..6401d4264 --- /dev/null +++ b/man/flags.Rd @@ -0,0 +1,37 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/flags.R +\name{flags} +\alias{flags} +\alias{RcppParallelLibs} +\alias{LdFlags} +\alias{CxxFlags} +\title{Compilation flags for RcppParallel} +\usage{ +CxxFlags() + +LdFlags() + +RcppParallelLibs() +} +\value{ +Returns \code{NULL}, invisibly. These functions are called for +their side effects (writing the associated flags to stdout). +} +\description{ +Output the compiler or linker flags required to build against RcppParallel. +} +\details{ +These functions are typically called from \code{Makevars} as follows:\preformatted{PKG_LIBS += $(shell "$\{R_HOME\}/bin/Rscript" -e "RcppParallel::LdFlags()") +} + +On Windows, the flags ensure that the package links with the built-in TBB +library. On Linux and macOS, the output is empty, because TBB is loaded +dynamically on load by \code{RcppParallel}. + +\R packages using RcppParallel should also add the following to their +\code{NAMESPACE} file:\preformatted{importFrom(RcppParallel, RcppParallelLibs) +} + +This is necessary to ensure that \pkg{RcppParallel} (and so, TBB) is loaded +and available. +} diff --git a/man/setThreadOptions.Rd b/man/setThreadOptions.Rd index 3cb874c71..5e68a3c35 100644 --- a/man/setThreadOptions.Rd +++ b/man/setThreadOptions.Rd @@ -1,48 +1,49 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/options.R \name{setThreadOptions} \alias{setThreadOptions} \alias{defaultNumThreads} -\title{ -Thread options for RcppParallel -} -\description{ -Set thread options (number of threads to use for task scheduling and stack -size per-thread) for RcppParallel. -} +\title{Thread options for RcppParallel} \usage{ -setThreadOptions(numThreads = "auto", - stackSize = "auto") +setThreadOptions(numThreads = "auto", stackSize = "auto") + defaultNumThreads() } \arguments{ - \item{numThreads}{ - Number of threads to use for task scheduling (call - \code{defaultNumThreads} to determine the the default - value used for "auto"). - } - \item{stackSize}{ - Stack size (in bytes) to use for worker threads. The - default used for "auto" is 2MB on 32-bit systems and - 4MB on 64-bit systems (note that this parameter has - no effect on Windows). - } -} -\details{ - RcppParallel is automatically initialized with the default number - of threads and thread stack size when it loads. You can call - \code{setThreadOptions} at any time to change the defaults. -} +\item{numThreads}{Number of threads to use for task scheduling. Call \code{defaultNumThreads()} +to determine the the default value used for "auto".} +\item{stackSize}{Stack size (in bytes) to use for worker threads. The +default used for "auto" is 2MB on 32-bit systems and 4MB on 64-bit systems +(note that this parameter has no effect on Windows).} +} \value{ - The \code{defaultNumThreads} returns the default number of threads - that are used by RcppParallel if another value isn't specified using - \code{setThreadOptions}. +\code{defaultNumThreads()} returns the default number of threads used by +RcppParallel, if another value isn't specified either via +\code{setThreadOptions()} or explicitly in calls to \code{parallelFor()} and +\code{parallelReduce()}. } +\description{ +Set thread options (number of threads to use for task scheduling and stack +size per-thread) for RcppParallel. +} +\details{ +RcppParallel is automatically initialized with the default number of threads +and thread stack size when it loads. You can call \code{setThreadOptions()} at +any time to change the defaults. +The \code{parallelFor()} and \code{parallelReduce()} also accept \code{numThreads} as +an argument, if you'd like to control the number of threads specifically +to be made available for a particular parallel function call. Note that +this value is advisory, and TBB may choose a smaller number of threads +if the number of requested threads cannot be honored on the system. +} \examples{ -library(RcppParallel) +\dontrun{ +library(RcppParallel) setThreadOptions(numThreads = 4) - defaultNumThreads() } +} diff --git a/man/tbbLibraryPath.Rd b/man/tbbLibraryPath.Rd new file mode 100644 index 000000000..3989c45e1 --- /dev/null +++ b/man/tbbLibraryPath.Rd @@ -0,0 +1,18 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/tbb.R +\name{tbbLibraryPath} +\alias{tbbLibraryPath} +\title{Get the Path to a TBB Library} +\usage{ +tbbLibraryPath(name = NULL) +} +\arguments{ +\item{name}{The name of the TBB library to be resolved. Normally, this is one of +\code{tbb}, \code{tbbmalloc}, or \code{tbbmalloc_proxy}. When \code{NULL}, the library +path containing the TBB libraries is returned instead.} +} +\description{ +Retrieve the path to a TBB library. This can be useful for \R packages +using RcppParallel that wish to use, or re-use, the version of TBB that +RcppParallel has been configured to use. +} diff --git a/patches/windows_arm64.diff b/patches/windows_arm64.diff new file mode 100644 index 000000000..514f62756 --- /dev/null +++ b/patches/windows_arm64.diff @@ -0,0 +1,47 @@ +diff --git a/src/tbb/build/Makefile.tbb b/src/tbb/build/Makefile.tbb +index 8d155f80..c58f4fb1 100644 +--- a/src/tbb/build/Makefile.tbb ++++ b/src/tbb/build/Makefile.tbb +@@ -91,7 +91,11 @@ ifneq (,$(TBB.DEF)) + tbb.def: $(TBB.DEF) $(TBB.LST) + $(CPLUS) $(PREPROC_ONLY) $< $(CPLUS_FLAGS) $(INCLUDES) > $@ + +-LIB_LINK_FLAGS += $(EXPORT_KEY)tbb.def ++# LLVM on Windows doesn't need --version-script export ++# https://reviews.llvm.org/D63743 ++ifeq (, $(WINARM64_CLANG)) ++ LIB_LINK_FLAGS += $(EXPORT_KEY)tbb.def ++endif + $(TBB.DLL): tbb.def + endif + +diff --git a/src/tbb/build/Makefile.tbbmalloc b/src/tbb/build/Makefile.tbbmalloc +index 421e95c5..e7c38fa4 100644 +--- a/src/tbb/build/Makefile.tbbmalloc ++++ b/src/tbb/build/Makefile.tbbmalloc +@@ -74,7 +74,11 @@ ifneq (,$(MALLOC.DEF)) + tbbmalloc.def: $(MALLOC.DEF) + $(CPLUS) $(PREPROC_ONLY) $< $(M_CPLUS_FLAGS) $(WARNING_SUPPRESS) $(INCLUDES) > $@ + +-MALLOC_LINK_FLAGS += $(EXPORT_KEY)tbbmalloc.def ++# LLVM on Windows doesn't need --version-script export ++# https://reviews.llvm.org/D63743 ++ifeq (, $(WINARM64_CLANG)) ++ MALLOC_LINK_FLAGS += $(EXPORT_KEY)tbbmalloc.def ++endif + $(MALLOC.DLL): tbbmalloc.def + endif + +diff --git a/src/tbb/src/tbbmalloc/TypeDefinitions.h b/src/tbb/src/tbbmalloc/TypeDefinitions.h +index 3178442e..fd4b7956 100644 +--- a/src/tbb/src/tbbmalloc/TypeDefinitions.h ++++ b/src/tbb/src/tbbmalloc/TypeDefinitions.h +@@ -25,7 +25,7 @@ + # define __ARCH_ipf 1 + # elif defined(_M_IX86)||defined(__i386__) // the latter for MinGW support + # define __ARCH_x86_32 1 +-# elif defined(_M_ARM) ++# elif defined(_M_ARM) || defined(__aarch64__) + # define __ARCH_other 1 + # else + # error Unknown processor architecture for Windows diff --git a/revdep/README.md b/revdep/README.md deleted file mode 100644 index 61c9d606a..000000000 --- a/revdep/README.md +++ /dev/null @@ -1,520 +0,0 @@ -# Setup - -## Platform - -|setting |value | -|:--------|:----------------------------| -|version |R version 3.4.2 (2017-09-28) | -|system |x86_64, mingw32 | -|ui |RStudio (1.1.379) | -|language |(EN) | -|collate |English_United States.1252 | -|tz |Europe/Berlin | -|date |2017-10-05 | - -## Packages - -|package |* |version |date |source | -|:------------|:--|:--------|:----------|:------------------------------| -|BH | |1.65.0-1 |2017-08-24 |CRAN (R 3.4.1) | -|knitr | |1.17 |2017-08-10 |CRAN (R 3.4.2) | -|Rcpp | |0.12.13 |2017-09-28 |CRAN (R 3.4.2) | -|RcppParallel | |2018.1.0 |2017-10-05 |local (jolars/RcppParallel@NA) | -|rmarkdown | |1.6 |2017-06-15 |CRAN (R 3.4.2) | -|RUnit | |0.4.31 |2015-11-06 |CRAN (R 3.4.1) | - -# Check results - -24 packages - -|package |version | errors| warnings| notes| -|:------------|:---------|------:|--------:|-----:| -|catSurv |1.0.3 | 0| 0| 2| -|Cyclops |1.3.0 | 0| 0| 1| -|dbmss |2.4-0 | 0| 0| 1| -|FLSSS |5.2 | 0| 0| 1| -|gaston |1.5 | 0| 0| 2| -|GERGM |0.11.2 | 0| 0| 1| -|lamW |1.3.0 | 0| 0| 3| -|markovchain |0.6.9.8-1 | 0| 0| 1| -|microclass |1.1 | 0| 0| 2| -|milr |0.3.0 | 0| 0| 1| -|MultiBD |0.2.0 | 0| 0| 1| -|parallelDist |0.2.0 | 0| 0| 1| -|patternplot |0.1 | 0| 0| 2| -|qualpalr |0.4.2 | 0| 0| 2| -|quanteda |0.99.9 | 0| 0| 1| -|re2r |0.2.0 | 0| 0| 2| -|rENA |0.1.0 | 1| 0| 0| -|roll |1.0.7 | 0| 0| 2| -|rPref |1.2 | 0| 0| 3| -|skm |0.1.5.4 | 0| 0| 2| -|SpatMCA |1.0.0.2 | 0| 0| 2| -|SpatPCA |1.1.1.2 | 0| 0| 2| -|StMoSim |3.0 | 0| 0| 3| -|text2vec |0.5.0 | 0| 0| 3| - -## catSurv (1.0.3) -Maintainer: Erin Rossiter -Bug reports: https://github.com/erossiter/catSurv/issues - -0 errors | 0 warnings | 2 notes - -``` -checking for GNU extensions in Makefiles ... NOTE -GNU make is a SystemRequirements. - -checking compiled code ... NOTE -File 'catSurv/libs/x64/catSurv.dll': - Found no calls to: 'R_registerRoutines', 'R_useDynamicSymbols' - -It is good practice to register native routines and to disable symbol -search. - -See 'Writing portable packages' in the 'Writing R Extensions' manual. -``` - -## Cyclops (1.3.0) -Maintainer: Marc A. Suchard -Bug reports: https://github.com/ohdsi/cyclops/issues - -0 errors | 0 warnings | 1 note - -``` -checking compiled code ... NOTE -File 'Cyclops/libs/x64/Cyclops.dll': - Found no calls to: 'R_registerRoutines', 'R_useDynamicSymbols' - -It is good practice to register native routines and to disable symbol -search. - -See 'Writing portable packages' in the 'Writing R Extensions' manual. -``` - -## dbmss (2.4-0) -Maintainer: Eric Marcon - -0 errors | 0 warnings | 1 note - -``` -checking compiled code ... NOTE -File 'dbmss/libs/x64/dbmss.dll': - Found no calls to: 'R_registerRoutines', 'R_useDynamicSymbols' - -It is good practice to register native routines and to disable symbol -search. - -See 'Writing portable packages' in the 'Writing R Extensions' manual. -``` - -## FLSSS (5.2) -Maintainer: Charlie Wusuo Liu - -0 errors | 0 warnings | 1 note - -``` -checking compiled code ... NOTE -File 'FLSSS/libs/x64/FLSSS.dll': - Found no calls to: 'R_registerRoutines', 'R_useDynamicSymbols' - -It is good practice to register native routines and to disable symbol -search. - -See 'Writing portable packages' in the 'Writing R Extensions' manual. -``` - -## gaston (1.5) -Maintainer: Hervé Perdry - -0 errors | 0 warnings | 2 notes - -``` -checking Rd cross-references ... NOTE -Package unavailable to check Rd xrefs: 'WhopGenome' - -checking compiled code ... NOTE -File 'gaston/libs/x64/gaston.dll': - Found no calls to: 'R_registerRoutines', 'R_useDynamicSymbols' - -It is good practice to register native routines and to disable symbol -search. - -See 'Writing portable packages' in the 'Writing R Extensions' manual. -``` - -## GERGM (0.11.2) -Maintainer: Matthew J. Denny - -0 errors | 0 warnings | 1 note - -``` -checking compiled code ... NOTE -File 'GERGM/libs/x64/GERGM.dll': - Found no calls to: 'R_registerRoutines', 'R_useDynamicSymbols' - -It is good practice to register native routines and to disable symbol -search. - -See 'Writing portable packages' in the 'Writing R Extensions' manual. -``` - -## lamW (1.3.0) -Maintainer: Avraham Adler -Bug reports: https://bitbucket.org/aadler/lamw/issues - -0 errors | 0 warnings | 3 notes - -``` -checking Rd cross-references ... NOTE -Package unavailable to check Rd xrefs: 'gsl' - -checking for GNU extensions in Makefiles ... NOTE -GNU make is a SystemRequirements. - -checking compiled code ... NOTE -File 'lamW/libs/x64/lamW.dll': - Found no calls to: 'R_registerRoutines', 'R_useDynamicSymbols' - -It is good practice to register native routines and to disable symbol -search. - -See 'Writing portable packages' in the 'Writing R Extensions' manual. -``` - -## markovchain (0.6.9.8-1) -Maintainer: Giorgio Alfredo Spedicato -Bug reports: http://github.com/spedygiorgio/markovchain/issues - -0 errors | 0 warnings | 1 note - -``` -checking compiled code ... NOTE -File 'markovchain/libs/x64/markovchain.dll': - Found no calls to: 'R_registerRoutines', 'R_useDynamicSymbols' - -It is good practice to register native routines and to disable symbol -search. - -See 'Writing portable packages' in the 'Writing R Extensions' manual. -``` - -## microclass (1.1) -Maintainer: Lars Snipen - -0 errors | 0 warnings | 2 notes - -``` -checking for GNU extensions in Makefiles ... NOTE -GNU make is a SystemRequirements. - -checking compiled code ... NOTE -File 'microclass/libs/x64/microclass.dll': - Found no calls to: 'R_registerRoutines', 'R_useDynamicSymbols' - -It is good practice to register native routines and to disable symbol -search. - -See 'Writing portable packages' in the 'Writing R Extensions' manual. -``` - -## milr (0.3.0) -Maintainer: Ping-Yang Chen -Bug reports: https://github.com/PingYangChen/milr/issues - -0 errors | 0 warnings | 1 note - -``` -checking compiled code ... NOTE -File 'milr/libs/x64/milr.dll': - Found no calls to: 'R_registerRoutines', 'R_useDynamicSymbols' - -It is good practice to register native routines and to disable symbol -search. - -See 'Writing portable packages' in the 'Writing R Extensions' manual. -``` - -## MultiBD (0.2.0) -Maintainer: Marc A. Suchard - -0 errors | 0 warnings | 1 note - -``` -checking compiled code ... NOTE -File 'MultiBD/libs/x64/MultiBD.dll': - Found no calls to: 'R_registerRoutines', 'R_useDynamicSymbols' - -It is good practice to register native routines and to disable symbol -search. - -See 'Writing portable packages' in the 'Writing R Extensions' manual. -``` - -## parallelDist (0.2.0) -Maintainer: Alexander Eckert -Bug reports: https://github.com/alexeckert/parallelDist/issues - -0 errors | 0 warnings | 1 note - -``` -checking compiled code ... NOTE -File 'parallelDist/libs/x64/parallelDist.dll': - Found no calls to: 'R_registerRoutines', 'R_useDynamicSymbols' - -It is good practice to register native routines and to disable symbol -search. - -See 'Writing portable packages' in the 'Writing R Extensions' manual. -``` - -## patternplot (0.1) -Maintainer: Chunqiao Luo - -0 errors | 0 warnings | 2 notes - -``` -checking for GNU extensions in Makefiles ... NOTE -GNU make is a SystemRequirements. - -checking compiled code ... NOTE -File 'patternplot/libs/x64/patternplot.dll': - Found no calls to: 'R_registerRoutines', 'R_useDynamicSymbols' - -It is good practice to register native routines and to disable symbol -search. - -See 'Writing portable packages' in the 'Writing R Extensions' manual. -``` - -## qualpalr (0.4.2) -Maintainer: Johan Larsson -Bug reports: https://github.com/jolars/qualpalr/issues - -0 errors | 0 warnings | 2 notes - -``` -checking for GNU extensions in Makefiles ... NOTE -GNU make is a SystemRequirements. - -checking compiled code ... NOTE -File 'qualpalr/libs/x64/qualpalr.dll': - Found no calls to: 'R_registerRoutines', 'R_useDynamicSymbols' - -It is good practice to register native routines and to disable symbol -search. - -See 'Writing portable packages' in the 'Writing R Extensions' manual. -``` - -## quanteda (0.99.9) -Maintainer: Kenneth Benoit -Bug reports: https://github.com/kbenoit/quanteda/issues - -0 errors | 0 warnings | 1 note - -``` -checking compiled code ... NOTE -File 'quanteda/libs/x64/quanteda.dll': - Found no calls to: 'R_registerRoutines', 'R_useDynamicSymbols' - -It is good practice to register native routines and to disable symbol -search. - -See 'Writing portable packages' in the 'Writing R Extensions' manual. -``` - -## re2r (0.2.0) -Maintainer: Qin Wenfeng -Bug reports: https://github.com/qinwf/re2r/issues - -0 errors | 0 warnings | 2 notes - -``` -checking for GNU extensions in Makefiles ... NOTE -GNU make is a SystemRequirements. - -checking compiled code ... NOTE -File 're2r/libs/x64/re2r.dll': - Found no calls to: 'R_registerRoutines', 'R_useDynamicSymbols' - -It is good practice to register native routines and to disable symbol -search. - -See 'Writing portable packages' in the 'Writing R Extensions' manual. -``` - -## rENA (0.1.0) -Maintainer: Cody L Marquart - -1 error | 0 warnings | 0 notes - -``` -checking whether package 'rENA' can be installed ... ERROR -Installation failed. -See 'C:/Users/johan/Documents/GitHub/RcppParallel/revdep/checks/rENA.Rcheck/00install.out' for details. -``` - -## roll (1.0.7) -Maintainer: Jason Foster -Bug reports: https://github.com/jjf234/roll/issues - -0 errors | 0 warnings | 2 notes - -``` -checking for GNU extensions in Makefiles ... NOTE -GNU make is a SystemRequirements. - -checking compiled code ... NOTE -File 'roll/libs/x64/roll.dll': - Found no calls to: 'R_registerRoutines', 'R_useDynamicSymbols' - -It is good practice to register native routines and to disable symbol -search. - -See 'Writing portable packages' in the 'Writing R Extensions' manual. -``` - -## rPref (1.2) -Maintainer: Patrick Roocks - -0 errors | 0 warnings | 3 notes - -``` -checking package dependencies ... NOTE -Package suggested but not available for checking: 'Rgraphviz' - -checking for GNU extensions in Makefiles ... NOTE -GNU make is a SystemRequirements. - -checking compiled code ... NOTE -File 'rPref/libs/x64/rPref.dll': - Found no calls to: 'R_registerRoutines', 'R_useDynamicSymbols' - -It is good practice to register native routines and to disable symbol -search. - -See 'Writing portable packages' in the 'Writing R Extensions' manual. -``` - -## skm (0.1.5.4) -Maintainer: Guang Yang -Bug reports: http://github.com/gyang274/skm/issues - -0 errors | 0 warnings | 2 notes - -``` -checking for GNU extensions in Makefiles ... NOTE -GNU make is a SystemRequirements. - -checking compiled code ... NOTE -File 'skm/libs/x64/skm.dll': - Found no calls to: 'R_registerRoutines', 'R_useDynamicSymbols' - -It is good practice to register native routines and to disable symbol -search. - -See 'Writing portable packages' in the 'Writing R Extensions' manual. -``` - -## SpatMCA (1.0.0.2) -Maintainer: Wen-Ting Wang -Bug reports: https://github.com/egpivo/SpatMCA/issues - -0 errors | 0 warnings | 2 notes - -``` -checking for GNU extensions in Makefiles ... NOTE -GNU make is a SystemRequirements. - -checking compiled code ... NOTE -File 'SpatMCA/libs/x64/SpatMCA.dll': - Found no calls to: 'R_registerRoutines', 'R_useDynamicSymbols' - -It is good practice to register native routines and to disable symbol -search. - -See 'Writing portable packages' in the 'Writing R Extensions' manual. -``` - -## SpatPCA (1.1.1.2) -Maintainer: Wen-Ting Wang -Bug reports: https://github.com/egpivo/SpatPCA/issues - -0 errors | 0 warnings | 2 notes - -``` -checking for GNU extensions in Makefiles ... NOTE -GNU make is a SystemRequirements. - -checking compiled code ... NOTE -File 'SpatPCA/libs/x64/SpatPCA.dll': - Found no calls to: 'R_registerRoutines', 'R_useDynamicSymbols' - -It is good practice to register native routines and to disable symbol -search. - -See 'Writing portable packages' in the 'Writing R Extensions' manual. -``` - -## StMoSim (3.0) -Maintainer: Matthias Salvisberg - -0 errors | 0 warnings | 3 notes - -``` -checking DESCRIPTION meta-information ... NOTE -Malformed Title field: should not end in a period. - -checking R code for possible problems ... NOTE -myQQNorm: no visible global function definition for 'mad' -myQQNorm: no visible global function definition for 'qnorm' -myQQNorm: no visible global function definition for 'plot' -myQQNorm: no visible global function definition for 'matlines' -myQQNorm: no visible global function definition for 'points' -myQQNorm: no visible global function definition for 'qqline' -myQQNorm: no visible global function definition for 'box' -qqnormSim,lm: no visible global function definition for 'resid' -Undefined global functions or variables: - box mad matlines plot points qnorm qqline resid -Consider adding - importFrom("graphics", "box", "matlines", "plot", "points") - importFrom("stats", "mad", "qnorm", "qqline", "resid") -to your NAMESPACE file. - -checking compiled code ... NOTE -File 'StMoSim/libs/x64/StMoSim.dll': - Found no calls to: 'R_registerRoutines', 'R_useDynamicSymbols' - -It is good practice to register native routines and to disable symbol -search. - -See 'Writing portable packages' in the 'Writing R Extensions' manual. -``` - -## text2vec (0.5.0) -Maintainer: Dmitriy Selivanov -Bug reports: https://github.com/dselivanov/text2vec/issues - -0 errors | 0 warnings | 3 notes - -``` -checking installed package size ... NOTE - installed size is 7.5Mb - sub-directories of 1Mb or more: - data 2.6Mb - doc 3.5Mb - libs 1.0Mb - -checking for GNU extensions in Makefiles ... NOTE -GNU make is a SystemRequirements. - -checking compiled code ... NOTE -File 'text2vec/libs/x64/text2vec.dll': - Found no calls to: 'R_registerRoutines', 'R_useDynamicSymbols' - -It is good practice to register native routines and to disable symbol -search. - -See 'Writing portable packages' in the 'Writing R Extensions' manual. -``` - diff --git a/revdep/checks.rds b/revdep/checks.rds deleted file mode 100644 index 1651e1b8a..000000000 Binary files a/revdep/checks.rds and /dev/null differ diff --git a/revdep/data.sqlite b/revdep/data.sqlite deleted file mode 100644 index f36a21a78..000000000 Binary files a/revdep/data.sqlite and /dev/null differ diff --git a/revdep/problems.md b/revdep/problems.md deleted file mode 100644 index bc638ead0..000000000 --- a/revdep/problems.md +++ /dev/null @@ -1,44 +0,0 @@ -# Setup - -## Platform - -|setting |value | -|:--------|:----------------------------| -|version |R version 3.4.2 (2017-09-28) | -|system |x86_64, mingw32 | -|ui |RStudio (1.1.379) | -|language |(EN) | -|collate |English_United States.1252 | -|tz |Europe/Berlin | -|date |2017-10-05 | - -## Packages - -|package |* |version |date |source | -|:------------|:--|:--------|:----------|:------------------------------| -|BH | |1.65.0-1 |2017-08-24 |CRAN (R 3.4.1) | -|knitr | |1.17 |2017-08-10 |CRAN (R 3.4.2) | -|Rcpp | |0.12.13 |2017-09-28 |CRAN (R 3.4.2) | -|RcppParallel | |2018.1.0 |2017-10-05 |local (jolars/RcppParallel@NA) | -|rmarkdown | |1.6 |2017-06-15 |CRAN (R 3.4.2) | -|RUnit | |0.4.31 |2015-11-06 |CRAN (R 3.4.1) | - -# Check results - -1 packages with problems - -|package |version | errors| warnings| notes| -|:-------|:-------|------:|--------:|-----:| -|rENA |0.1.0 | 1| 0| 0| - -## rENA (0.1.0) -Maintainer: Cody L Marquart - -1 error | 0 warnings | 0 notes - -``` -checking whether package 'rENA' can be installed ... ERROR -Installation failed. -See 'C:/Users/johan/Documents/GitHub/RcppParallel/revdep/checks/rENA.Rcheck/00install.out' for details. -``` - diff --git a/revdep/timing.md b/revdep/timing.md deleted file mode 100644 index 8bf7780fb..000000000 --- a/revdep/timing.md +++ /dev/null @@ -1,30 +0,0 @@ -# Check times - -| |package |version | check_time| -|:--|:------------|:---------|----------:| -|20 |skm |0.1.5.4 | 892.5| -|10 |milr |0.3.0 | 687.9| -|5 |gaston |1.5 | 577.1| -|6 |GERGM |0.11.2 | 490| -|15 |quanteda |0.99.9 | 488.9| -|8 |markovchain |0.6.9.8-1 | 423.6| -|12 |parallelDist |0.2.0 | 337| -|24 |text2vec |0.5.0 | 332.1| -|16 |re2r |0.2.0 | 256.4| -|1 |catSurv |1.0.3 | 218.5| -|2 |Cyclops |1.3.0 | 201.5| -|3 |dbmss |2.4-0 | 181.7| -|11 |MultiBD |0.2.0 | 163.4| -|9 |microclass |1.1 | 149.3| -|17 |rENA |0.1.0 | 110.3| -|19 |rPref |1.2 | 91.5| -|21 |SpatMCA |1.0.0.2 | 73| -|13 |patternplot |0.1 | 72.9| -|14 |qualpalr |0.4.2 | 67.6| -|22 |SpatPCA |1.1.1.2 | 66.7| -|18 |roll |1.0.7 | 61.1| -|4 |FLSSS |5.2 | 59.5| -|7 |lamW |1.3.0 | 53.1| -|23 |StMoSim |3.0 | 42.3| - - diff --git a/src/Makevars.in b/src/Makevars.in index e2b27ec3c..7158b7322 100644 --- a/src/Makevars.in +++ b/src/Makevars.in @@ -1,102 +1,40 @@ -PKG_CXXFLAGS += @CXX11STD@ -I../inst/include/ -ifeq ($(OS), Windows_NT) +# This needs to expand to something the shell will accept as '$ORIGIN', +# including a literal '$' (no variable expansion) +ORIGIN = \$$ORIGIN - USE_TBB=Windows - TBB_COPY_PATTERN=tbb*.dll - - MAKEFLAGS = -j1 - MAKE = make -e -s - MAKE_CMD = \ - CYGWIN=nodosfilewarning \ - CONLY="@WINDOWS_CC@" \ - CPLUS="@WINDOWS_CXX11@" \ - CXXFLAGS="@CXX11FLAGS@ -DTBB_NO_LEGACY=1" \ - PIC_KEY="@CXX11PICFLAGS@" \ - WARNING_SUPPRESS="" \ - $(MAKE) +CMAKE = @CMAKE@ +R = @R@ -else - - UNAME := $(shell uname) - TBB_COPY_PATTERN=libtbb*.* - ifeq ($(UNAME), Darwin) - USE_TBB=Mac - endif - ifeq ($(UNAME), Linux) - USE_TBB=Linux - endif - ifeq ($(UNAME), SunOS) - SUNOS_ARCH = $(shell uname -p) - ifeq ($(SUNOS_ARCH), i386) - USE_TBB=SunOS - endif - endif +TBB_LIB = @TBB_LIB@ +TBB_INC = @TBB_INC@ +TBB_NAME = @TBB_NAME@ +TBB_MALLOC_NAME = @TBB_MALLOC_NAME@ - MAKE += -e -s - MAKE_CMD = \ - CONLY="@CC@" \ - CPLUS="@CXX11@" \ - CXXFLAGS="@CXX11FLAGS@ -DTBB_NO_LEGACY=1" \ - PIC_KEY="@CXX11PICFLAGS@" \ - WARNING_SUPPRESS="" \ - $(MAKE) +PKG_CPPFLAGS = @PKG_CPPFLAGS@ +PKG_CXXFLAGS = @PKG_CXXFLAGS@ -endif +PKG_LIBS = @PKG_LIBS@ @PKG_LIBS_EXTRA@ -ifdef USE_TBB - -PKG_CXXFLAGS += -DRCPP_PARALLEL_USE_TBB=1 - -MAKE_ARGS := tbb_release tbbmalloc_release tbb_build_prefix=lib - -ifeq ($(USE_TBB), Windows) - - # rtools: turn on hacks to compensate for make and shell differences rtools<=>MinGW - # compiler: overwrite default (which is cl = MS compiler) - MAKE_ARGS += rtools=true compiler=gcc - ifeq ("$(WIN)", "64") - MAKE_ARGS += arch=intel64 - ARCH_DIR=x64/ - else - MAKE_ARGS += arch=ia32 - ARCH_DIR=i386/ - endif - - # Linker needs access to the tbb dll; otherwise you get errors such as: - # "undefined reference to `tbb::task_scheduler_init::terminate()'" - PKG_LIBS += -L../inst/lib/$(ARCH_DIR) -ltbb -ltbbmalloc - -endif - -# For Solaris detect if this is 32-bit R on x86 and if so forward that to TBB -ifeq ($(USE_TBB), SunOS) - R_32BIT = $(shell ${R_HOME}/bin/Rscript -e 'cat(.Machine$$sizeof.pointer == 4)') - ifeq ($(R_32BIT), TRUE) - MAKE_ARGS += arch=ia32 - endif -endif - -.PHONY: all tbb - -# Order is important in Windows' case. See PKG_LIBS above all: tbb $(SHLIB) -tbb: - mkdir -p ../inst/lib/$(ARCH_DIR); \ - cd tbb/src; \ - if [ -n "$(shell echo $(CC) | grep clang)" ]; then \ - $(MAKE_CMD) stdver=@STDVER@ compiler=clang $(MAKE_ARGS); \ - elif [ -n "$(shell echo $(CC) | grep gcc)" ]; then \ - $(MAKE_CMD) stdver=@STDVER@ compiler=gcc $(MAKE_ARGS); \ - else \ - $(MAKE_CMD) stdver=@STDVER@ $(MAKE_ARGS); \ - fi; \ - cd ../..; \ - cp tbb/build/lib_release/$(TBB_COPY_PATTERN) ../inst/lib/$(ARCH_DIR) - -clean: - (cd tbb/src; make clean) - -endif - +# TBB needs to be built before our C++ sources are built, so that +# headers are copied and available from the expected locations. +$(OBJECTS): tbb + +# NOTE: TBB libraries are installed via install.libs.R. +# However, we need to copy headers here so that they are visible during compilation. +tbb: tbb-clean + @TBB_LIB="$(TBB_LIB)" TBB_INC="$(TBB_INC)" \ + TBB_NAME="$(TBB_NAME)" TBB_MALLOC_NAME="$(TBB_MALLOC_NAME)" \ + CC="$(CC)" CFLAGS="$(CFLAGS)" CPPFLAGS="$(CPPFLAGS)" \ + CXX="$(CXX)" CXXFLAGS="$(CXXFLAGS)" LDFLAGS="$(LDFLAGS)" \ + CMAKE="$(CMAKE)" "@R@" -s -f install.libs.R --args build + +# NOTE: we do not want to clean ../inst/lib or ../inst/libs here, +# as we may be writing to those locations in multiarch builds +tbb-clean: + @rm -rf ../inst/include/tbb + @rm -rf ../inst/include/oneapi + @rm -rf ../inst/include/tbb_local + @rm -rf ../inst/include/serial diff --git a/src/init.cpp b/src/init.cpp index b30f7f201..b95aebed1 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -6,11 +6,9 @@ /* .Call calls */ extern "C" SEXP defaultNumThreads(); -extern "C" SEXP setThreadOptions(SEXP, SEXP); static const R_CallMethodDef CallEntries[] = { {"defaultNumThreads", (DL_FUNC) &defaultNumThreads, 0}, - {"setThreadOptions", (DL_FUNC) &setThreadOptions, 2}, {NULL, NULL, 0} }; diff --git a/src/install.libs.R b/src/install.libs.R new file mode 100644 index 000000000..4dcbac274 --- /dev/null +++ b/src/install.libs.R @@ -0,0 +1,273 @@ + +# !diagnostics suppress=R_PACKAGE_DIR,SHLIB_EXT,R_ARCH +.install.libs <- function(tbbLib) { + + # copy default library + files <- Sys.glob(paste0("*", SHLIB_EXT)) + dest <- file.path(R_PACKAGE_DIR, paste0("libs", R_ARCH)) + dir.create(dest, recursive = TRUE, showWarnings = FALSE) + file.copy(files, dest, overwrite = TRUE) + + # copy symbols if available + if (file.exists("symbols.rds")) + file.copy("symbols.rds", dest, overwrite = TRUE) + + # also copy to package 'libs' folder, for devtools + libsDest <- paste0("../libs", R_ARCH) + dir.create(libsDest, recursive = TRUE, showWarnings = FALSE) + file.copy(files, libsDest, overwrite = TRUE) + + # copy tbb (NOTE: do not use inst/ folder as R will resolve symlinks, + # behavior which we do _not_ want here!) + tbbDest <- file.path(R_PACKAGE_DIR, paste0("lib", R_ARCH)) + dir.create(tbbDest, recursive = TRUE, showWarnings = FALSE) + + # note: on Linux, TBB gets compiled with extensions like + # '.so.2', so be ready to handle those + shlibPattern <- switch( + Sys.info()[["sysname"]], + Windows = "^tbb.*\\.dll$", + Darwin = "^libtbb.*\\.dylib$", + "^libtbb.*\\.so.*$" + ) + # WASM only supports static libraries + if (R.version$os == "emscripten") { + shlibPattern <- "^libtbb.*\\.a$" + } + + if (!nzchar(tbbLib)) { + + # using bundled TBB + tbbLibs <- list.files( + path = "tbb/build/lib_release", + pattern = shlibPattern, + full.names = TRUE + ) + + for (tbbLib in tbbLibs) { + system2("cp", c("-P", shQuote(tbbLib), shQuote(tbbDest))) + } + + } else { + + # using system tbb + tbbLibs <- list.files( + path = tbbLib, + pattern = shlibPattern, + full.names = TRUE + ) + + # don't copy symlinks + tbbLibs <- tbbLibs[!nzchar(Sys.readlink(tbbLibs))] + + # copy / link the libraries + useSymlinks <- Sys.getenv("TBB_USE_SYMLINKS", unset = .Platform$OS.type != "windows") + if (useSymlinks) { + file.symlink(tbbLibs, tbbDest) + } else { + for (tbbLib in tbbLibs) { + system2("cp", c("-P", shQuote(tbbLib), shQuote(tbbDest))) + } + } + + } + + # on Windows, we create a stub library that links to us so that + # older binaries (like rstan) can still load + if (.Platform$OS.type == "windows") { + tbbDll <- file.path(tbbDest, "tbb.dll") + if (!file.exists(tbbDll)) { + writeLines("** creating tbb stub library") + status <- system("R CMD SHLIB -o tbb-compat/tbb.dll tbb-compat/tbb-compat.cpp") + if (status != 0) + stop("error building tbb stub library") + file.copy("tbb-compat/tbb.dll", file.path(tbbDest, "tbb.dll")) + } + } + +} + +useTbbPreamble <- function(tbbInc) { + dir.create("../inst/include", recursive = TRUE, showWarnings = FALSE) + for (suffix in c("oneapi", "serial", "tbb")) { + tbbPath <- file.path(tbbInc, suffix) + if (file.exists(tbbPath)) { + file.copy(tbbPath, "../inst/include", recursive = TRUE) + } + } +} + +useSystemTbb <- function(tbbLib, tbbInc) { + useTbbPreamble(tbbInc) +} + +useBundledTbb <- function() { + + useTbbPreamble("tbb/include") + dir.create("tbb/build-tbb", showWarnings = FALSE) + + cmake <- Sys.getenv("CMAKE", unset = "cmake") + buildType <- Sys.getenv("CMAKE_BUILD_TYPE", unset = "Release") + verbose <- Sys.getenv("VERBOSE", unset = "0") + + splitCompilerVar("CC", "CFLAGS") + splitCompilerVar("CXX", "CXXFLAGS") + + prependFlags("CPPFLAGS", "CFLAGS") + prependFlags("CPPFLAGS", "CXXFLAGS") + + cmakeFlags <- c( + forwardEnvVar("CC", "CMAKE_C_COMPILER"), + forwardEnvVar("CXX", "CMAKE_CXX_COMPILER"), + forwardEnvVar("CFLAGS", "CMAKE_C_FLAGS"), + forwardEnvVar("CXXFLAGS", "CMAKE_CXX_FLAGS"), + forwardEnvVar("CMAKE_BUILD_TYPE", "CMAKE_BUILD_TYPE"), + "-DTBB_TEST=0", + "-DTBB_EXAMPLES=0", + "-DTBB_STRICT=0", + ".." + ) + + if (R.version$os == "emscripten") { + cmakeFlags <- c( + "-DEMSCRIPTEN=1", + "-DTBBMALLOC_BUILD=0", + "-DBUILD_SHARED_LIBS=0", + cmakeFlags + ) + } + + writeLines("*** configuring tbb") + owd <- setwd("tbb/build-tbb") + output <- system2(cmake, shQuote(cmakeFlags), stdout = TRUE, stderr = TRUE) + status <- attr(output, "status") + if (is.numeric(status) && status != 0L) { + writeLines(output) + stop("error configuring tbb (status code ", status, ")") + } else if (!identical(verbose, "0")) { + writeLines(output) + } + setwd(owd) + + if (!identical(verbose, "0")) { + writeLines("*** dumping CMakeCache.txt") + writeLines(readLines("tbb/build-tbb/CMakeCache.txt")) + } + + writeLines("*** building tbb") + owd <- setwd("tbb/build-tbb") + output <- system2(cmake, c("--build", ".", "--config", "release"), stdout = TRUE, stderr = TRUE) + status <- attr(output, "status") + if (is.numeric(status) && status != 0L) { + writeLines(output) + stop("error building tbb (status code ", status, ")") + } else if (!identical(verbose, "0")) { + writeLines(output) + } + setwd(owd) + + shlibPattern <- switch( + Sys.info()[["sysname"]], + Windows = "^tbb.*\\.dll$", + Darwin = "^libtbb.*\\.dylib$", + "^libtbb.*\\.so.*$" + ) + + # WASM only supports static libraries + if (R.version$os == "emscripten") { + shlibPattern <- "^libtbb.*\\.a$" + } + + tbbFiles <- list.files( + file.path(getwd(), "tbb/build-tbb"), + pattern = shlibPattern, + recursive = TRUE, + full.names = TRUE + ) + + dir.create("tbb/build/lib_release", recursive = TRUE, showWarnings = FALSE) + file.copy(tbbFiles, "tbb/build/lib_release", overwrite = TRUE) + unlink("tbb/build-tbb", recursive = TRUE) + writeLines("*** finished building tbb") + +} + +getenv <- function(key, unset = "") { + Sys.getenv(key, unset = unset) +} + + +setenv <- function(key, value) { + args <- list(paste(value, collapse = " ")) + names(args) <- key + do.call(Sys.setenv, args) +} + + +# CMake doesn't support flags specified directly as part of the compiler +# definition, so manually split it here. +splitCompilerVar <- function(compilerVar, flagsVar) { + + compiler <- Sys.getenv(compilerVar, unset = NA) + if (is.na(compiler)) + return(FALSE) + + tokens <- scan(text = compiler, what = character(), quiet = TRUE) + if (length(tokens) < 2L) + return(FALSE) + + setenv(compilerVar, tokens[[1L]]) + + oldFlags <- Sys.getenv(flagsVar) + newFlags <- c(tokens[-1L], oldFlags) + setenv(flagsVar, newFlags[nzchar(newFlags)]) + + TRUE + +} + + +# Given an environment variable like 'CC', forward that to CMake +# via the corresponding CMAKE_C_COMPILER flag. +forwardEnvVar <- function(envVar, cmakeVar) { + envVal <- Sys.getenv(envVar, unset = NA) + if (!is.na(envVal)) { + sprintf("-D%s=%s", cmakeVar, envVal) + } +} + +prependFlags <- function(prependFlags, toFlags) { + + prependVal <- Sys.getenv(prependFlags, unset = NA) + if (is.na(prependVal)) + return(FALSE) + + oldVal <- Sys.getenv(toFlags, unset = NA) + if (is.na(oldVal)) { + setenv(toFlags, prependVal) + } else { + setenv(toFlags, paste(prependVal, oldVal)) + } + + TRUE + +} + +# Main ---- + +tbbLib <- Sys.getenv("TBB_LIB") +tbbInc <- Sys.getenv("TBB_INC") + +args <- commandArgs(trailingOnly = TRUE) +if (identical(args, "build")) { + if (nzchar(tbbLib) && nzchar(tbbInc)) { + useSystemTbb(tbbLib, tbbInc) + } else if (.Platform$OS.type == "windows") { + writeLines("** building RcppParallel without tbb backend") + } else { + useBundledTbb() + } +} else { + source("../R/tbb-autodetected.R") + .install.libs(tbbLib) +} diff --git a/src/options.cpp b/src/options.cpp index 741643d62..4f9de2e2c 100644 --- a/src/options.cpp +++ b/src/options.cpp @@ -8,40 +8,13 @@ #include #include -#include - -extern "C" SEXP setThreadOptions(SEXP numThreadsSEXP, SEXP stackSizeSEXP) { - - static tbb::task_scheduler_init* s_pTaskScheduler = NULL; - - int numThreads = Rf_asInteger(numThreadsSEXP); - - int stackSize = Rf_asInteger(stackSizeSEXP); - - try - { - if (!s_pTaskScheduler) { - s_pTaskScheduler = new tbb::task_scheduler_init(numThreads, stackSize); - } else { - s_pTaskScheduler->terminate(); - s_pTaskScheduler->initialize(numThreads, stackSize); - } - } - catch(const std::exception& e) - { - Rf_error(("Error loading TBB: " + std::string(e.what())).c_str()); - } - catch(...) - { - Rf_error("Error loading TBB: (Unknown error)"); - } - - return R_NilValue; -} - extern "C" SEXP defaultNumThreads() { SEXP threadsSEXP = Rf_allocVector(INTSXP, 1); +#ifdef __TBB_task_arena_H + INTEGER(threadsSEXP)[0] = tbb::this_task_arena::max_concurrency(); +#else INTEGER(threadsSEXP)[0] = tbb::task_scheduler_init::default_num_threads(); +#endif return threadsSEXP; } @@ -49,10 +22,6 @@ extern "C" SEXP defaultNumThreads() { #include -extern "C" SEXP setThreadOptions(SEXP numThreadsSEXP, SEXP stackSizeSEXP) { - return R_NilValue; -} - extern "C" SEXP defaultNumThreads() { SEXP threadsSEXP = Rf_allocVector(INTSXP, 1); INTEGER(threadsSEXP)[0] = tthread::thread::hardware_concurrency(); diff --git a/src/tbb-compat/tbb-compat.cpp b/src/tbb-compat/tbb-compat.cpp new file mode 100644 index 000000000..cbde70b4d --- /dev/null +++ b/src/tbb-compat/tbb-compat.cpp @@ -0,0 +1,143 @@ + +#include + +#include "../tbb/include/oneapi/tbb/detail/_namespace_injection.h" +#include "../tbb/include/oneapi/tbb/task_arena.h" + +#include "../tbb/src/tbb/observer_proxy.h" +#include "../tbb/src/tbb/main.h" +#include "../tbb/src/tbb/thread_data.h" + +#ifdef _WIN32 +# define DLL_EXPORT __declspec(dllexport) +#else +# define DLL_EXPORT +#endif + +namespace tbb { + +namespace interface6 { +class task_scheduler_observer; +} + +namespace internal { + +class task_scheduler_observer_v3 { + friend class tbb::detail::r1::observer_proxy; + friend class tbb::detail::r1::observer_list; + friend class interface6::task_scheduler_observer; + + //! Pointer to the proxy holding this observer. + /** Observers are proxied by the scheduler to maintain persistent lists of them. **/ + tbb::detail::r1::observer_proxy* my_proxy; + + //! Counter preventing the observer from being destroyed while in use by the scheduler. + /** Valid only when observation is on. **/ + std::atomic my_busy_count; + +public: + //! Enable or disable observation + /** For local observers the method can be used only when the current thread + has the task scheduler initialized or is attached to an arena. + Repeated calls with the same state are no-ops. **/ + void DLL_EXPORT __TBB_EXPORTED_METHOD observe( bool state=true ); + + //! Returns true if observation is enabled, false otherwise. + bool is_observing() const {return my_proxy!=NULL;} + + //! Construct observer with observation disabled. + task_scheduler_observer_v3() : my_proxy(NULL) { my_busy_count.store(0); } + + //! Entry notification + /** Invoked from inside observe(true) call and whenever a worker enters the arena + this observer is associated with. If a thread is already in the arena when + the observer is activated, the entry notification is called before it + executes the first stolen task. + Obsolete semantics. For global observers it is called by a thread before + the first steal since observation became enabled. **/ + virtual void on_scheduler_entry( bool /*is_worker*/ ) {} + + //! Exit notification + /** Invoked from inside observe(false) call and whenever a worker leaves the + arena this observer is associated with. + Obsolete semantics. For global observers it is called by a thread before + the first steal since observation became enabled. **/ + virtual void on_scheduler_exit( bool /*is_worker*/ ) {} + + //! Destructor automatically switches observation off if it is enabled. + virtual ~task_scheduler_observer_v3() { if(my_proxy) observe(false);} +}; + +} // namespace internal + +namespace interface6 { + +class task_scheduler_observer : public internal::task_scheduler_observer_v3 { + friend class internal::task_scheduler_observer_v3; + friend class tbb::detail::r1::observer_proxy; + friend class tbb::detail::r1::observer_list; + + /** Negative numbers with the largest absolute value to minimize probability + of coincidence in case of a bug in busy count usage. **/ + // TODO: take more high bits for version number + static const intptr_t v6_trait = (intptr_t)((~(uintptr_t)0 >> 1) + 1); + + //! contains task_arena pointer or tag indicating local or global semantics of the observer + intptr_t my_context_tag; + enum { global_tag = 0, implicit_tag = 1 }; + +public: + //! Construct local or global observer in inactive state (observation disabled). + /** For a local observer entry/exit notifications are invoked whenever a worker + thread joins/leaves the arena of the observer's owner thread. If a thread is + already in the arena when the observer is activated, the entry notification is + called before it executes the first stolen task. **/ + /** TODO: Obsolete. + Global observer semantics is obsolete as it violates master thread isolation + guarantees and is not composable. Thus the current default behavior of the + constructor is obsolete too and will be changed in one of the future versions + of the library. **/ + explicit task_scheduler_observer( bool local = false ) { + my_context_tag = local? implicit_tag : global_tag; + } + + //! Construct local observer for a given arena in inactive state (observation disabled). + /** entry/exit notifications are invoked whenever a thread joins/leaves arena. + If a thread is already in the arena when the observer is activated, the entry notification + is called before it executes the first stolen task. **/ + explicit task_scheduler_observer( task_arena & a) { + my_context_tag = (intptr_t)&a; + } + + /** Destructor protects instance of the observer from concurrent notification. + It is recommended to disable observation before destructor of a derived class starts, + otherwise it can lead to concurrent notification callback on partly destroyed object **/ + virtual ~task_scheduler_observer() { if(my_proxy) observe(false); } + + //! Enable or disable observation + /** Warning: concurrent invocations of this method are not safe. + Repeated calls with the same state are no-ops. **/ + void observe( bool state=true ) { + if( state && !my_proxy ) { + __TBB_ASSERT( !my_busy_count, "Inconsistent state of task_scheduler_observer instance"); + my_busy_count.store(v6_trait); + } + internal::task_scheduler_observer_v3::observe(state); + } +}; + +} // namespace interface6 + +} // namespace tbb + +namespace tbb { +namespace internal { + +DLL_EXPORT +void __TBB_EXPORTED_FUNC task_scheduler_observer_v3::observe( bool enable ) { + auto* tso = (tbb::detail::d1::task_scheduler_observer*) (this); + tbb::detail::r1::observe(*tso, enable); +} + +} // namespace internal +} // namespace tbb diff --git a/src/tbb.cpp b/src/tbb.cpp new file mode 100644 index 000000000..008e4ad4a --- /dev/null +++ b/src/tbb.cpp @@ -0,0 +1,245 @@ + +#if RCPP_PARALLEL_USE_TBB + +#include +#include + +#include +#include +#include +#include + +namespace RcppParallel { + +tbb::global_control* s_globalControl = nullptr; + +// TBB Tools ---- + +struct TBBWorker +{ + explicit TBBWorker(Worker& worker) : worker_(worker) {} + + void operator()(const tbb::blocked_range& r) const { + worker_(r.begin(), r.end()); + } + +private: + Worker& worker_; +}; + +ThreadStackSizeControl::ThreadStackSizeControl() +{ + int stackSize = resolveValue("RCPP_PARALLEL_STACK_SIZE", 0, 0); + if (stackSize > 0) + { + s_globalControl = new tbb::global_control( + tbb::global_control::thread_stack_size, + stackSize + ); + } +} + +ThreadStackSizeControl::~ThreadStackSizeControl() +{ + if (s_globalControl != nullptr) + { + delete s_globalControl; + s_globalControl = nullptr; + } +} + + +// TBB Parallel For ---- + +class TBBParallelForExecutor +{ +public: + + TBBParallelForExecutor(Worker& worker, + std::size_t begin, + std::size_t end, + std::size_t grainSize) + : worker_(worker), + begin_(begin), + end_(end), + grainSize_(grainSize) + { + } + + void operator()() const + { + TBBWorker tbbWorker(worker_); + tbb::parallel_for( + tbb::blocked_range(begin_, end_, grainSize_), + tbbWorker + ); + } + +private: + Worker& worker_; + std::size_t begin_; + std::size_t end_; + std::size_t grainSize_; +}; + +class TBBArenaParallelForExecutor +{ +public: + + TBBArenaParallelForExecutor(tbb::task_group& group, + Worker& worker, + std::size_t begin, + std::size_t end, + std::size_t grainSize) + : group_(group), + worker_(worker), + begin_(begin), + end_(end), + grainSize_(grainSize) + { + } + + void operator()() const + { + TBBParallelForExecutor executor(worker_, begin_, end_, grainSize_); + group_.run_and_wait(executor); + } + +private: + + tbb::task_group& group_; + Worker& worker_; + std::size_t begin_; + std::size_t end_; + std::size_t grainSize_; +}; + +void tbbParallelFor(std::size_t begin, + std::size_t end, + Worker& worker, + std::size_t grainSize, + int numThreads) +{ + ThreadStackSizeControl control; + + tbb::task_group group; + TBBArenaParallelForExecutor executor(group, worker, begin, end, grainSize); + + tbb::task_arena arena(numThreads); + arena.execute(executor); +} + + +// TBB Parallel Reduce ---- + +struct TBBReducer +{ + explicit TBBReducer(ReducerWrapper& reducer) + : pSplitReducer_(NULL), reducer_(reducer) + { + } + + TBBReducer(TBBReducer& tbbReducer, tbb::split) + : pSplitReducer_(new ReducerWrapper(tbbReducer.reducer_, RcppParallel::Split())), + reducer_(*pSplitReducer_) + { + } + + virtual ~TBBReducer() { delete pSplitReducer_; } + + void operator()(const tbb::blocked_range& r) + { + reducer_(r.begin(), r.end()); + } + + void join(const TBBReducer& tbbReducer) + { + reducer_.join(tbbReducer.reducer_); + } + +private: + ReducerWrapper* pSplitReducer_; + ReducerWrapper& reducer_; +}; + +class TBBParallelReduceExecutor +{ +public: + + TBBParallelReduceExecutor(ReducerWrapper& reducer, + std::size_t begin, + std::size_t end, + std::size_t grainSize) + : reducer_(reducer), + begin_(begin), + end_(end), + grainSize_(grainSize) + { + } + + void operator()() const + { + TBBReducer tbbReducer(reducer_); + tbb::parallel_reduce( + tbb::blocked_range(begin_, end_, grainSize_), + tbbReducer + ); + } + +private: + ReducerWrapper& reducer_; + std::size_t begin_; + std::size_t end_; + std::size_t grainSize_; +}; + +class TBBArenaParallelReduceExecutor +{ +public: + + TBBArenaParallelReduceExecutor(tbb::task_group& group, + ReducerWrapper& reducer, + std::size_t begin, + std::size_t end, + std::size_t grainSize) + : group_(group), + reducer_(reducer), + begin_(begin), + end_(end), + grainSize_(grainSize) + { + } + + void operator()() const + { + TBBParallelReduceExecutor executor(reducer_, begin_, end_, grainSize_); + group_.run_and_wait(executor); + } + +private: + + tbb::task_group& group_; + ReducerWrapper& reducer_; + std::size_t begin_; + std::size_t end_; + std::size_t grainSize_; +}; + +void tbbParallelReduceImpl(std::size_t begin, + std::size_t end, + ReducerWrapper& reducer, + std::size_t grainSize, + int numThreads) +{ + ThreadStackSizeControl control; + + tbb::task_group group; + TBBArenaParallelReduceExecutor executor(group, reducer, begin, end, grainSize); + + tbb::task_arena arena(numThreads); + arena.execute(executor); +} + +} // end namespace RcppParallel + +#endif /* RCPP_PARALLEL_USE_TBB */ diff --git a/src/tbb/.gitignore b/src/tbb/.gitignore new file mode 100644 index 000000000..a4d040468 --- /dev/null +++ b/src/tbb/.gitignore @@ -0,0 +1,62 @@ +# -------- C++ -------- +# Prerequisites +*.d + +# Compiled Object files +*.slo +*.lo +*.o +*.obj + +# Precompiled Headers +*.gch +*.pch + +# Compiled Dynamic libraries +*.so +*.so.* +*.dylib +*.dll + +# Fortran module files +*.mod +*.smod + +# Compiled Static libraries +*.lai +*.la +*.a +*.lib + +# Executables +*.exe +*.out +*.app + +# -------- CMake -------- +CMakeCache.txt +CMakeFiles +CMakeScripts +Testing +Makefile +cmake_install.cmake +install_manifest.txt +compile_commands.json +CTestTestfile.cmake +build/* + +# -------- Python -------- +__pycache__/ +*.py[cod] +*$py.class + +# -------- IDE -------- +.vscode/* +.vs/* +out/* +CMakeSettings.json + +# -------- CTags -------- +.tags +.ctags + diff --git a/src/tbb/CHANGES b/src/tbb/CHANGES deleted file mode 100644 index aa7f126fd..000000000 --- a/src/tbb/CHANGES +++ /dev/null @@ -1,2443 +0,0 @@ ------------------------------------------------------------------------- -The list of most significant changes made over time in -Intel(R) Threading Building Blocks (Intel(R) TBB). - -Intel TBB 2018 Update 1 -TBB_INTERFACE_VERSION == 10001 - -Changes (w.r.t. Intel TBB 2018): - -- Added lambda-friendly overloads for parallel_scan. -- Added support of static and simple partitioners in - parallel_deterministic_reduce. - -Preview Features: - -- Added initial support for Flow Graph Analyzer to parallel_for. -- Added reservation support in overwrite_node and write_once_node. - -Bugs fixed: - -- Fixed a potential deadlock scenario in the flow graph that affected - Intel TBB 2018. - ------------------------------------------------------------------------- -Intel TBB 2018 -TBB_INTERFACE_VERSION == 10000 - -Changes (w.r.t. Intel TBB 2017 Update 7): - -- Introduced Parallel STL, an implementation of the C++ standard - library algorithms with support for execution policies. For more - information, see Getting Started with Parallel STL - (https://software.intel.com/en-us/get-started-with-pstl). -- this_task_arena::isolate() function is now a fully supported feature. -- this_task_arena::isolate() function and task_arena::execute() method - were extended to pass on the value returned by the executed functor - (requires C++11). -- task_arena::enqueue() and task_group::run() methods extended to accept - move-only functors. -- A flow graph now spawns all tasks into the same task arena, - and waiting for graph completion also happens in that arena. -- Improved support for Flow Graph Analyzer in async_node, opencl_node, - and composite_node. -- Added support for Android* NDK r15, r15b. -- Added support for Universal Windows Platform. -- Increased minimally supported version of macOS* - (MACOSX_DEPLOYMENT_TARGET) to 10.11. - -Changes affecting backward compatibility: - -- Internal layout changes in some flow graph classes; -- Several undocumented methods are removed from class graph, - including set_active() and is_active(). -- Due to incompatible changes, the namespace version is updated - for the flow graph; recompilation is recommended for all - binaries that use the flow graph classes. - -Preview Features: - -- opencl_node can be used with any graph object; class opencl_graph - is removed. -- graph::wait_for_all() now automatically waits for all not yet consumed - async_msg objects. -- Improved concurrent_lru_cache::handle_object to support C++11 move - semantics, default construction, and conversion to bool. - -Bugs fixed: - -- Fixed a bug preventing use of streaming_node and opencl_node with - Clang; inspired by a contribution from Francisco Facioni. -- Fixed this_task_arena::isolate() function to work correctly with - parallel_invoke and parallel_do algorithms. -- Fixed a memory leak in composite_node. -- Fixed an assertion failure in debug tbbmalloc binaries when - TBBMALLOC_CLEAN_ALL_BUFFERS is used. - ------------------------------------------------------------------------- -Intel TBB 2017 Update 8 -TBB_INTERFACE_VERSION == 9108 - -Changes (w.r.t. Intel TBB 2017 Update 7): - -Bugs fixed: - -- Fixed an assertion failure in debug tbbmalloc binaries when - TBBMALLOC_CLEAN_ALL_BUFFERS is used. - ------------------------------------------------------------------------- -Intel TBB 2017 Update 7 -TBB_INTERFACE_VERSION == 9107 - -Changes (w.r.t. Intel TBB 2017 Update 6): - -- In the huge pages mode, the memory allocator now is also able to use - transparent huge pages. - -Preview Features: - -- Added support for Intel TBB integration into CMake-aware - projects, with valuable guidance and feedback provided by Brad King - (Kitware). - -Bugs fixed: - -- Fixed scalable_allocation_command(TBBMALLOC_CLEAN_ALL_BUFFERS, 0) - to process memory left after exited threads. - ------------------------------------------------------------------------- -Intel TBB 2017 Update 6 -TBB_INTERFACE_VERSION == 9106 - -Changes (w.r.t. Intel TBB 2017 Update 5): - -- Added support for Android* NDK r14. - -Preview Features: - -- Added a blocking terminate extension to the task_scheduler_init class - that allows an object to wait for termination of worker threads. - -Bugs fixed: - -- Fixed compilation and testing issues with MinGW (GCC 6). -- Fixed compilation with /std:c++latest option of VS 2017 - (https://github.com/01org/tbb/issues/13). - ------------------------------------------------------------------------- -Intel TBB 2017 Update 5 -TBB_INTERFACE_VERSION == 9105 - -Changes (w.r.t. Intel TBB 2017 Update 4): - -- Added support for Microsoft* Visual Studio* 2017. -- Added graph/matmult example to demonstrate support for compute offload - to Intel(R) Graphics Technology in the flow graph API. -- The "compiler" build option now allows to specify a full path to the - compiler. - -Changes affecting backward compatibility: - -- Constructors for many classes, including graph nodes, concurrent - containers, thread-local containers, etc., are declared explicit and - cannot be used for implicit conversions anymore. - -Bugs fixed: - -- Added a workaround for bug 16657 in the GNU C Library (glibc) - affecting the debug version of tbb::mutex. -- Fixed a crash in pool_identify() called for an object allocated in - another thread. - ------------------------------------------------------------------------- -Intel TBB 2017 Update 4 -TBB_INTERFACE_VERSION == 9104 - -Changes (w.r.t. Intel TBB 2017 Update 3): - -- Added support for C++11 move semantics in parallel_do. -- Added support for FreeBSD* 11. - -Changes affecting backward compatibility: - -- Minimal compiler versions required for support of C++11 move semantics - raised to GCC 4.5, VS 2012, and Intel(R) C++ Compiler 14.0. - -Bugs fixed: - -- The workaround for crashes in the library compiled with GCC 6 - (-flifetime-dse=1) was extended to Windows*. - ------------------------------------------------------------------------- -Intel TBB 2017 Update 3 -TBB_INTERFACE_VERSION == 9103 - -Changes (w.r.t. Intel TBB 2017 Update 2): - -- Added support for Android* 7.0 and Android* NDK r13, r13b. - -Preview Features: - -- Added template class gfx_factory to the flow graph API. It implements - the Factory concept for streaming_node to offload computations to - Intel(R) processor graphics. - -Bugs fixed: - -- Fixed a possible deadlock caused by missed wakeup signals in - task_arena::execute(). - -Open-source contributions integrated: - -- A build fix for Linux* s390x platform by Jerry J. - ------------------------------------------------------------------------- -Intel TBB 2017 Update 2 -TBB_INTERFACE_VERSION == 9102 - -Changes (w.r.t. Intel TBB 2017 Update 1): - -- Removed the long-outdated support for Xbox* consoles. - -Bugs fixed: - -- Fixed the issue with task_arena::execute() not being processed when - the calling thread cannot join the arena. -- Fixed dynamic memory allocation replacement failure on macOS* 10.12. - ------------------------------------------------------------------------- -Intel TBB 2017 Update 1 -TBB_INTERFACE_VERSION == 9101 - -Changes (w.r.t. Intel TBB 2017): - -Bugs fixed: - -- Fixed dynamic memory allocation replacement failures on Windows* 10 - Anniversary Update. -- Fixed emplace() method of concurrent unordered containers to not - require a copy constructor. - ------------------------------------------------------------------------- -Intel TBB 2017 -TBB_INTERFACE_VERSION == 9100 - -Changes (w.r.t. Intel TBB 4.4 Update 5): - -- static_partitioner class is now a fully supported feature. -- async_node class is now a fully supported feature. -- Improved dynamic memory allocation replacement on Windows* OS to skip - DLLs for which replacement cannot be done, instead of aborting. -- Intel TBB no longer performs dynamic memory allocation replacement - for Microsoft* Visual Studio* 2008. -- For 64-bit platforms, quadrupled the worst-case limit on the amount - of memory the Intel TBB allocator can handle. -- Added TBB_USE_GLIBCXX_VERSION macro to specify the version of GNU - libstdc++ when it cannot be properly recognized, e.g. when used - with Clang on Linux* OS. Inspired by a contribution from David A. -- Added graph/stereo example to demostrate tbb::flow::async_msg. -- Removed a few cases of excessive user data copying in the flow graph. -- Reworked split_node to eliminate unnecessary overheads. -- Added support for C++11 move semantics to the argument of - tbb::parallel_do_feeder::add() method. -- Added C++11 move constructor and assignment operator to - tbb::combinable template class. -- Added tbb::this_task_arena::max_concurrency() function and - max_concurrency() method of class task_arena returning the maximal - number of threads that can work inside an arena. -- Deprecated tbb::task_arena::current_thread_index() static method; - use tbb::this_task_arena::current_thread_index() function instead. -- All examples for commercial version of library moved online: - https://software.intel.com/en-us/product-code-samples. Examples are - available as a standalone package or as a part of Intel(R) Parallel - Studio XE or Intel(R) System Studio Online Samples packages. - -Changes affecting backward compatibility: - -- Renamed following methods and types in async_node class: - Old New - async_gateway_type => gateway_type - async_gateway() => gateway() - async_try_put() => try_put() - async_reserve() => reserve_wait() - async_commit() => release_wait() -- Internal layout of some flow graph nodes has changed; recompilation - is recommended for all binaries that use the flow graph. - -Preview Features: - -- Added template class streaming_node to the flow graph API. It allows - a flow graph to offload computations to other devices through - streaming or offloading APIs. -- Template class opencl_node reimplemented as a specialization of - streaming_node that works with OpenCL*. -- Added tbb::this_task_arena::isolate() function to isolate execution - of a group of tasks or an algorithm from other tasks submitted - to the scheduler. - -Bugs fixed: - -- Added a workaround for GCC bug #62258 in std::rethrow_exception() - to prevent possible problems in case of exception propagation. -- Fixed parallel_scan to provide correct result if the initial value - of an accumulator is not the operation identity value. -- Fixed a memory corruption in the memory allocator when it meets - internal limits. -- Fixed the memory allocator on 64-bit platforms to align memory - to 16 bytes by default for all allocations bigger than 8 bytes. -- As a workaround for crashes in the Intel TBB library compiled with - GCC 6, added -flifetime-dse=1 to compilation options on Linux* OS. -- Fixed a race in the flow graph implementation. - -Open-source contributions integrated: - -- Enabling use of C++11 'override' keyword by Raf Schietekat. - ------------------------------------------------------------------------- -Intel TBB 4.4 Update 6 -TBB_INTERFACE_VERSION == 9006 - -Changes (w.r.t. Intel TBB 4.4 Update 5): - -- For 64-bit platforms, quadrupled the worst-case limit on the amount - of memory the Intel TBB allocator can handle. - -Bugs fixed: - -- Fixed a memory corruption in the memory allocator when it meets - internal limits. -- Fixed the memory allocator on 64-bit platforms to align memory - to 16 bytes by default for all allocations bigger than 8 bytes. -- Fixed parallel_scan to provide correct result if the initial value - of an accumulator is not the operation identity value. -- As a workaround for crashes in the Intel TBB library compiled with - GCC 6, added -flifetime-dse=1 to compilation options on Linux* OS. - ------------------------------------------------------------------------- -Intel TBB 4.4 Update 5 -TBB_INTERFACE_VERSION == 9005 - -Changes (w.r.t. Intel TBB 4.4 Update 4): - -- Modified graph/fgbzip2 example to remove unnecessary data queuing. - -Preview Features: - -- Added a Python* module which is able to replace Python's thread pool - class with the implementation based on Intel TBB task scheduler. - -Bugs fixed: - -- Fixed the implementation of 64-bit tbb::atomic for IA-32 architecture - to work correctly with GCC 5.2 in C++11/14 mode. -- Fixed a possible crash when tasks with affinity (e.g. specified via - affinity_partitioner) are used simultaneously with task priority - changes. - ------------------------------------------------------------------------- -Intel TBB 4.4 Update 4 -TBB_INTERFACE_VERSION == 9004 - -Changes (w.r.t. Intel TBB 4.4 Update 3): - -- Removed a few cases of excessive user data copying in the flow graph. -- Improved robustness of concurrent_bounded_queue::abort() in case of - simultaneous push and pop operations. - -Preview Features: - -- Added tbb::flow::async_msg, a special message type to support - communications between the flow graph and external asynchronous - activities. -- async_node modified to support use with C++03 compilers. - -Bugs fixed: - -- Fixed a bug in dynamic memory allocation replacement for Windows* OS. -- Fixed excessive memory consumption on Linux* OS caused by enabling - zero-copy realloc. -- Fixed performance regression on Intel(R) Xeon Phi(tm) coprocessor with - auto_partitioner. - ------------------------------------------------------------------------- -Intel TBB 4.4 Update 3 -TBB_INTERFACE_VERSION == 9003 - -Changes (w.r.t. Intel TBB 4.4 Update 2): - -- Modified parallel_sort to not require a default constructor for values - and to use iter_swap() for value swapping. -- Added support for creating or initializing a task_arena instance that - is connected to the arena currently used by the thread. -- graph/binpack example modified to use multifunction_node. -- For performance analysis, use Intel(R) VTune(TM) Amplifier XE 2015 - and higher; older versions are no longer supported. -- Improved support for compilation with disabled RTTI, by omitting its use - in auxiliary code, such as assertions. However some functionality, - particularly the flow graph, does not work if RTTI is disabled. -- The tachyon example for Android* can be built using Android Studio 1.5 - and higher with experimental Gradle plugin 0.4.0. - -Preview Features: - -- Added class opencl_subbufer that allows using OpenCL* sub-buffer - objects with opencl_node. -- Class global_control supports the value of 1 for - max_allowed_parallelism. - -Bugs fixed: - -- Fixed a race causing "TBB Warning: setaffinity syscall failed" message. -- Fixed a compilation issue on OS X* with Intel(R) C++ Compiler 15.0. -- Fixed a bug in queuing_rw_mutex::downgrade() that could temporarily - block new readers. -- Fixed speculative_spin_rw_mutex to stop using the lazy subscription - technique due to its known flaws. -- Fixed memory leaks in the tool support code. - ------------------------------------------------------------------------- -Intel TBB 4.4 Update 2 -TBB_INTERFACE_VERSION == 9002 - -Changes (w.r.t. Intel TBB 4.4 Update 1): - -- Improved interoperability with Intel(R) OpenMP RTL (libiomp) on Linux: - OpenMP affinity settings do not affect the default number of threads - used in the task scheduler. Intel(R) C++ Compiler 16.0 Update 1 - or later is required. -- Added a new flow graph example with different implementations of the - Cholesky Factorization algorithm. - -Preview Features: - -- Added template class opencl_node to the flow graph API. It allows a - flow graph to offload computations to OpenCL* devices. -- Extended join_node to use type-specified message keys. It simplifies - the API of the node by obtaining message keys via functions - associated with the message type (instead of node ports). -- Added static_partitioner that minimizes overhead of parallel_for and - parallel_reduce for well-balanced workloads. -- Improved template class async_node in the flow graph API to support - user settable concurrency limits. - -Bugs fixed: - -- Fixed a possible crash in the GUI layer for library examples on Linux. - ------------------------------------------------------------------------- -Intel TBB 4.4 Update 1 -TBB_INTERFACE_VERSION == 9001 - -Changes (w.r.t. Intel TBB 4.4): - -- Added support for Microsoft* Visual Studio* 2015. -- Intel TBB no longer performs dynamic replacement of memory allocation - functions for Microsoft Visual Studio 2005 and earlier versions. -- For GCC 4.7 and higher, the intrinsics-based platform isolation layer - uses __atomic_* built-ins instead of the legacy __sync_* ones. - This change is inspired by a contribution from Mathieu Malaterre. -- Improvements in task_arena: - Several application threads may join a task_arena and execute tasks - simultaneously. The amount of concurrency reserved for application - threads at task_arena construction can be set to any value between - 0 and the arena concurrency limit. -- The fractal example was modified to demonstrate class task_arena - and moved to examples/task_arena/fractal. - -Bugs fixed: - -- Fixed a deadlock during destruction of task_scheduler_init objects - when one of destructors is set to wait for worker threads. -- Added a workaround for a possible crash on OS X* when dynamic memory - allocator replacement (libtbbmalloc_proxy) is used and memory is - released during application startup. -- Usage of mutable functors with task_group::run_and_wait() and - task_arena::enqueue() is disabled. An attempt to pass a functor - which operator()() is not const will produce compilation errors. -- Makefiles and environment scripts now properly recognize GCC 5.0 and - higher. - -Open-source contributions integrated: - -- Improved performance of parallel_for_each for inputs allowing random - access, by Raf Schietekat. - ------------------------------------------------------------------------- -Intel TBB 4.4 -TBB_INTERFACE_VERSION == 9000 - -Changes (w.r.t. Intel TBB 4.3 Update 6): - -- The following features are now fully supported: - tbb::flow::composite_node; - additional policies of tbb::flow::graph_node::reset(). -- Platform abstraction layer for Windows* OS updated to use compiler - intrinsics for most atomic operations. -- The tbb/compat/thread header updated to automatically include - C++11 where available. -- Fixes and refactoring in the task scheduler and class task_arena. -- Added key_matching policy to tbb::flow::join_node, which removes - the restriction on the type that can be compared-against. -- For tag_matching join_node, tag_value is redefined to be 64 bits - wide on all architectures. -- Expanded the documentation for the flow graph with details about - node semantics and behavior. -- Added dynamic replacement of C11 standard function aligned_alloc() - under Linux* OS. -- Added C++11 move constructors and assignment operators to - tbb::enumerable_thread_specific container. -- Added hashing support for tbb::tbb_thread::id. -- On OS X*, binaries that depend on libstdc++ are not provided anymore. - In the makefiles, libc++ is now used by default; for building with - libstdc++, specify stdlib=libstdc++ in the make command line. - -Preview Features: - -- Added a new example, graph/fgbzip2, that shows usage of - tbb::flow::async_node. -- Modification to the low-level API for memory pools: - added a function for finding a memory pool by an object allocated - from that pool. -- tbb::memory_pool now does not request memory till the first allocation - from the pool. - -Changes affecting backward compatibility: - -- Internal layout of flow graph nodes has changed; recompilation is - recommended for all binaries that use the flow graph. -- Resetting a tbb::flow::source_node will immediately activate it, - unless it was created in inactive state. - -Bugs fixed: - -- Failure at creation of a memory pool will not cause process - termination anymore. - -Open-source contributions integrated: - -- Supported building TBB with Clang on AArch64 with use of built-in - intrinsics by David A. - ------------------------------------------------------------------------- -Intel TBB 4.3 Update 6 -TBB_INTERFACE_VERSION == 8006 - -Changes (w.r.t. Intel TBB 4.3 Update 5): - -- Supported zero-copy realloc for objects >1MB under Linux* via - mremap system call. -- C++11 move-aware insert and emplace methods have been added to - concurrent_hash_map container. -- install_name is set to @rpath/ on OS X*. - -Preview Features: - -- Added template class async_node to the flow graph API. It allows a - flow graph to communicate with an external activity managed by - the user or another runtime. -- Improved speed of flow::graph::reset() clearing graph edges. - rf_extract flag has been renamed rf_clear_edges. -- extract() method of graph nodes now takes no arguments. - -Bugs fixed: - -- concurrent_unordered_{set,map} behaves correctly for degenerate - hashes. -- Fixed a race condition in the memory allocator that may lead to - excessive memory consumption under high multithreading load. - ------------------------------------------------------------------------- -Intel TBB 4.3 Update 5 -TBB_INTERFACE_VERSION == 8005 - -Changes (w.r.t. Intel TBB 4.3 Update 4): - -- Added add_ref_count() method of class tbb::task. - -Preview Features: - -- Added class global_control for application-wide control of allowed - parallelism and thread stack size. -- memory_pool_allocator now throws the std::bad_alloc exception on - allocation failure. -- Exceptions thrown for by memory pool constructors changed from - std::bad_alloc to std::invalid_argument and std::runtime_error. - -Bugs fixed: - -- scalable_allocator now throws the std::bad_alloc exception on - allocation failure. -- Fixed a race condition in the memory allocator that may lead to - excessive memory consumption under high multithreading load. -- A new scheduler created right after destruction of the previous one - might be unable to modify the number of worker threads. - -Open-source contributions integrated: - -- (Added but not enabled) push_front() method of class tbb::task_list - by Raf Schietekat. - ------------------------------------------------------------------------- -Intel TBB 4.3 Update 4 -TBB_INTERFACE_VERSION == 8004 - -Changes (w.r.t. Intel TBB 4.3 Update 3): - -- Added a C++11 variadic constructor for enumerable_thread_specific. - The arguments from this constructor are used to construct - thread-local values. -- Improved exception safety for enumerable_thread_specific. -- Added documentation for tbb::flow::tagged_msg class and - tbb::flow::output_port function. -- Fixed build errors for systems that do not support dynamic linking. -- C++11 move-aware insert and emplace methods have been added to - concurrent unordered containers. - -Preview Features: - -- Interface-breaking change: typedefs changed for node predecessor and - successor lists, affecting copy_predecessors and copy_successors - methods. -- Added template class composite_node to the flow graph API. It packages - a subgraph to represent it as a first-class flow graph node. -- make_edge and remove_edge now accept multiport nodes as arguments, - automatically using the node port with index 0 for an edge. - -Open-source contributions integrated: - -- Draft code for enumerable_thread_specific constructor with multiple - arguments (see above) by Adrien Guinet. -- Fix for GCC invocation on IBM* Blue Gene* - by Jeff Hammond and Raf Schietekat. -- Extended testing with smart pointers for Clang & libc++ - by Raf Schietekat. - ------------------------------------------------------------------------- -Intel TBB 4.3 Update 3 -TBB_INTERFACE_VERSION == 8003 - -Changes (w.r.t. Intel TBB 4.3 Update 2): - -- Move constructor and assignment operator were added to unique_lock. - -Preview Features: - -- Time overhead for memory pool destruction was reduced. - -Open-source contributions integrated: - -- Build error fix for iOS* by Raf Schietekat. - ------------------------------------------------------------------------- -Intel TBB 4.3 Update 2 -TBB_INTERFACE_VERSION == 8002 - -Changes (w.r.t. Intel TBB 4.3 Update 1): - -- Binary files for 64-bit Android* applications were added as part of the - Linux* OS package. -- Exact exception propagation is enabled for Intel C++ Compiler on OS X*. -- concurrent_vector::shrink_to_fit was optimized for types that support - C++11 move semantics. - -Bugs fixed: - -- Fixed concurrent unordered containers to insert elements much faster - in debug mode. -- Fixed concurrent priority queue to support types that do not have - copy constructors. -- Fixed enumerable_thread_specific to forbid copying from an instance - with a different value type. - -Open-source contributions integrated: - -- Support for PathScale* EKOPath* Compiler by Erik Lindahl. - ------------------------------------------------------------------------- -Intel TBB 4.3 Update 1 -TBB_INTERFACE_VERSION == 8001 - -Changes (w.r.t. Intel TBB 4.3): - -- The ability to split blocked_ranges in a proportion, used by - affinity_partitioner since version 4.2 Update 4, became a formal - extension of the Range concept. -- More checks for an incorrect address to release added to the debug - version of the memory allocator. -- Different kind of solutions for each TBB example were merged. - -Preview Features: - -- Task priorities are re-enabled in preview binaries. - -Bugs fixed: - -- Fixed a duplicate symbol when TBB_PREVIEW_VARIADIC_PARALLEL_INVOKE is - used in multiple compilation units. -- Fixed a crash in __itt_fini_ittlib seen on Ubuntu 14.04. -- Fixed a crash in memory release after dynamic replacement of the - OS X* memory allocator. -- Fixed incorrect indexing of arrays in seismic example. -- Fixed a data race in lazy initialization of task_arena. - -Open-source contributions integrated: - -- Fix for dumping information about gcc and clang compiler versions - by Misty De Meo. - ------------------------------------------------------------------------- -Intel TBB 4.3 -TBB_INTERFACE_VERSION == 8000 - -Changes (w.r.t. Intel TBB 4.2 Update 5): - -- The following features are now fully supported: flow::indexer_node, - task_arena, speculative_spin_rw_mutex. -- Compatibility with C++11 standard improved for tbb/compat/thread - and tbb::mutex. -- C++11 move constructors have been added to concurrent_queue and - concurrent_bounded_queue. -- C++11 move constructors and assignment operators have been added to - concurrent_vector, concurrent_hash_map, concurrent_priority_queue, - concurrent_unordered_{set,multiset,map,multimap}. -- C++11 move-aware emplace/push/pop methods have been added to - concurrent_vector, concurrent_queue, concurrent_bounded_queue, - concurrent_priority_queue. -- Methods to insert a C++11 initializer list have been added: - concurrent_vector::grow_by(), concurrent_hash_map::insert(), - concurrent_unordered_{set,multiset,map,multimap}::insert(). -- Testing for compatibility of containers with some C++11 standard - library types has been added. -- Dynamic replacement of standard memory allocation routines has been - added for OS X*. -- Microsoft* Visual Studio* projects for Intel TBB examples updated - to VS 2010. -- For open-source packages, debugging information (line numbers) in - precompiled binaries now matches the source code. -- Debug information was added to release builds for OS X*, Solaris*, - FreeBSD* operating systems and MinGW*. -- Various improvements in documentation, debug diagnostics and examples. - -Preview Features: - -- Additional actions on reset of graphs, and extraction of individual - nodes from a graph (TBB_PREVIEW_FLOW_GRAPH_FEATURES). -- Support for an arbitrary number of arguments in parallel_invoke - (TBB_PREVIEW_VARIADIC_PARALLEL_INVOKE). - -Changes affecting backward compatibility: - -- For compatibility with C++11 standard, copy and move constructors and - assignment operators are disabled for all mutex classes. To allow - the old behavior, use TBB_DEPRECATED_MUTEX_COPYING macro. -- flow::sequencer_node rejects messages with repeating sequence numbers. -- Changed internal interface between tbbmalloc and tbbmalloc_proxy. -- Following deprecated functionality has been removed: - old debugging macros TBB_DO_ASSERT & TBB_DO_THREADING_TOOLS; - no-op depth-related methods in class task; - tbb::deprecated::concurrent_queue; - deprecated variants of concurrent_vector methods. -- register_successor() and remove_successor() are deprecated as methods - to add and remove edges in flow::graph; use make_edge() and - remove_edge() instead. - -Bugs fixed: - -- Fixed incorrect scalable_msize() implementation for aligned objects. -- Flow graph buffering nodes now destroy their copy of forwarded items. -- Multiple fixes in task_arena implementation, including for: - inconsistent task scheduler state inside executed functions; - incorrect floating-point settings and exception propagation; - possible stalls in concurrent invocations of execute(). -- Fixed floating-point settings propagation when the same instance of - task_group_context is used in different arenas. -- Fixed compilation error in pipeline.h with Intel Compiler on OS X*. -- Added missed headers for individual components to tbb.h. - -Open-source contributions integrated: - -- Range interface addition to parallel_do, parallel_for_each and - parallel_sort by Stephan Dollberg. -- Variadic template implementation of parallel_invoke - by Kizza George Mbidde (see Preview Features). -- Improvement in Seismic example for MacBook Pro* with Retina* display - by Raf Schietekat. - ------------------------------------------------------------------------- -Intel TBB 4.2 Update 5 -TBB_INTERFACE_VERSION == 7005 - -Changes (w.r.t. Intel TBB 4.2 Update 4): - -- The second template argument of class aligned_space now is set - to 1 by default. - -Preview Features: - -- Better support for exception safety, task priorities and floating - point settings in class task_arena. -- task_arena::current_slot() has been renamed to - task_arena::current_thread_index(). - -Bugs fixed: - -- Task priority change possibly ignored by a worker thread entering - a nested parallel construct. -- Memory leaks inside the task scheduler when running on - Intel(R) Xeon Phi(tm) coprocessor. - -Open-source contributions integrated: - -- Improved detection of X Window support for Intel TBB examples - and other feedback by Raf Schietekat. - ------------------------------------------------------------------------- -Intel TBB 4.2 Update 4 -TBB_INTERFACE_VERSION == 7004 - -Changes (w.r.t. Intel TBB 4.2 Update 3): - -- Added possibility to specify floating-point settings at invocation - of most parallel algorithms (including flow::graph) via - task_group_context. -- Added dynamic replacement of malloc_usable_size() under - Linux*/Android* and dlmalloc_usable_size() under Android*. -- Added new methods to concurrent_vector: - grow_by() that appends a sequence between two given iterators; - grow_to_at_least() that initializes new elements with a given value. -- Improved affinity_partitioner for better performance on balanced - workloads. -- Improvements in the task scheduler, including better scalability - when threads search for a task arena, and better diagnostics. -- Improved allocation performance for workloads that do intensive - allocation/releasing of same-size objects larger than ~8KB from - multiple threads. -- Exception support is enabled by default for 32-bit MinGW compilers. -- The tachyon example for Android* can be built for all targets - supported by the installed NDK. -- Added Windows Store* version of the tachyon example. -- GettingStarted/sub_string_finder example ported to offload execution - on Windows* for Intel(R) Many Integrated Core Architecture. - -Preview Features: - -- Removed task_scheduler_observer::on_scheduler_leaving() callback. -- Added task_scheduler_observer::may_sleep() callback. -- The CPF or_node has been renamed indexer_node. The input to - indexer_node is now a list of types. The output of indexer_node is - a tagged_msg type composed of a tag and a value. For indexer_node, - the tag is a size_t. - -Bugs fixed: - -- Fixed data races in preview extensions of task_scheduler_observer. -- Added noexcept(false) for destructor of task_group_base to avoid - crash on cancellation of structured task group in C++11. - -Open-source contributions integrated: - -- Improved concurrency detection for BG/Q, and other improvements - by Raf Schietekat. -- Fix for crashes in enumerable_thread_specific in case if a contained - object is too big to be constructed on the stack by Adrien Guinet. - ------------------------------------------------------------------------- -Intel TBB 4.2 Update 3 -TBB_INTERFACE_VERSION == 7003 - -Changes (w.r.t. Intel TBB 4.2 Update 2): - -- Added support for Microsoft* Visual Studio* 2013. -- Improved Microsoft* PPL-compatible form of parallel_for for better - support of auto-vectorization. -- Added a new example for cancellation and reset in the flow graph: - Kohonen self-organizing map (examples/graph/som). -- Various improvements in source code, tests, and makefiles. - -Bugs fixed: - -- Added dynamic replacement of _aligned_msize() previously missed. -- Fixed task_group::run_and_wait() to throw invalid_multiple_scheduling - exception if the specified task handle is already scheduled. - -Open-source contributions integrated: - -- A fix for ARM* processors by Steve Capper. -- Improvements in std::swap calls by Robert Maynard. - ------------------------------------------------------------------------- -Intel TBB 4.2 Update 2 -TBB_INTERFACE_VERSION == 7002 - -Changes (w.r.t. Intel TBB 4.2 Update 1): - -- Enable C++11 features for Microsoft* Visual Studio* 2013 Preview. -- Added a test for compatibility of TBB containers with C++11 - range-based for loop. - -Changes affecting backward compatibility: - -- Internal layout changed for class tbb::flow::limiter_node. - -Preview Features: - -- Added speculative_spin_rw_mutex, a read-write lock class which uses - Intel(R) Transactional Synchronization Extensions. - -Bugs fixed: - -- When building for Intel(R) Xeon Phi(tm) coprocessor, TBB programs - no longer require explicit linking with librt and libpthread. - -Open-source contributions integrated: - -- Fixes for ARM* processors by Steve Capper, Leif Lindholm - and Steven Noonan. -- Support for Clang on Linux by Raf Schietekat. -- Typo correction in scheduler.cpp by Julien Schueller. - ------------------------------------------------------------------------- -Intel TBB 4.2 Update 1 -TBB_INTERFACE_VERSION == 7001 - -Changes (w.r.t. Intel TBB 4.2): - -- Added project files for Microsoft* Visual Studio* 2010. -- Initial support of Microsoft* Visual Studio* 2013 Preview. -- Enable C++11 features available in Intel(R) C++ Compiler 14.0. -- scalable_allocation_mode(TBBMALLOC_SET_SOFT_HEAP_LIMIT, ) can be - used to urge releasing memory from tbbmalloc internal buffers when - the given limit is exceeded. - -Preview Features: - -- Class task_arena no longer requires linking with a preview library, - though still remains a community preview feature. -- The method task_arena::wait_until_empty() is removed. -- The method task_arena::current_slot() now returns -1 if - the task scheduler is not initialized in the thread. - -Changes affecting backward compatibility: - -- Because of changes in internal layout of graph nodes, the namespace - interface number of flow::graph has been incremented from 6 to 7. - -Bugs fixed: - -- Fixed a race in lazy initialization of task_arena. -- Fixed flow::graph::reset() to prevent situations where tasks would be - spawned in the process of resetting the graph to its initial state. -- Fixed decrement bug in limiter_node. -- Fixed a race in arc deletion in the flow graph. - -Open-source contributions integrated: - -- Improved support for IBM* Blue Gene* by Raf Schietekat. - ------------------------------------------------------------------------- -Intel TBB 4.2 -TBB_INTERFACE_VERSION == 7000 - -Changes (w.r.t. Intel TBB 4.1 Update 4): - -- Added speculative_spin_mutex, which uses Intel(R) Transactional - Synchronization Extensions when they are supported by hardware. -- Binary files linked with libc++ (the C++ standard library in Clang) - were added on OS X*. -- For OS X* exact exception propagation is supported with Clang; - it requires use of libc++ and corresponding Intel TBB binaries. -- Support for C++11 initializer lists in constructor and assigment - has been added to concurrent_hash_map, concurrent_unordered_set, - concurrent_unordered_multiset, concurrent_unordered_map, - concurrent_unordered_multimap. -- The memory allocator may now clean its per-thread memory caches - when it cannot get more memory. -- Added the scalable_allocation_command() function for on-demand - cleaning of internal memory caches. -- Reduced the time overhead for freeing memory objects smaller than ~8K. -- Simplified linking with the debug library for applications that use - Intel TBB in code offloaded to Intel(R) Xeon Phi(tm) coprocessors. - See an example in - examples/GettingStarted/sub_string_finder/Makefile. -- Various improvements in source code, scripts and makefiles. - -Changes affecting backward compatibility: - -- tbb::flow::graph has been modified to spawn its tasks; - the old behaviour (task enqueuing) is deprecated. This change may - impact applications that expected a flow graph to make progress - without calling wait_for_all(), which is no longer guaranteed. See - the documentation for more details. -- Changed the return values of the scalable_allocation_mode() function. - -Bugs fixed: - -- Fixed a leak of parallel_reduce body objects when execution is - cancelled or an exception is thrown, as suggested by Darcy Harrison. -- Fixed a race in the task scheduler which can lower the effective - priority despite the existence of higher priority tasks. -- On Linux an error during destruction of the internal thread local - storage no longer results in an exception. - -Open-source contributions integrated: - -- Fixed task_group_context state propagation to unrelated context trees - by Raf Schietekat. - ------------------------------------------------------------------------- -Intel TBB 4.1 Update 4 -TBB_INTERFACE_VERSION == 6105 - -Changes (w.r.t. Intel TBB 4.1 Update 3): - -- Use /volatile:iso option with VS 2012 to disable extended - semantics for volatile variables. -- Various improvements in affinity_partitioner, scheduler, - tests, examples, makefiles. -- Concurrent_priority_queue class now supports initialization/assignment - via C++11 initializer list feature (std::initializer_list). - -Bugs fixed: - -- Fixed more possible stalls in concurrent invocations of - task_arena::execute(), especially waiting for enqueued tasks. -- Fixed requested number of workers for task_arena(P,0). -- Fixed interoperability with Intel(R) VTune(TM) Amplifier XE in - case of using task_arena::enqueue() from a terminating thread. - -Open-source contributions integrated: - -- Type fixes, cleanups, and code beautification by Raf Schietekat. -- Improvements in atomic operations for big endian platforms - by Raf Schietekat. - ------------------------------------------------------------------------- -Intel TBB 4.1 Update 3 -TBB_INTERFACE_VERSION == 6103 - -Changes (w.r.t. Intel TBB 4.1 Update 2): - -- Binary files for Android* applications were added to the Linux* OS - package. -- Binary files for Windows Store* applications were added to the - Windows* OS package. -- Exact exception propagation (exception_ptr) support on Linux OS is - now turned on by default for GCC 4.4 and higher. -- Stopped implicit use of large memory pages by tbbmalloc (Linux-only). - Now use of large pages must be explicitly enabled with - scalable_allocation_mode() function or TBB_MALLOC_USE_HUGE_PAGES - environment variable. - -Community Preview Features: - -- Extended class task_arena constructor and method initialize() to - allow some concurrency to be reserved strictly for application - threads. -- New methods terminate() and is_active() were added to class - task_arena. - -Bugs fixed: - -- Fixed initialization of hashing helper constant in the hash - containers. -- Fixed possible stalls in concurrent invocations of - task_arena::execute() when no worker thread is available to make - progress. -- Fixed incorrect calculation of hardware concurrency in the presence - of inactive processor groups, particularly on systems running - Windows* 8 and Windows* Server 2012. - -Open-source contributions integrated: - -- The fix for the GUI examples on OS X* systems by Raf Schietekat. -- Moved some power-of-2 calculations to functions to improve readability - by Raf Schietekat. -- C++11/Clang support improvements by arcata. -- ARM* platform isolation layer by Steve Capper, Leif Lindholm, Leo Lara - (ARM). - ------------------------------------------------------------------------- -Intel TBB 4.1 Update 2 -TBB_INTERFACE_VERSION == 6102 - -Changes (w.r.t. Intel TBB 4.1 Update 1): - -- Objects up to 128 MB are now cached by the tbbmalloc. Previously - the threshold was 8MB. Objects larger than 128 MB are still - processed by direct OS calls. -- concurrent_unordered_multiset and concurrent_unordered_multimap - have been added, based on Microsoft* PPL prototype. -- Ability to value-initialize a tbb::atomic variable on construction - in C++11, with const expressions properly supported. - -Community Preview Features: - -- Added a possibility to wait until all worker threads terminate. - This is necessary before calling fork() from an application. - -Bugs fixed: - -- Fixed data race in tbbmalloc that might lead to memory leaks - for large object allocations. -- Fixed task_arena::enqueue() to use task_group_context of target arena. -- Improved implementation of 64 bit atomics on ia32. - ------------------------------------------------------------------------- -Intel TBB 4.1 Update 1 -TBB_INTERFACE_VERSION == 6101 - -Changes (w.r.t. Intel TBB 4.1): - -- concurrent_vector class now supports initialization/assignment - via C++11 initializer list feature (std::initializer_list) -- Added implementation of the platform isolation layer based on - Intel compiler atomic built-ins; it is supposed to work on - any platform supported by compiler version 12.1 and newer. -- Using GetNativeSystemInfo() instead of GetSystemInfo() to support - more than 32 processors for 32-bit applications under WOW64. -- The following form of parallel_for: - parallel_for(first, last, [step,] f[, context]) now accepts an - optional partitioner parameter after the function f. - -Backward-incompatible API changes: - -- The library no longer injects tuple in to namespace std. - In previous releases, tuple was injected into namespace std by - flow_graph.h when std::tuple was not available. In this release, - flow_graph.h now uses tbb::flow::tuple. On platforms where - std::tuple is available, tbb::flow::tuple is typedef'ed to - std::tuple. On all other platforms, tbb::flow::tuple provides - a subset of the functionality defined by std::tuple. Users of - flow_graph.h may need to change their uses of std::tuple to - tbb::flow::tuple to ensure compatibility with non-C++11 compliant - compilers. - -Bugs fixed: - -- Fixed local observer to be able to override propagated CPU state and - to provide correct value of task_arena::current_slot() in callbacks. - ------------------------------------------------------------------------- -Intel TBB 4.1 -TBB_INTERFACE_VERSION == 6100 - -Changes (w.r.t. Intel TBB 4.0 Update 5): - -- _WIN32_WINNT must be set to 0x0501 or greater in order to use TBB - on Microsoft* Windows*. -- parallel_deterministic_reduce template function is fully supported. -- TBB headers can be used with C++0x/C++11 mode (-std=c++0x) of GCC - and Intel(R) Compiler. -- C++11 std::make_exception_ptr is used where available, instead of - std::copy_exception from earlier C++0x implementations. -- Improvements in the TBB allocator to reduce extra memory consumption. -- Partial refactoring of the task scheduler data structures. -- TBB examples allow more flexible specification of the thread number, - including arithmetic and geometric progression. - -Bugs fixed: - -- On Linux & OS X*, pre-built TBB binaries do not yet support exact - exception propagation via C++11 exception_ptr. To prevent run time - errors, by default TBB headers disable exact exception propagation - even if the C++ implementation provides exception_ptr. - -Community Preview Features: - -- Added: class task_arena, for work submission by multiple application - threads with thread-independent control of concurrency level. -- Added: task_scheduler_observer can be created as local to a master - thread, to observe threads that work on behalf of that master. - Local observers may have new on_scheduler_leaving() callback. - ------------------------------------------------------------------------- -Intel TBB 4.0 Update 5 -TBB_INTERFACE_VERSION == 6005 - -Changes (w.r.t. Intel TBB 4.0 Update 4): - -- Parallel pipeline optimization (directly storing small objects in the - interstage data buffers) limited to trivially-copyable types for - C++11 and a short list of types for earlier compilers. -- _VARIADIC_MAX switch is honored for TBB tuple implementation - and flow::graph nodes based on tuple. -- Support of Cocoa framework was added to the GUI examples on OS X* - systems. - -Bugs fixed: - -- Fixed a tv_nsec overflow bug in condition_variable::wait_for. -- Fixed execution order of enqueued tasks with different priorities. -- Fixed a bug with task priority changes causing lack of progress - for fire-and-forget tasks when TBB was initialized to use 1 thread. -- Fixed duplicate symbol problem when linking multiple compilation - units that include flow_graph.h on VC 10. - ------------------------------------------------------------------------- -Intel TBB 4.0 Update 4 -TBB_INTERFACE_VERSION == 6004 - -Changes (w.r.t. Intel TBB 4.0 Update 3): - -- The TBB memory allocator transparently supports large pages on Linux. -- A new flow_graph example, logic_sim, was added. -- Support for DirectX* 9 was added to GUI examples. - -Community Preview Features: - -- Added: aggregator, a new concurrency control mechanism. - -Bugs fixed: - -- The abort operation on concurrent_bounded_queue now leaves the queue - in a reusable state. If a bad_alloc or bad_last_alloc exception is - thrown while the queue is recovering from an abort, that exception - will be reported instead of user_abort on the thread on which it - occurred, and the queue will not be reusable. -- Steal limiting heuristic fixed to avoid premature stealing disabling - when large amount of __thread data is allocated on thread stack. -- Fixed a low-probability leak of arenas in the task scheduler. -- In STL-compatible allocator classes, the method construct() was fixed - to comply with C++11 requirements. -- Fixed a bug that prevented creation of fixed-size memory pools - smaller than 2M. -- Significantly reduced the amount of warnings from various compilers. - -Open-source contributions integrated: - -- Multiple improvements by Raf Schietekat. -- Basic support for Clang on OS X* by Blas Rodriguez Somoza. -- Fixes for warnings and corner-case bugs by Blas Rodriguez Somoza - and Edward Lam. - ------------------------------------------------------------------------- -Intel TBB 4.0 Update 3 -TBB_INTERFACE_VERSION == 6003 - -Changes (w.r.t. Intel TBB 4.0 Update 2): - -- Modifications to the low-level API for memory pools: - added support for aligned allocations; - pool policies reworked to allow backward-compatible extensions; - added a policy to not return memory space till destruction; - pool_reset() does not return memory space anymore. -- Class tbb::flow::graph_iterator added to iterate over all nodes - registered with a graph instance. -- multioutput_function_node has been renamed multifunction_node. - multifunction_node and split_node are now fully-supported features. -- For the tagged join node, the policy for try_put of an item with - already existing tag has been defined: the item will be rejected. -- Matching the behavior on Windows, on other platforms the optional - shared libraries (libtbbmalloc, libirml) now are also searched - only in the directory where libtbb is located. -- The platform isolation layer based on GCC built-ins is extended. - -Backward-incompatible API changes: - -- a graph reference parameter is now required to be passed to the - constructors of the following flow graph nodes: overwrite_node, - write_once_node, broadcast_node, and the CPF or_node. -- the following tbb::flow node methods and typedefs have been renamed: - Old New - join_node and or_node: - inputs() -> input_ports() - input_ports_tuple_type -> input_ports_type - multifunction_node and split_node: - ports_type -> output_ports_type - -Bugs fixed: - -- Not all logical processors were utilized on systems with more than - 64 cores split by Windows into several processor groups. - ------------------------------------------------------------------------- -Intel TBB 4.0 Update 2 commercial-aligned release -TBB_INTERFACE_VERSION == 6002 - -Changes (w.r.t. Intel TBB 4.0 Update 1 commercial-aligned release): - -- concurrent_bounded_queue now has an abort() operation that releases - threads involved in pending push or pop operations. The released - threads will receive a tbb::user_abort exception. -- Added Community Preview Feature: concurrent_lru_cache container, - a concurrent implementation of LRU (least-recently-used) cache. - -Bugs fixed: - -- fixed a race condition in the TBB scalable allocator. -- concurrent_queue counter wraparound bug was fixed, which occurred when - the number of push and pop operations exceeded ~>4 billion on IA32. -- fixed races in the TBB scheduler that could put workers asleep too - early, especially in presence of affinitized tasks. - ------------------------------------------------------------------------- -Intel TBB 4.0 Update 1 commercial-aligned release -TBB_INTERFACE_VERSION == 6000 (forgotten to increment) - -Changes (w.r.t. Intel TBB 4.0 commercial-aligned release): - -- Memory leaks fixed in binpack example. -- Improvements and fixes in the TBB allocator. - ------------------------------------------------------------------------- -Intel TBB 4.0 commercial-aligned release -TBB_INTERFACE_VERSION == 6000 - -Changes (w.r.t. Intel TBB 3.0 Update 8 commercial-aligned release): - -- concurrent_priority_queue is now a fully supported feature. - Capacity control methods were removed. -- Flow graph is now a fully supported feature. -- A new memory backend has been implemented in the TBB allocator. - It can reuse freed memory for both small and large objects, and - returns unused memory blocks to the OS more actively. -- Improved partitioning algorithms for parallel_for and parallel_reduce - to better handle load imbalance. -- The convex_hull example has been refactored for reproducible - performance results. -- The major interface version has changed from 5 to 6. - Deprecated interfaces might be removed in future releases. - -Community Preview Features: - -- Added: serial subset, i.e. sequential implementations of TBB generic - algorithms (currently, only provided for parallel_for). -- Preview of new flow graph nodes: - or_node (accepts multiple inputs, forwards each input separately - to all successors), - split_node (accepts tuples, and forwards each element of a tuple - to a corresponding successor), and - multioutput_function_node (accepts one input, and passes the input - and a tuple of output ports to the function body to support outputs - to multiple successors). -- Added: memory pools for more control on memory source, grouping, - and collective deallocation. - ------------------------------------------------------------------------- -Intel TBB 3.0 Update 8 commercial-aligned release -TBB_INTERFACE_VERSION == 5008 - -Changes (w.r.t. Intel TBB 3.0 Update 7 commercial-aligned release): - -- Task priorities become an official feature of TBB, - not community preview as before. -- Atomics API extended, and implementation refactored. -- Added task::set_parent() method. -- Added concurrent_unordered_set container. - -Open-source contributions integrated: - -- PowerPC support by Raf Schietekat. -- Fix of potential task pool overrun and other improvements - in the task scheduler by Raf Schietekat. -- Fix in parallel_for_each to work with std::set in Visual* C++ 2010. - -Community Preview Features: - -- Graph community preview feature was renamed to flow graph. - Multiple improvements in the implementation. - Binpack example was added for the feature. -- A number of improvements to concurrent_priority_queue. - Shortpath example was added for the feature. -- TBB runtime loaded functionality was added (Windows*-only). - It allows to specify which versions of TBB should be used, - as well as to set directories for the library search. -- parallel_deterministic_reduce template function was added. - ------------------------------------------------------------------------- -Intel TBB 3.0 Update 7 commercial-aligned release -TBB_INTERFACE_VERSION == 5006 (forgotten to increment) - -Changes (w.r.t. Intel TBB 3.0 Update 6 commercial-aligned release): - -- Added implementation of the platform isolation layer based on - GCC atomic built-ins; it is supposed to work on any platform - where GCC has these built-ins. - -Community Preview Features: - -- Graph's dining_philosophers example added. -- A number of improvements to graph and concurrent_priority_queue. - - ------------------------------------------------------------------------- -Intel TBB 3.0 Update 6 commercial-aligned release -TBB_INTERFACE_VERSION == 5006 - -Changes (w.r.t. Intel TBB 3.0 Update 5 commercial-aligned release): - -- Added Community Preview feature: task and task group priority, and - Fractal example demonstrating it. -- parallel_pipeline optimized for data items of small and large sizes. -- Graph's join_node is now parametrized with a tuple of up to 10 types. -- Improved performance of concurrent_priority_queue. - -Open-source contributions integrated: - -- Initial NetBSD support by Aleksej Saushev. - -Bugs fixed: - -- Failure to enable interoperability with Intel(R) Cilk(tm) Plus runtime - library, and a crash caused by invoking the interoperability layer - after one of the libraries was unloaded. -- Data race that could result in concurrent_unordered_map structure - corruption after call to clear() method. -- Stack corruption caused by PIC version of 64-bit CAS compiled by Intel - compiler on Linux. -- Inconsistency of exception propagation mode possible when application - built with Microsoft* Visual Studio* 2008 or earlier uses TBB built - with Microsoft* Visual Studio* 2010. -- Affinitizing master thread to a subset of available CPUs after TBB - scheduler was initialized tied all worker threads to the same CPUs. -- Method is_stolen_task() always returned 'false' for affinitized tasks. -- write_once_node and overwrite_node did not immediately send buffered - items to successors - ------------------------------------------------------------------------- -Intel TBB 3.0 Update 5 commercial-aligned release -TBB_INTERFACE_VERSION == 5005 - -Changes (w.r.t. Intel TBB 3.0 Update 4 commercial-aligned release): - -- Added Community Preview feature: graph. -- Added automatic propagation of master thread FPU settings to - TBB worker threads. -- Added a public function to perform a sequentially consistent full - memory fence: tbb::atomic_fence() in tbb/atomic.h. - -Bugs fixed: - -- Data race that could result in scheduler data structures corruption - when using fire-and-forget tasks. -- Potential referencing of destroyed concurrent_hash_map element after - using erase(accessor&A) method with A acquired as const_accessor. -- Fixed a correctness bug in the convex hull example. - -Open-source contributions integrated: - -- Patch for calls to internal::atomic_do_once() by Andrey Semashev. - ------------------------------------------------------------------------- -Intel TBB 3.0 Update 4 commercial-aligned release -TBB_INTERFACE_VERSION == 5004 - -Changes (w.r.t. Intel TBB 3.0 Update 3 commercial-aligned release): - -- Added Community Preview feature: concurrent_priority_queue. -- Fixed library loading to avoid possibility for remote code execution, - see http://www.microsoft.com/technet/security/advisory/2269637.mspx. -- Added support of more than 64 cores for appropriate Microsoft* - Windows* versions. For more details, see - http://msdn.microsoft.com/en-us/library/dd405503.aspx. -- Default number of worker threads is adjusted in accordance with - process affinity mask. - -Bugs fixed: - -- Calls of scalable_* functions from inside the allocator library - caused issues if the functions were overridden by another module. -- A crash occurred if methods run() and wait() were called concurrently - for an empty tbb::task_group (1736). -- The tachyon example exhibited build problems associated with - bug 554339 on Microsoft* Visual Studio* 2010. Project files were - modified as a partial workaround to overcome the problem. See - http://connect.microsoft.com/VisualStudio/feedback/details/554339. - ------------------------------------------------------------------------- -Intel TBB 3.0 Update 3 commercial-aligned release -TBB_INTERFACE_VERSION == 5003 - -Changes (w.r.t. Intel TBB 3.0 Update 2 commercial-aligned release): - -- cache_aligned_allocator class reworked to use scalable_aligned_malloc. -- Improved performance of count() and equal_range() methods - in concurrent_unordered_map. -- Improved implementation of 64-bit atomic loads and stores on 32-bit - platforms, including compilation with VC 7.1. -- Added implementation of atomic operations on top of OSAtomic API - provided by OS X*. -- Removed gratuitous try/catch blocks surrounding thread function calls - in tbb_thread. -- Xcode* projects were added for sudoku and game_of_life examples. -- Xcode* projects were updated to work without TBB framework. - -Bugs fixed: - -- Fixed a data race in task scheduler destruction that on rare occasion - could result in memory corruption. -- Fixed idle spinning in thread bound filters in tbb::pipeline (1670). - -Open-source contributions integrated: - -- MinGW-64 basic support by brsomoza (partially). -- Patch for atomic.h by Andrey Semashev. -- Support for AIX & GCC on PowerPC by Giannis Papadopoulos. -- Various improvements by Raf Schietekat. - ------------------------------------------------------------------------- -Intel TBB 3.0 Update 2 commercial-aligned release -TBB_INTERFACE_VERSION == 5002 - -Changes (w.r.t. Intel TBB 3.0 Update 1 commercial-aligned release): - -- Destructor of tbb::task_group class throws missing_wait exception - if there are tasks running when it is invoked. -- Interoperability layer with Intel Cilk Plus runtime library added - to protect TBB TLS in case of nested usage with Intel Cilk Plus. -- Compilation fix for dependent template names in concurrent_queue. -- Memory allocator code refactored to ease development and maintenance. - -Bugs fixed: - -- Improved interoperability with other Intel software tools on Linux in - case of dynamic replacement of memory allocator (1700) -- Fixed install issues that prevented installation on - Mac OS* X 10.6.4 (1711). - ------------------------------------------------------------------------- -Intel TBB 3.0 Update 1 commercial-aligned release -TBB_INTERFACE_VERSION == 5000 (forgotten to increment) - -Changes (w.r.t. Intel TBB 3.0 commercial-aligned release): - -- Decreased memory fragmentation by allocations bigger than 8K. -- Lazily allocate worker threads, to avoid creating unnecessary stacks. - -Bugs fixed: - -- TBB allocator used much more memory than malloc (1703) - see above. -- Deadlocks happened in some specific initialization scenarios - of the TBB allocator (1701, 1704). -- Regression in enumerable_thread_specific: excessive requirements - for object constructors. -- A bug in construction of parallel_pipeline filters when body instance - was a temporary object. -- Incorrect usage of memory fences on PowerPC and XBOX360 platforms. -- A subtle issue in task group context binding that could result - in cancellation signal being missed by nested task groups. -- Incorrect construction of concurrent_unordered_map if specified - number of buckets is not power of two. -- Broken count() and equal_range() of concurrent_unordered_map. -- Return type of postfix form of operator++ for hash map's iterators. - ------------------------------------------------------------------------- -Intel TBB 3.0 commercial-aligned release -TBB_INTERFACE_VERSION == 5000 - -Changes (w.r.t. Intel TBB 2.2 Update 3 commercial-aligned release): - -- All open-source-release changes down to TBB 2.2 U3 below - were incorporated into this release. - ------------------------------------------------------------------------- -20100406 open-source release - -Changes (w.r.t. 20100310 open-source release): - -- Added support for Microsoft* Visual Studio* 2010, including binaries. -- Added a PDF file with recommended Design Patterns for TBB. -- Added parallel_pipeline function and companion classes and functions - that provide a strongly typed lambda-friendly pipeline interface. -- Reworked enumerable_thread_specific to use a custom implementation of - hash map that is more efficient for ETS usage models. -- Added example for class task_group; see examples/task_group/sudoku. -- Removed two examples, as they were long outdated and superceded: - pipeline/text_filter (use pipeline/square); - parallel_while/parallel_preorder (use parallel_do/parallel_preorder). -- PDF documentation updated. -- Other fixes and changes in code, tests, and examples. - -Bugs fixed: - -- Eliminated build errors with MinGW32. -- Fixed post-build step and other issues in VS projects for examples. -- Fixed discrepancy between scalable_realloc and scalable_msize that - caused crashes with malloc replacement on Windows. - ------------------------------------------------------------------------- -20100310 open-source release - -Changes (w.r.t. Intel TBB 2.2 Update 3 commercial-aligned release): - -- Version macros changed in anticipation of a future release. -- Directory structure aligned with Intel(R) C++ Compiler; - now TBB binaries reside in //[bin|lib] - (in TBB 2.x, it was [bin|lib]//). -- Visual Studio projects changed for examples: instead of separate set - of files for each VS version, now there is single 'msvs' directory - that contains workspaces for MS C++ compiler (_cl.sln) and - Intel C++ compiler (_icl.sln). Works with VS 2005 and above. -- The name versioning scheme for backward compatibility was improved; - now compatibility-breaking changes are done in a separate namespace. -- Added concurrent_unordered_map implementation based on a prototype - developed in Microsoft for a future version of PPL. -- Added PPL-compatible writer-preference RW lock (reader_writer_lock). -- Added TBB_IMPLEMENT_CPP0X macro to control injection of C++0x names - implemented in TBB into namespace std. -- Added almost-C++0x-compatible std::condition_variable, plus a bunch - of other C++0x classes required by condition_variable. -- With TBB_IMPLEMENT_CPP0X, tbb_thread can be also used as std::thread. -- task.cpp was split into several translation units to structure - TBB scheduler sources layout. Static data layout and library - initialization logic were also updated. -- TBB scheduler reworked to prevent master threads from stealing - work belonging to other masters. -- Class task was extended with enqueue() method, and slightly changed - semantics of methods spawn() and destroy(). For exact semantics, - refer to TBB Reference manual. -- task_group_context now allows for destruction by non-owner threads. -- Added TBB_USE_EXCEPTIONS macro to control use of exceptions in TBB - headers. It turns off (i.e. sets to 0) automatically if specified - compiler options disable exception handling. -- TBB is enabled to run on top of Microsoft's Concurrency Runtime - on Windows* 7 (via our worker dispatcher known as RML). -- Removed old unused busy-waiting code in concurrent_queue. -- Described the advanced build & test options in src/index.html. -- Warning level for GCC raised with -Wextra and a few other options. -- Multiple fixes and improvements in code, tests, examples, and docs. - -Open-source contributions integrated: - -- Xbox support by Roman Lut (Deep Shadows), though further changes are - required to make it working; e.g. post-2.1 entry points are missing. -- "Eventcount" by Dmitry Vyukov evolved into concurrent_monitor, - an internal class used in the implementation of concurrent_queue. - ------------------------------------------------------------------------- -Intel TBB 2.2 Update 3 commercial-aligned release -TBB_INTERFACE_VERSION == 4003 - -Changes (w.r.t. Intel TBB 2.2 Update 2 commercial-aligned release): - -- PDF documentation updated. - -Bugs fixed: - -- concurrent_hash_map compatibility issue exposed on Linux in case - two versions of the container were used by different modules. -- enforce 16 byte stack alignment for consistence with GCC; required - to work correctly with 128-bit variables processed by SSE. -- construct() methods of allocator classes now use global operator new. - ------------------------------------------------------------------------- -Intel TBB 2.2 Update 2 commercial-aligned release -TBB_INTERFACE_VERSION == 4002 - -Changes (w.r.t. Intel TBB 2.2 Update 1 commercial-aligned release): - -- parallel_invoke and parallel_for_each now take function objects - by const reference, not by value. -- Building TBB with /MT is supported, to avoid dependency on particular - versions of Visual C++* runtime DLLs. TBB DLLs built with /MT - are located in vc_mt directory. -- Class critical_section introduced. -- Improvements in exception support: new exception classes introduced, - all exceptions are thrown via an out-of-line internal method. -- Improvements and fixes in the TBB allocator and malloc replacement, - including robust memory identification, and more reliable dynamic - function substitution on Windows*. -- Method swap() added to class tbb_thread. -- Methods rehash() and bucket_count() added to concurrent_hash_map. -- Added support for Visual Studio* 2010 Beta2. No special binaries - provided, but CRT-independent DLLs (vc_mt) should work. -- Other fixes and improvements in code, tests, examples, and docs. - -Open-source contributions integrated: - -- The fix to build 32-bit TBB on Mac OS* X 10.6. -- GCC-based port for SPARC Solaris by Michailo Matijkiw, with use of - earlier work by Raf Schietekat. - -Bugs fixed: - -- 159 - TBB build for PowerPC* running Mac OS* X. -- 160 - IBM* Java segfault if used with TBB allocator. -- crash in concurrent_queue (1616). - ------------------------------------------------------------------------- -Intel TBB 2.2 Update 1 commercial-aligned release -TBB_INTERFACE_VERSION == 4001 - -Changes (w.r.t. Intel TBB 2.2 commercial-aligned release): - -- Incorporates all changes from open-source releases below. -- Documentation was updated. -- TBB scheduler auto-initialization now covers all possible use cases. -- concurrent_queue: made argument types of sizeof used in paddings - consistent with those actually used. -- Memory allocator was improved: supported corner case of user's malloc - calling scalable_malloc (non-Windows), corrected processing of - memory allocation requests during tbb memory allocator startup - (Linux). -- Windows malloc replacement has got better support for static objects. -- In pipeline setups that do not allow actual parallelism, execution - by a single thread is guaranteed, idle spinning eliminated, and - performance improved. -- RML refactoring and clean-up. -- New constructor for concurrent_hash_map allows reserving space for - a number of items. -- Operator delete() added to the TBB exception classes. -- Lambda support was improved in parallel_reduce. -- gcc 4.3 warnings were fixed for concurrent_queue. -- Fixed possible initialization deadlock in modules using TBB entities - during construction of global static objects. -- Copy constructor in concurrent_hash_map was fixed. -- Fixed a couple of rare crashes in the scheduler possible before - in very specific use cases. -- Fixed a rare crash in the TBB allocator running out of memory. -- New tests were implemented, including test_lambda.cpp that checks - support for lambda expressions. -- A few other small changes in code, tests, and documentation. - ------------------------------------------------------------------------- -20090809 open-source release - -Changes (w.r.t. Intel TBB 2.2 commercial-aligned release): - -- Fixed known exception safety issues in concurrent_vector. -- Better concurrency of simultaneous grow requests in concurrent_vector. -- TBB allocator further improves performance of large object allocation. -- Problem with source of text relocations was fixed on Linux -- Fixed bugs related to malloc replacement under Windows -- A few other small changes in code and documentation. - ------------------------------------------------------------------------- -Intel TBB 2.2 commercial-aligned release -TBB_INTERFACE_VERSION == 4000 - -Changes (w.r.t. Intel TBB 2.1 U4 commercial-aligned release): - -- Incorporates all changes from open-source releases below. -- Architecture folders renamed from em64t to intel64 and from itanium - to ia64. -- Major Interface version changed from 3 to 4. Deprecated interfaces - might be removed in future releases. -- Parallel algorithms that use partitioners have switched to use - the auto_partitioner by default. -- Improved memory allocator performance for allocations bigger than 8K. -- Added new thread-bound filters functionality for pipeline. -- New implementation of concurrent_hash_map that improves performance - significantly. -- A few other small changes in code and documentation. - ------------------------------------------------------------------------- -20090511 open-source release - -Changes (w.r.t. previous open-source release): - -- Basic support for MinGW32 development kit. -- Added tbb::zero_allocator class that initializes memory with zeros. - It can be used as an adaptor to any STL-compatible allocator class. -- Added tbb::parallel_for_each template function as alias to parallel_do. -- Added more overloads for tbb::parallel_for. -- Added support for exact exception propagation (can only be used with - compilers that support C++0x std::exception_ptr). -- tbb::atomic template class can be used with enumerations. -- mutex, recursive_mutex, spin_mutex, spin_rw_mutex classes extended - with explicit lock/unlock methods. -- Fixed size() and grow_to_at_least() methods of tbb::concurrent_vector - to provide space allocation guarantees. More methods added for - compatibility with std::vector, including some from C++0x. -- Preview of a lambda-friendly interface for low-level use of tasks. -- scalable_msize function added to the scalable allocator (Windows only). -- Rationalized internal auxiliary functions for spin-waiting and backoff. -- Several tests undergo decent refactoring. - -Changes affecting backward compatibility: - -- Improvements in concurrent_queue, including limited API changes. - The previous version is deprecated; its functionality is accessible - via methods of the new tbb::concurrent_bounded_queue class. -- grow* and push_back methods of concurrent_vector changed to return - iterators; old semantics is deprecated. - ------------------------------------------------------------------------- -Intel TBB 2.1 Update 4 commercial-aligned release -TBB_INTERFACE_VERSION == 3016 - -Changes (w.r.t. Intel TBB 2.1 U3 commercial-aligned release): - -- Added tests for aligned memory allocations and malloc replacement. -- Several improvements for better bundling with Intel(R) C++ Compiler. -- A few other small changes in code and documentaion. - -Bugs fixed: - -- 150 - request to build TBB examples with debug info in release mode. -- backward compatibility issue with concurrent_queue on Windows. -- dependency on VS 2005 SP1 runtime libraries removed. -- compilation of GUI examples under Xcode* 3.1 (1577). -- On Windows, TBB allocator classes can be instantiated with const types - for compatibility with MS implementation of STL containers (1566). - ------------------------------------------------------------------------- -20090313 open-source release - -Changes (w.r.t. 20081109 open-source release): - -- Includes all changes introduced in TBB 2.1 Update 2 & Update 3 - commercial-aligned releases (see below for details). -- Added tbb::parallel_invoke template function. It runs up to 10 - user-defined functions in parallel and waits for them to complete. -- Added a special library providing ability to replace the standard - memory allocation routines in Microsoft* C/C++ RTL (malloc/free, - global new/delete, etc.) with the TBB memory allocator. - Usage details are described in include/tbb/tbbmalloc_proxy.h file. -- Task scheduler switched to use new implementation of its core - functionality (deque based task pool, new structure of arena slots). -- Preview of Microsoft* Visual Studio* 2005 project files for - building the library is available in build/vsproject folder. -- Added tests for aligned memory allocations and malloc replacement. -- Added parallel_for/game_of_life.net example (for Windows only) - showing TBB usage in a .NET application. -- A number of other fixes and improvements to code, tests, makefiles, - examples and documents. - -Bugs fixed: - -- The same list as in TBB 2.1 Update 4 right above. - ------------------------------------------------------------------------- -Intel TBB 2.1 Update 3 commercial-aligned release -TBB_INTERFACE_VERSION == 3015 - -Changes (w.r.t. Intel TBB 2.1 U2 commercial-aligned release): - -- Added support for aligned allocations to the TBB memory allocator. -- Added a special library to use with LD_PRELOAD on Linux* in order to - replace the standard memory allocation routines in C/C++ with the - TBB memory allocator. -- Added null_mutex and null_rw_mutex: no-op classes interface-compliant - to other TBB mutexes. -- Improved performance of parallel_sort, to close most of the serial gap - with std::sort, and beat it on 2 and more cores. -- A few other small changes. - -Bugs fixed: - -- the problem where parallel_for hanged after exception throw - if affinity_partitioner was used (1556). -- get rid of VS warnings about mbstowcs deprecation (1560), - as well as some other warnings. -- operator== for concurrent_vector::iterator fixed to work correctly - with different vector instances. - ------------------------------------------------------------------------- -Intel TBB 2.1 Update 2 commercial-aligned release -TBB_INTERFACE_VERSION == 3014 - -Changes (w.r.t. Intel TBB 2.1 U1 commercial-aligned release): - -- Incorporates all open-source-release changes down to TBB 2.1 U1, - except for: - - 20081019 addition of enumerable_thread_specific; -- Warning level for Microsoft* Visual C++* compiler raised to /W4 /Wp64; - warnings found on this level were cleaned or suppressed. -- Added TBB_runtime_interface_version API function. -- Added new example: pipeline/square. -- Added exception handling and cancellation support - for parallel_do and pipeline. -- Added copy constructor and [begin,end) constructor to concurrent_queue. -- Added some support for beta version of Intel(R) Parallel Amplifier. -- Added scripts to set environment for cross-compilation of 32-bit - applications on 64-bit Linux with Intel(R) C++ Compiler. -- Fixed semantics of concurrent_vector::clear() to not deallocate - internal arrays. Fixed compact() to perform such deallocation later. -- Fixed the issue with atomic when T is incomplete type. -- Improved support for PowerPC* Macintosh*, including the fix - for a bug in masked compare-and-swap reported by a customer. -- As usual, a number of other improvements everywhere. - ------------------------------------------------------------------------- -20081109 open-source release - -Changes (w.r.t. previous open-source release): - -- Added new serial out of order filter for tbb::pipeline. -- Fixed the issue with atomic::operator= reported at the forum. -- Fixed the issue with using tbb::task::self() in task destructor - reported at the forum. -- A number of other improvements to code, tests, makefiles, examples - and documents. - -Open-source contributions integrated: -- Changes in the memory allocator were partially integrated. - ------------------------------------------------------------------------- -20081019 open-source release - -Changes (w.r.t. previous open-source release): - -- Introduced enumerable_thread_specific. This new class provides a - wrapper around native thread local storage as well as iterators and - ranges for accessing the thread local copies (1533). -- Improved support for Intel(R) Threading Analysis Tools - on Intel(R) 64 architecture. -- Dependency from Microsoft* CRT was integrated to the libraries using - manifests, to avoid issues if called from code that uses different - version of Visual C++* runtime than the library. -- Introduced new defines TBB_USE_ASSERT, TBB_USE_DEBUG, - TBB_USE_PERFORMANCE_WARNINGS, TBB_USE_THREADING_TOOLS. -- A number of other improvements to code, tests, makefiles, examples - and documents. - -Open-source contributions integrated: - -- linker optimization: /incremental:no . - ------------------------------------------------------------------------- -20080925 open-source release - -Changes (w.r.t. previous open-source release): - -- Same fix for a memory leak in the memory allocator as in TBB 2.1 U1. -- Improved support for lambda functions. -- Fixed more concurrent_queue issues reported at the forum. -- A number of other improvements to code, tests, makefiles, examples - and documents. - ------------------------------------------------------------------------- -Intel TBB 2.1 Update 1 commercial-aligned release -TBB_INTERFACE_VERSION == 3013 - -Changes (w.r.t. Intel TBB 2.1 commercial-aligned release): - -- Fixed small memory leak in the memory allocator. -- Incorporates all open-source-release changes since TBB 2.1, - except for: - - 20080825 changes for parallel_do; - ------------------------------------------------------------------------- -20080825 open-source release - -Changes (w.r.t. previous open-source release): - -- Added exception handling and cancellation support for parallel_do. -- Added default HashCompare template argument for concurrent_hash_map. -- Fixed concurrent_queue.clear() issues due to incorrect assumption - about clear() being private method. -- Added the possibility to use TBB in applications that change - default calling conventions (Windows* only). -- Many improvements to code, tests, examples, makefiles and documents. - -Bugs fixed: - -- 120, 130 - memset declaration missed in concurrent_hash_map.h - ------------------------------------------------------------------------- -20080724 open-source release - -Changes (w.r.t. previous open-source release): - -- Inline assembly for atomic operations improved for gcc 4.3 -- A few more improvements to the code. - ------------------------------------------------------------------------- -20080709 open-source release - -Changes (w.r.t. previous open-source release): - -- operator=() was added to the tbb_thread class according to - the current working draft for std::thread. -- Recognizing SPARC* in makefiles for Linux* and Sun Solaris*. - -Bugs fixed: - -- 127 - concurrent_hash_map::range fixed to split correctly. - -Open-source contributions integrated: - -- fix_set_midpoint.diff by jyasskin -- SPARC* support in makefiles by Raf Schietekat - ------------------------------------------------------------------------- -20080622 open-source release - -Changes (w.r.t. previous open-source release): - -- Fixed a hang that rarely happened on Linux - during deinitialization of the TBB scheduler. -- Improved support for Intel(R) Thread Checker. -- A few more improvements to the code. - ------------------------------------------------------------------------- -Intel TBB 2.1 commercial-aligned release -TBB_INTERFACE_VERSION == 3011 - -Changes (w.r.t. Intel TBB 2.0 U3 commercial-aligned release): - -- All open-source-release changes down to, and including, TBB 2.0 below, - were incorporated into this release. - ------------------------------------------------------------------------- -20080605 open-source release - -Changes (w.r.t. previous open-source release): - -- Explicit control of exported symbols by version scripts added on Linux. -- Interfaces polished for exception handling & algorithm cancellation. -- Cache behavior improvements in the scalable allocator. -- Improvements in text_filter, polygon_overlay, and other examples. -- A lot of other stability improvements in code, tests, and makefiles. -- First release where binary packages include headers/docs/examples, so - binary packages are now self-sufficient for using TBB. - -Open-source contributions integrated: - -- atomics patch (partially). -- tick_count warning patch. - -Bugs fixed: - -- 118 - fix for boost compatibility. -- 123 - fix for tbb_machine.h. - ------------------------------------------------------------------------- -20080512 open-source release - -Changes (w.r.t. previous open-source release): - -- Fixed a problem with backward binary compatibility - of debug Linux builds. -- Sun* Studio* support added. -- soname support added on Linux via linker script. To restore backward - binary compatibility, *.so -> *.so.2 softlinks should be created. -- concurrent_hash_map improvements - added few new forms of insert() - method and fixed precondition and guarantees of erase() methods. - Added runtime warning reporting about bad hash function used for - the container. Various improvements for performance and concurrency. -- Cancellation mechanism reworked so that it does not hurt scalability. -- Algorithm parallel_do reworked. Requirement for Body::argument_type - definition removed, and work item argument type can be arbitrarily - cv-qualified. -- polygon_overlay example added. -- A few more improvements to code, tests, examples and Makefiles. - -Open-source contributions integrated: - -- Soname support patch for Bugzilla #112. - -Bugs fixed: - -- 112 - fix for soname support. - ------------------------------------------------------------------------- -Intel TBB 2.0 U3 commercial-aligned release (package 017, April 20, 2008) - -Corresponds to commercial 019 (for Linux*, 020; for Mac OS* X, 018) -packages. - -Changes (w.r.t. Intel TBB 2.0 U2 commercial-aligned release): - -- Does not contain open-source-release changes below; this release is - only a minor update of TBB 2.0 U2. -- Removed spin-waiting in pipeline and concurrent_queue. -- A few more small bug fixes from open-source releases below. - ------------------------------------------------------------------------- -20080408 open-source release - -Changes (w.r.t. previous open-source release): - -- count_strings example reworked: new word generator implemented, hash - function replaced, and tbb_allocator is used with std::string class. -- Static methods of spin_rw_mutex were replaced by normal member - functions, and the class name was versioned. -- tacheon example was renamed to tachyon. -- Improved support for Intel(R) Thread Checker. -- A few more minor improvements. - -Open-source contributions integrated: - -- Two sets of Sun patches for IA Solaris support. - ------------------------------------------------------------------------- -20080402 open-source release - -Changes (w.r.t. previous open-source release): - -- Exception handling and cancellation support for tasks and algorithms - fully enabled. -- Exception safety guaranties defined and fixed for all concurrent - containers. -- User-defined memory allocator support added to all concurrent - containers. -- Performance improvement of concurrent_hash_map, spin_rw_mutex. -- Critical fix for a rare race condition during scheduler - initialization/de-initialization. -- New methods added for concurrent containers to be closer to STL, - as well as automatic filters removal from pipeline - and __TBB_AtomicAND function. -- The volatile keyword dropped from where it is not really needed. -- A few more minor improvements. - ------------------------------------------------------------------------- -20080319 open-source release - -Changes (w.r.t. previous open-source release): - -- Support for gcc version 4.3 was added. -- tbb_thread class, near compatible with std::thread expected in C++0x, - was added. - -Bugs fixed: - -- 116 - fix for compilation issues with gcc version 4.2.1. -- 120 - fix for compilation issues with gcc version 4.3. - ------------------------------------------------------------------------- -20080311 open-source release - -Changes (w.r.t. previous open-source release): - -- An enumerator added for pipeline filter types (serial vs. parallel). -- New task_scheduler_observer class introduced, to observe when - threads start and finish interacting with the TBB task scheduler. -- task_scheduler_init reverted to not use internal versioned class; - binary compatibility guaranteed with stable releases only. -- Various improvements to code, tests, examples and Makefiles. - ------------------------------------------------------------------------- -20080304 open-source release - -Changes (w.r.t. previous open-source release): - -- Task-to-thread affinity support, previously kept under a macro, - now fully legalized. -- Work-in-progress on cache_aligned_allocator improvements. -- Pipeline really supports parallel input stage; it's no more serialized. -- Various improvements to code, tests, examples and Makefiles. - -Bugs fixed: - -- 119 - fix for scalable_malloc sometimes failing to return a big block. -- TR575 - fixed a deadlock occurring on Windows in startup/shutdown - under some conditions. - ------------------------------------------------------------------------- -20080226 open-source release - -Changes (w.r.t. previous open-source release): - -- Introduced tbb_allocator to select between standard allocator and - tbb::scalable_allocator when available. -- Removed spin-waiting in pipeline and concurrent_queue. -- Improved performance of concurrent_hash_map by using tbb_allocator. -- Improved support for Intel(R) Thread Checker. -- Various improvements to code, tests, examples and Makefiles. - ------------------------------------------------------------------------- -Intel TBB 2.0 U2 commercial-aligned release (package 017, February 14, 2008) - -Corresponds to commercial 017 (for Linux*, 018; for Mac OS* X, 016) -packages. - -Changes (w.r.t. Intel TBB 2.0 U1 commercial-aligned release): - -- Does not contain open-source-release changes below; this release is - only a minor update of TBB 2.0 U1. -- Add support for Microsoft* Visual Studio* 2008, including binary - libraries and VS2008 projects for examples. -- Use SwitchToThread() not Sleep() to yield threads on Windows*. -- Enhancements to Doxygen-readable comments in source code. -- A few more small bug fixes from open-source releases below. - -Bugs fixed: - -- TR569 - Memory leak in concurrent_queue. - ------------------------------------------------------------------------- -20080207 open-source release - -Changes (w.r.t. previous open-source release): - -- Improvements and minor fixes in VS2008 projects for examples. -- Improvements in code for gating worker threads that wait for work, - previously consolidated under #if IMPROVED_GATING, now legalized. -- Cosmetic changes in code, examples, tests. - -Bugs fixed: - -- 113 - Iterators and ranges should be convertible to their const - counterparts. -- TR569 - Memory leak in concurrent_queue. - ------------------------------------------------------------------------- -20080122 open-source release - -Changes (w.r.t. previous open-source release): - -- Updated examples/parallel_for/seismic to improve the visuals and to - use the affinity_partitioner (20071127 and forward) for better - performance. -- Minor improvements to unittests and performance tests. - ------------------------------------------------------------------------- -20080115 open-source release - -Changes (w.r.t. previous open-source release): - -- Cleanup, simplifications and enhancements to the Makefiles for - building the libraries (see build/index.html for high-level - changes) and the examples. -- Use SwitchToThread() not Sleep() to yield threads on Windows*. -- Engineering work-in-progress on exception safety/support. -- Engineering work-in-progress on affinity_partitioner for - parallel_reduce. -- Engineering work-in-progress on improved gating for worker threads - (idle workers now block in the OS instead of spinning). -- Enhancements to Doxygen-readable comments in source code. - -Bugs fixed: - -- 102 - Support for parallel build with gmake -j -- 114 - /Wp64 build warning on Windows*. - ------------------------------------------------------------------------- -20071218 open-source release - -Changes (w.r.t. previous open-source release): - -- Full support for Microsoft* Visual Studio* 2008 in open-source. - Binaries for vc9/ will be available in future stable releases. -- New recursive_mutex class. -- Full support for 32-bit PowerMac including export files for builds. -- Improvements to parallel_do. - ------------------------------------------------------------------------- -20071206 open-source release - -Changes (w.r.t. previous open-source release): - -- Support for Microsoft* Visual Studio* 2008 in building libraries - from source as well as in vc9/ projects for examples. -- Small fixes to the affinity_partitioner first introduced in 20071127. -- Small fixes to the thread-stack size hook first introduced in 20071127. -- Engineering work in progress on concurrent_vector. -- Engineering work in progress on exception behavior. -- Unittest improvements. - ------------------------------------------------------------------------- -20071127 open-source release - -Changes (w.r.t. previous open-source release): - -- Task-to-thread affinity support (affinity partitioner) first appears. -- More work on concurrent_vector. -- New parallel_do algorithm (function-style version of parallel while) - and parallel_do/parallel_preorder example. -- New task_scheduler_init() hooks for getting default_num_threads() and - for setting thread stack size. -- Support for weak memory consistency models in the code base. -- Futex usage in the task scheduler (Linux). -- Started adding 32-bit PowerMac support. -- Intel(R) 9.1 compilers are now the base supported Intel(R) compiler - version. -- TBB libraries added to link line automatically on Microsoft Windows* - systems via #pragma comment linker directives. - -Open-source contributions integrated: - -- FreeBSD platform support patches. -- AIX weak memory model patch. - -Bugs fixed: - -- 108 - Removed broken affinity.h reference. -- 101 - Does not build on Debian Lenny (replaced arch with uname -m). - ------------------------------------------------------------------------- -20071030 open-source release - -Changes (w.r.t. previous open-source release): - -- More work on concurrent_vector. -- Better support for building with -Wall -Werror (or not) as desired. -- A few fixes to eliminate extraneous warnings. -- Begin introduction of versioning hooks so that the internal/API - version is tracked via TBB_INTERFACE_VERSION. The newest binary - libraries should always work with previously-compiled code when- - ever possible. -- Engineering work in progress on using futex inside the mutexes (Linux). -- Engineering work in progress on exception behavior. -- Engineering work in progress on a new parallel_do algorithm. -- Unittest improvements. - ------------------------------------------------------------------------- -20070927 open-source release - -Changes (w.r.t. Intel TBB 2.0 U1 commercial-aligned release): - -- Minor update to TBB 2.0 U1 below. -- Begin introduction of new concurrent_vector interfaces not released - with TBB 2.0 U1. - ------------------------------------------------------------------------- -Intel TBB 2.0 U1 commercial-aligned release (package 014, October 1, 2007) - -Corresponds to commercial 014 (for Linux*, 016) packages. - -Changes (w.r.t. Intel TBB 2.0 commercial-aligned release): - -- All open-source-release changes down to, and including, TBB 2.0 - below, were incorporated into this release. -- Made a number of changes to the officially supported OS list: - Added Linux* OSs: - Asianux* 3, Debian* 4.0, Fedora Core* 6, Fedora* 7, - Turbo Linux* 11, Ubuntu* 7.04; - Dropped Linux* OSs: - Asianux* 2, Fedora Core* 4, Haansoft* Linux 2006 Server, - Mandriva/Mandrake* 10.1, Miracle Linux* 4.0, - Red Flag* DC Server 5.0; - Only Mac OS* X 10.4.9 (and forward) and Xcode* tool suite 2.4.1 (and - forward) are now supported. -- Commercial installers on Linux* fixed to recommend the correct - binaries to use in more cases, with less unnecessary warnings. -- Changes to eliminate spurious build warnings. - -Open-source contributions integrated: - -- Two small header guard macro patches; it also fixed bug #94. -- New blocked_range3d class. - -Bugs fixed: - -- 93 - Removed misleading comments in task.h. -- 94 - See above. - ------------------------------------------------------------------------- -20070815 open-source release - -Changes: - -- Changes to eliminate spurious build warnings. -- Engineering work in progress on concurrent_vector allocator behavior. -- Added hooks to use the Intel(R) compiler code coverage tools. - -Open-source contributions integrated: - -- Mac OS* X build warning patch. - -Bugs fixed: - -- 88 - Fixed TBB compilation errors if both VS2005 and Windows SDK are - installed. - ------------------------------------------------------------------------- -20070719 open-source release - -Changes: - -- Minor update to TBB 2.0 commercial-aligned release below. -- Changes to eliminate spurious build warnings. - ------------------------------------------------------------------------- -Intel TBB 2.0 commercial-aligned release (package 010, July 19, 2007) - -Corresponds to commercial 010 (for Linux*, 012) packages. - -- TBB open-source debut release. - ------------------------------------------------------------------------- -Intel TBB 1.1 commercial release (April 10, 2007) - -Changes (w.r.t. Intel TBB 1.0 commercial release): - -- auto_partitioner which offered an automatic alternative to specifying - a grain size parameter to estimate the best granularity for tasks. -- The release was added to the Intel(R) C++ Compiler 10.0 Pro. - ------------------------------------------------------------------------- -Intel TBB 1.0 Update 2 commercial release - -Changes (w.r.t. Intel TBB 1.0 Update 1 commercial release): - -- Mac OS* X 64-bit support added. -- Source packages for commercial releases introduced. - ------------------------------------------------------------------------- -Intel TBB 1.0 Update 1 commercial-aligned release - -Changes (w.r.t. Intel TBB 1.0 commercial release): - -- Fix for critical package issue on Mac OS* X. - ------------------------------------------------------------------------- -Intel TBB 1.0 commercial release (August 29, 2006) - -Changes (w.r.t. Intel TBB 1.0 beta commercial release): - -- New namespace (and compatibility headers for old namespace). - Namespaces are tbb and tbb::internal and all classes are in the - underscore_style not the WindowsStyle. -- New class: scalable_allocator (and cache_aligned_allocator using that - if it exists). -- Added parallel_for/tacheon example. -- Removed C-style casts from headers for better C++ compliance. -- Bug fixes. -- Documentation improvements. -- Improved performance of the concurrent_hash_map class. -- Upgraded parallel_sort() to support STL-style random-access iterators - instead of just pointers. -- The Windows vs7_1 directories renamed to vs7.1 in examples. -- New class: spin version of reader-writer lock. -- Added push_back() interface to concurrent_vector(). - ------------------------------------------------------------------------- -Intel TBB 1.0 beta commercial release - -Initial release. - -Features / APIs: - -- Concurrent containers: ConcurrentHashTable, ConcurrentVector, - ConcurrentQueue. -- Parallel algorithms: ParallelFor, ParallelReduce, ParallelScan, - ParallelWhile, Pipeline, ParallelSort. -- Support: AlignedSpace, BlockedRange (i.e., 1D), BlockedRange2D -- Task scheduler with multi-master support. -- Atomics: read, write, fetch-and-store, fetch-and-add, compare-and-swap. -- Locks: spin, reader-writer, queuing, OS-wrapper. -- Memory allocation: STL-style memory allocator that avoids false - sharing. -- Timers. - -Tools Support: -- Intel(R) Thread Checker 3.0. -- Intel(R) Thread Profiler 3.0. - -Documentation: -- First Use Documents: README.txt, INSTALL.txt, Release_Notes.txt, - Doc_Index.html, Getting_Started.pdf, Tutorial.pdf, Reference.pdf. -- Class hierarchy HTML pages (Doxygen). -- Tree of index.html pages for navigating the installed package, esp. - for the examples. - -Examples: -- One for each of these TBB features: ConcurrentHashTable, ParallelFor, - ParallelReduce, ParallelWhile, Pipeline, Task. -- Live copies of examples from Getting_Started.pdf. -- TestAll example that exercises every class and header in the package - (i.e., a "liveness test"). -- Compilers: see Release_Notes.txt. -- APIs: OpenMP, WinThreads, Pthreads. - -Packaging: -- Package for Windows installs IA-32 and EM64T bits. -- Package for Linux installs IA-32, EM64T and IPF bits. -- Package for Mac OS* X installs IA-32 bits. -- All packages support Intel(R) software setup assistant (ISSA) and - install-time FLEXlm license checking. -- ISSA support allows license file to be specified directly in case of - no Internet connection or problems with IRC or serial #s. -- Linux installer allows root or non-root, RPM or non-RPM installs. -- FLEXlm license servers (for those who need floating/counted licenses) - are provided separately on Intel(R) Premier. - ------------------------------------------------------------------------- -Intel, the Intel logo, Xeon, Intel Xeon Phi, and Cilk are registered -trademarks or trademarks of Intel Corporation or its subsidiaries in -the United States and other countries. - -* Other names and brands may be claimed as the property of others. diff --git a/src/tbb/CMakeLists.txt b/src/tbb/CMakeLists.txt new file mode 100644 index 000000000..811a3a554 --- /dev/null +++ b/src/tbb/CMakeLists.txt @@ -0,0 +1,348 @@ +# Copyright (c) 2020-2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cmake_minimum_required(VERSION 3.5) + +# Enable CMake policies + +if (POLICY CMP0068) + # RPATH settings do not affect install_name on macOS since CMake 3.9 + cmake_policy(SET CMP0068 NEW) +endif() + +if (POLICY CMP0091) + # The NEW behavior for this policy is to not place MSVC runtime library flags in the default + # CMAKE__FLAGS_ cache entries and use CMAKE_MSVC_RUNTIME_LIBRARY abstraction instead. + cmake_policy(SET CMP0091 NEW) +elseif (DEFINED CMAKE_MSVC_RUNTIME_LIBRARY) + message(FATAL_ERROR "CMAKE_MSVC_RUNTIME_LIBRARY was defined while policy CMP0091 is not available. Use CMake 3.15 or newer.") +endif() + +if (TBB_WINDOWS_DRIVER AND (NOT ("${CMAKE_MSVC_RUNTIME_LIBRARY}" STREQUAL MultiThreaded OR "${CMAKE_MSVC_RUNTIME_LIBRARY}" STREQUAL MultiThreadedDebug))) + message(FATAL_ERROR "Enabled TBB_WINDOWS_DRIVER requires CMAKE_MSVC_RUNTIME_LIBRARY to be set to MultiThreaded or MultiThreadedDebug.") +endif() + +# Enable support of minimum supported macOS version flag +if (APPLE) + if (NOT CMAKE_CXX_OSX_DEPLOYMENT_TARGET_FLAG) + set(CMAKE_CXX_OSX_DEPLOYMENT_TARGET_FLAG "-mmacosx-version-min=" CACHE STRING "Minimum macOS version flag") + endif() + if (NOT CMAKE_C_OSX_DEPLOYMENT_TARGET_FLAG) + set(CMAKE_C_OSX_DEPLOYMENT_TARGET_FLAG "-mmacosx-version-min=" CACHE STRING "Minimum macOS version flag") + endif() +endif() + +file(READ include/oneapi/tbb/version.h _tbb_version_info) +string(REGEX REPLACE ".*#define TBB_VERSION_MAJOR ([0-9]+).*" "\\1" _tbb_ver_major "${_tbb_version_info}") +string(REGEX REPLACE ".*#define TBB_VERSION_MINOR ([0-9]+).*" "\\1" _tbb_ver_minor "${_tbb_version_info}") +string(REGEX REPLACE ".*#define TBB_VERSION_PATCH ([0-9]+).*" "\\1" _tbb_ver_patch "${_tbb_version_info}") +string(REGEX REPLACE ".*#define TBB_INTERFACE_VERSION ([0-9]+).*" "\\1" TBB_INTERFACE_VERSION "${_tbb_version_info}") +string(REGEX REPLACE ".*#define __TBB_BINARY_VERSION ([0-9]+).*" "\\1" TBB_BINARY_VERSION "${_tbb_version_info}") +string(REGEX REPLACE "..(..)." "\\1" TBB_BINARY_MINOR_VERSION "${TBB_INTERFACE_VERSION}") +set(TBBMALLOC_BINARY_VERSION 2) +set(TBBBIND_BINARY_VERSION 3) + +project(TBB VERSION ${_tbb_ver_major}.${_tbb_ver_minor}.${_tbb_ver_patch} LANGUAGES CXX) +unset(_tbb_ver_major) +unset(_tbb_ver_minor) + +include(CheckCXXCompilerFlag) +include(GNUInstallDirs) +include(CMakeDependentOption) + +# --------------------------------------------------------------------------------------------------------- +# Handle C++ standard version. +if (NOT MSVC) # no need to cover MSVC as it uses C++14 by default. + if (NOT CMAKE_CXX_STANDARD) + set(CMAKE_CXX_STANDARD 11) + endif() + + if (CMAKE_CXX${CMAKE_CXX_STANDARD}_STANDARD_COMPILE_OPTION) # if standard option was detected by CMake + set(CMAKE_CXX_STANDARD_REQUIRED ON) + else() # if standard option wasn't detected by CMake (e.g. for Intel Compiler with CMake 3.1) + # TBB_CXX_STD_FLAG should be added to targets via target_compile_options + set(TBB_CXX_STD_FLAG -std=c++${CMAKE_CXX_STANDARD}) + + check_cxx_compiler_flag(${TBB_CXX_STD_FLAG} c++${CMAKE_CXX_STANDARD}) + if (NOT c++${CMAKE_CXX_STANDARD}) + message(FATAL_ERROR "C++${CMAKE_CXX_STANDARD} (${TBB_CXX_STD_FLAG}) support is required") + endif() + unset(c++${CMAKE_CXX_STANDARD}) + endif() +endif() + +set(CMAKE_CXX_EXTENSIONS OFF) # use -std=c++... instead of -std=gnu++... +# --------------------------------------------------------------------------------------------------------- + +# Detect architecture (bitness). +if (CMAKE_SIZEOF_VOID_P EQUAL 4) + set(TBB_ARCH 32) +else() + set(TBB_ARCH 64) +endif() + +option(TBB_TEST "Enable testing" ON) +option(TBB_EXAMPLES "Enable examples" OFF) +option(TBB_STRICT "Treat compiler warnings as errors" ON) +option(TBB_WINDOWS_DRIVER "Build as Universal Windows Driver (UWD)" OFF) +option(TBB_NO_APPCONTAINER "Apply /APPCONTAINER:NO (for testing binaries for Windows Store)" OFF) +option(TBB4PY_BUILD "Enable tbb4py build" OFF) +option(TBB_BUILD "Enable tbb build" ON) +option(TBBMALLOC_BUILD "Enable tbbmalloc build" ON) +cmake_dependent_option(TBBMALLOC_PROXY_BUILD "Enable tbbmalloc_proxy build" ON "TBBMALLOC_BUILD" OFF) +option(TBB_CPF "Enable preview features of the library" OFF) +option(TBB_FIND_PACKAGE "Enable search for external oneTBB using find_package instead of build from sources" OFF) +option(TBB_DISABLE_HWLOC_AUTOMATIC_SEARCH "Disable HWLOC automatic search by pkg-config tool" ${CMAKE_CROSSCOMPILING}) +option(TBB_ENABLE_IPO "Enable Interprocedural Optimization (IPO) during the compilation" ON) +option(TBB_FUZZ_TESTING "Enable fuzz testing" OFF) +option(TBB_INSTALL "Enable installation" ON) +if(LINUX) +option(TBB_LINUX_SEPARATE_DBG "Enable separation of the debug symbols during the build" OFF) +endif() +if(APPLE) +option(TBB_BUILD_APPLE_FRAMEWORKS "Build as Apple Frameworks" OFF) +endif() + +if (NOT DEFINED BUILD_SHARED_LIBS) + set(BUILD_SHARED_LIBS ON) +endif() + +if (NOT BUILD_SHARED_LIBS) + if(NOT DEFINED CMAKE_POSITION_INDEPENDENT_CODE) + set(CMAKE_POSITION_INDEPENDENT_CODE ON) + endif() + message(WARNING "You are building oneTBB as a static library. This is highly discouraged and such configuration is not supported. Consider building a dynamic library to avoid unforeseen issues.") +endif() + +if (NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) + set(CMAKE_BUILD_TYPE RelWithDebInfo CACHE STRING "Build type" FORCE) + message(STATUS "CMAKE_BUILD_TYPE is not specified. Using default: ${CMAKE_BUILD_TYPE}") + # Possible values of build type for cmake-gui + set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo") +endif() + +if (CMAKE_BUILD_TYPE) + string(TOLOWER ${CMAKE_BUILD_TYPE} _tbb_build_type) + if (_tbb_build_type STREQUAL "debug") + set(TBB_ENABLE_IPO OFF) + endif() + unset(_tbb_build_type) +endif() + +# ------------------------------------------------------------------- +# Files and folders naming +set(CMAKE_DEBUG_POSTFIX _debug) + +if (NOT DEFINED TBB_OUTPUT_DIR_BASE) + if (MSVC) + if (NOT DEFINED CMAKE_MSVC_RUNTIME_LIBRARY OR CMAKE_MSVC_RUNTIME_LIBRARY MATCHES DLL) + set(_tbb_msvc_runtime _md) + else() + set(_tbb_msvc_runtime _mt) + endif() + + if (WINDOWS_STORE) + if (TBB_NO_APPCONTAINER) + set(_tbb_win_store _wsnoappcont) + else() + set(_tbb_win_store _ws) + endif() + elseif(TBB_WINDOWS_DRIVER) + set(_tbb_win_store _wd) + endif() + endif() + + string(REGEX MATCH "^([0-9]+\.[0-9]+|[0-9]+)" _tbb_compiler_version_short ${CMAKE_CXX_COMPILER_VERSION}) + string(TOLOWER ${CMAKE_CXX_COMPILER_ID}_${_tbb_compiler_version_short}_cxx${CMAKE_CXX_STANDARD}_${TBB_ARCH}${_tbb_msvc_runtime}${_tbb_win_store} TBB_OUTPUT_DIR_BASE) + unset(_tbb_msvc_runtime) + unset(_tbb_win_store) + unset(_tbb_compiler_version_short) +endif() + +foreach(output_type LIBRARY ARCHIVE PDB RUNTIME) + if (CMAKE_BUILD_TYPE) + string(TOLOWER ${CMAKE_BUILD_TYPE} _tbb_build_type_lower) + set(CMAKE_${output_type}_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/${TBB_OUTPUT_DIR_BASE}_${_tbb_build_type_lower}) + unset(_tbb_build_type_lower) + endif() + + if (CMAKE_CONFIGURATION_TYPES) + foreach(suffix ${CMAKE_CONFIGURATION_TYPES}) + string(TOUPPER ${suffix} _tbb_suffix_upper) + string(TOLOWER ${suffix} _tbb_suffix_lower) + set(CMAKE_${output_type}_OUTPUT_DIRECTORY_${_tbb_suffix_upper} ${CMAKE_BINARY_DIR}/${TBB_OUTPUT_DIR_BASE}_${_tbb_suffix_lower}) + endforeach() + unset(_tbb_suffix_lower) + unset(_tbb_suffix_upper) + endif() +endforeach() + +if (CMAKE_CONFIGURATION_TYPES) + # We can't use generator expressions in a cmake variable name. + set(TBB_TEST_WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/${TBB_OUTPUT_DIR_BASE}_$>) +else() + set(TBB_TEST_WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) +endif() + +# ------------------------------------------------------------------- + +# ------------------------------------------------------------------- +# Common dependencies +#force -pthread during compilation for Emscripten +if (EMSCRIPTEN AND NOT EMSCRIPTEN_WITHOUT_PTHREAD) + set(THREADS_HAVE_PTHREAD_ARG TRUE) +endif() + +set(THREADS_PREFER_PTHREAD_FLAG TRUE) +find_package(Threads REQUIRED) +# ------------------------------------------------------------------- + +file(GLOB FILES_WITH_EXTRA_TARGETS ${CMAKE_CURRENT_SOURCE_DIR}/cmake/*.cmake) +foreach(FILE_WITH_EXTRA_TARGETS ${FILES_WITH_EXTRA_TARGETS}) + include(${FILE_WITH_EXTRA_TARGETS}) +endforeach() + +# - Enabling LTO on Android causes the NDK bug. +# NDK throws the warning: "argument unused during compilation: '-Wa,--noexecstack'" +# - For some reason GCC does not instrument code with Thread Sanitizer when lto is enabled and C linker is used. +if (TBB_ENABLE_IPO AND BUILD_SHARED_LIBS AND NOT ANDROID_PLATFORM AND NOT TBB_SANITIZE MATCHES "thread") + if (NOT CMAKE_VERSION VERSION_LESS 3.9) + cmake_policy(SET CMP0069 NEW) + include(CheckIPOSupported) + check_ipo_supported(RESULT TBB_IPO_PROPERTY) + else() + set(TBB_IPO_FLAGS TRUE) + endif() + if (TBB_IPO_PROPERTY OR TBB_IPO_FLAGS) + message(STATUS "IPO enabled") + endif() +endif() + +set(TBB_COMPILER_SETTINGS_FILE ${CMAKE_CURRENT_SOURCE_DIR}/cmake/compilers/${CMAKE_CXX_COMPILER_ID}.cmake) +if (EXISTS ${TBB_COMPILER_SETTINGS_FILE}) + include(${TBB_COMPILER_SETTINGS_FILE}) +else() + message(WARNING "TBB compiler settings not found ${TBB_COMPILER_SETTINGS_FILE}") +endif() + +if (TBB_FIND_PACKAGE AND TBB_DIR) + # Allow specifying external TBB to test with. + # Do not add main targets and installation instructions in that case. + message(STATUS "Using external TBB for testing") + find_package(TBB REQUIRED) +else() + if (TBB_BUILD) + add_subdirectory(src/tbb) + endif() + if (TBBMALLOC_BUILD) + add_subdirectory(src/tbbmalloc) + if(TBBMALLOC_PROXY_BUILD AND NOT "${MSVC_CXX_ARCHITECTURE_ID}" MATCHES "ARM64") + add_subdirectory(src/tbbmalloc_proxy) + endif() + endif() + if (NOT BUILD_SHARED_LIBS) + message(STATUS "TBBBind build targets are disabled due to unsupported environment") + else() + add_subdirectory(src/tbbbind) + endif() + if (TBB_INSTALL) + # ------------------------------------------------------------------- + # Installation instructions + include(CMakePackageConfigHelpers) + + install(DIRECTORY include/ + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} + COMPONENT devel) + + install(EXPORT ${PROJECT_NAME}Targets + NAMESPACE TBB:: + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME} + COMPONENT devel) + file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config.cmake + "include(\${CMAKE_CURRENT_LIST_DIR}/${PROJECT_NAME}Targets.cmake)\n") + if (NOT BUILD_SHARED_LIBS) + file(APPEND ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config.cmake + "include(CMakeFindDependencyMacro)\nfind_dependency(Threads)\n") + endif() + + write_basic_package_version_file("${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake" + COMPATIBILITY AnyNewerVersion) + + install(FILES "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config.cmake" + "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake" + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME} + COMPONENT devel) + + install(FILES "README.md" + DESTINATION ${CMAKE_INSTALL_DOCDIR} + COMPONENT devel) + # ------------------------------------------------------------------- + endif() +endif() + +if (TBB_TEST) + enable_testing() + add_subdirectory(test) +endif() + +if (TBB_EXAMPLES) + add_subdirectory(examples) +endif() + +if (TBB_BENCH) + if (NOT EXISTS ${CMAKE_CURRENT_LIST_DIR}/benchmark) + message(FATAL_ERROR "Benchmarks are not supported yet") + endif() + + enable_testing() + add_subdirectory(benchmark) +endif() + +if (ANDROID_PLATFORM) + if ("${ANDROID_STL}" STREQUAL "c++_shared") + if (${ANDROID_NDK_MAJOR} GREATER_EQUAL "25") + if(ANDROID_ABI STREQUAL "arm64-v8a") + set(ANDROID_TOOLCHAIN_NAME "aarch64-linux-android") + elseif(ANDROID_ABI STREQUAL "x86_64") + set(ANDROID_TOOLCHAIN_NAME "x86_64-linux-android") + elseif(ANDROID_ABI STREQUAL "armeabi-v7a") + set(ANDROID_TOOLCHAIN_NAME "arm-linux-androideabi") + elseif(ANDROID_ABI STREQUAL "x86") + set(ANDROID_TOOLCHAIN_NAME "i686-linux-android") + endif() + + configure_file( + "${ANDROID_TOOLCHAIN_ROOT}/sysroot/usr/lib/${ANDROID_TOOLCHAIN_NAME}/libc++_shared.so" + "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/libc++_shared.so" + COPYONLY) + else() + configure_file( + "${ANDROID_NDK}/sources/cxx-stl/llvm-libc++/libs/${ANDROID_ABI}/libc++_shared.so" + "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/libc++_shared.so" + COPYONLY) + endif() + endif() + # This custom target may be implemented without separate CMake script, but it requires + # ADB(Android Debug Bridge) executable file availability, so to incapsulate this requirement + # only for corresponding custom target, it was implemented by this way. + add_custom_target(device_environment_cleanup COMMAND ${CMAKE_COMMAND} + -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/android/device_environment_cleanup.cmake) +endif() + +if (TBB4PY_BUILD) + add_subdirectory(python) +endif() + +# Keep it the last instruction. +add_subdirectory(cmake/post_install) diff --git a/src/tbb/CODEOWNERS b/src/tbb/CODEOWNERS new file mode 100644 index 000000000..78105ac7e --- /dev/null +++ b/src/tbb/CODEOWNERS @@ -0,0 +1,27 @@ +# Where component owners are known, add them here. + +/oneTBB/src/tbb/ @pavelkumbrasev +/oneTBB/src/tbb/ @dnmokhov +/oneTBB/src/tbb/ @JhaShweta1 +/oneTBB/src/tbb/ @sarathnandu +/oneTBB/include/oneapi/tbb/parallel_* @pavelkumbrasev +/oneTBB/include/oneapi/tbb/concurrent_* @kboyarinov +/oneTBB/include/oneapi/tbb/flow_graph* @kboyarinov +/oneTBB/include/oneapi/tbb/flow_graph* @aleksei-fedotov +/oneTBB/include/oneapi/tbb/detail/_flow_graph* @kboyarinov +/oneTBB/include/oneapi/tbb/detail/_flow_graph* @aleksei-fedotov +/oneTBB/include/oneapi/tbb/detail/_concurrent* @kboyarinov +/oneTBB/src/doc @aepanchi +/oneTBB/src/tbbbind/ @isaevil +/oneTBB/src/tbbmalloc/ @lplewa +/oneTBB/src/tbbmalloc_proxy/ @lplewa +/oneTBB/cmake/ @isaevil +/oneTBB/*CMakeLists.txt @isaevil +/oneTBB/python/ @sarathnandu +/oneTBB/python/ @isaevil + +# Bazel build related files. +/oneTBB/.bazelversion @Vertexwahn +/oneTBB/Bazel.md @Vertexwahn +/oneTBB/BUILD.bazel @Vertexwahn +/oneTBB/MODULE.bazel @Vertexwahn diff --git a/src/tbb/CODE_OF_CONDUCT.md b/src/tbb/CODE_OF_CONDUCT.md new file mode 100644 index 000000000..c16970739 --- /dev/null +++ b/src/tbb/CODE_OF_CONDUCT.md @@ -0,0 +1,134 @@ + +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, caste, color, religion, or sexual +identity and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our +community include: + +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +* Focusing on what is best not just for us as individuals, but for the overall + community + +Examples of unacceptable behavior include: + +* The use of sexualized language or imagery, and sexual attention or advances of + any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or email address, + without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official email address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement at +oneTBBCodeOfConduct At intel DOT com. +All complaints will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series of +actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or permanent +ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within the +community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.1, available at +[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1]. + +Community Impact Guidelines were inspired by +[Mozilla's code of conduct enforcement ladder][Mozilla CoC]. + +For answers to common questions about this code of conduct, see the FAQ at +[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at +[https://www.contributor-covenant.org/translations][translations]. + +[homepage]: https://www.contributor-covenant.org +[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html +[Mozilla CoC]: https://github.com/mozilla/diversity +[FAQ]: https://www.contributor-covenant.org/faq +[translations]: https://www.contributor-covenant.org/translations + diff --git a/src/tbb/CONTRIBUTING.md b/src/tbb/CONTRIBUTING.md new file mode 100644 index 000000000..b2b6a968c --- /dev/null +++ b/src/tbb/CONTRIBUTING.md @@ -0,0 +1,45 @@ + + +# How to Contribute +As an open source project, we welcome community contributions to oneAPI Threading Building Blocks (oneTBB). This document explains how to participate in project conversations, log bugs and enhancement requests, and submit code patches to the project. + +## Licensing + +Licensing is very important to open source projects. It helps ensure the software continues to be available under the terms that the author desired. The oneTBB project uses the [Apache 2.0 License](https://github.com/oneapi-src/oneTBB/blob/master/LICENSE.txt), a permissive open source license that allows you to freely use, modify, and distribute your own products that include Apache 2.0 licensed software. By contributing to the oneTBB project, you agree to the license and copyright terms therein and release your own contributions under these terms. + +Some imported or reused components within oneTBB use other licenses, as described in [third-party-programs.txt](https://github.com/oneapi-src/oneTBB/blob/master/third-party-programs.txt). By carefully reviewing potential contributions, we can ensure that the community can develop products with oneTBB without concerns over patent or copyright issues. + +## Prerequisites + +As a contributor, you’ll want to be familiar with the oneTBB project and the repository layout. You should also know how to use it as explained in the [oneTBB documentation](https://oneapi-src.github.io/oneTBB/) and how to set up your build development environment to configure, build, and test oneTBB as explained in the [oneTBB Build System Description](cmake/README.md). + +## Pull Requests + +You can find all [open oneTBB pull requests](https://github.com/oneapi-src/oneTBB/pulls) on GitHub. + +### Before contributing changes directly to the oneTBB repository + +* Make sure you can build the product and run all the tests with your patch. +* For a larger feature, provide a relevant test. +* Document your code. The oneTBB project uses reStructuredText for documentation. +* Update the copyright year in the first line of the changing file(s). + For example, if you commit your changes in 2022: + * the copyright year should be `2005-2022` for existing files + * the copyright year should be `2022` for new files +* Submit a pull request into the master branch. You can submit changes with a pull request (preferred) or by sending an email patch. + +Continuous Integration (CI) testing is enabled for the repository. Your pull request must pass all checks before it can be merged. We will review your contribution and may provide feedback to guide you if any additional fixes or modifications are necessary. When reviewed and accepted, your pull request will be merged into our GitHub repository. diff --git a/src/tbb/INSTALL.md b/src/tbb/INSTALL.md new file mode 100644 index 000000000..0ac95f875 --- /dev/null +++ b/src/tbb/INSTALL.md @@ -0,0 +1,135 @@ +# Installation from Sources + + +## Prerequisites + + - Make sure you have installed CMake version 3.1 (or newer) on your system. oneTBB uses CMake build configuration. + - Configure and build oneTBB. To work with build configurations, see [Build System Description](cmake/README.md). + + +## Configure oneTBB + +At the command prompt, type: +``` +cmake +``` + +You may want to use some additional options for configuration: + +| Option | Purpose | Description | +| ------ |------ | ------ | +| `-G ` | Specify project generator | For more information, run cmake `–help`. | +|`-DCMAKE_BUILD_TYPE=Debug` | Specify for Debug build | Not applicable for multi-configuration generators such as Visual Studio generator. | + + +## Build oneTBB + +To build the system, run: +``` +cmake --build . +``` + +Some useful build options: +- `--target ` - specific target, "all" is default. +- `--config ` - build configuration, applicable only for multi-config generators such as Visual Studio generator. + + +## Install and Pack oneTBB + +--- +**NOTE** + +Be careful about installing prefix. It defaults to `/usr/local` on UNIX* and `c:/Program Files/${PROJECT_NAME}` on Windows* OS. +You can define custom `CMAKE_INSTALL_PREFIX` during configuration: + +``` +cmake -DCMAKE_INSTALL_PREFIX=/my/install/prefix .. +``` + +--- + +Installation can also be done using: + +``` +cmake --install +``` + +Special ``--install`` target can alternatively be used for installation, e.g. ``make install``. + +You can use the ``install`` components for partial installation. + +The following install components are supported: +- `runtime` - oneTBB runtime package (core shared libraries and `.dll` files on Windows* OS). +- `devel` - oneTBB development package (header files, CMake integration files, library symbolic links, and `.lib` files on Windows* OS). +- `tbb4py` - [oneTBB Module for Python](https://github.com/oneapi-src/oneTBB/blob/master/python/README.md). + +If you want to install specific components after configuration and build, run: + +```bash +cmake -DCOMPONENT= [-DBUILD_TYPE=] -P cmake_install.cmake +``` + +Simple packaging using CPack is supported. +The following commands allow you to create a simple portable package that includes header files, libraries, and integration files for CMake: + +```bash +cmake .. +cpack +``` + +## Installation from vcpkg + +You can download and install oneTBB using the [vcpkg](https://github.com/Microsoft/vcpkg) dependency manager: +```sh + git clone https://github.com/Microsoft/vcpkg.git + cd vcpkg + ./bootstrap-vcpkg.sh #.\bootstrap-vcpkg.bat(for Windows) + ./vcpkg integrate install + ./vcpkg install tbb +``` + +The oneTBB port in vcpkg is kept up to date by Microsoft* team members and community contributors. If the version is out of date, create an issue or pull request on the [vcpkg repository](https://github.com/Microsoft/vcpkg). + +## Example of Installation + +### Single-configuration generators + +The following example demonstrates how to install oneTBB for single-configuration generators (e.g. GNU Make, Ninja, etc.). +```bash +# Do our experiments in /tmp +cd /tmp +# Clone oneTBB repository +git clone https://github.com/oneapi-src/oneTBB.git +cd oneTBB +# Create binary directory for out-of-source build +mkdir build && cd build +# Configure: customize CMAKE_INSTALL_PREFIX and disable TBB_TEST to avoid tests build +cmake -DCMAKE_INSTALL_PREFIX=/tmp/my_installed_onetbb -DTBB_TEST=OFF .. +# Build +cmake --build . +# Install +cmake --install . +# Well done! Your installed oneTBB is in /tmp/my_installed_onetbb +``` + +### Multi-configuration generators + +The following example demonstrates how to install oneTBB for multi-configuration generators such as Visual Studio*. + +Choose the configuration during the build and install steps: +```batch +REM Do our experiments in %TMP% +cd %TMP% +REM Clone oneTBB repository +git clone https://github.com/oneapi-src/oneTBB.git +cd oneTBB +REM Create binary directory for out-of-source build +mkdir build && cd build +REM Configure: customize CMAKE_INSTALL_PREFIX and disable TBB_TEST to avoid tests build +cmake -DCMAKE_INSTALL_PREFIX=%TMP%\my_installed_onetbb -DTBB_TEST=OFF .. +REM Build "release with debug information" configuration +cmake --build . --config relwithdebinfo +REM Install "release with debug information" configuration +cmake --install . --config relwithdebinfo +REM Well done! Your installed oneTBB is in %TMP%\my_installed_onetbb +``` diff --git a/src/tbb/COPYING b/src/tbb/LICENSE.txt similarity index 100% rename from src/tbb/COPYING rename to src/tbb/LICENSE.txt diff --git a/src/tbb/Makefile b/src/tbb/Makefile deleted file mode 100644 index d7dd56f97..000000000 --- a/src/tbb/Makefile +++ /dev/null @@ -1,80 +0,0 @@ -# Copyright (c) 2005-2017 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -# -# - -tbb_root?=. -include $(tbb_root)/build/common.inc -.PHONY: default all tbb tbbmalloc tbbproxy test examples - -#workaround for non-depend targets tbb and tbbmalloc which both depend on version_string.ver -#According to documentation, recursively invoked make commands can process their targets in parallel -.NOTPARALLEL: tbb tbbmalloc tbbproxy - -default: tbb tbbmalloc $(if $(use_proxy),tbbproxy) - -all: tbb tbbmalloc tbbproxy test examples - -tbb: mkdir - $(MAKE) -C "$(work_dir)_debug" -r -f $(tbb_root)/build/Makefile.tbb cfg=debug - $(MAKE) -C "$(work_dir)_release" -r -f $(tbb_root)/build/Makefile.tbb cfg=release - -tbbmalloc: mkdir - $(MAKE) -C "$(work_dir)_debug" -r -f $(tbb_root)/build/Makefile.tbbmalloc cfg=debug malloc - $(MAKE) -C "$(work_dir)_release" -r -f $(tbb_root)/build/Makefile.tbbmalloc cfg=release malloc - -tbbproxy: mkdir - $(MAKE) -C "$(work_dir)_debug" -r -f $(tbb_root)/build/Makefile.tbbproxy cfg=debug tbbproxy - $(MAKE) -C "$(work_dir)_release" -r -f $(tbb_root)/build/Makefile.tbbproxy cfg=release tbbproxy - -test: tbb tbbmalloc $(if $(use_proxy),tbbproxy) - -$(MAKE) -C "$(work_dir)_debug" -r -f $(tbb_root)/build/Makefile.tbbmalloc cfg=debug malloc_test - -$(MAKE) -C "$(work_dir)_debug" -r -f $(tbb_root)/build/Makefile.test cfg=debug - -$(MAKE) -C "$(work_dir)_release" -r -f $(tbb_root)/build/Makefile.tbbmalloc cfg=release malloc_test - -$(MAKE) -C "$(work_dir)_release" -r -f $(tbb_root)/build/Makefile.test cfg=release - -rml: mkdir - $(MAKE) -C "$(work_dir)_debug" -r -f $(tbb_root)/build/Makefile.rml cfg=debug - $(MAKE) -C "$(work_dir)_release" -r -f $(tbb_root)/build/Makefile.rml cfg=release - - -examples: tbb tbbmalloc - $(MAKE) -C examples -r -f Makefile tbb_root=.. release test - -.PHONY: clean clean_examples mkdir info - -clean: clean_examples - $(shell $(RM) $(work_dir)_release$(SLASH)*.* >$(NUL) 2>$(NUL)) - $(shell $(RD) $(work_dir)_release >$(NUL) 2>$(NUL)) - $(shell $(RM) $(work_dir)_debug$(SLASH)*.* >$(NUL) 2>$(NUL)) - $(shell $(RD) $(work_dir)_debug >$(NUL) 2>$(NUL)) - @echo clean done - -clean_examples: - $(shell $(MAKE) -s -i -r -C examples -f Makefile tbb_root=.. clean >$(NUL) 2>$(NUL)) - -mkdir: - $(shell $(MD) "$(work_dir)_release" >$(NUL) 2>$(NUL)) - $(shell $(MD) "$(work_dir)_debug" >$(NUL) 2>$(NUL)) - @echo Created $(work_dir)_release and ..._debug directories - -info: - @echo OS: $(tbb_os) - @echo arch=$(arch) - @echo compiler=$(compiler) - @echo runtime=$(runtime) - @echo tbb_build_prefix=$(tbb_build_prefix) - diff --git a/src/tbb/README b/src/tbb/README deleted file mode 100644 index fcc87af0c..000000000 --- a/src/tbb/README +++ /dev/null @@ -1,11 +0,0 @@ -Intel(R) Threading Building Blocks - README - -See index.html for directions and documentation. - -If source is present (./Makefile and src/ directories), -type 'gmake' in this directory to build and test. - -See examples/index.html for runnable examples and directions. - -See http://threadingbuildingblocks.org for full documentation -and software information. diff --git a/src/tbb/README.md b/src/tbb/README.md new file mode 100644 index 000000000..2e7c2e81b --- /dev/null +++ b/src/tbb/README.md @@ -0,0 +1,70 @@ +# oneAPI Threading Building Blocks (oneTBB) +[![Apache License Version 2.0](https://img.shields.io/badge/license-Apache_2.0-green.svg)](LICENSE.txt) [![oneTBB CI](https://github.com/oneapi-src/oneTBB/actions/workflows/ci.yml/badge.svg)](https://github.com/oneapi-src/oneTBB/actions/workflows/ci.yml?query=branch%3Amaster) +[![Join the community on GitHub Discussions](https://badgen.net/badge/join%20the%20discussion/on%20github/blue?icon=github)](https://github.com/oneapi-src/oneTBB/discussions) +[![OpenSSF Best Practices](https://www.bestpractices.dev/projects/9125/badge)](https://www.bestpractices.dev/projects/9125) +[![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/oneapi-src/oneTBB/badge)](https://securityscorecards.dev/viewer/?uri=github.com/oneapi-src/oneTBB) + +oneTBB is a flexible C++ library that simplifies the work of adding parallelism +to complex applications, even if you are not a threading expert. + +The library lets you easily write parallel programs that take full advantage of the multi-core performance. Such programs are portable, +composable and have a future-proof scalability. oneTBB provides you with functions, interfaces, and classes to parallelize and scale the code. +All you have to do is to use the templates. + +The library differs from typical threading packages in the following ways: +* oneTBB enables you to specify logical parallelism instead of threads. +* oneTBB targets threading for performance. +* oneTBB is compatible with other threading packages. +* oneTBB emphasizes scalable, data parallel programming. +* oneTBB relies on generic programming. + + +Refer to oneTBB [examples](examples) and [samples](https://github.com/oneapi-src/oneAPI-samples/tree/master/Libraries/oneTBB) to see how you can use the library. + +oneTBB is a part of the [UXL Foundation](http://www.uxlfoundation.org) and is an implementation of [oneAPI specification](https://oneapi.io). + +> **_NOTE:_** Threading Building Blocks (TBB) is now called oneAPI Threading Building Blocks (oneTBB) to highlight that the tool is a part of the oneAPI ecosystem. + +## Release Information + +See [Release Notes](RELEASE_NOTES.md) and [System Requirements](SYSTEM_REQUIREMENTS.md). + +## Documentation +* [oneTBB Specification](https://spec.oneapi.com/versions/latest/elements/oneTBB/source/nested-index.html) +* [oneTBB Developer Guide and Reference](https://oneapi-src.github.io/oneTBB) +* [Migrating from TBB to oneTBB](https://oneapi-src.github.io/oneTBB/main/tbb_userguide/Migration_Guide.html) +* [README for the CMake build system](cmake/README.md) +* [oneTBB Testing Approach](https://oneapi-src.github.io/oneTBB/main/intro/testing_approach.html) +* [Basic support for the Bazel build system](Bazel.md) +* [oneTBB Discussions](https://github.com/oneapi-src/oneTBB/discussions) +* [WASM Support](WASM_Support.md) + +## Installation +See [Installation from Sources](INSTALL.md) to learn how to install oneTBB. + +## Governance + +The oneTBB project is governed by the UXL Foundation. +You can get involved in this project in following ways: +* Join the [Open Source and Specification Working Group](https://github.com/uxlfoundation/foundation/tree/main?tab=readme-ov-file#working-groups) meetings. +* Join the mailing lists for the [UXL Foundation](https://lists.uxlfoundation.org/g/main/subgroups) to receive meetings schedule and latest updates. +* Contribute to oneTBB project or oneTBB specification. Read [CONTRIBUTING](./CONTRIBUTING.md) for more information. + +## Support +See our [documentation](./SUPPORT.md) to learn how to request help. + +## How to Contribute +We welcome community contributions, so check our [Contributing Guidelines](CONTRIBUTING.md) +to learn more. + +Use GitHub Issues for feature requests, bug reports, and minor inquiries. For broader questions and development-related discussions, use GitHub Discussions. + +## License +oneAPI Threading Building Blocks is licensed under [Apache License, Version 2.0](LICENSE.txt). +By its terms, contributions submitted to the project are also done under that license. + +## Engineering team contacts +* [Email us.](mailto:inteltbbdevelopers@intel.com) + +------------------------------------------------------------------------ +\* All names and brands may be claimed as the property of others. diff --git a/src/tbb/RELEASE_NOTES.md b/src/tbb/RELEASE_NOTES.md new file mode 100644 index 000000000..c9b8e9713 --- /dev/null +++ b/src/tbb/RELEASE_NOTES.md @@ -0,0 +1,42 @@ + + +# Release Notes +This document contains changes of oneTBB compared to the last release. + +## Table of Contents +- [Known Limitations](#known-limitations) +- [Fixed Issues](#fixed-issues) + +## :rotating_light: Known Limitations +- The ``oneapi::tbb::info`` namespace interfaces might unexpectedly change the process affinity mask on Windows* OS systems (see https://github.com/open-mpi/hwloc/issues/366 for details) when using hwloc version lower than 2.5. +- Using a hwloc version other than 1.11, 2.0, or 2.5 may cause an undefined behavior on Windows OS. See https://github.com/open-mpi/hwloc/issues/477 for details. +- The NUMA topology may be detected incorrectly on Windows* OS machines where the number of NUMA node threads exceeds the size of 1 processor group. +- On Windows OS on ARM64*, when compiling an application using oneTBB with the Microsoft* Compiler, the compiler issues a warning C4324 that a structure was padded due to the alignment specifier. Consider suppressing the warning by specifying /wd4324 to the compiler command line. +- C++ exception handling mechanism on Windows* OS on ARM64* might corrupt memory if an exception is thrown from any oneTBB parallel algorithm (see Windows* OS on ARM64* compiler issue: https://developercommunity.visualstudio.com/t/ARM64-incorrect-stack-unwinding-for-alig/1544293. +- When CPU resource coordination is enabled, tasks from a lower-priority ``task_arena`` might be executed before tasks from a higher-priority ``task_arena``. + +> **_NOTE:_** To see known limitations that impact all versions of oneTBB, refer to [oneTBB Documentation](https://oneapi-src.github.io/oneTBB/main/intro/limitations.html). + + +## :hammer: Fixed Issues +- Fixed ``parallel_for_each`` algorithm behavior for iterators defining ``iterator_concept`` trait instead of ``iterator_category``. +- Fixed the redefinition issue for ``std::min`` and ``std::max`` on Windows* OS ([GitHub* #832](https://github.com/oneapi-src/oneTBB/issues/832)). +- Fixed the incorrect binary search order in ``TBBConfig.cmake``. +- Enabled the oneTBB library search using the pkg-config tool in Conda packages. + +## :octocat: Open-source Contributions Integrated +- Fixed the compiler warning for missing virtual destructor. Contributed by Elias Engelbert Plank (https://github.com/oneapi-src/oneTBB/pull/1215). diff --git a/src/tbb/SECURITY.md b/src/tbb/SECURITY.md new file mode 100644 index 000000000..4926041fc --- /dev/null +++ b/src/tbb/SECURITY.md @@ -0,0 +1,66 @@ +# Security Policy +As an open-source project, we understand the importance of and responsibility +for security. This Security Policy outlines our guidelines and procedures to +ensure the highest level of security and trust for oneTBB users. + +## Supported Versions +Security vulnerabilities are fixed in the [latest version][1] +and delivered as a patch release. We don't guarantee security fixes to be +back-ported to older oneTBB versions. + +## Report a Vulnerability +We are very grateful to the security researchers and users that report back +security vulnerabilities. We investigate every report thoroughly. +We strongly encourage you to report security vulnerabilities to us privately, +before disclosing them on public forums or opening a public GitHub* issue. + +Report a vulnerability to us in one of two ways: +* Open a draft **[GitHub* Security Advisory][2]** +* Send an e-mail to: **security@uxlfoundation.org**. +Along with the report, provide the following info: + * A descriptive title. + * Your name and affiliation (if any). + * A description of the technical details of the vulnerabilities. + * A minimal example of the vulnerability so we can reproduce your findings. + * An explanation of who can exploit this vulnerability, and what they gain + doing so. + * Whether this vulnerability is public or known to third parties. If it is, + provide details. + +### When Should I Report a Vulnerability? +* You think you discovered a potential security vulnerability in oneTBB. +* You are unsure how the potential vulnerability affects oneTBB. +* You think you discovered a vulnerability in another project or 3rd party +component on which oneTBB depends. If the issue is not fixed in the 3rd party +component, try to report directly there first. + +### When Should I NOT Report a Vulnerability? +* You got an automated scan hit and are unable to provide details. +* You need help using oneTBB for security. +* You need help applying security-related updates. +* Your issue is not security-related. + +## Security Reports Review Process +We aim to respond quickly to your inquiry and coordinate a fix and +disclosure with you. All confirmed security vulnerabilities will be addressed +according to severity level and impact on oneTBB. Normally, security issues +are fixed in the next planned release. + +## Disclosure Policy +We will publish security advisories using the +[**GitHub Security Advisories feature**][3] +to keep our community well-informed, and will credit you for your findings +unless you prefer to stay anonymous. We request that you refrain from +exploiting the vulnerability or making it public before the official disclosure. + +We will disclose the vulnerabilities and bugs as soon as possible once +mitigation is implemented and available. + +## Feedback on This Policy +If you have any suggestions on how this Policy could be improved, submit +an issue or a pull request to this repository. **Do not** report +potential vulnerabilities or security flaws via a pull request. + +[1]: https://github.com/oneapi-src/oneTBB/releases/latest +[2]: https://github.com/oneapi-src/oneTBB/security/advisories/new +[3]: https://github.com/oneapi-src/oneTBB/security/advisories diff --git a/src/tbb/SUPPORT.md b/src/tbb/SUPPORT.md new file mode 100644 index 000000000..47bb60a53 --- /dev/null +++ b/src/tbb/SUPPORT.md @@ -0,0 +1,35 @@ + + +# oneTBB Support + +We are committed to providing support and assistance to help you make the most out of oneTBB. +Use the following methods if you face any challenges. + +## Issues + +If you have a problem, check out the [GitHub Issues](https://github.com/oneapi-src/oneTBB/issues) to see if the issue you want to address is already reported. +You may find users that have encountered the same bug or have similar ideas for changes or updates. + +You can use issues to report a problem, make a feature request, or add comments on an existing issue. + +## Discussions + +Visit the [GitHub Discussions](https://github.com/oneapi-src/oneTBB/discussions) to engage with the community, ask questions, or help others. + +## Email + +Reach out to us privately via [email](mailto:inteltbbdevelopers@intel.com). \ No newline at end of file diff --git a/src/tbb/SYSTEM_REQUIREMENTS.md b/src/tbb/SYSTEM_REQUIREMENTS.md new file mode 100644 index 000000000..7f9d81616 --- /dev/null +++ b/src/tbb/SYSTEM_REQUIREMENTS.md @@ -0,0 +1,86 @@ + + +# System Requirements +This document provides details about hardware, operating system, and software prerequisites for the oneAPI Threading Building Blocks (oneTBB). + +## Table of Contents +- [Supported Hardware](#supported-hardware) +- [Software](#software) + - [Supported Operating Systems](#supported-operating-systems) + - [Community-Supported Platforms](#community-supported-platforms) + - [Supported Compilers](#supported-compilers) +- [Limitations](#limitations) + + +## Supported Hardware +- Intel(R) Celeron(R) processor family +- Intel(R) Core* processor family +- Intel(R) Xeon(R) processor family +- Intel(R) Atom* processor family +- Non-Intel(R) processors compatible with the processors listed above + + +## Software + +### Supported Operating Systems +- Systems with Microsoft* Windows* operating systems: + - Microsoft* Windows* 10 + - Microsoft* Windows* 11 + - Microsoft* Windows* Server 2019 + - Microsoft* Windows* Server 2022 +- Systems with Linux* operating systems: + - Oracle Linux* 8 + - Amazon* Linux 2, 2022 + - Debian* 9, 10, 11 + - Fedora* 36, 37, 38 + - Rocky* Linux* 8, 9 + - Red Hat* Enterprise Linux* 8, 9 + - SuSE* Linux* Enterprise Server 15 + - Ubuntu* 20.04, 22.04 +- Systems with macOS* operating systems: + - macOS* 12.x, 13.x +- Systems with Android* operating systems: + - Android* 9 + +### Community-Supported Platforms +- MinGW* +- FreeBSD* +- Microsoft* Windows* on ARM*/ARM64* +- macOS* on ARM64* + +### Supported Compilers +- Intel* oneAPI DPC++/C++ Compiler +- Intel® C++ Compiler Classic 2021.1 - 2021.9 +- Microsoft* Visual C++ 14.2 (Microsoft* Visual Studio* 2019, Windows* OS only) +- Microsoft* Visual C++ 14.3 (Microsoft* Visual Studio* 2022, Windows* OS only) +- For each supported Linux* operating system, the standard gcc version provided with that operating system is supported: + - GNU Compilers (gcc) 8.x – 12.x + - GNU C Library (glibc) version 2.28 – 2.36 + - Clang* 6.0.0 - 13.0.0 + +## Limitations +There are some cases where we cannot provide support for your platforms. It includes: + +1. The platform is out of official support (met end of life). When you use an unsupported platform, you can face a security risk that can be difficult to resolve. +2. We do not have the infrastructure to test a platform. Therefore we cannot guarantee that oneTBB works correctly on that platform. +3. Changes affect more code than just platform-specific macros. +4. The platform is incompatible with oneTBB. Some platforms may have limitations that prevent oneTBB from working correctly. We cannot provide support in these cases as the issue is beyond our control. +5. The platform is modified or customized. If you made significant updates to your platform, it might be hard for us to find the root cause of the issue. Therefore, we may not be able to provide support as the modification could affect the oneTBB functionality. + + +We understand that these limitations can be frustrating. Thus, we suggest creating a branch specifically for the unsupported platform, allowing other users to contribute to or use your implementation. + diff --git a/src/tbb/WASM_Support.md b/src/tbb/WASM_Support.md new file mode 100644 index 000000000..6306620d7 --- /dev/null +++ b/src/tbb/WASM_Support.md @@ -0,0 +1,81 @@ + + +# WASM Support + +oneTBB extends its capabilities by offering robust support for ``WASM`` (see ``Limitation`` sections). + +``WASM`` stands for WebAssembly, a low-level binary format for executing code in web browsers. +It is designed to be a portable target for compilers and efficient to parse and execute. + +Using oneTBB with WASM, you can take full advantage of parallelism and concurrency while working on web-based applications, interactive websites, and a variety of other WASM-compatible platforms. + +oneTBB offers WASM support through the integration with [Emscripten*](https://emscripten.org/docs/introducing_emscripten/index.html), a powerful toolchain for compiling C and C++ code into WASM-compatible runtimes. + +## Build + +**Prerequisites:** Download and install Emscripten*. See the [instructions](https://emscripten.org/docs/getting_started/downloads.html). + +To build the system, run: + +``` +mkdir build && cd build +emcmake cmake .. -DCMAKE_CXX_COMPILER=em++ -DCMAKE_C_COMPILER=emcc -DTBB_STRICT=OFF -DCMAKE_CXX_FLAGS=-Wno-unused-command-line-argument -DTBB_DISABLE_HWLOC_AUTOMATIC_SEARCH=ON -DBUILD_SHARED_LIBS=ON -DTBB_EXAMPLES=ON -DTBB_TEST=ON +``` +To compile oneTBB without ``pthreads``, set the flag ``-DEMSCRIPTEN_WITHOUT_PTHREAD=true`` in the command above. By default, oneTBB uses the ``pthreads``. +``` +cmake --build . +cmake --install . +``` +Where: + +* ``emcmake`` - a tool that sets up the environment for Emscripten*. +* ``-DCMAKE_CXX_COMPILER=em++`` - specifies the C++ compiler as Emscripten* C++ compiler. +* ``-DCMAKE_C_COMPILER=emcc`` - specifies the C compiler as Emscripten* C compiler. + + +> **_NOTE:_** See [CMake documentation](https://github.com/oneapi-src/oneTBB/blob/master/cmake/README.md) to learn about other options. + + +## Run Test + +To run tests, use: + +``` +ctest +``` + +# Limitations + +You can successfully build your application with oneTBB using WASM, but you may not achieve optimal performance immediately. This is due to the limitation with nested Web Workers: a Web Worker cannot schedule another worker without help from a browser thread. This can lead to unexpected performance outcomes, such as the application running in serial. +Find more information in the [issue](https://github.com/emscripten-core/emscripten/discussions/21963) in the Emscripten repository. +To workaround this issue, try one of the following ways: +1. **Recommended Solution: Use the ``-sPROXY_TO_PTHREAD`` Flag**. +This flag splits the initial thread into a browser thread and a main thread (proxied by a Web Worker), effectively resolving the issue as the browser thread is always present in the event loop and can participate in Web Workers scheduling. Refer to the [Emscripten documentation](https://emscripten.org/docs/porting/pthreads.html) for more details about ``-sPROXY_TO_PTHREAD`` since using this flag may require refactoring the code. +2. **Alternative Solution: Warm Up the oneTBB Thread Pool** +Initialize the oneTBB thread pool before making the first call to oneTBB. This approach forces the browser thread to participate in Web Workers scheduling. +```cpp + int num_threads = tbb::this_task_arena::max_concurrency(); + std::atomic barrier{num_threads}; + tbb::parallel_for(0, num_threads, [&barrier] (int) { + barrier--; + while (barrier > 0) { + // Send browser thread to event loop + std::this_thread::yield(); + } + }, tbb::static_partitioner{}); +``` +> **_NOTE:_** Be aware that it might cause delays on the browser side. diff --git a/src/tbb/build/.gitignore b/src/tbb/build/.gitignore deleted file mode 100644 index 53dc09fc1..000000000 --- a/src/tbb/build/.gitignore +++ /dev/null @@ -1 +0,0 @@ -lib_*/ diff --git a/src/tbb/build/AIX.gcc.inc b/src/tbb/build/AIX.gcc.inc deleted file mode 100644 index 18e6e6ad9..000000000 --- a/src/tbb/build/AIX.gcc.inc +++ /dev/null @@ -1,75 +0,0 @@ -# Copyright (c) 2005-2017 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -# -# - -COMPILE_ONLY = -c -MMD -PREPROC_ONLY = -E -x c++ -INCLUDE_KEY = -I -DEFINE_KEY = -D -OUTPUT_KEY = -o # -OUTPUTOBJ_KEY = -o # -PIC_KEY = -fPIC -WARNING_AS_ERROR_KEY = -Werror -WARNING_KEY = -Wall -DYLIB_KEY = -shared -LIBDL = -ldl - -CPLUS = g++ -CONLY = gcc -LIB_LINK_FLAGS = -shared -LIBS = -lpthread -ldl -C_FLAGS = $(CPLUS_FLAGS) -x c - -ifeq ($(cfg), release) - CPLUS_FLAGS = -O2 -DUSE_PTHREAD -pthread -endif -ifeq ($(cfg), debug) - CPLUS_FLAGS = -DTBB_USE_DEBUG -g -O0 -DUSE_PTHREAD -pthread -endif - -ASM= -ASM_FLAGS= - -TBB_ASM.OBJ= - -ifeq (powerpc,$(arch)) - CPLUS_FLAGS += -maix64 -Wl,-G - LIB_LINK_FLAGS += -maix64 -Wl,-b64 -Wl,-brtl -Wl,-G -endif - -#------------------------------------------------------------------------------ -# Setting assembler data. -#------------------------------------------------------------------------------ - -ASSEMBLY_SOURCE=ibm_aix51 -ifeq (powerpc,$(arch)) - TBB_ASM.OBJ = atomic_support.o -endif - -#------------------------------------------------------------------------------ -# End of setting assembler data. -#------------------------------------------------------------------------------ - -#------------------------------------------------------------------------------ -# Setting tbbmalloc data. -#------------------------------------------------------------------------------ - -M_CPLUS_FLAGS = $(CPLUS_FLAGS) -fno-rtti -fno-exceptions - -#------------------------------------------------------------------------------ -# End of setting tbbmalloc data. -#------------------------------------------------------------------------------ diff --git a/src/tbb/build/AIX.inc b/src/tbb/build/AIX.inc deleted file mode 100644 index abe12d52a..000000000 --- a/src/tbb/build/AIX.inc +++ /dev/null @@ -1,66 +0,0 @@ -# Copyright (c) 2005-2017 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -# -# - -ifndef arch - arch:=$(shell uname -p) - export arch -endif - -ifndef runtime - gcc_version:=$(shell gcc -dumpversion) - os_version:=$(shell uname -r) - os_kernel_version:=$(shell uname -r | sed -e 's/-.*$$//') - export runtime:=cc$(gcc_version)_kernel$(os_kernel_version) -endif - -native_compiler := gcc -export compiler ?= gcc -debugger ?= gdb - -CMD=$(SHELL) -c -CWD=$(shell pwd) -RM?=rm -f -RD?=rmdir -MD?=mkdir -p -NUL= /dev/null -SLASH=/ -MAKE_VERSIONS=sh $(tbb_root)/build/version_info_aix.sh $(VERSION_FLAGS) >version_string.ver -MAKE_TBBVARS=sh $(tbb_root)/build/generate_tbbvars.sh - -ifdef LIBPATH - export LIBPATH := .:$(LIBPATH) -else - export LIBPATH := . -endif - -####### Build settings ######################################################## - -OBJ = o -DLL = so - -TBB.LST = -TBB.DEF = -TBB.DLL = libtbb$(CPF_SUFFIX)$(DEBUG_SUFFIX).$(DLL) -TBB.LIB = $(TBB.DLL) -LINK_TBB.LIB = $(TBB.LIB) - -MALLOC.DLL = libtbbmalloc$(DEBUG_SUFFIX).$(DLL) -MALLOC.LIB = $(MALLOC.DLL) -LINK_MALLOC.LIB = $(MALLOC.LIB) - -TEST_LAUNCHER=sh $(tbb_root)/build/test_launcher.sh $(largs) diff --git a/src/tbb/build/FreeBSD.clang.inc b/src/tbb/build/FreeBSD.clang.inc deleted file mode 100644 index 3579603db..000000000 --- a/src/tbb/build/FreeBSD.clang.inc +++ /dev/null @@ -1,110 +0,0 @@ -# Copyright (c) 2005-2017 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -# -# - -COMPILE_ONLY = -c -MMD -PREPROC_ONLY = -E -x c++ -INCLUDE_KEY = -I -DEFINE_KEY = -D -OUTPUT_KEY = -o # -OUTPUTOBJ_KEY = -o # -PIC_KEY = -fPIC -WARNING_AS_ERROR_KEY = -Werror -WARNING_KEY = -Wall -TEST_WARNING_KEY = -Wextra -Wshadow -Wcast-qual -Woverloaded-virtual -Wnon-virtual-dtor -WARNING_SUPPRESS = -Wno-parentheses -Wno-non-virtual-dtor -Wno-dangling-else -DYLIB_KEY = -shared -EXPORT_KEY = -Wl,--version-script, -LIBDL = - -CPLUS = clang++ -CONLY = clang -LIB_LINK_FLAGS = $(DYLIB_KEY) -Wl,-soname=$(BUILDING_LIBRARY) -LIBS += -lpthread -lrt -LINK_FLAGS = -Wl,-rpath-link=. -Wl,-rpath=. -rdynamic -C_FLAGS = $(CPLUS_FLAGS) - -ifeq ($(cfg), release) - CPLUS_FLAGS = $(ITT_NOTIFY) -g -O2 -DUSE_PTHREAD -endif -ifeq ($(cfg), debug) - CPLUS_FLAGS = -DTBB_USE_DEBUG $(ITT_NOTIFY) -g -O0 -DUSE_PTHREAD -endif - -ifneq (,$(stdlib)) - CPLUS_FLAGS += -stdlib=$(stdlib) - LIB_LINK_FLAGS += -stdlib=$(stdlib) -endif - -TBB_ASM.OBJ= -MALLOC_ASM.OBJ= - -ifeq (intel64,$(arch)) - ITT_NOTIFY = -DDO_ITT_NOTIFY - CPLUS_FLAGS += -m64 - LIB_LINK_FLAGS += -m64 -endif - -ifeq (ia32,$(arch)) - ITT_NOTIFY = -DDO_ITT_NOTIFY - CPLUS_FLAGS += -m32 -march=pentium4 - LIB_LINK_FLAGS += -m32 -endif - -ifeq (ppc64,$(arch)) - CPLUS_FLAGS += -m64 - LIB_LINK_FLAGS += -m64 -endif - -ifeq (ppc32,$(arch)) - CPLUS_FLAGS += -m32 - LIB_LINK_FLAGS += -m32 -endif - -ifeq (bg,$(arch)) - CPLUS = bgclang++ - CONLY = bgclang -endif - -#------------------------------------------------------------------------------ -# Setting assembler data. -#------------------------------------------------------------------------------ -ASM = as -ifeq (intel64,$(arch)) - ASM_FLAGS += --64 -endif -ifeq (ia32,$(arch)) - ASM_FLAGS += --32 -endif -ifeq ($(cfg),debug) - ASM_FLAGS += -g -endif - -ASSEMBLY_SOURCE=$(arch)-gas -#------------------------------------------------------------------------------ -# End of setting assembler data. -#------------------------------------------------------------------------------ - -#------------------------------------------------------------------------------ -# Setting tbbmalloc data. -#------------------------------------------------------------------------------ - -M_CPLUS_FLAGS = $(CPLUS_FLAGS) -fno-rtti -fno-exceptions - -#------------------------------------------------------------------------------ -# End of setting tbbmalloc data. -#------------------------------------------------------------------------------ diff --git a/src/tbb/build/FreeBSD.gcc.inc b/src/tbb/build/FreeBSD.gcc.inc deleted file mode 100644 index 794cb7c41..000000000 --- a/src/tbb/build/FreeBSD.gcc.inc +++ /dev/null @@ -1,93 +0,0 @@ -# Copyright (c) 2005-2017 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -# -# - -COMPILE_ONLY = -c -MMD -PREPROC_ONLY = -E -x c++ -INCLUDE_KEY = -I -DEFINE_KEY = -D -OUTPUT_KEY = -o # -OUTPUTOBJ_KEY = -o # -PIC_KEY = -fPIC -WARNING_AS_ERROR_KEY = -Werror -WARNING_KEY = -Wall -DYLIB_KEY = -shared -WARNING_SUPPRESS = -Wno-parentheses - -CPLUS = g++ -CONLY = gcc -LIB_LINK_FLAGS = -shared -LIBS = -lpthread -C_FLAGS = $(CPLUS_FLAGS) - -# gcc 6.0 and later have -flifetime-dse option that controls -# elimination of stores done outside the object lifetime -ifneq (,$(shell gcc -dumpversion | egrep "^([6-9])")) - # keep pre-contruction stores for zero initialization - DSE_KEY = -flifetime-dse=1 -endif - -ifeq ($(cfg), release) - CPLUS_FLAGS = -g -O2 -DUSE_PTHREAD -endif -ifeq ($(cfg), debug) - CPLUS_FLAGS = -DTBB_USE_DEBUG -g -O0 -DUSE_PTHREAD -endif - -ASM= -ASM_FLAGS= - -TBB_ASM.OBJ= -MALLOC_ASM.OBJ= - -ifeq (ia64,$(arch)) -# Position-independent code (PIC) is a must on IA-64 architecture, even for regular (not shared) executables - CPLUS_FLAGS += $(PIC_KEY) -endif - -ifeq (intel64,$(arch)) - CPLUS_FLAGS += -m64 - LIB_LINK_FLAGS += -m64 -endif - -ifeq (ia32,$(arch)) - CPLUS_FLAGS += -m32 - LIB_LINK_FLAGS += -m32 -endif - -#------------------------------------------------------------------------------ -# Setting assembler data. -#------------------------------------------------------------------------------ -ASSEMBLY_SOURCE=$(arch)-gas -ifeq (ia64,$(arch)) - ASM=as - TBB_ASM.OBJ = atomic_support.o lock_byte.o log2.o pause.o - MALLOC_ASM.OBJ = atomic_support.o lock_byte.o pause.o -endif -#------------------------------------------------------------------------------ -# End of setting assembler data. -#------------------------------------------------------------------------------ - -#------------------------------------------------------------------------------ -# Setting tbbmalloc data. -#------------------------------------------------------------------------------ - -M_CPLUS_FLAGS = $(CPLUS_FLAGS) -fno-rtti -fno-exceptions - -#------------------------------------------------------------------------------ -# End of setting tbbmalloc data. -#------------------------------------------------------------------------------ diff --git a/src/tbb/build/FreeBSD.inc b/src/tbb/build/FreeBSD.inc deleted file mode 100644 index 3bd6a0640..000000000 --- a/src/tbb/build/FreeBSD.inc +++ /dev/null @@ -1,74 +0,0 @@ -# Copyright (c) 2005-2017 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -# -# - -ifndef arch - ifeq ($(shell uname -m),i386) - export arch:=ia32 - endif - ifeq ($(shell uname -m),ia64) - export arch:=ia64 - endif - ifeq ($(shell uname -m),amd64) - export arch:=intel64 - endif -endif - -ifndef runtime - clang_version:=$(shell clang -v 2>&1 | grep version | sed "s/.*version \([0-9]*\.[0-9]*\).*/\1/") - os_version:=$(shell uname -r) - os_kernel_version:=$(shell uname -r | sed -e 's/-.*$$//') - export runtime:=cc$(clang_version)_kernel$(os_kernel_version) -endif - -native_compiler := clang -export compiler ?= clang -debugger ?= gdb - -CMD=$(SHELL) -c -CWD=$(shell pwd) -RM?=rm -f -RD?=rmdir -MD?=mkdir -p -NUL= /dev/null -SLASH=/ -MAKE_VERSIONS=sh $(tbb_root)/build/version_info_linux.sh $(VERSION_FLAGS) >version_string.ver -MAKE_TBBVARS=sh $(tbb_root)/build/generate_tbbvars.sh - -ifdef LD_LIBRARY_PATH - export LD_LIBRARY_PATH := .:$(LD_LIBRARY_PATH) -else - export LD_LIBRARY_PATH := . -endif - -####### Build settings ######################################################## - -OBJ = o -DLL = so -LIBEXT=so - -TBB.LST = -TBB.DEF = -TBB.DLL = libtbb$(CPF_SUFFIX)$(DEBUG_SUFFIX).$(DLL) -TBB.LIB = $(TBB.DLL) -LINK_TBB.LIB = $(TBB.LIB) - -MALLOC.DLL = libtbbmalloc$(DEBUG_SUFFIX).$(DLL) -MALLOC.LIB = $(MALLOC.DLL) -LINK_MALLOC.LIB = $(MALLOC.LIB) - -TEST_LAUNCHER=sh $(tbb_root)/build/test_launcher.sh $(largs) diff --git a/src/tbb/build/Makefile.rml b/src/tbb/build/Makefile.rml deleted file mode 100644 index 241d4bfbb..000000000 --- a/src/tbb/build/Makefile.rml +++ /dev/null @@ -1,166 +0,0 @@ -# Copyright (c) 2005-2017 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -# -# - -# TODO: investigate why version_string.ver is not complete when $(RML_SERVER.OBJ) is being compiled. -.NOTPARALLEL: - -tbb_root ?= $(TBBROOT) -BUILDING_PHASE=1 -TEST_RESOURCE = $(RML.RES) -include $(tbb_root)/build/common.inc -DEBUG_SUFFIX=$(findstring _debug,_$(cfg)) - -ifeq (android,$(target)) -$(error "RML is not supported on Android") -endif - -# default target -default_rml: rml rml_test - -RML_ROOT ?= $(tbb_root)/src/rml -RML_SERVER_ROOT = $(RML_ROOT)/server - -VPATH = $(tbb_root)/src/tbb $(tbb_root)/src/tbb/$(ASSEMBLY_SOURCE) -VPATH += $(RML_ROOT)/server $(RML_ROOT)/client $(RML_ROOT)/test $(tbb_root)/src/test - -include $(tbb_root)/build/common_rules.inc - -#-------------------------------------------------------------------------- -# Define rules for making the RML server shared library and client objects. -#-------------------------------------------------------------------------- - -# Object files that make up RML server -RML_SERVER.OBJ = rml_server.$(OBJ) - -# Object files that RML clients need -RML_TBB_CLIENT.OBJ ?= rml_tbb.$(OBJ) dynamic_link_rml.$(OBJ) -RML_OMP_CLIENT.OBJ ?= rml_omp.$(OBJ) omp_dynamic_link.$(OBJ) - -RML.OBJ = $(RML_SERVER.OBJ) $(RML_TBB_CLIENT.OBJ) $(RML_OMP_CLIENT.OBJ) -ifeq (windows,$(tbb_os)) -RML_ASM.OBJ = $(if $(findstring intel64,$(arch)),$(TBB_ASM.OBJ)) -endif -ifeq (linux,$(tbb_os)) -RML_ASM.OBJ = $(if $(findstring ia64,$(arch)),$(TBB_ASM.OBJ)) -endif - -RML_TBB_DEP= cache_aligned_allocator_rml.$(OBJ) dynamic_link_rml.$(OBJ) concurrent_vector_rml.$(OBJ) semaphore_rml.$(OBJ) tbb_misc_rml.$(OBJ) tbb_misc_ex_rml.$(OBJ) -TBB_DEP_NON_RML_TEST?= cache_aligned_allocator_rml.$(OBJ) dynamic_link_rml.$(OBJ) $(RML_ASM.OBJ) tbb_misc_rml.$(OBJ) tbb_misc_ex_rml.$(OBJ) -ifeq ($(cfg),debug) -RML_TBB_DEP+= spin_mutex_rml.$(OBJ) -TBB_DEP_RML_TEST?= $(RML_ASM.OBJ) tbb_misc_rml.$(OBJ) -else -TBB_DEP_RML_TEST?= $(RML_ASM.OBJ) -endif -LIBS += $(LIBDL) - -INCLUDES += $(INCLUDE_KEY)$(RML_ROOT)/include $(INCLUDE_KEY). -T_INCLUDES = $(INCLUDES) $(INCLUDE_KEY)$(tbb_root)/src/test $(INCLUDE_KEY)$(RML_SERVER_ROOT) - -ifeq ($(rml_wcrm),1) -CPLUS_FLAGS+=/DRML_USE_WCRM -endif - -# Suppress superfluous warnings for RML compilation -R_CPLUS_FLAGS = $(subst DO_ITT_NOTIFY,DO_ITT_NOTIFY=0,$(CPLUS_FLAGS)) $(WARNING_SUPPRESS) \ - $(DEFINE_KEY)TBB_USE_THREADING_TOOLS=0 $(DEFINE_KEY)__TBB_RML_STATIC=1 $(DEFINE_KEY)__TBB_NO_IMPLICIT_LINKAGE=1 - -%.$(OBJ): %.cpp - $(CPLUS) $(COMPILE_ONLY) $(R_CPLUS_FLAGS) $(PIC_KEY) $(DSE_KEY) $(INCLUDES) $< - -ifeq (linux,$(tbb_os)) -omp_dynamic_link.$(OBJ): CPLUS_FLAGS+=-fno-exceptions -endif - -tbb_misc_rml.$(OBJ) $(RML_SERVER.OBJ): version_string.ver - -RML_TEST.OBJ = test_job_automaton.$(OBJ) test_thread_monitor.$(OBJ) test_rml_tbb.$(OBJ) test_rml_omp.$(OBJ) test_rml_mixed.$(OBJ) - -$(RML_TBB_DEP): %_rml.$(OBJ): %.cpp - $(CPLUS) $(COMPILE_ONLY) $(OUTPUTOBJ_KEY)$@ $(R_CPLUS_FLAGS) $(PIC_KEY) $(DSE_KEY) $(INCLUDES) $< - -$(RML_TEST.OBJ): %.$(OBJ): %.cpp - $(CPLUS) $(COMPILE_ONLY) $(R_CPLUS_FLAGS) $(PIC_KEY) $(T_INCLUDES) $< - -ifneq (,$(RML.DEF)) -rml.def: $(RML.DEF) - $(CPLUS) $(PREPROC_ONLY) $< $(CPLUS_FLAGS) $(INCLUDES) > $@ - -LIB_LINK_FLAGS += $(EXPORT_KEY)rml.def -$(RML.DLL): rml.def -endif - -$(RML.DLL): CPLUS_FLAGS += $(SDL_FLAGS) -$(RML.DLL): BUILDING_LIBRARY = $(RML.DLL) -$(RML.DLL): $(RML_TBB_DEP) $(RML_SERVER.OBJ) $(RML.RES) $(RML_NO_VERSION.DLL) $(RML_ASM.OBJ) - $(LIB_LINK_CMD) $(LIB_OUTPUT_KEY)$(RML.DLL) $(RML_SERVER.OBJ) $(RML_TBB_DEP) $(RML_ASM.OBJ) $(RML.RES) $(LIB_LINK_LIBS) $(LIB_LINK_FLAGS) - -ifneq (,$(RML_NO_VERSION.DLL)) -$(RML_NO_VERSION.DLL): - echo "INPUT ($(RML.DLL))" > $(RML_NO_VERSION.DLL) -endif - -rml: $(RML.DLL) $(RML_TBB_CLIENT.OBJ) $(RML_OMP_CLIENT.OBJ) - -#------------------------------------------------------ -# End of rules for making the RML server shared library -#------------------------------------------------------ - -#------------------------------------------------------ -# Define rules for making the RML unit tests -#------------------------------------------------------ - -add_debug=$(basename $(1))_debug$(suffix $(1)) -cross_suffix=$(if $(crosstest),$(if $(DEBUG_SUFFIX),$(subst _debug,,$(1)),$(call add_debug,$(1))),$(1)) - -RML_TESTS = test_job_automaton.$(TEST_EXT) test_thread_monitor.$(TEST_EXT) -RML_CUSTOM_TESTS = test_rml_tbb.$(TEST_EXT) test_rml_omp.$(TEST_EXT) test_rml_mixed.$(TEST_EXT) test_rml_omp_c_linkage.$(TEST_EXT) - -test_rml_tbb.$(TEST_EXT): test_rml_tbb.$(OBJ) $(RML_TBB_CLIENT.OBJ) $(TBB_DEP_RML_TEST) - $(CPLUS) $(OUTPUT_KEY)$@ $(CPLUS_FLAGS) test_rml_tbb.$(OBJ) $(RML_TBB_CLIENT.OBJ) $(TBB_DEP_RML_TEST) $(LIBS) $(LINK_FLAGS) - -test_rml_omp.$(TEST_EXT): test_rml_omp.$(OBJ) $(RML_OMP_CLIENT.OBJ) $(TBB_DEP_NON_RML_TEST) - $(CPLUS) $(OUTPUT_KEY)$@ $(CPLUS_FLAGS) test_rml_omp.$(OBJ) $(RML_OMP_CLIENT.OBJ) $(TBB_DEP_NON_RML_TEST) $(LIBS) $(LINK_FLAGS) - -test_rml_mixed.$(TEST_EXT): test_rml_mixed.$(OBJ) $(RML_TBB_CLIENT.OBJ) $(RML_OMP_CLIENT.OBJ) $(TBB_DEP_RML_TEST) - $(CPLUS) $(OUTPUT_KEY)$@ $(CPLUS_FLAGS) test_rml_mixed.$(OBJ) $(RML_TBB_CLIENT.OBJ) $(RML_OMP_CLIENT.OBJ) $(TBB_DEP_RML_TEST) $(LIBS) $(LINK_FLAGS) - -rml_omp_stub.$(OBJ): rml_omp_stub.cpp - $(CPLUS) $(COMPILE_ONLY) $(M_CPLUS_FLAGS) $(WARNING_SUPPRESS) $(T_INCLUDES) $(PIC_KEY) $< - -test_rml_omp_c_linkage.$(TEST_EXT): test_rml_omp_c_linkage.$(OBJ) rml_omp_stub.$(OBJ) omp_dynamic_link.$(OBJ) - $(CONLY) $(C_FLAGS) $(OUTPUT_KEY)$@ test_rml_omp_c_linkage.$(OBJ) rml_omp_stub.$(OBJ) omp_dynamic_link.$(OBJ) $(LIBS) $(LINK_FLAGS) - -$(RML_TESTS): %.$(TEST_EXT): %.$(OBJ) $(TBB_DEP_NON_RML_TEST) - $(CPLUS) $(OUTPUT_KEY)$@ $(CPLUS_FLAGS) $< $(TBB_DEP_NON_RML_TEST) $(LIBS) $(LINK_FLAGS) - -### run_cmd is usually empty -rml_test: $(call cross_suffix,$(RML.DLL)) $(TEST_PREREQUISITE) $(RML_TESTS) $(RML_CUSTOM_TESTS) - $(run_cmd) ./test_job_automaton.$(TEST_EXT) $(args) - $(run_cmd) ./test_thread_monitor.$(TEST_EXT) $(args) - $(run_cmd) ./test_rml_tbb.$(TEST_EXT) $(args) - $(run_cmd) ./test_rml_omp.$(TEST_EXT) $(args) - $(run_cmd) ./test_rml_mixed.$(TEST_EXT) $(args) - $(run_cmd) ./test_rml_omp_c_linkage.$(TEST_EXT) $(args) - -#------------------------------------------------------ -# End of rules for making the TBBMalloc unit tests -#------------------------------------------------------ - -# Include automatically generated dependencies --include *.d diff --git a/src/tbb/build/Makefile.tbb b/src/tbb/build/Makefile.tbb deleted file mode 100644 index b66c309b5..000000000 --- a/src/tbb/build/Makefile.tbb +++ /dev/null @@ -1,118 +0,0 @@ -# Copyright (c) 2005-2017 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -# -# - -#------------------------------------------------------------------------------ -# Define rules for making the TBB shared library. -#------------------------------------------------------------------------------ - -tbb_root ?= "$(TBBROOT)" -BUILDING_PHASE=1 -include $(tbb_root)/build/common.inc -CPLUS_FLAGS += $(SDL_FLAGS) -DEBUG_SUFFIX=$(findstring _debug,_$(cfg)) - -#------------------------------------------------------------ -# Define static pattern rules dealing with .cpp source files -#------------------------------------------------------------ -$(warning CONFIG: cfg=$(cfg) arch=$(arch) compiler=$(compiler) target=$(target) runtime=$(runtime)) - -default_tbb: $(TBB.DLL) -.PHONY: default_tbb tbbvars clean -.PRECIOUS: %.$(OBJ) - -VPATH = $(tbb_root)/src/tbb/$(ASSEMBLY_SOURCE) $(tbb_root)/src/tbb $(tbb_root)/src/old $(tbb_root)/src/rml/client - -CPLUS_FLAGS += $(PIC_KEY) $(DSE_KEY) $(DEFINE_KEY)__TBB_BUILD=1 - -# Object files (that were compiled from C++ code) that gmake up TBB -TBB_CPLUS.OBJ = concurrent_hash_map.$(OBJ) \ - concurrent_queue.$(OBJ) \ - concurrent_vector.$(OBJ) \ - dynamic_link.$(OBJ) \ - itt_notify.$(OBJ) \ - cache_aligned_allocator.$(OBJ) \ - pipeline.$(OBJ) \ - queuing_mutex.$(OBJ) \ - queuing_rw_mutex.$(OBJ) \ - reader_writer_lock.$(OBJ) \ - spin_rw_mutex.$(OBJ) \ - x86_rtm_rw_mutex.$(OBJ) \ - spin_mutex.$(OBJ) \ - critical_section.$(OBJ) \ - mutex.$(OBJ) \ - recursive_mutex.$(OBJ) \ - condition_variable.$(OBJ) \ - tbb_thread.$(OBJ) \ - concurrent_monitor.$(OBJ) \ - semaphore.$(OBJ) \ - private_server.$(OBJ) \ - rml_tbb.$(OBJ) \ - tbb_misc.$(OBJ) \ - tbb_misc_ex.$(OBJ) \ - task.$(OBJ) \ - task_group_context.$(OBJ) \ - governor.$(OBJ) \ - market.$(OBJ) \ - arena.$(OBJ) \ - scheduler.$(OBJ) \ - observer_proxy.$(OBJ) \ - tbb_statistics.$(OBJ) \ - tbb_main.$(OBJ) - -# OLD/Legacy object files for backward binary compatibility -ifeq (,$(findstring $(DEFINE_KEY)TBB_NO_LEGACY,$(CXXFLAGS))) -TBB_CPLUS_OLD.OBJ = \ - concurrent_vector_v2.$(OBJ) \ - concurrent_queue_v2.$(OBJ) \ - spin_rw_mutex_v2.$(OBJ) \ - task_v2.$(OBJ) -endif - -# Object files that gmake up TBB (TBB_ASM.OBJ is platform-specific) -TBB.OBJ = $(TBB_CPLUS.OBJ) $(TBB_CPLUS_OLD.OBJ) $(TBB_ASM.OBJ) - -# Suppress superfluous warnings for TBB compilation -WARNING_KEY += $(WARNING_SUPPRESS) - -include $(tbb_root)/build/common_rules.inc - -ifneq (,$(TBB.DEF)) -tbb.def: $(TBB.DEF) $(TBB.LST) - $(CPLUS) $(PREPROC_ONLY) $< $(CPLUS_FLAGS) $(INCLUDES) > $@ - -LIB_LINK_FLAGS += $(EXPORT_KEY)tbb.def -$(TBB.DLL): tbb.def -endif - -tbbvars.sh: - $(MAKE_TBBVARS) - -$(TBB.DLL): BUILDING_LIBRARY = $(TBB.DLL) -$(TBB.DLL): $(TBB.OBJ) $(TBB.RES) tbbvars.sh $(TBB_NO_VERSION.DLL) - $(LIB_LINK_CMD) $(LIB_OUTPUT_KEY)$(TBB.DLL) $(TBB.OBJ) $(TBB.RES) $(LIB_LINK_LIBS) $(LIB_LINK_FLAGS) - -ifneq (,$(TBB_NO_VERSION.DLL)) -$(TBB_NO_VERSION.DLL): - echo "INPUT ($(TBB.DLL))" > $(TBB_NO_VERSION.DLL) -endif - -#clean: -# $(RM) *.$(OBJ) *.$(DLL) *.res *.map *.ilk *.pdb *.exp *.manifest *.tmp *.d core core.*[0-9][0-9] *.ver - -# Include automatically generated dependencies --include *.d diff --git a/src/tbb/build/Makefile.tbbmalloc b/src/tbb/build/Makefile.tbbmalloc deleted file mode 100644 index dedcfa566..000000000 --- a/src/tbb/build/Makefile.tbbmalloc +++ /dev/null @@ -1,240 +0,0 @@ -# Copyright (c) 2005-2017 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -# -# - -# default target -default_malloc: malloc malloc_test - -tbb_root ?= $(TBBROOT) -BUILDING_PHASE=1 -TEST_RESOURCE = $(MALLOC.RES) -TESTFILE=tbbmalloc -include $(tbb_root)/build/common.inc -DEBUG_SUFFIX=$(findstring _debug,$(call cross_cfg,_$(cfg))) - -MALLOC_ROOT ?= $(tbb_root)/src/tbbmalloc -MALLOC_SOURCE_ROOT ?= $(MALLOC_ROOT) - -VPATH = $(tbb_root)/src/tbb/$(ASSEMBLY_SOURCE) $(tbb_root)/src/tbb $(tbb_root)/src/test -VPATH += $(MALLOC_ROOT) $(MALLOC_SOURCE_ROOT) - -CPLUS_FLAGS += $(if $(crosstest),$(DEFINE_KEY)__TBBMALLOC_NO_IMPLICIT_LINKAGE=1) - -TEST_SUFFIXES=proxy -TEST_PREREQUISITE+=$(MALLOC.LIB) -LINK_FILES+=$(LINK_MALLOC.LIB) -include $(tbb_root)/build/common_rules.inc - -ORIG_CPLUS_FLAGS:=$(CPLUS_FLAGS) -ORIG_INCLUDES:=$(INCLUDES) -ORIG_LINK_MALLOC.LIB:=$(LINK_MALLOC.LIB) - -#------------------------------------------------------ -# Define rules for making the TBBMalloc shared library. -#------------------------------------------------------ - -# Object files that make up TBBMalloc -MALLOC_CPLUS.OBJ = backend.$(OBJ) large_objects.$(OBJ) backref.$(OBJ) tbbmalloc.$(OBJ) -MALLOC.OBJ := $(MALLOC_CPLUS.OBJ) $(MALLOC_ASM.OBJ) itt_notify_malloc.$(OBJ) frontend.$(OBJ) -PROXY.OBJ := proxy.$(OBJ) tbb_function_replacement.$(OBJ) -M_CPLUS_FLAGS := $(subst $(WARNING_KEY),,$(M_CPLUS_FLAGS)) $(DEFINE_KEY)__TBBMALLOC_BUILD=1 -M_INCLUDES := $(INCLUDES) $(INCLUDE_KEY)$(MALLOC_ROOT) $(INCLUDE_KEY)$(MALLOC_SOURCE_ROOT) - -# Suppress superfluous warnings for TBBMalloc compilation -$(MALLOC.OBJ): M_CPLUS_FLAGS += $(WARNING_SUPPRESS) -# Suppress superfluous warnings for TBBMalloc proxy compilation -$(PROXY.OBJ): CPLUS_FLAGS += $(WARNING_SUPPRESS) - -frontend.$(OBJ): frontend.cpp version_string.ver - $(CPLUS) $(COMPILE_ONLY) $(M_CPLUS_FLAGS) $(PIC_KEY) $(DSE_KEY) $(M_INCLUDES) $(INCLUDE_KEY). $< - -$(PROXY.OBJ): %.$(OBJ): %.cpp - $(CPLUS) $(COMPILE_ONLY) $(CPLUS_FLAGS) $(PIC_KEY) $(DSE_KEY) $(DEFINE_KEY)__TBBMALLOC_BUILD=1 $(M_INCLUDES) $< - -$(MALLOC_CPLUS.OBJ): %.$(OBJ): %.cpp - $(CPLUS) $(COMPILE_ONLY) $(M_CPLUS_FLAGS) $(PIC_KEY) $(DSE_KEY) $(M_INCLUDES) $< - -itt_notify_malloc.$(OBJ): itt_notify.cpp - $(CPLUS) $(COMPILE_ONLY) $(M_CPLUS_FLAGS) $(PIC_KEY) $(DSE_KEY) $(OUTPUTOBJ_KEY)$@ $(INCLUDES) $< - -MALLOC_LINK_FLAGS = $(LIB_LINK_FLAGS) -PROXY_LINK_FLAGS = $(LIB_LINK_FLAGS) - -ifneq (,$(MALLOC.DEF)) -tbbmalloc.def: $(MALLOC.DEF) - $(CPLUS) $(PREPROC_ONLY) $< $(M_CPLUS_FLAGS) $(WARNING_SUPPRESS) $(INCLUDES) > $@ - -MALLOC_LINK_FLAGS += $(EXPORT_KEY)tbbmalloc.def -$(MALLOC.DLL): tbbmalloc.def -endif - -$(MALLOC.DLL) $(MALLOCPROXY.DLL): CPLUS_FLAGS += $(SDL_FLAGS) -$(MALLOC.DLL) $(MALLOCPROXY.DLL): M_CPLUS_FLAGS += $(SDL_FLAGS) -$(MALLOC.DLL): BUILDING_LIBRARY = $(MALLOC.DLL) -$(MALLOC.DLL): $(MALLOC.OBJ) $(MALLOC.RES) $(MALLOC_NO_VERSION.DLL) - $(subst $(CPLUS),$(CONLY),$(LIB_LINK_CMD)) $(LIB_OUTPUT_KEY)$(MALLOC.DLL) $(MALLOC.OBJ) $(MALLOC.RES) $(LIB_LINK_LIBS) $(MALLOC_LINK_FLAGS) - -ifneq (,$(MALLOCPROXY.DEF)) -tbbmallocproxy.def: $(MALLOCPROXY.DEF) - $(CPLUS) $(PREPROC_ONLY) $< $(CPLUS_FLAGS) $(WARNING_SUPPRESS) $(INCLUDES) > $@ - -PROXY_LINK_FLAGS += $(EXPORT_KEY)tbbmallocproxy.def -$(MALLOCPROXY.DLL): tbbmallocproxy.def -endif - -ifneq (,$(MALLOCPROXY.DLL)) -$(MALLOCPROXY.DLL): BUILDING_LIBRARY = $(MALLOCPROXY.DLL) -$(MALLOCPROXY.DLL): $(PROXY.OBJ) $(MALLOCPROXY_NO_VERSION.DLL) $(MALLOC.DLL) $(MALLOC.RES) - $(LIB_LINK_CMD) $(LIB_OUTPUT_KEY)$(MALLOCPROXY.DLL) $(PROXY.OBJ) $(MALLOC.RES) $(LIB_LINK_LIBS) $(LINK_MALLOC.LIB) $(PROXY_LINK_FLAGS) -endif - -ifneq (,$(MALLOC_NO_VERSION.DLL)) -$(MALLOC_NO_VERSION.DLL): - echo "INPUT ($(MALLOC.DLL))" > $(MALLOC_NO_VERSION.DLL) -endif - -ifneq (,$(MALLOCPROXY_NO_VERSION.DLL)) -$(MALLOCPROXY_NO_VERSION.DLL): - echo "INPUT ($(MALLOCPROXY.DLL))" > $(MALLOCPROXY_NO_VERSION.DLL) -endif - -malloc: $(MALLOC.DLL) $(MALLOCPROXY.DLL) - -malloc_dll: $(MALLOC.DLL) - -malloc_proxy_dll: $(MALLOCPROXY.DLL) - -.PHONY: malloc malloc_dll malloc_proxy_dll - -#------------------------------------------------------ -# End of rules for making the TBBMalloc shared library -#------------------------------------------------------ - -#------------------------------------------------------ -# Define rules for making the TBBMalloc unit tests -#------------------------------------------------------ - -# --------- The list of TBBMalloc unit tests ---------- -MALLOC_TESTS = test_ScalableAllocator.$(TEST_EXT) \ - test_ScalableAllocator_STL.$(TEST_EXT) \ - test_malloc_compliance.$(TEST_EXT) \ - test_malloc_regression.$(TEST_EXT) \ - test_malloc_init_shutdown.$(TEST_EXT) \ - test_malloc_pools.$(TEST_EXT) \ - test_malloc_pure_c.$(TEST_EXT) \ - test_malloc_whitebox.$(TEST_EXT) \ - test_malloc_used_by_lib.$(TEST_EXT) \ - test_malloc_lib_unload.$(TEST_EXT) -ifneq (,$(MALLOCPROXY.DLL)) -MALLOC_TESTS += test_malloc_overload.$(TEST_EXT) \ - test_malloc_overload_proxy.$(TEST_EXT) \ - test_malloc_atexit.$(TEST_EXT) -endif -# ----------------------------------------------------- - -# ------------ Set test specific variables ------------ -# TODO: implement accurate warning suppression for tests to unify with Makefile.test. -$(MALLOC_TESTS): CPLUS_FLAGS += $(TEST_WARNING_KEY) $(if $(no_exceptions),$(DEFINE_KEY)__TBB_TEST_NO_EXCEPTIONS=1) -$(MALLOC_TESTS): M_CPLUS_FLAGS += $(TEST_WARNING_KEY) $(if $(no_exceptions),$(DEFINE_KEY)__TBB_TEST_NO_EXCEPTIONS=1) -$(MALLOC_TESTS): INCLUDES += $(INCLUDE_TEST_HEADERS) -$(MALLOC_TESTS): M_INCLUDES += $(INCLUDE_TEST_HEADERS) - -ifeq (windows.gcc,$(tbb_os).$(compiler)) -test_malloc_overload.$(TEST_EXT): LIBS += $(MALLOCPROXY.LIB) -endif - -MALLOC_M_CPLUS_TESTS = test_malloc_whitebox.$(TEST_EXT) test_malloc_lib_unload.$(TEST_EXT) \ - test_malloc_used_by_lib.$(TEST_EXT) -MALLOC_NO_LIB_TESTS = test_malloc_whitebox.$(TEST_EXT) test_malloc_lib_unload.$(TEST_EXT) \ - test_malloc_used_by_lib.$(TEST_EXT) test_malloc_overload.$(TEST_EXT) -MALLOC_LINK_PROXY_TESTS = test_malloc_overload_proxy.$(TEST_EXT) -MALLOC_ADD_DLL_TESTS = test_malloc_lib_unload.$(TEST_EXT) test_malloc_used_by_lib.$(TEST_EXT) \ - test_malloc_atexit.$(TEST_EXT) - -$(MALLOC_M_CPLUS_TESTS): CPLUS_FLAGS=$(M_CPLUS_FLAGS) -$(MALLOC_M_CPLUS_TESTS): INCLUDES=$(M_INCLUDES) -$(MALLOC_NO_LIB_TESTS): LINK_MALLOC.LIB= -$(MALLOC_NO_LIB_TESTS): LINK_FLAGS+=$(LIBDL) -$(MALLOC_LINK_PROXY_TESTS): LINK_MALLOC.LIB=$(LINK_MALLOCPROXY.LIB) -ifneq (,$(DYLIB_KEY)) -$(MALLOC_ADD_DLL_TESTS): %.$(TEST_EXT): %_dll.$(DLL) -$(MALLOC_ADD_DLL_TESTS): TEST_LIBS+=$(@:.$(TEST_EXT)=_dll.$(LIBEXT)) -endif - -test_malloc_over%.$(TEST_EXT): CPLUS_FLAGS=$(subst /MT,/MD,$(M_CPLUS_FLAGS)) -test_malloc_over%.$(TEST_EXT): INCLUDES=$(M_INCLUDES) -test_malloc_overload_proxy.$(TEST_EXT): LINK_FLAGS+=$(LIBDL) - -test_malloc_atexit_dll.$(DLL): CPLUS_FLAGS=$(subst /MT,/MD,$(M_CPLUS_FLAGS)) -test_malloc_atexit.$(TEST_EXT): CPLUS_FLAGS=$(subst /MT,/MD,$(M_CPLUS_FLAGS)) -test_malloc_atexit.$(TEST_EXT): LINK_FLAGS+=$(LIBDL) -# on Ubuntu 11.10 linker called with --as-needed, so dependency on libtbbmalloc_proxy -# is not created, and malloc overload via linking with -ltbbmalloc_proxy is not working. -# Overcome with --no-as-needed. -ifeq (linux.gcc,$(tbb_os).$(compiler)) -test_malloc_atexit.$(TEST_EXT): MALLOCPROXY.LIB := -Wl,--no-as-needed $(MALLOCPROXY.LIB) -endif -# The test isn't added to MALLOC_LINK_PROXY_TESTS, because we need both -# tbbmalloc and proxy libs. For platforms other than Android it's enough -# to modify LINK_MALLOC.LIB for TEST_EXT target only. But under Android build -# of DLL and TEST_EXT can be requested independently, so there is no chance -# to set LINK_MALLOC.LIB in TEST_EXT build rule, and affect DLL build. -test_malloc_atexit.$(TEST_EXT): LINK_MALLOC.LIB := $(LINK_MALLOC.LIB) $(LINK_MALLOCPROXY.LIB) -test_malloc_atexit_dll.$(DLL): LINK_MALLOC.LIB := $(LINK_MALLOC.LIB) $(LINK_MALLOCPROXY.LIB) - -test_malloc_whitebox.$(TEST_EXT): $(MALLOC_ASM.OBJ) version_string.ver -test_malloc_whitebox.$(TEST_EXT): INCLUDES+=$(INCLUDE_KEY). -test_malloc_whitebox.$(TEST_EXT): LINK_FILES+=$(MALLOC_ASM.OBJ) - -# Some _dll targets need to restore variables since they are changed by parent -# target-specific rule of its .exe targets -test_malloc_lib_unload_dll.$(DLL): CPLUS_FLAGS=$(ORIG_CPLUS_FLAGS) $(if $(no_exceptions),$(DEFINE_KEY)__TBB_TEST_NO_EXCEPTIONS=1) -test_malloc_lib_unload_dll.$(DLL): INCLUDES=$(ORIG_INCLUDES) $(INCLUDE_TEST_HEADERS) - -test_malloc_used_by_lib_dll.$(DLL): CPLUS_FLAGS=$(subst /MT,/LD,$(M_CPLUS_FLAGS)) -test_malloc_used_by_lib_dll.$(DLL): LINK_FILES+=$(ORIG_LINK_MALLOC.LIB) -test_malloc_used_by_lib_dll.$(DLL): LIBDL= -# ----------------------------------------------------- - -# ---- The list of TBBMalloc test running commands ---- -# run_cmd is usually empty -malloc_test: $(MALLOC.DLL) malloc_test_no_depends - -malloc_test_no_depends: $(TEST_PREREQUISITE) $(MALLOC_TESTS) - $(run_cmd) ./test_malloc_pools.$(TEST_EXT) $(args) 1:4 -ifneq (,$(MALLOCPROXY.DLL)) - $(run_cmd) ./test_malloc_atexit.$(TEST_EXT) $(args) - $(run_cmd) $(TEST_LAUNCHER) -l $(MALLOCPROXY.DLL) ./test_malloc_overload.$(TEST_EXT) $(args) - $(run_cmd) $(TEST_LAUNCHER) ./test_malloc_overload_proxy.$(TEST_EXT) $(args) -endif - $(run_cmd) $(TEST_LAUNCHER) ./test_malloc_lib_unload.$(TEST_EXT) $(args) - $(run_cmd) $(TEST_LAUNCHER) ./test_malloc_used_by_lib.$(TEST_EXT) - $(run_cmd) ./test_malloc_whitebox.$(TEST_EXT) $(args) 1:4 - $(run_cmd) $(TEST_LAUNCHER) -u ./test_malloc_compliance.$(TEST_EXT) $(args) 1:4 - $(run_cmd) ./test_ScalableAllocator.$(TEST_EXT) $(args) - $(run_cmd) ./test_ScalableAllocator_STL.$(TEST_EXT) $(args) - $(run_cmd) ./test_malloc_regression.$(TEST_EXT) $(args) - $(run_cmd) ./test_malloc_init_shutdown.$(TEST_EXT) $(args) - $(run_cmd) ./test_malloc_pure_c.$(TEST_EXT) $(args) -# ----------------------------------------------------- - -#------------------------------------------------------ -# End of rules for making the TBBMalloc unit tests -#------------------------------------------------------ - -# Include automatically generated dependencies --include *.d diff --git a/src/tbb/build/Makefile.tbbproxy b/src/tbb/build/Makefile.tbbproxy deleted file mode 100644 index eb2e0fd65..000000000 --- a/src/tbb/build/Makefile.tbbproxy +++ /dev/null @@ -1,109 +0,0 @@ -# Copyright (c) 2005-2017 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -# -# - -# default target -default_tbbproxy: tbbproxy tbbproxy_test - -tbb_root ?= $(TBBROOT) -BUILDING_PHASE=1 -include $(tbb_root)/build/common.inc -DEBUG_SUFFIX=$(findstring _debug,_$(cfg)) - -PROXY_ROOT ?= $(tbb_root)/src/tbbproxy -PROXY_SOURCE_ROOT ?= $(PROXY_ROOT) - -VPATH = $(tbb_root)/src/tbb/$(ASSEMBLY_SOURCE) $(tbb_root)/src/tbb $(tbb_root)/src/test -VPATH += $(PROXY_ROOT) $(PROXY_SOURCE_ROOT) - -CPLUS_FLAGS += $(DEFINE_KEY)__TBB_DLL_NAME=$(TBB.DLL) -CPLUS_FLAGS += $(DEFINE_KEY)__TBB_LST=$(TBB.LST) -CPLUS_FLAGS += $(foreach dir,$(VPATH),$(INCLUDE_KEY)$(dir)) -CPLUS_FLAGS += $(PIC_KEY) $(DSE_KEY) - -include $(tbb_root)/build/common_rules.inc - -#------------------------------------------------------ -# Define rules for making the TBB Proxy static library. -#------------------------------------------------------ - -# Object files that make up TBB Proxy -PROXY_CPLUS.OBJ = tbbproxy.$(OBJ) -PROXY_ASM.OBJ = tbbproxy-asm.$(OBJ) -PROXY.OBJ := $(PROXY_CPLUS.OBJ) $(PROXY_ASM.OBJ) - -# Not using intrinsics prevents undesired dependence on ICL libraries (e.g. libirc). -# Not using default libs prevents link issues caused by different CRT versions in tbbproxy and in an app. -$(PROXY.OBJ): CPLUS_FLAGS += $(DEFINE_KEY)ARCH_$(arch) $(DEFINE_KEY)OS_$(tbb_os) $(NOINTRINSIC_KEY) $(NODEFAULTLIB_KEY) - -$(PROXY_CPLUS.OBJ): CPLUS_FLAGS+=$(if $(filter windows.%cl,$(tbb_os).$(compiler)),/Fdtbbproxy$(DEBUG_SUFFIX).pdb) -$(PROXY_CPLUS.OBJ): %.$(OBJ): %.cpp - $(CPLUS) $(COMPILE_ONLY) $(CPLUS_FLAGS) $(INCLUDES) $< - -$(PROXY.LIB): $(PROXY.OBJ) - $(AR) $(AR_FLAGS) $(AR_OUTPUT_KEY)$@ $^ - -.PRECIOUS : %.$(ASMEXT) -tbbproxy-asm.$(ASMEXT) : tbbproxy-$(tbb_os).$(ASMEXT) $(TBB.LST) $(TBB-OBJECTS.LST) - $(CPLUS) $(PREPROC_ONLY) $< $(INCLUDES) $(CPLUS_FLAGS) $(DEFINE_KEY)__TBB_BUILD=1 > $@ - -.PHONY: tbbproxy -ifeq (windows,$(tbb_os)) -tbbproxy: $(PROXY.LIB) -else -tbbproxy: -endif - -#------------------------------------------------------ -# End of rules for making the TBB Proxy static library -#------------------------------------------------------ - -#------------------------------------------------------ -# Define rules for making the TBB Proxy unit tests -#------------------------------------------------------ - -add_debug=$(basename $(1))_debug$(suffix $(1)) -cross_suffix=$(if $(crosstest),$(if $(DEBUG_SUFFIX),$(subst _debug,,$(1)),$(call add_debug,$(1))),$(1)) - -PROXY_LIB = $(call cross_suffix,$(PROXY.LIB)) -PROXY_TESTS_SRCS = test_runtime_loader.cpp -PROXY_TESTS_OBJS = $(PROXY_TESTS_SRCS:.cpp=.$(OBJ)) -PROXY_TESTS_EXES = $(PROXY_TESTS_OBJS:.$(OBJ)=.$(TEST_EXT)) - -# Run rules. -.PHONY: tbbproxy_test -ifeq (windows,$(tbb_os)) -tbbproxy_test: $(call cross_suffix,$(PROXY.LIB)) $(TEST_PREREQUISITE) $(PROXY_TESTS_EXES) - $(run_cmd) ./test_runtime_loader.$(TEST_EXT) $(args) -else -tbbproxy_test: -endif - -# Link rules. -$(PROXY_TESTS_EXES): %.$(TEST_EXT): %.$(OBJ) $(PROXY_LIB) - $(CPLUS) $(OUTPUT_KEY)$@ $(CPLUS_FLAGS) $< $(PROXY_LIB) $(LIBS) $(LIBDL) $(LINK_FLAGS) - -# Compilation rules. -$(PROXY_TESTS_OBJS): %.$(OBJ): %.cpp - $(CPLUS) $(COMPILE_ONLY) $(CPLUS_FLAGS) $(CXX_ONLY_FLAGS) $(CXX_WARN_SUPPRESS) $(INCLUDES) $(OUTPUT_KEY)$@ $< - -#------------------------------------------------------ -# End of rules for making the TBB Proxy unit tests -#------------------------------------------------------ - -# Include automatically generated dependencies --include *.d diff --git a/src/tbb/build/Makefile.test b/src/tbb/build/Makefile.test deleted file mode 100644 index 9de7860d6..000000000 --- a/src/tbb/build/Makefile.test +++ /dev/null @@ -1,317 +0,0 @@ -# Copyright (c) 2005-2017 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -# -# - -#------------------------------------------------------------------------------ -# Define rules for making the TBB tests. -#------------------------------------------------------------------------------ -.PHONY: default test_tbb_plain test_tbb_openmp test_tbb_cilk test_tbb_old clean - -default: test_tbb_plain test_tbb_openmp test_tbb_cilk test_tbb_old - -tbb_root ?= $(TBBROOT) -BUILDING_PHASE=1 -TEST_RESOURCE = $(TBB.RES) -TESTFILE=test -include $(tbb_root)/build/common.inc -DEBUG_SUFFIX=$(findstring _debug,$(call cross_cfg,_$(cfg))) - -#------------------------------------------------------------ -# Define static pattern rules dealing with .cpp source files -#------------------------------------------------------------ - -VPATH = $(tbb_root)/src/tbb/$(ASSEMBLY_SOURCE) $(tbb_root)/src/tbb $(tbb_root)/src/rml/client $(tbb_root)/src/old $(tbb_root)/src/test $(tbb_root)/src/perf -CPLUS_FLAGS += $(if $(crosstest),$(DEFINE_KEY)__TBB_NO_IMPLICIT_LINKAGE=1) \ - $(if $(no_exceptions),$(DEFINE_KEY)__TBB_TEST_NO_EXCEPTIONS=1) \ - $(if $(LINK_TBB.LIB),$(DEFINE_KEY)TEST_USES_TBB=1) - -TEST_PREREQUISITE+=$(TBB.LIB) -LINK_FILES+=$(LINK_TBB.LIB) - -ifdef use_proxy - USE_PROXY_FLAG = $(DEFINE_KEY)HARNESS_USE_RUNTIME_LOADER - CPLUS_FLAGS += $(USE_PROXY_FLAG) - LINK_TBB.LIB = $(PROXY.LIB) - LIBS += $(LIBDL) -endif - -TEST_SUFFIXES=secondary compiler_builtins pic -include $(tbb_root)/build/common_rules.inc - -# Rules for the tests, which use TBB in a dynamically loadable library -test_model_plugin.$(TEST_EXT): LINK_TBB.LIB = -test_model_plugin.$(TEST_EXT): CPLUS_FLAGS := $(CPLUS_FLAGS:$(USE_PROXY_FLAG)=) -test_model_plugin.$(TEST_EXT): LIBS += $(LIBDL) -ifneq (,$(DYLIB_KEY)) -test_model_plugin.$(TEST_EXT): test_model_plugin_dll.$(DLL) -endif - -# tbb_misc.$(OBJ) has to be specified here (instead of harness_inject_scheduler.h) because it carries dependency on version_string.ver -SCHEDULER_DEPENDENCIES = $(TBB_ASM.OBJ) tbb_misc.$(OBJ) - -# These executables don't depend on the TBB library, but include core .cpp files directly -SCHEDULER_DIRECTLY_INCLUDED = test_task_leaks.$(TEST_EXT) \ - test_task_assertions.$(TEST_EXT) \ - test_fast_random.$(TEST_EXT) \ - test_global_control_whitebox.$(TEST_EXT) \ - test_concurrent_queue_whitebox.$(TEST_EXT) - -# Necessary to locate version_string.ver referenced from directly included tbb_misc.cpp -INCLUDES += $(INCLUDE_KEY). $(INCLUDE_TEST_HEADERS) - -$(SCHEDULER_DIRECTLY_INCLUDED): CPLUS_FLAGS += $(DSE_KEY) -$(SCHEDULER_DIRECTLY_INCLUDED): WARNING_KEY += $(WARNING_SUPPRESS) -$(SCHEDULER_DIRECTLY_INCLUDED): LIBS += $(LIBDL) -#tbb.lib must not be linked to scheduler white box tests in order to not violate ODR -$(SCHEDULER_DIRECTLY_INCLUDED): LINK_TBB.LIB = -$(SCHEDULER_DIRECTLY_INCLUDED): LINK_FILES += $(SCHEDULER_DEPENDENCIES) -$(SCHEDULER_DIRECTLY_INCLUDED): $(SCHEDULER_DEPENDENCIES) - -# test_tbb_header detects "multiple definition" linker error using the test that covers the whole library -TWICE_LINKED_TESTS = test_tbb_header.$(TEST_EXT) \ - test_concurrent_unordered_set.$(TEST_EXT) - -%_secondary.$(OBJ): CPLUS_FLAGS+=$(DEFINE_KEY)__TBB_TEST_SECONDARY=1 - -# Detecting "multiple definition" linker error using the test that covers the whole library -$(TWICE_LINKED_TESTS): %.$(TEST_EXT): %.$(OBJ) %_secondary.$(OBJ) -$(TWICE_LINKED_TESTS): LINK_FILES+=$(@:.$(TEST_EXT)=_secondary.$(OBJ)) - -# Checks that TBB works correctly in position independent code -%_pic.$(OBJ): CPLUS_FLAGS+=$(PIC_KEY) -%_pic.$(OBJ): CPLUS_FLAGS+=$(DEFINE_KEY)__TBB_TEST_PIC=1 - -# Test of generic gcc port and icc intrinsics port -%_compiler_builtins.$(TEST_EXT): LINK_TBB.LIB = -%_compiler_builtins.$(OBJ): CPLUS_FLAGS+=$(DEFINE_KEY)__TBB_TEST_BUILTINS=1 $(DEFINE_KEY)TBB_USE_ASSERT=0 - -# dynamic_link tests don't depend on the TBB library -test_dynamic_link%.$(TEST_EXT): LINK_TBB.LIB = -test_dynamic_link.$(TEST_EXT): LIBS += $(LIBDL) - -# Resolving issue with the number of sections that an object file can contain -ifneq (,$(BIGOBJ_KEY)) -TEST_BIGOBJ = test_opencl_node.$(TEST_EXT) \ - test_atomic.$(TEST_EXT) \ - test_concurrent_unordered_set.$(TEST_EXT) \ - test_concurrent_unordered_map.$(TEST_EXT) \ - test_join_node_key_matching.$(TEST_EXT) \ - test_join_node_msg_key_matching.$(TEST_EXT) \ - test_join_node.$(TEST_EXT) -$(TEST_BIGOBJ): override CXXFLAGS += $(BIGOBJ_KEY) -endif - -# TODO: remove repetition of .$(TEST_EXT) in the list bellow -# The main list of TBB tests -TEST_TBB_PLAIN.EXE = test_assembly.$(TEST_EXT) \ - test_global_control.$(TEST_EXT) \ - test_tbb_fork.$(TEST_EXT) \ - test_assembly_compiler_builtins.$(TEST_EXT) \ - test_aligned_space.$(TEST_EXT) \ - test_atomic.$(TEST_EXT) \ - test_atomic_pic.$(TEST_EXT) \ - test_atomic_compiler_builtins.$(TEST_EXT) \ - test_blocked_range.$(TEST_EXT) \ - test_blocked_range2d.$(TEST_EXT) \ - test_blocked_range3d.$(TEST_EXT) \ - test_concurrent_queue.$(TEST_EXT) \ - test_concurrent_vector.$(TEST_EXT) \ - test_concurrent_unordered_set.$(TEST_EXT) \ - test_concurrent_unordered_map.$(TEST_EXT) \ - test_concurrent_hash_map.$(TEST_EXT) \ - test_enumerable_thread_specific.$(TEST_EXT) \ - test_handle_perror.$(TEST_EXT) \ - test_halt.$(TEST_EXT) \ - test_model_plugin.$(TEST_EXT) \ - test_mutex.$(TEST_EXT) \ - test_mutex_native_threads.$(TEST_EXT) \ - test_rwm_upgrade_downgrade.$(TEST_EXT) \ - test_cache_aligned_allocator.$(TEST_EXT) \ - test_cache_aligned_allocator_STL.$(TEST_EXT) \ - test_parallel_for.$(TEST_EXT) \ - test_parallel_reduce.$(TEST_EXT) \ - test_parallel_sort.$(TEST_EXT) \ - test_parallel_scan.$(TEST_EXT) \ - test_parallel_while.$(TEST_EXT) \ - test_parallel_do.$(TEST_EXT) \ - test_pipeline.$(TEST_EXT) \ - test_pipeline_with_tbf.$(TEST_EXT) \ - test_parallel_pipeline.$(TEST_EXT) \ - test_lambda.$(TEST_EXT) \ - test_task_scheduler_init.$(TEST_EXT) \ - test_task_scheduler_observer.$(TEST_EXT) \ - test_task.$(TEST_EXT) \ - test_tbb_thread.$(TEST_EXT) \ - test_std_thread.$(TEST_EXT) \ - test_tick_count.$(TEST_EXT) \ - test_inits_loop.$(TEST_EXT) \ - test_yield.$(TEST_EXT) \ - test_eh_tasks.$(TEST_EXT) \ - test_eh_algorithms.$(TEST_EXT) \ - test_eh_flow_graph.$(TEST_EXT) \ - test_parallel_invoke.$(TEST_EXT) \ - test_task_group.$(TEST_EXT) \ - test_ittnotify.$(TEST_EXT) \ - test_parallel_for_each.$(TEST_EXT) \ - test_tbb_header.$(TEST_EXT) \ - test_combinable.$(TEST_EXT) \ - test_task_auto_init.$(TEST_EXT) \ - test_task_arena.$(TEST_EXT) \ - test_concurrent_monitor.$(TEST_EXT) \ - test_semaphore.$(TEST_EXT) \ - test_critical_section.$(TEST_EXT) \ - test_reader_writer_lock.$(TEST_EXT) \ - test_tbb_condition_variable.$(TEST_EXT) \ - test_intrusive_list.$(TEST_EXT) \ - test_concurrent_priority_queue.$(TEST_EXT) \ - test_task_priority.$(TEST_EXT) \ - test_task_enqueue.$(TEST_EXT) \ - test_task_steal_limit.$(TEST_EXT) \ - test_hw_concurrency.$(TEST_EXT) \ - test_fp.$(TEST_EXT) \ - test_tuple.$(TEST_EXT) \ - test_flow_graph.$(TEST_EXT) \ - test_broadcast_node.$(TEST_EXT) \ - test_continue_node.$(TEST_EXT) \ - test_function_node.$(TEST_EXT) \ - test_limiter_node.$(TEST_EXT) \ - test_join_node.$(TEST_EXT) \ - test_join_node_key_matching.$(TEST_EXT) \ - test_join_node_msg_key_matching.$(TEST_EXT) \ - test_buffer_node.$(TEST_EXT) \ - test_queue_node.$(TEST_EXT) \ - test_priority_queue_node.$(TEST_EXT) \ - test_sequencer_node.$(TEST_EXT) \ - test_source_node.$(TEST_EXT) \ - test_overwrite_node.$(TEST_EXT) \ - test_write_once_node.$(TEST_EXT) \ - test_indexer_node.$(TEST_EXT) \ - test_multifunction_node.$(TEST_EXT) \ - test_split_node.$(TEST_EXT) \ - test_static_assert.$(TEST_EXT) \ - test_aggregator.$(TEST_EXT) \ - test_concurrent_lru_cache.$(TEST_EXT) \ - test_examples_common_utility.$(TEST_EXT) \ - test_dynamic_link.$(TEST_EXT) \ - test_parallel_for_vectorization.$(TEST_EXT) \ - test_tagged_msg.$(TEST_EXT) \ - test_partitioner_whitebox.$(TEST_EXT) \ - test_flow_graph_whitebox.$(TEST_EXT) \ - test_composite_node.$(TEST_EXT) \ - test_async_node.$(TEST_EXT) \ - test_async_msg.$(TEST_EXT) \ - test_tbb_version.$(TEST_EXT) # insert new files right above - -# These tests depend on other technologies -TEST_TBB_SPECIAL.EXE = test_openmp.$(TEST_EXT) \ - test_cilk_interop.$(TEST_EXT) \ - test_gfx_factory.$(TEST_EXT) \ - test_opencl_node.$(TEST_EXT) - -# skip mode_plugin for now -skip_tests += test_model_plugin - -ifdef OPENMP_FLAG -test_openmp.$(TEST_EXT): CPLUS_FLAGS += $(OPENMP_FLAG) - -test_tbb_openmp: $(TEST_PREREQUISITE) test_openmp.$(TEST_EXT) - $(run_cmd) ./test_openmp.$(TEST_EXT) 1:4 -else -test_tbb_openmp: - @echo "OpenMP is not available" -endif - -ifdef CILK_AVAILABLE -# Workaround on cilkrts linkage known issue (see Intel(R) C++ Composer XE 2011 Release Notes) -# The issue reveals itself if a version of binutils is prior to 2.17 -ifeq (linux_icc,$(tbb_os)_$(compiler)) -test_cilk_interop.$(TEST_EXT): LIBS += -lcilkrts -test_gfx_factory.$(TEST_EXT): LIBS += -lcilkrts -endif -test_tbb_cilk: test_cilk_interop.$(TEST_EXT) - $(run_cmd) ./test_cilk_interop.$(TEST_EXT) $(args) -else -test_tbb_cilk: - @echo "Intel(R) Cilk(TM) Plus is not available" -endif - -test_opencl_node.$(TEST_EXT): LIBS += $(OPENCL.LIB) - -$(TEST_TBB_PLAIN.EXE) $(TEST_TBB_SPECIAL.EXE): WARNING_KEY += $(TEST_WARNING_KEY) - -# Run tests that are in SCHEDULER_DIRECTLY_INCLUDED and TEST_TBB_PLAIN.EXE but not in skip_tests (which is specified by user) -TESTS_TO_RUN := $(filter-out $(addsuffix .$(TEST_EXT),$(skip_tests)),$(TEST_TBB_PLAIN.EXE) $(SCHEDULER_DIRECTLY_INCLUDED)) - -# This definition intentionally consists of two blank lines -define eol - - -endef - -# First build the targets, then run them -# Form a list of commands separated with end of line -# Note that usually run_cmd is empty, and tests run directly - -test_tbb_plain: $(TEST_PREREQUISITE) $(TESTS_TO_RUN) - $(foreach test, $(TESTS_TO_RUN), $(run_cmd) ./$(test) $(args) $(eol)) - - -# For deprecated files, we don't mind warnings etc., thus compilation rules are most relaxed -CPLUS_FLAGS_DEPRECATED = $(DEFINE_KEY)__TBB_TEST_DEPRECATED=1 $(subst $(WARNING_KEY),,$(CPLUS_FLAGS)) $(WARNING_SUPPRESS) $(INCLUDE_KEY)$(tbb_root)/src/test -TEST_TBB_OLD.OBJ = test_concurrent_vector_v2.$(OBJ) test_concurrent_queue_v2.$(OBJ) test_mutex_v2.$(OBJ) test_task_scheduler_observer_v3.$(OBJ) - -$(TEST_TBB_OLD.OBJ): CPLUS_FLAGS := $(CPLUS_FLAGS_DEPRECATED) - -TEST_TBB_OLD.EXE = $(subst .$(OBJ),.$(TEST_EXT),$(TEST_TBB_OLD.OBJ)) - -ifeq (,$(NO_LEGACY_TESTS)) -test_tbb_old: $(TEST_PREREQUISITE) $(TEST_TBB_OLD.EXE) - $(run_cmd) ./test_concurrent_vector_v2.$(TEST_EXT) $(args) 1:4 - $(run_cmd) ./test_concurrent_queue_v2.$(TEST_EXT) $(args) 1:4 - $(run_cmd) ./test_mutex_v2.$(TEST_EXT) $(args) 1 - $(run_cmd) ./test_mutex_v2.$(TEST_EXT) $(args) 2 - $(run_cmd) ./test_mutex_v2.$(TEST_EXT) $(args) 4 - $(run_cmd) ./test_task_scheduler_observer_v3.$(TEST_EXT) $(args) 1:4 -else -test_tbb_old: - @echo Legacy tests skipped -endif - -ifneq (,$(codecov)) -codecov_gen: - profmerge - codecov $(if $(findstring -,$(codecov)),$(codecov),) -demang -comp $(tbb_root)/build/codecov.txt -endif - -time_%: time_%.$(TEST_EXT) $(TEST_PREREQUISITE) - $(run_cmd) ./$< $(args) - - -# for some reason, "perf_%.$(TEST_EXT): perf_dll.$(DLL)" does not work TODO: find out how to apply pattern here -perf_sched.$(TEST_EXT): perf_dll.$(DLL) -perf_%.$(TEST_EXT): TEST_LIBS = perf_dll.$(LIBEXT) -perf_%: perf_%.$(TEST_EXT) $(TEST_PREREQUISITE) - $(run_cmd) ./$< $(args) - -clean_%: - $(RM) $*.$(OBJ) $*.exe $*.$(DLL) $*.$(LIBEXT) $*.res $*.map $*.ilk $*.pdb $*.exp $*.*manifest $*.tmp $*.d *.ver - -clean: - $(RM) *.$(OBJ) *.exe *.$(DLL) *.$(LIBEXT) *.res *.map *.ilk *.pdb *.exp *.manifest *.tmp *.d pgopti.* *.dyn core core.*[0-9][0-9] *.ver - -# Include automatically generated dependencies --include *.d diff --git a/src/tbb/build/SunOS.gcc.inc b/src/tbb/build/SunOS.gcc.inc deleted file mode 100644 index 2ebed62fc..000000000 --- a/src/tbb/build/SunOS.gcc.inc +++ /dev/null @@ -1,95 +0,0 @@ -# Copyright (c) 2005-2017 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -# -# - -COMPILE_ONLY = -c -MMD -PREPROC_ONLY = -E -x c++ -INCLUDE_KEY = -I -DEFINE_KEY = -D -OUTPUT_KEY = -o # -OUTPUTOBJ_KEY = -o # -PIC_KEY = -fPIC -WARNING_AS_ERROR_KEY = -Werror -WARNING_KEY = -Wall -TEST_WARNING_KEY = -Wshadow -Wcast-qual -Woverloaded-virtual -Wnon-virtual-dtor $(if $(findstring cc4., $(runtime)),-Wextra) -WARNING_SUPPRESS = -Wno-parentheses -Wno-non-virtual-dtor -DYLIB_KEY = -shared -LIBDL = -ldl - -CPLUS = g++ -CONLY = gcc -LIB_LINK_FLAGS = -shared -LIBS = -lpthread -lrt -ldl -C_FLAGS = $(CPLUS_FLAGS) -x c - -ifeq ($(cfg), release) - CPLUS_FLAGS = -g -O2 -DUSE_PTHREAD -endif -ifeq ($(cfg), debug) - CPLUS_FLAGS = -DTBB_USE_DEBUG -g -O0 -DUSE_PTHREAD -endif - -ASM= -ASM_FLAGS= - -TBB_ASM.OBJ= - -ifeq (ia64,$(arch)) -# Position-independent code (PIC) is a must for IA-64 - CPLUS_FLAGS += $(PIC_KEY) -endif - -ifeq (intel64,$(arch)) - CPLUS_FLAGS += -m64 - LIB_LINK_FLAGS += -m64 -endif - -ifeq (ia32,$(arch)) - CPLUS_FLAGS += -m32 - LIB_LINK_FLAGS += -m32 -endif - -# for some gcc versions on Solaris, -m64 may imply V9, but perhaps not everywhere (TODO: verify) -# RcppParallel: CRAN uses 32-bit SPARC so we force that here -ifeq (sparc,$(arch)) - CPLUS_FLAGS += -m32 - LIB_LINK_FLAGS += -m32 -# CPLUS_FLAGS += -mcpu=v9 -m64 -# LIB_LINK_FLAGS += -mcpu=v9 -m64 -endif - -#------------------------------------------------------------------------------ -# Setting assembler data. -#------------------------------------------------------------------------------ -ASSEMBLY_SOURCE=$(arch)-gas -ifeq (ia64,$(arch)) - ASM=ias - TBB_ASM.OBJ = atomic_support.o lock_byte.o log2.o pause.o -endif -#------------------------------------------------------------------------------ -# End of setting assembler data. -#------------------------------------------------------------------------------ - -#------------------------------------------------------------------------------ -# Setting tbbmalloc data. -#------------------------------------------------------------------------------ - -M_CPLUS_FLAGS = $(CPLUS_FLAGS) -fno-rtti -fno-exceptions - -#------------------------------------------------------------------------------ -# End of setting tbbmalloc data. -#------------------------------------------------------------------------------ diff --git a/src/tbb/build/SunOS.inc b/src/tbb/build/SunOS.inc deleted file mode 100644 index 447e059e1..000000000 --- a/src/tbb/build/SunOS.inc +++ /dev/null @@ -1,83 +0,0 @@ -# Copyright (c) 2005-2017 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -# -# - -ifndef arch - arch:=$(shell uname -p) - ifeq ($(arch),i386) - ifeq ($(shell isainfo -b),64) - arch:=intel64 - else - arch:=ia32 - endif - endif - export arch -# For non-IA systems running Sun OS, 'arch' will contain whatever is printed by uname -p. -# In particular, for SPARC architecture it will contain "sparc". -endif - -ifndef runtime - gcc_version:=$(shell gcc -dumpversion) - os_version:=$(shell uname -r) - os_kernel_version:=$(shell uname -r | sed -e 's/-.*$$//') - export runtime:=cc$(gcc_version)_kernel$(os_kernel_version) -endif - -ifeq ($(arch),sparc) - native_compiler := gcc - export compiler ?= gcc -else - native_compiler := suncc - export compiler ?= suncc -endif -# debugger ?= gdb - -CMD=$(SHELL) -c -CWD=$(shell pwd) -RM?=rm -f -RD?=rmdir -MD?=mkdir -p -NUL= /dev/null -SLASH=/ -MAKE_VERSIONS=bash $(tbb_root)/build/version_info_sunos.sh $(VERSION_FLAGS) >version_string.ver -MAKE_TBBVARS=bash $(tbb_root)/build/generate_tbbvars.sh - -ifdef LD_LIBRARY_PATH - export LD_LIBRARY_PATH := .:$(LD_LIBRARY_PATH) -else - export LD_LIBRARY_PATH := . -endif - -####### Build settings ######################################################## - -OBJ = o -DLL = so -LIBEXT=so - -TBB.LST = -TBB.DEF = -TBB.DLL = libtbb$(CPF_SUFFIX)$(DEBUG_SUFFIX).$(DLL) -TBB.LIB = $(TBB.DLL) -LINK_TBB.LIB = $(TBB.LIB) - -MALLOC.DLL = libtbbmalloc$(DEBUG_SUFFIX).$(DLL) -MALLOC.LIB = $(MALLOC.DLL) -LINK_MALLOC.LIB = $(MALLOC.LIB) - -MALLOCPROXY.DLL = libtbbmalloc_proxy$(DEBUG_SUFFIX).$(DLL) - -TEST_LAUNCHER=sh $(tbb_root)/build/test_launcher.sh $(largs) diff --git a/src/tbb/build/SunOS.suncc.inc b/src/tbb/build/SunOS.suncc.inc deleted file mode 100644 index b9cc946ef..000000000 --- a/src/tbb/build/SunOS.suncc.inc +++ /dev/null @@ -1,93 +0,0 @@ -# Copyright (c) 2005-2017 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -# -# - -COMPILE_ONLY = -c -xMMD -errtags -PREPROC_ONLY = -E -xMMD -INCLUDE_KEY = -I -DEFINE_KEY = -D -OUTPUT_KEY = -o # -OUTPUTOBJ_KEY = -o # -PIC_KEY = -KPIC -DYLIB_KEY = -G -LIBDL = -ldl -# WARNING_AS_ERROR_KEY = -errwarn=%all -WARNING_AS_ERROR_KEY = Warning as error -# Supported Solaris Studio* 12.2 and above, remove ',inlasmpnu' in the line below to build by compiler prior Solaris Studio* 12.2 -WARNING_SUPPRESS = -erroff=unassigned,attrskipunsup,badargtype2w,badbinaryopw,wbadasg,wvarhidemem,inlasmpnu -tbb_strict=0 - -# RcppParallel patch: -library=stlport4 -CPLUS = CC -library=stlport4 -CONLY = cc - -OPENMP_FLAG = -xopenmp -LIB_LINK_FLAGS = -G -R . -M$(tbb_root)/build/suncc.map.pause -LINK_FLAGS += -M$(tbb_root)/build/suncc.map.pause -LIBS = -lpthread -lrt -R . -C_FLAGS = $(CPLUS_FLAGS) - -#TODO: the $(stdlib) instead of hard-wiring STLPort -ifeq ($(cfg), release) - CPLUS_FLAGS = -mt -xO2 -g -library=stlport4 -DUSE_PTHREAD $(WARNING_SUPPRESS) -endif -ifeq ($(cfg), debug) - CPLUS_FLAGS = -mt -DTBB_USE_DEBUG -g -library=stlport4 -DUSE_PTHREAD $(WARNING_SUPPRESS) -endif - -ASM= -ASM_FLAGS= - -TBB_ASM.OBJ= - -ifeq (intel64,$(arch)) - CPLUS_FLAGS += -m64 - ASM_FLAGS += -m64 - LIB_LINK_FLAGS += -m64 -endif - -ifeq (ia32,$(arch)) - CPLUS_FLAGS += -m32 - LIB_LINK_FLAGS += -m32 -endif - -# TODO: verify whether -m64 implies V9 on relevant Sun Studio versions -# (those that handle gcc assembler syntax) -ifeq (sparc,$(arch)) - CPLUS_FLAGS += -m64 - LIB_LINK_FLAGS += -m64 -endif - -export TBB_CUSTOM_VARS_SH=export CXXFLAGS="-I$${TBBROOT}/include -library=stlport4 $(CXXFLAGS) -M$${TBBROOT}/build/suncc.map.pause" -export TBB_CUSTOM_VARS_CSH=setenv CXXFLAGS "-I$${TBBROOT}/include -library=stlport4 $(CXXFLAGS) -M$${TBBROOT}/build/suncc.map.pause" - -#------------------------------------------------------------------------------ -# Setting assembler data. -#------------------------------------------------------------------------------ -ASSEMBLY_SOURCE=$(arch)-fbe -#------------------------------------------------------------------------------ -# End of setting assembler data. -#------------------------------------------------------------------------------ - -#------------------------------------------------------------------------------ -# Setting tbbmalloc data. -#------------------------------------------------------------------------------ -M_INCLUDES = $(INCLUDES) -I$(MALLOC_ROOT) -I$(MALLOC_SOURCE_ROOT) -M_CPLUS_FLAGS = $(CPLUS_FLAGS) -#------------------------------------------------------------------------------ -# End of setting tbbmalloc data. -#------------------------------------------------------------------------------ diff --git a/src/tbb/build/android.clang.inc b/src/tbb/build/android.clang.inc deleted file mode 100644 index a935968c9..000000000 --- a/src/tbb/build/android.clang.inc +++ /dev/null @@ -1,118 +0,0 @@ -# Copyright (c) 2005-2017 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -# -# - - -COMPILE_ONLY = -c -MMD -PREPROC_ONLY = -E -x c++ -INCLUDE_KEY = -I -DEFINE_KEY = -D -OUTPUT_KEY = -o # -OUTPUTOBJ_KEY = -o # -PIC_KEY = -fPIC -WARNING_AS_ERROR_KEY = -Werror -WARNING_KEY = -Wall -SDL_FLAGS = -fPIE -fPIC -fstack-protector -Wformat -Wformat-security -TEST_WARNING_KEY = -Wshadow -Wcast-qual -Woverloaded-virtual -Wnon-virtual-dtor -Wextra - -WARNING_SUPPRESS = -Wno-parentheses -Wno-non-virtual-dtor -DYLIB_KEY = -shared -EXPORT_KEY = -Wl,--version-script, -LIBDL = -ldl - -CPLUS = $(TARGET_CXX) -CONLY = $(TARGET_CC) - -# -soname is necessary for proper linkage to TBB prebuilt libraries when building application with Android SDK -LIB_LINK_FLAGS = $(DYLIB_KEY) -Wl,-soname=$(BUILDING_LIBRARY) -z relro -z now - -# pie is necessary for test executables to work and might be removed if newer NDK will add it implicitly -PIE_FLAG = -pie -ifeq ($(APP_PIE), false) - PIE_FLAG= -endif - -LINK_FLAGS = -Wl,-rpath-link=. -rdynamic -C_FLAGS = $(CPLUS_FLAGS) - -ifeq ($(cfg), release) - SDL_FLAGS += -D_FORTIFY_SOURCE=2 - CPLUS_FLAGS = -O2 -endif -ifeq ($(cfg), debug) - CPLUS_FLAGS = -g -O0 $(DEFINE_KEY)TBB_USE_DEBUG -endif - -CPLUS_FLAGS += $(DEFINE_KEY)USE_PTHREAD $(DEFINE_KEY)_GLIBCXX_HAVE_FENV_H - -ifneq (,$(findstring $(arch),ia32 intel64)) - CPLUS_FLAGS += $(DEFINE_KEY)DO_ITT_NOTIFY -endif - -ifeq (0, $(dynamic_load)) - CPLUS_FLAGS += $(DEFINE_KEY)__TBB_DYNAMIC_LOAD_ENABLED=0 -endif - -# Paths to the NDK prebuilt tools and libraries -CPLUS_FLAGS += --sysroot=$(SYSROOT) -LIB_LINK_FLAGS += --sysroot=$(SYSROOT) -LIBS = -L$(CPLUS_LIB_PATH) -lc++_shared -ifeq (,$(findstring $(ndk_version),$(foreach v, 7 8 9 10 11,r$(v) r$(v)b r$(v)c r$(v)d r$(v)e))) - LIBS += -lc++abi - ifeq (arm,$(arch)) - LIBS += -lunwind - endif -endif - -ifeq (arm,$(arch)) - CPLUS_FLAGS += $(DEFINE_KEY)__TBB_64BIT_ATOMICS=0 -endif - -CPLUS_FLAGS += $(TARGET_CFLAGS) -LIB_LINK_FLAGS += $(TARGET_CFLAGS) $(TARGET_LDFLAGS) -L$(CPLUS_LIB_PATH) - -#------------------------------------------------------------------------------ -# Setting assembler data. -#------------------------------------------------------------------------------ -TBB_ASM.OBJ= -MALLOC_ASM.OBJ= - -ASM = $(tbb_tool_prefix)as -ifeq (intel64,$(arch)) - ASM_FLAGS += --64 -endif -ifeq (ia32,$(arch)) - ASM_FLAGS += --32 -endif -ifeq ($(cfg),debug) - ASM_FLAGS += -g -endif - -ASSEMBLY_SOURCE=$(arch)-gas -#------------------------------------------------------------------------------ -# End of setting assembler data. -#------------------------------------------------------------------------------ - -#------------------------------------------------------------------------------ -# Setting tbbmalloc data. -#------------------------------------------------------------------------------ - -M_CPLUS_FLAGS = $(CPLUS_FLAGS) -fno-rtti -fno-exceptions - -#------------------------------------------------------------------------------ -# End of setting tbbmalloc data. -#------------------------------------------------------------------------------ diff --git a/src/tbb/build/android.gcc.inc b/src/tbb/build/android.gcc.inc deleted file mode 100644 index 973ca3f0a..000000000 --- a/src/tbb/build/android.gcc.inc +++ /dev/null @@ -1,117 +0,0 @@ -# Copyright (c) 2005-2017 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -# -# - - -COMPILE_ONLY = -c -MMD -PREPROC_ONLY = -E -x c++ -INCLUDE_KEY = -I -DEFINE_KEY = -D -OUTPUT_KEY = -o # -OUTPUTOBJ_KEY = -o # -PIC_KEY = -fPIC -WARNING_AS_ERROR_KEY = -Werror -WARNING_KEY = -Wall -TEST_WARNING_KEY = -Wshadow -Wcast-qual -Woverloaded-virtual -Wnon-virtual-dtor -Wextra - -WARNING_SUPPRESS = -Wno-parentheses -Wno-non-virtual-dtor -DYLIB_KEY = -shared -EXPORT_KEY = -Wl,--version-script, -LIBDL = -ldl - -CPLUS = $(tbb_tool_prefix)g++ -CONLY = $(tbb_tool_prefix)gcc - -# -soname is necessary for proper linkage to TBB prebuilt libraries when building application with Android SDK -LIB_LINK_FLAGS = $(DYLIB_KEY) -Wl,-soname=$(BUILDING_LIBRARY) - -# pie is necessary for test executables to work and might be removed if newer NDK will add it implicitly -PIE_FLAG = -pie -ifeq ($(APP_PIE), false) - PIE_FLAG= -endif - -LINK_FLAGS = -Wl,-rpath-link=. -rdynamic -C_FLAGS = $(CPLUS_FLAGS) - -ifeq ($(cfg), release) - CPLUS_FLAGS = -O2 -endif -ifeq ($(cfg), debug) - CPLUS_FLAGS = -g -O0 $(DEFINE_KEY)TBB_USE_DEBUG -endif - -CPLUS_FLAGS += $(DEFINE_KEY)USE_PTHREAD $(DEFINE_KEY)_GLIBCXX_HAVE_FENV_H - -ifneq (,$(findstring $(arch),ia32 intel64)) - CPLUS_FLAGS += $(DEFINE_KEY)DO_ITT_NOTIFY -endif - -ifeq (0, $(dynamic_load)) - CPLUS_FLAGS += $(DEFINE_KEY)__TBB_DYNAMIC_LOAD_ENABLED=0 -endif - - -# Paths to the NDK prebuilt tools and libraries -CPLUS_FLAGS += --sysroot=$(SYSROOT) -LIB_LINK_FLAGS += --sysroot=$(SYSROOT) -LIBS = -L$(CPLUS_LIB_PATH) -lgnustl_shared - -ifeq (ia32,$(arch)) - # TODO: Determine best setting of -march and add to CPLUS_FLAGS - CPLUS_FLAGS += -m32 - LIB_LINK_FLAGS += -m32 -else ifeq (intel64,$(arch)) - CPLUS_FLAGS += -m64 - LIB_LINK_FLAGS += -m64 -else ifeq (arm,$(arch)) - CPLUS_FLAGS += -march=armv7-a $(DEFINE_KEY)TBB_USE_GCC_BUILTINS=1 $(DEFINE_KEY)__TBB_64BIT_ATOMICS=0 -else ifeq (arm64,$(arch)) - CPLUS_FLAGS += -march=armv8-a -endif - -#------------------------------------------------------------------------------ -# Setting assembler data. -#------------------------------------------------------------------------------ -TBB_ASM.OBJ= -MALLOC_ASM.OBJ= - -ASM = $(tbb_tool_prefix)as -ifeq (intel64,$(arch)) - ASM_FLAGS += --64 -endif -ifeq (ia32,$(arch)) - ASM_FLAGS += --32 -endif -ifeq ($(cfg),debug) - ASM_FLAGS += -g -endif - -ASSEMBLY_SOURCE=$(arch)-gas -#------------------------------------------------------------------------------ -# End of setting assembler data. -#------------------------------------------------------------------------------ - -#------------------------------------------------------------------------------ -# Setting tbbmalloc data. -#------------------------------------------------------------------------------ - -M_CPLUS_FLAGS = $(CPLUS_FLAGS) -fno-rtti -fno-exceptions - -#------------------------------------------------------------------------------ -# End of setting tbbmalloc data. -#------------------------------------------------------------------------------ diff --git a/src/tbb/build/android.icc.inc b/src/tbb/build/android.icc.inc deleted file mode 100644 index 64da9fd16..000000000 --- a/src/tbb/build/android.icc.inc +++ /dev/null @@ -1,120 +0,0 @@ -# Copyright (c) 2005-2017 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -# -# - - -COMPILE_ONLY = -c -MMD -PREPROC_ONLY = -E -x c++ -INCLUDE_KEY = -I -DEFINE_KEY = -D -OUTPUT_KEY = -o # -OUTPUTOBJ_KEY = -o # -PIC_KEY = -fPIC -WARNING_AS_ERROR_KEY = -Werror -WARNING_KEY = -TEST_WARNING_KEY = -Wshadow -Woverloaded-virtual -Wextra - -WARNING_SUPPRESS = -Wno-parentheses -Wno-non-virtual-dtor -DYLIB_KEY = -shared -EXPORT_KEY = -Wl,--version-script, -LIBDL = -ldl - -CPLUS = icpc -CONLY = icc - -# -soname is necessary for proper linkage to TBB prebuilt libraries when building application with Android SDK -LIB_LINK_FLAGS = $(DYLIB_KEY) -Wl,-soname=$(BUILDING_LIBRARY) - -# pie is necessary for test executables to work and might be removed if newer NDK will add it implicitly -PIE_FLAG = -pie -ifeq ($(APP_PIE), false) - PIE_FLAG= -endif - -LINK_FLAGS = -Wl,-rpath-link=. -rdynamic -C_FLAGS = $(CPLUS_FLAGS) - -ifeq ($(cfg), release) - CPLUS_FLAGS = -O2 -endif -ifeq ($(cfg), debug) - CPLUS_FLAGS = -g -O0 $(DEFINE_KEY)TBB_USE_DEBUG -endif - -CPLUS_FLAGS += $(DEFINE_KEY)USE_PTHREAD $(DEFINE_KEY)_GLIBCXX_HAVE_FENV_H - -ifneq (,$(findstring $(arch),ia32 intel64)) - CPLUS_FLAGS += $(DEFINE_KEY)DO_ITT_NOTIFY -endif - -ifeq (0, $(dynamic_load)) - CPLUS_FLAGS += $(DEFINE_KEY)__TBB_DYNAMIC_LOAD_ENABLED=0 -endif - - -# Paths to the NDK prebuilt tools and libraries -CPLUS_FLAGS += --sysroot=$(SYSROOT) -LIB_LINK_FLAGS += --sysroot=$(SYSROOT) -# the -static-intel flag is to remove the need to copy Intel-specific libs to the device. -LIBS = -L$(CPLUS_LIB_PATH) -lgnustl_shared -static-intel - -ifeq (ia32,$(arch)) - # TODO: Determine best setting of -march and add to CPLUS_FLAGS - CPLUS_FLAGS += -m32 -march=pentium4 -falign-stack=maintain-16-byte - LIB_LINK_FLAGS += -m32 -else - ifeq (intel64,$(arch)) - CPLUS_FLAGS += -m64 - LIB_LINK_FLAGS += -m64 - endif -endif - -ifeq (arm,$(findstring arm,$(arch))) - $(error "Unsupported architecture $(arch) for icc compiler") -endif - -#------------------------------------------------------------------------------ -# Setting assembler data. -#------------------------------------------------------------------------------ -TBB_ASM.OBJ= -MALLOC_ASM.OBJ= - -ASM = $(tbb_tool_prefix)as -ifeq (intel64,$(arch)) - ASM_FLAGS += --64 -endif -ifeq (ia32,$(arch)) - ASM_FLAGS += --32 -endif -ifeq ($(cfg),debug) - ASM_FLAGS += -g -endif - -ASSEMBLY_SOURCE=$(arch)-gas -#------------------------------------------------------------------------------ -# End of setting assembler data. -#------------------------------------------------------------------------------ - -#------------------------------------------------------------------------------ -# Setting tbbmalloc data. -#------------------------------------------------------------------------------ - -M_CPLUS_FLAGS = $(CPLUS_FLAGS) -fno-rtti -fno-exceptions - -#------------------------------------------------------------------------------ -# End of setting tbbmalloc data. -#------------------------------------------------------------------------------ diff --git a/src/tbb/build/android.inc b/src/tbb/build/android.inc deleted file mode 100644 index aeeb68c5f..000000000 --- a/src/tbb/build/android.inc +++ /dev/null @@ -1,63 +0,0 @@ -# Copyright (c) 2005-2017 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -# -# - -# -# Extra gmake command-line parameters for use with Android: -# -# dlopen_workaround: Some OS versions need workaround for dlopen to avoid recursive calls. -# - -####### Detections and Commands ############################################### - -ifeq (android,$(findstring android,$(tbb_os))) - $(error TBB only supports cross-compilation for Android. Specify "target=android" instead.) -endif - -ifndef BUILDING_PHASE - ifneq ("command line","$(origin arch)") - ifeq (icc,$(compiler)) - export COMPILER_VERSION := ICC: $(shell icc -V &1 | grep 'Version') - ifneq (,$(findstring running on IA-32, $(COMPILER_VERSION))) - export arch:=ia32 - else ifneq (,$(findstring running on Intel(R) 64, $(COMPILER_VERSION))) - export arch:=intel64 - else - $(error "No support for Android in $(COMPILER_VERSION)") - endif - - else - ifdef ANDROID_SERIAL - uname_m:=$(shell adb shell uname -m) - ifeq (i686,$(uname_m)) - export arch:=ia32 - else - export arch:=$(uname_m) - endif - endif - endif - endif -endif - -ifeq ("$(arch)","") - $(error "No target architecture specified and \'ANDROID_SERIAL\' environment variable specifying target device not set") -endif - -# Android platform only supported from TBB 4.1 forward -NO_LEGACY_TESTS = 1 - - diff --git a/src/tbb/build/android.linux.inc b/src/tbb/build/android.linux.inc deleted file mode 100644 index 7d022434a..000000000 --- a/src/tbb/build/android.linux.inc +++ /dev/null @@ -1,67 +0,0 @@ -# Copyright (c) 2005-2017 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -# -# - -####### Detections and Commands ############################################### - -# Must set def_prefix according to target architecture detected above -ifeq (ia32,$(arch)) - def_prefix = lin32 -endif -ifeq (arm,$(findstring arm,$(arch))) - def_prefix = lin32 -endif -ifeq (64,$(findstring 64,$(arch))) - def_prefix = lin64 -endif - -ifdef ndk_version - $(warning "NDK version $(ndk_version)") -else - $(warning "NDK version not set in environment, using \'unknown\' instead.") - ndk_version:=unknown -endif - -export runtime:=$(target)_NDK$(ndk_version)_version_$(target_os_version) - -AR = $(tbb_tool_prefix)ar -MAKE_VERSIONS=sh $(tbb_root)/build/version_info_android.sh $(VERSION_FLAGS) >version_string.ver - -####### Build settings ######################################################## - -# No SONAME_SUFFIX for Android allowed in library names -TBB.LST = $(tbb_root)/src/tbb/$(def_prefix)-tbb-export.lst -TBB.DEF = $(TBB.LST:.lst=.def) -TBB.DLL = libtbb$(CPF_SUFFIX)$(DEBUG_SUFFIX).$(DLL) -TBB.LIB = $(TBB.DLL) -TBB_NO_VERSION.DLL= -LINK_TBB.LIB = $(TBB.LIB) - -MALLOC.DEF = $(MALLOC_ROOT)/$(def_prefix)-tbbmalloc-export.def -MALLOC.DLL = libtbbmalloc$(DEBUG_SUFFIX).$(DLL) -MALLOC.LIB = $(MALLOC.DLL) -MALLOC_NO_VERSION.DLL= -LINK_MALLOC.LIB = $(MALLOC.LIB) - -MALLOCPROXY.DEF = $(MALLOC_ROOT)/$(def_prefix)-proxy-export.def -MALLOCPROXY.DLL = libtbbmalloc_proxy$(DEBUG_SUFFIX).$(DLL) -MALLOCPROXY_NO_VERSION.DLL= -MALLOCPROXY.LIB = $(MALLOCPROXY.DLL) -LINK_MALLOCPROXY.LIB = $(MALLOCPROXY.LIB) - -TEST_LAUNCHER= -run_cmd ?= -sh $(tbb_root)/build/android.linux.launcher.sh $(largs) diff --git a/src/tbb/build/android.linux.launcher.sh b/src/tbb/build/android.linux.launcher.sh deleted file mode 100644 index b3cf87645..000000000 --- a/src/tbb/build/android.linux.launcher.sh +++ /dev/null @@ -1,148 +0,0 @@ -#!/bin/sh -# -# Copyright (c) 2005-2017 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -# -# - -# Usage: -# android.linux.launcher.sh [-v] [-q] [-s] [-r ] [-u] [-l ] -# where: -v enables verbose output -# where: -q enables quiet mode -# where: -s runs the test in stress mode (until non-zero exit code or ctrl-c pressed) -# where: -r specifies number of times to repeat execution -# where: -u is ignored on Android -# where: -l specifies the library name to be assigned to LD_PRELOAD -# -# Libs and executable necessary for testing should be present in the current directory before running. -# ANDROID_SERIAL must be set to the connected Android target device name for file transfer and test runs. -# ANDROID_TEST_DIRECTORY may be set to the directory used for testing on the Android target device; otherwise, -# the default directory used is "/data/local/tmp/$(basename $PWD)". -# Note: Do not remove the redirections to '/dev/null' in the script, otherwise the nightly test system will fail. - -do_cleanup() # -{ # - adb pull $targetdir/events.txt events.txt > /dev/null 2>&1 # - # Remove target directory on the device - adb shell "rm -r ${targetdir}; mkdir -p ${targetdir}" > /dev/null 2>&1 # -} # -do_trap_cleanup() # -{ # - do_cleanup # - exit -1 # -} # -while getopts "qvsr:ul:" flag # -do case $flag in # - s ) # Stress testing mode - echo Doing stress testing. Press Ctrl-C to terminate - run_env='stressed() { while $*; do :; done; }; ' # - run_prefix="stressed $run_prefix" ;; # - r ) # Repeats test n times - run_env="repeated() { for i in $(seq -s ' ' 1 $OPTARG) ; do echo \$i of $OPTARG:; \$*; done; }; " # - run_prefix="repeated $run_prefix" ;; # - l ) # Additional library - ldpreload="$OPTARG " ;; # - u ) # Stack limit - ;; # - q ) # Quiet mode, removes 'done' but prepends any other output by test name - OUTPUT='2>&1 | sed -e "s/done//;/^[[:space:]]*$/d;s!^!$exename: !"' ;; # - v ) # Verbose mode - SUPPRESS='' # - verbose=1 ;; # -esac done # -shift `expr $OPTIND - 1` # -[ -z "$OUTPUT" ] && OUTPUT='| sed -e "s/\\r$//"' # -[ $verbose ] || SUPPRESS='>/dev/null' # -# Collect the executable name -exename=$(basename $1) # -shift # -# Prepare the target directory on the device -currentdir=$(basename $PWD) # -targetdir=${ANDROID_TEST_DIRECTORY:-/data/local/tmp/$currentdir} # -do_cleanup # -trap do_trap_cleanup INT # if someone hits control-c, cleanup the device -# Collect the list of files to transfer to the target device, starting with executable itself. -fnamelist="$exename" # -# Add the C++ standard library from the NDK, which is required for all tests on Android. -if [ ! -z "${LIB_STL_ANDROID}" ]; then # - fnamelist="$fnamelist ${LIB_STL_ANDROID}" # -else # - fnamelist="$fnamelist libc++_shared.so" # -fi # -# Find the TBB libraries and add them to the list. -# Add TBB libraries from the current directory that contains libtbb* files -files="$(ls libtbb* 2> /dev/null)" # -[ -z "$files" ] || fnamelist="$fnamelist $files" # -# Add any libraries built for specific tests. -exeroot=${exename%\.*} # -files="$(ls ${exeroot}*.so ${exeroot}*.so.* 2> /dev/null)" # -[ -z "$files" ] || fnamelist="$fnamelist $files" # -# TODO: Add extra libraries from the Intel(R) Compiler for certain tests -# found=$(echo $exename | egrep 'test_malloc_atexit\|test_malloc_lib_unload' 2> /dev/null) -# if [ ! -z $found ] ; then -# fnamelist="$fnamelist ${compiler_path_lib}/libimf.so \ -# ${compiler_path_lib}/libsvml.so \ -# ${compiler_path_lib}/libintlc.so.5" -# fi - -# Transfer collected executable and library files to the target device. -transfers_ok=1 # -for fullname in $fnamelist; do { # - if [ -r $fullname ]; then { # - # Transfer the executable and libraries to top-level target directory - [ $verbose ] && echo -n "Pushing $fullname: " # - eval "adb push $fullname ${targetdir}/$(basename $fullname) $SUPPRESS 2>&1" # - }; else { # - echo "Error: required file ${currentdir}/${fullname} for test $exename not available for transfer." # - transfers_ok=0 # - }; fi # -}; done # -if [ "${transfers_ok}" = "0" ]; then { # - do_cleanup # - exit -1 # -}; fi # -# Transfer input files used by example codes by scanning the executable argument list. -for fullname in "$@"; do { # - if [ -r $fullname ]; then { # - directory=$(dirname $fullname) # - filename=$(basename $fullname) # - # strip leading "." from fullname if present - if [ "$directory" = "\." ]; then { # - directory="" # - fullname=$filename # - }; fi # - # Create the target directory to hold input file if necessary - if [ ! -z $directory ]; then { # - eval "adb shell 'mkdir $directory' $SUPPRESS 2>&1" # - }; fi # - # Transfer the input file to corresponding directory on target device - [ $verbose ] && echo -n "Pushing $fullname: " # - eval "adb push $fullname ${targetdir}/$fullname $SUPPRESS 2>&1" # - }; fi # -}; done # -# Set LD_PRELOAD if necessary -[ -z "$ldpreload" ] || run_prefix="LD_PRELOAD='$ldpreload' $run_prefix" # -[ $verbose ] && echo Running $run_prefix ./$exename $* # -run_env="$run_env cd $targetdir; export LD_LIBRARY_PATH=." # -[ -z "$VIRTUAL_MACHINE" ] || run_env="$run_env; export VIRTUAL_MACHINE=$VIRTUAL_MACHINE" # -# The return_code file is the best way found to return the status of the test execution when using adb shell. -eval 'adb shell "$run_env; $run_prefix ./$exename $* || echo -n \$? >error_code"' "${OUTPUT}" # -# Capture the return code string and remove the trailing \r from the return_code file contents -err=`adb shell "cat $targetdir/error_code 2>/dev/null"` # -[ -z $err ] || echo $exename: exited with error $err # -do_cleanup # -# Return the exit code of the test. -exit $err # diff --git a/src/tbb/build/android.macos.inc b/src/tbb/build/android.macos.inc deleted file mode 100644 index bf84578a4..000000000 --- a/src/tbb/build/android.macos.inc +++ /dev/null @@ -1,76 +0,0 @@ -# Copyright (c) 2005-2017 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -# -# - -####### Detections and Commands ############################################### - -# Must set def_prefix according to target architecture detected above -ifeq (ia32,$(arch)) - def_prefix = lin32 -endif -ifeq (arm,$(findstring arm,$(arch))) - def_prefix = lin32 -endif -ifeq (64,$(findstring 64,$(arch))) - def_prefix = lin64 -endif - -ifdef ndk_version - $(warning "NDK version $(ndk_version)") -else - $(warning "NDK version not set in environment, using \'unknown\' instead.") - ndk_version:=unknown -endif - -export runtime:=$(target)_NDK$(ndk_version)_version_$(target_os_version) - -AR = $(tbb_tool_prefix)ar -MAKE_VERSIONS=sh $(tbb_root)/build/version_info_android.sh $(VERSION_FLAGS) >version_string.ver - -####### Build settings ######################################################## - -# No SONAME_SUFFIX for Android allowed in library names -TBB.LST = $(tbb_root)/src/tbb/$(def_prefix)-tbb-export.lst -TBB.DEF = $(TBB.LST:.lst=.def) -TBB.DLL = libtbb$(CPF_SUFFIX)$(DEBUG_SUFFIX).$(DLL) -TBB.LIB = $(TBB.DLL) -TBB_NO_VERSION.DLL= -LINK_TBB.LIB = $(TBB.LIB) - -MALLOC.DEF = $(MALLOC_ROOT)/$(def_prefix)-tbbmalloc-export.def -MALLOC.DLL = libtbbmalloc$(DEBUG_SUFFIX).$(DLL) -MALLOC.LIB = $(MALLOC.DLL) -MALLOC_NO_VERSION.DLL= -LINK_MALLOC.LIB = $(MALLOC.LIB) - -MALLOCPROXY.DEF = $(MALLOC_ROOT)/$(def_prefix)-proxy-export.def -MALLOCPROXY.DLL = libtbbmalloc_proxy$(DEBUG_SUFFIX).$(DLL) -MALLOCPROXY_NO_VERSION.DLL= -MALLOCPROXY.LIB = $(MALLOCPROXY.DLL) -LINK_MALLOCPROXY.LIB = $(MALLOCPROXY.LIB) - -TBB.RES = -MALLOC.RES = -RML.RES = -TBB.MANIFEST = -MALLOC.MANIFEST = -RML.MANIFEST = -OBJ = o -DLL = so - -TEST_LAUNCHER= -run_cmd ?= -sh $(tbb_root)/build/android.linux.launcher.sh $(largs) diff --git a/src/tbb/build/android.windows.inc b/src/tbb/build/android.windows.inc deleted file mode 100644 index f58e5d52f..000000000 --- a/src/tbb/build/android.windows.inc +++ /dev/null @@ -1,78 +0,0 @@ -# Copyright (c) 2005-2017 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -# -# - -####### Detections and Commands ############################################### - -# Must set def_prefix according to target architecture detected above -ifeq (ia32,$(arch)) - def_prefix = lin32 -endif -ifeq (arm,$(findstring arm,$(arch))) - def_prefix = lin32 -endif -ifeq (64,$(findstring 64,$(arch))) - def_prefix = lin64 -endif - -ifdef ndk_version - $(warning "NDK version $(ndk_version)") -else - $(warning "NDK version not set in environment, using \'unknown\' instead.") - ndk_version:=unknown -endif - -export runtime:=$(target)_NDK$(ndk_version)_version_$(target_os_version) - -AR = $(tbb_tool_prefix)ar -MAKE_VERSIONS = cmd /C cscript /nologo /E:jscript $(subst \,/,$(tbb_root))/build/version_info_windows.js $(CONLY) $(arch) $(subst \,/,"$(VERSION_FLAGS)") > version_string.ver - -####### Build settings ######################################################## - -# No SONAME_SUFFIX for Android allowed in library names -TBB.LST = $(tbb_root)/src/tbb/$(def_prefix)-tbb-export.lst -TBB.DEF = $(TBB.LST:.lst=.def) -TBB.DLL = libtbb$(CPF_SUFFIX)$(DEBUG_SUFFIX).$(DLL) -TBB.LIB = $(TBB.DLL) -TBB_NO_VERSION.DLL= -LINK_TBB.LIB = $(TBB.LIB) - -MALLOC.DEF = $(MALLOC_ROOT)/$(def_prefix)-tbbmalloc-export.def -MALLOC.DLL = libtbbmalloc$(DEBUG_SUFFIX).$(DLL) -MALLOC.LIB = $(MALLOC.DLL) -MALLOC_NO_VERSION.DLL= -LINK_MALLOC.LIB = $(MALLOC.LIB) - -MALLOCPROXY.DEF = $(MALLOC_ROOT)/$(def_prefix)-proxy-export.def -MALLOCPROXY.DLL = libtbbmalloc_proxy$(DEBUG_SUFFIX).$(DLL) -MALLOCPROXY_NO_VERSION.DLL= -MALLOCPROXY.LIB = $(MALLOCPROXY.DLL) - -TBB.RES = -MALLOC.RES = -RML.RES = -TBB.MANIFEST = -MALLOC.MANIFEST = -RML.MANIFEST = -OBJ = o -DLL = so - -TEST_LAUNCHER= -run_cmd ?= -sh $(tbb_root)/build/android.linux.launcher.sh $(largs) -export UNIXMODE = 1 -# Clang for Android* uses the INCLUDE variable (instead of CPATH) -export USE_INCLUDE_ENV = 1 diff --git a/src/tbb/build/big_iron.inc b/src/tbb/build/big_iron.inc deleted file mode 100644 index efed21253..000000000 --- a/src/tbb/build/big_iron.inc +++ /dev/null @@ -1,76 +0,0 @@ -# Copyright (c) 2005-2017 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -# -# - -#------------------------------------------------------------------------------ -# Defines settings for building the TBB run-time as a static library. -# Use these only on platforms where dynamic linking is impractical. -# -# IF YOU USE TBB AS A STATIC LIBRARY, YOU MUST GUARANTEE THAT ONLY ONE COPY OF -# THE TBB RUN-TIME IS LINKED INTO AN APPLICATION! LINKING IN MULTIPLE COPIES -# OF THE TBB RUN-TIME, DIRECTLY OR INDIRECTLY, MAY CAUSE PROGRAM FAILURE! -#------------------------------------------------------------------------------ - -# Note that ITT_NOTIFY allows to selectively remove the definition of -# DO_ITT_NOTIFY without sabotaging deferred expansion of CPLUS_FLAGS. -# TODO: currently only in linux.{gcc,xl}.inc - -# Note that -pthread with xl gives "1501-210 (W) command option t contains an incorrect subargument"; -# multithreading is instead achieved by using the _r affix in the compiler name. -# TODO: is -lpthread still relevant/needed with XL and _r affix? - -# Note that usage of dynamic (shared) libraries is disabled -# (via -D__TBB_DYNAMIC_LOAD_ENABLED=0 and LIBDL emptied) primarily for performance. - -# OS specific settings => - LIB_LINK_CMD = ar rcs - LIB_LINK_FLAGS = - LIB_LINK_LIBS = - LIB_OUTPUT_KEY = - DYLIB_KEY = - ifeq ($(tbb_os),linux) - ifeq ($(compiler),clang) - LIBS = -pthread -lrt - endif - ifeq ($(compiler),gcc) - LIBS = -pthread -lrt - endif - ifeq ($(compiler),xl) - LIBS = -lpthread -lrt - endif - LINK_FLAGS = - endif - override CXXFLAGS += -D__TBB_DYNAMIC_LOAD_ENABLED=0 -D__TBB_SOURCE_DIRECTLY_INCLUDED=1 - ITT_NOTIFY = - DLL = a - LIBEXT = a - LIBPREF = lib - LIBDL = -# <= OS specific settings - -TBB.DLL = $(LIBPREF)tbb$(DEBUG_SUFFIX).$(LIBEXT) -LINK_TBB.LIB = $(TBB.DLL) -TBB.LST = -TBB.DEF = -TBB_NO_VERSION.DLL = - -MALLOC.DLL = $(LIBPREF)tbbmalloc$(DEBUG_SUFFIX).$(LIBEXT) -LINK_MALLOC.LIB = $(MALLOC.DLL) -MALLOC.DEF = -MALLOC_NO_VERSION.DLL = -MALLOCPROXY.DLL = -MALLOCPROXY.DEF = diff --git a/src/tbb/build/codecov.txt b/src/tbb/build/codecov.txt deleted file mode 100644 index e22f8059a..000000000 --- a/src/tbb/build/codecov.txt +++ /dev/null @@ -1,7 +0,0 @@ -src/tbb -src/tbbmalloc -include/tbb -src/rml/server -src/rml/client -src/rml/include -source/malloc diff --git a/src/tbb/build/common.inc b/src/tbb/build/common.inc deleted file mode 100644 index b2c20de69..000000000 --- a/src/tbb/build/common.inc +++ /dev/null @@ -1,166 +0,0 @@ -# Copyright (c) 2005-2017 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -# -# - -ifndef tbb_os - - # Windows sets environment variable OS; for other systems, ask uname - ifeq ($(OS),) - OS:=$(shell uname) - ifeq ($(OS),) - $(error "Cannot detect operating system") - endif - export tbb_os=$(OS) - endif - - ifeq ($(OS), Windows_NT) - export tbb_os=windows - endif - ifeq ($(OS), Linux) - export tbb_os=linux - endif - ifeq ($(OS), Darwin) - export tbb_os=macos - endif - -endif # !tbb_os - -ifeq (1,$(tbb_cpf)) - export CPF_SUFFIX ?=_preview -endif - -ifeq (0,$(exceptions)) -# Inverse the value, for simplicity of use - export no_exceptions=1 -endif - -ifdef cpp0x - $(warning "Warning: deprecated cpp0x=$(cpp0x) is used, stdver must be used instead. Building in stdver=c++0x mode.") - export stdver?=c++0x - override cpp0x= -endif - -ifneq (,$(stdver)) - ifeq (,$(findstring ++, $(stdver))) - $(warning "Warning: unexpected stdver=$(stdver) is used.") - endif - CXX_STD_FLAGS=-std=$(stdver) -endif - -# The requested option is added unconditionally. -# If it is not supported, a compiler warning or error is expected. -# Note that CXX_STD_FLAGS can be changed in ..inc. -CXX_ONLY_FLAGS+=$(CXX_STD_FLAGS) - -ifeq (,$(wildcard $(tbb_root)/build/$(tbb_os).inc)) - $(error "$(tbb_os)" is not supported. Add build/$(tbb_os).inc file with os-specific settings ) -endif - -# detect arch and runtime versions, provide common host-specific definitions -include $(tbb_root)/build/$(tbb_os).inc - -ifeq ($(arch),) - $(error Architecture not detected) -endif -ifeq ($(runtime),) - $(error Runtime version not detected) -endif - -# process target-dependent compilation and testing configurations -ifdef target - # optionally process target-dependent options for compilation and testing - ifneq (,$(wildcard $(tbb_root)/build/$(target).inc)) - include $(tbb_root)/build/$(target).inc - endif - - # optionally process host-dependent environment for target-dependent compilation and testing - ifneq (,$(wildcard $(tbb_root)/build/$(target).$(tbb_os).inc)) - include $(tbb_root)/build/$(target).$(tbb_os).inc - endif - - # insure at least one target-dependent configuration file was found for compilation and testing - ifeq (,$(wildcard $(tbb_root)/build/$(target).inc)$(wildcard $(tbb_root)/build/$(target).$(tbb_os).inc)) - $(error "$(target)" is not supported. Add build/$(target).inc or build/$(target).$(tbb_os).inc file) - endif -endif #target - -# Support for running debug tests to release library and vice versa -flip_cfg=$(subst _flipcfg,_release,$(subst _release,_debug,$(subst _debug,_flipcfg,$(1)))) -cross_cfg = $(if $(crosstest),$(call flip_cfg,$(1)),$(1)) -# Setting default configuration to release -cfg?=release - -compiler_name=$(notdir $(compiler)) -ifdef BUILDING_PHASE - ifndef target - target:=$(tbb_os) - endif - # process host/target compiler-dependent build configuration - ifeq (,$(wildcard $(tbb_root)/build/$(target).$(compiler_name).inc)) - $(error "$(compiler_name)" is not supported on $(target). Add build/$(target).$(compiler_name).inc file with compiler-specific settings. ) - endif - include $(tbb_root)/build/$(target).$(compiler_name).inc -endif - -ifneq ($(BUILDING_PHASE),1) - # definitions for top-level Makefiles - origin_build_dir:=$(origin tbb_build_dir) - tbb_build_dir?=$(tbb_root)$(SLASH)build - export tbb_build_prefix?=$(tbb_os)_$(arch)_$(compiler_name)_$(runtime)$(CPF_SUFFIX) - work_dir=$(tbb_build_dir)$(SLASH)$(tbb_build_prefix) -endif # BUILDING_PHASE != 1 - -ifdef offload - extra_inc=$(offload).offload.inc -endif -ifdef extra_inc - ifneq (,$(wildcard $(tbb_root)/build/$(extra_inc))) - include $(tbb_root)/build/$(extra_inc) - else - $(error specified build file: "build/$(extra_inc)" is not found. ) - endif -endif - -ifndef BUILDING_PHASE - work_dir:=$(work_dir) - # assign new value for tbb_root if path is not absolute (the filter keeps only /* paths) - ifeq ($(filter /% $(SLASH)%, $(subst :, ,$(tbb_root)) ),) - full_tbb_root:=$(CURDIR)/$(tbb_root) - ifeq ($(origin_build_dir),undefined) - #relative path are needed here as a workaround to support whitespaces in path - override tbb_root:=../.. - else - override tbb_root:=$(full_tbb_root) - endif - export tbb_root - endif - endif # !BUILDING_PHASE - -.DELETE_ON_ERROR: # Make will delete target if error occurred when building it. - -# MAKEOVERRIDES contains the command line variable definitions. Reseting it to -# empty allows propogating all exported overridden variables to nested makes. -# NOTEs: -# 1. All variable set in command line are propagated to nested makes. -# 2. All variables declared with the "export" keyword are propagated to -# nested makes. -# 3. "override" allows changing variables set in command line. But it doesn't -# propagate new values to nested makes. For propagation, the "export" keyword -# should be used. -# 4. gmake v3.80 doesn't support exporting of target-specific variables using -# the "export" keyword -MAKEOVERRIDES = diff --git a/src/tbb/build/common_rules.inc b/src/tbb/build/common_rules.inc deleted file mode 100644 index faf6546f3..000000000 --- a/src/tbb/build/common_rules.inc +++ /dev/null @@ -1,174 +0,0 @@ -# Copyright (c) 2005-2017 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -# -# - - -ifeq ($(tbb_strict),1) - ifeq ($(WARNING_AS_ERROR_KEY),) - $(error WARNING_AS_ERROR_KEY is empty) - endif - # Do not remove line below! - WARNING_KEY += $(WARNING_AS_ERROR_KEY) -endif - -ifneq (,$(findstring s,$(MAKEFLAGS))) - override largs+=-q -endif -ifneq (,$(repeat)) - override largs+=-r $(repeat) -endif -ifneq (,$(largs)$(run_prefix)) - override run_cmd:=$(run_cmd) $(TEST_LAUNCHER) - TEST_LAUNCHER= - ifeq (,$(strip $(run_cmd))) - $(warning Test launcher is not defined for the platform, ignoring launcher arguments) - endif -endif - -ifndef TEST_EXT - TEST_EXT = exe -endif - -INCLUDES += $(INCLUDE_KEY)$(tbb_root)/src $(INCLUDE_KEY)$(tbb_root)/src/rml/include $(INCLUDE_KEY)$(tbb_root)/include - -CPLUS_FLAGS += $(WARNING_KEY) $(CXXFLAGS) -ifeq (1,$(tbb_cpf)) -CPLUS_FLAGS += $(DEFINE_KEY)__TBB_CPF_BUILD=1 -endif -ifeq (0,$(exceptions)) -CPLUS_FLAGS += $(DEFINE_KEY)TBB_USE_EXCEPTIONS=0 -endif -LINK_FLAGS += $(LDFLAGS) -LIB_LINK_FLAGS += $(LDFLAGS) - -LIB_LINK_CMD ?= $(CPLUS) $(PIC_KEY) -ifeq ($(origin LIB_OUTPUT_KEY), undefined) - LIB_OUTPUT_KEY = $(OUTPUT_KEY) -endif -ifeq ($(origin LIB_LINK_LIBS), undefined) - LIB_LINK_LIBS = $(LIBDL) $(LIBS) -endif - -CONLY ?= $(CPLUS) - -# The most generic rules -#$(1) - is the target pattern -define make-cxx-obj -$1: %.cpp - $$(CPLUS) $$(OUTPUTOBJ_KEY)$$@ $$(COMPILE_ONLY) $$(CPLUS_FLAGS) $$(CXX_ONLY_FLAGS) $$(CXX_WARN_SUPPRESS) $$(INCLUDES) $$< -endef - -TEST_AFFIXES_OBJS=$(addsuffix .$(OBJ),$(addprefix %_,$(TEST_SUFFIXES)) $(addsuffix _%,$(TEST_PREFIXES))) - -# Make will not process the same recipe for each test pattern (since the dependency on the same %.cpp) -# thus the separated recipes should be provided -$(foreach t,%.$(OBJ) $(TEST_AFFIXES_OBJS),$(eval $(call make-cxx-obj,$(t)))) - -.PRECIOUS: %.$(OBJ) %.$(TEST_EXT) %.res $(TEST_AFFIXES_OBJS) - -# Rules for generating a test DLL -%_dll.$(OBJ): %.cpp - $(CPLUS) $(COMPILE_ONLY) $(OUTPUTOBJ_KEY)$@ $(CPLUS_FLAGS) $(PIC_KEY) $(DEFINE_KEY)_USRDLL $(INCLUDES) $< - -#$(1) - is the binary name -#$(2) - is the input obj files and libraries -define make-test-binary - $(CPLUS) $(OUTPUT_KEY)$(strip $1) $(CPLUS_FLAGS) $(2) $(LIBS) $(LINK_FLAGS) -endef - -# LINK_FILES the list of options to link test specific files (libraries and object files) -LINK_FILES+=$(TEST_LIBS) -# Rule for generating executable test -%.$(TEST_EXT): %.$(OBJ) $(TEST_LIBS) $(TEST_PREREQUISITE) $(if $(use_proxy),$(PROXY.LIB)) - $(call make-test-binary,$@,$< $(LINK_FILES) $(PIE_FLAG)) - -# Rules for generating a test DLL -%_dll.$(DLL): LINK_FLAGS += $(PIC_KEY) $(DYLIB_KEY) -%_dll.$(DLL): TEST_LIBS := $(subst %_dll.$(DLL),,$(TEST_LIBS)) -%_dll.$(DLL): %_dll.$(OBJ) - $(call make-test-binary,$@,$< $(subst %_dll.$(DLL),,$(LINK_FILES))) -.PRECIOUS: %_dll.$(OBJ) %_dll.$(DLL) - -%.$(OBJ): %.c - $(CONLY) $(COMPILE_ONLY) $(OUTPUTOBJ_KEY)$@ $(C_FLAGS) $(INCLUDES) $< - -%.$(OBJ): %.asm - $(ASM) $(ASM_FLAGS) $< - -%.$(OBJ): %.s - cpp <$< | grep -v '^#' >$*.tmp - $(ASM) $(ASM_FLAGS) -o $@ $*.tmp - -ifdef rtools -# Line 70 doesn't work with rtool's version of make. The symptom being that the asm rule kicks off instead, and these rules are cl only -%.$(OBJ): %.cpp - $(CPLUS) $(OUTPUTOBJ_KEY)$@ $(COMPILE_ONLY) $(CPLUS_FLAGS) $(CXX_ONLY_FLAGS) $(CXX_WARN_SUPPRESS) $(INCLUDES) $< -endif - -# Rule for generating .E file if needed for visual inspection -# Note that ICL treats an argument after PREPROC_ONLY as a file to open, -# so all uses of PREPROC_ONLY should be immediately followed by a file name -%.E: %.cpp - $(CPLUS) $(CPLUS_FLAGS) $(CXX_ONLY_FLAGS) $(INCLUDES) $(PREPROC_ONLY) $< >$@ - -# TODO Rule for generating .asm file if needed for visual inspection -%.asm: %.cpp - $(CPLUS) /c /FAs /Fa $(CPLUS_FLAGS) $(CXX_ONLY_FLAGS) $(INCLUDES) $< - -# TODO Rule for generating .s file if needed for visual inspection -%.s: %.cpp - $(CPLUS) -S $(CPLUS_FLAGS) $(CXX_ONLY_FLAGS) $(INCLUDES) $< - -# Customizations -$(KNOWN_WARNINGS): %.$(OBJ): %.cpp - $(CPLUS) $(COMPILE_ONLY) $(subst $(WARNING_KEY),,$(CPLUS_FLAGS)) $(CXX_ONLY_FLAGS) $(CXX_WARN_SUPPRESS) $(INCLUDES) $< - -tbb_misc.$(OBJ): version_string.ver -tbb_misc.$(OBJ): INCLUDES+=$(INCLUDE_KEY). - -tbb_misc.E: tbb_misc.cpp version_string.ver - $(CPLUS) $(CPLUS_FLAGS) $(CXX_ONLY_FLAGS) $(INCLUDE_KEY). $(INCLUDES) $(PREPROC_ONLY) $< >$@ - -%.res: %.rc version_string.ver $(TBB.MANIFEST) - rc /Fo$@ $(INCLUDES) $(filter /D%,$(CPLUS_FLAGS)) $< - -# TODO: add $(LIB_LINK_LIBS) $(LIB_LINK_FLAGS) (in a separate line?) and remove useless $(INCLUDES) -VERSION_FLAGS=$(CPLUS) $(CPLUS_FLAGS) $(CXX_ONLY_FLAGS) $(INCLUDES) - -ifneq (,$(TBB.MANIFEST)) -$(TBB.MANIFEST): - cmd /C "echo #include ^ >tbbmanifest.c" - cmd /C "echo int main(){return 0;} >>tbbmanifest.c" - cl /nologo $(C_FLAGS) tbbmanifest.c - -version_string.ver: $(TBB.MANIFEST) - $(MAKE_VERSIONS) - cmd /C "echo #define TBB_MANIFEST 1 >> version_string.ver" -# TODO: fix parallel build by writing to a temporary file and rename it when complete -else -# TODO: make version strings directly representative for all the libraries -version_string.ver: - $(MAKE_VERSIONS) -endif - -test_% debug_%: test_%.$(TEST_EXT) $(TEST_PREREQUISITE) - $(run_cmd) ./$< $(args) -ifneq (,$(codecov)) - profmerge - codecov $(if $(findstring -,$(codecov)),$(codecov),) -demang -comp $(tbb_root)/build/codecov.txt -endif - diff --git a/src/tbb/build/detect.js b/src/tbb/build/detect.js deleted file mode 100644 index b2a0943bc..000000000 --- a/src/tbb/build/detect.js +++ /dev/null @@ -1,199 +0,0 @@ -// Copyright (c) 2005-2017 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// -// -// - -function readAllFromFile(fname) { - var fso = new ActiveXObject("Scripting.FileSystemObject"); - var file = null; - try { - file = fso.OpenTextFile(fname, 1, 0); - return (file.readAll()); - } finally { - // Close the file in the finally section to guarantee that it will be closed in any case - // (if the exception is thrown or not). - file.Close(); - } -} - -function doWork() { - var WshShell = WScript.CreateObject("WScript.Shell"); - - var tmpExec = WshShell.Run("cmd /c echo int main(){return 0;} >detect.c", 0, true); - - // The next block deals with GCC (MinGW) - if (WScript.Arguments.Count() > 1) { - var compilerPath = WScript.Arguments(1); - // The RegExp matches everything up to and including the last slash (it uses a greedy approach.) - var compilerName = compilerPath.replace(/^.*[\/\\]/, ""); - if (compilerName.match(/gcc/i) != null) { - if (WScript.Arguments(0) == "/arch") { - // Get predefined macros - tmpExec = WshShell.Run("cmd /C " + compilerPath + " -dM -E detect.c > detect.map", 0, true); - var defs = readAllFromFile("detect.map"); - //detect target architecture - var intel64 = /x86_64|amd64/mgi; - var ia32 = /i386/mgi; - if (defs.match(intel64)) { - WScript.Echo("intel64"); - } else if (defs.match(ia32)) { - WScript.Echo("ia32"); - } else { - WScript.Echo("unknown"); - } - } else { - tmpExec = WshShell.Exec(compilerPath + " -dumpversion"); - var gccVersion = tmpExec.StdOut.ReadLine(); - if (WScript.Arguments(0) == "/runtime") { - WScript.Echo("mingw" + gccVersion); - } - else if (WScript.Arguments(0) == "/minversion") { - // Comparing strings, not numbers; will not work for two-digit versions - if (gccVersion >= WScript.Arguments(2)) { - WScript.Echo("ok"); - } else { - WScript.Echo("fail"); - } - } - } - return; - } - } - - //Compile binary - tmpExec = WshShell.Exec("cl /MD detect.c /link /MAP"); - while (tmpExec.Status == 0) { - WScript.Sleep(100); - } - //compiler banner that includes version and target arch was printed to stderr - var clVersion = tmpExec.StdErr.ReadAll(); - - if (WScript.Arguments(0) == "/arch") { - //detect target architecture - var intel64 = /AMD64|EM64T|x64/mgi; - var ia32 = /[80|\s]x86/mgi; - var arm = /ARM/mgi; - if (clVersion.match(intel64)) { - WScript.Echo("intel64"); - } else if (clVersion.match(ia32)) { - WScript.Echo("ia32"); - } else if (clVersion.match(arm)) { - WScript.Echo("armv7"); - } else { - WScript.Echo("unknown"); - } - return; - } - - if (WScript.Arguments(0) == "/runtime") { - //read map-file - var mapContext = readAllFromFile("detect.map"); - //detect runtime - var vc71 = /MSVCR71\.DLL/mgi; - var vc80 = /MSVCR80\.DLL/mgi; - var vc90 = /MSVCR90\.DLL/mgi; - var vc100 = /MSVCR100\.DLL/mgi; - var vc110 = /MSVCR110\.DLL/mgi; - var vc120 = /MSVCR120\.DLL/mgi; - var vc140 = /VCRUNTIME140\.DLL/mgi; - var psdk = /MSVCRT\.DLL/mgi; - if (mapContext.match(vc71)) { - WScript.Echo("vc7.1"); - } else if (mapContext.match(vc80)) { - WScript.Echo("vc8"); - } else if (mapContext.match(vc90)) { - WScript.Echo("vc9"); - } else if (mapContext.match(vc100)) { - WScript.Echo("vc10"); - } else if (mapContext.match(vc110)) { - WScript.Echo("vc11"); - } else if (mapContext.match(vc120)) { - WScript.Echo("vc12"); - } else if (mapContext.match(vc140)) { - if (WshShell.ExpandEnvironmentStrings("%VisualStudioVersion%") == "15.0") - WScript.Echo("vc14.1"); - else - WScript.Echo("vc14"); - } else { - WScript.Echo("unknown"); - } - return; - } - - if (WScript.Arguments(0) == "/minversion") { - var compilerVersion; - var compilerUpdate; - if (WScript.Arguments(1) == "cl") { - compilerVersion = clVersion.match(/Compiler Version ([0-9.]+)\s/mi)[1]; - // compilerVersion is in xx.xx.xxxxx.xx format, i.e. a string. - // It will compare well with major.minor versions where major has two digits, - // which is sufficient as the versions of interest start from 13 (for VC7). - } else if (WScript.Arguments(1) == "icl") { - // Get predefined ICL macros - tmpExec = WshShell.Run("cmd /C icl /QdM /E detect.c > detect.map", 0, true); - var defs = readAllFromFile("detect.map"); - // In #define __INTEL_COMPILER XXYY, XX is the major ICL version, YY is minor - compilerVersion = defs.match(/__INTEL_COMPILER[ \t]*([0-9]+).*$/mi)[1] / 100; - compilerUpdate = defs.match(/__INTEL_COMPILER_UPDATE[ \t]*([0-9]+).*$/mi)[1]; - // compiler version is a number; it compares well with another major.minor - // version number, where major has one, two, and perhaps more digits (9.1, 11, etc). - } - var requestedVersion = WScript.Arguments(2); - var requestedUpdate = 0; - if (WScript.Arguments.Count() > 3) - requestedUpdate = WScript.Arguments(3); - if (compilerVersion < requestedVersion) { - WScript.Echo("fail"); - } else if (compilerVersion == requestedVersion && compilerUpdate < requestedUpdate) { - WScript.Echo("fail"); - } else { - WScript.Echo("ok"); - } - return; - } -} - -function doClean() { - var fso = new ActiveXObject("Scripting.FileSystemObject"); - // delete intermediate files - if (fso.FileExists("detect.c")) - fso.DeleteFile("detect.c", false); - if (fso.FileExists("detect.obj")) - fso.DeleteFile("detect.obj", false); - if (fso.FileExists("detect.map")) - fso.DeleteFile("detect.map", false); - if (fso.FileExists("detect.exe")) - fso.DeleteFile("detect.exe", false); - if (fso.FileExists("detect.exe.manifest")) - fso.DeleteFile("detect.exe.manifest", false); -} - -if (WScript.Arguments.Count() > 0) { - - try { - doWork(); - } catch (error) { - WScript.Echo("unknown"); - } - doClean(); - -} else { - WScript.Echo("Supported options:\n" - + "\t/arch [compiler]\n" - + "\t/runtime [compiler]\n" - + "\t/minversion compiler version"); -} - diff --git a/src/tbb/build/generate_tbbvars.bat b/src/tbb/build/generate_tbbvars.bat deleted file mode 100644 index 8925f577b..000000000 --- a/src/tbb/build/generate_tbbvars.bat +++ /dev/null @@ -1,66 +0,0 @@ -@echo off -REM -REM Copyright (c) 2005-2017 Intel Corporation -REM -REM Licensed under the Apache License, Version 2.0 (the "License"); -REM you may not use this file except in compliance with the License. -REM You may obtain a copy of the License at -REM -REM http://www.apache.org/licenses/LICENSE-2.0 -REM -REM Unless required by applicable law or agreed to in writing, software -REM distributed under the License is distributed on an "AS IS" BASIS, -REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -REM See the License for the specific language governing permissions and -REM limitations under the License. -REM -REM -REM -REM -REM -setlocal -for %%D in ("%tbb_root%") do set actual_root=%%~fD -set fslash_root=%actual_root:\=/% -set bin_dir=%CD% -set fslash_bin_dir=%bin_dir:\=/% -set _INCLUDE=INCLUDE& set _LIB=LIB -if not x%UNIXMODE%==x set _INCLUDE=CPATH& set _LIB=LIBRARY_PATH -if not x%USE_INCLUDE_ENV%==x set _INCLUDE=INCLUDE - -echo Generating local tbbvars.bat -echo @echo off>tbbvars.bat -echo SET TBBROOT=%actual_root%>>tbbvars.bat -echo SET TBB_ARCH_PLATFORM=%arch%\%runtime%>>tbbvars.bat -echo SET TBB_TARGET_ARCH=%arch%>>tbbvars.bat -echo SET %_INCLUDE%=%%TBBROOT%%\include;%%%_INCLUDE%%%>>tbbvars.bat -echo SET %_LIB%=%bin_dir%;%%%_LIB%%%>>tbbvars.bat -echo SET PATH=%bin_dir%;%%PATH%%>>tbbvars.bat -if not x%UNIXMODE%==x echo SET LD_LIBRARY_PATH=%bin_dir%;%%LD_LIBRARY_PATH%%>>tbbvars.bat - -echo Generating local tbbvars.sh -echo #!/bin/sh>tbbvars.sh -echo export TBBROOT="%fslash_root%">>tbbvars.sh -echo export TBB_ARCH_PLATFORM="%arch%\%runtime%">>tbbvars.sh -echo export TBB_TARGET_ARCH="%arch%">>tbbvars.sh -echo export %_INCLUDE%="${TBBROOT}/include;$%_INCLUDE%">>tbbvars.sh -echo export %_LIB%="%fslash_bin_dir%;$%_LIB%">>tbbvars.sh -echo export PATH="%fslash_bin_dir%;$PATH">>tbbvars.sh -if not x%UNIXMODE%==x echo export LD_LIBRARY_PATH="%fslash_bin_dir%;$LD_LIBRARY_PATH">>tbbvars.sh - -echo Generating local tbbvars.csh -echo #!/bin/csh>tbbvars.csh -echo setenv TBBROOT "%actual_root%">>tbbvars.csh -echo setenv TBB_ARCH_PLATFORM "%arch%\%runtime%">>tbbvars.csh -echo setenv TBB_TARGET_ARCH "%arch%">>tbbvars.csh -echo setenv %_INCLUDE% "${TBBROOT}\include;$%_INCLUDE%">>tbbvars.csh -echo setenv %_LIB% "%bin_dir%;$%_LIB%">>tbbvars.csh -echo setenv PATH "%bin_dir%;$PATH">>tbbvars.csh -if not x%UNIXMODE%==x echo setenv LD_LIBRARY_PATH "%bin_dir%;$LD_LIBRARY_PATH">>tbbvars.csh - -if not x%LIB_STL_ANDROID%==x ( -REM Workaround for copying Android* specific stl shared library to work folder -copy /Y "%LIB_STL_ANDROID:/=\%" . -) - -endlocal -exit diff --git a/src/tbb/build/generate_tbbvars.sh b/src/tbb/build/generate_tbbvars.sh deleted file mode 100644 index 0ca0965ac..000000000 --- a/src/tbb/build/generate_tbbvars.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/bin/bash -# -# Copyright (c) 2005-2017 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -# -# - -# Script used to generate tbbvars.[c]sh scripts -bin_dir="$PWD" # -cd "$tbb_root" # keep this comments here -tbb_root="$PWD" # to make it unsensible -cd "$bin_dir" # to EOL encoding -cat >./tbbvars.sh <./tbbvars.csh < - - -

Overview

-This directory contains the internal Makefile infrastructure for Intel® Threading Building Blocks (Intel® TBB). - -

-See below for how to build Intel TBB and how to port Intel TBB -to a new platform, operating system or architecture. -

- -

Files

-The files here are not intended to be used directly. See below for usage. -
-
Makefile.tbb -
Main Makefile to build the Intel TBB library. - Invoked via 'make tbb' from top-level Makefile. -
Makefile.tbbmalloc -
Main Makefile to build the Intel TBB scalable memory allocator library as well as its tests. - Invoked via 'make tbbmalloc' from top-level Makefile. -
Makefile.test -
Main Makefile to build and run the tests for the Intel TBB library. - Invoked via 'make test' from top-level Makefile. -
common.inc -
Main common included Makefile that includes OS-specific and compiler-specific Makefiles. -
<os>.inc -
OS-specific Makefile for a particular <os>. -
<os>.<compiler>.inc -
Compiler-specific Makefile for a particular <os> / <compiler> combination. -
*.sh -
Infrastructure utilities for Linux* OS, macOS*, and UNIX*-related operating systems. -
*.js, *.bat -
Infrastructure utilities for Windows* OS. -
- -

To Build

 -

-To port Intel TBB to a new platform, operating system or architecture, see the porting directions below. -

- -

Software prerequisites:

-
    -
  1. C++ compiler for the platform, operating system and architecture of interest. - Either the native compiler for your system, or, optionally, the appropriate Intel® C++ Compiler, may be used. -
  2. GNU make utility. On Windows OS, if a UNIX* emulator is used to run GNU make, - it should be able to run Windows OS utilities and commands. On Linux OS, macOS, etc., - shell commands issued by GNU make should execute in a Bourne or BASH compatible shell. - In the following examples, replace make with the correct GNU make command for - your system (for example, gmake). GNU make version 3.80 and more recent are supported. -
- -

-Intel TBB libraries can be built by performing the following steps. -On systems that support only one ABI (e.g., 32-bit), these steps build the libraries for that ABI. -On systems that support both 64-bit and 32-bit libraries, these steps build the 64-bit libraries -(Linux OS, macOS, and related systems) or whichever ABI is selected in the development environment (Windows OS). -

-
    -
  1. Change to the top-level directory of the installed software. -
  2. If using the Intel® C++ Compiler, make sure the appropriate compiler is available in your PATH - (e.g., by sourcing the appropriate iccvars script for the compiler to be used). -
  3. Invoke GNU make using no arguments, for example, make. -
- -

-To build Intel TBB libraries for other than the default ABI (e.g., to build 32-bit libraries on Linux OS, macOS, -or related systems that support both 64-bit and 32-bit libraries), perform the following steps: -

-
    -
  1. Change to the top-level directory of the installed software. -
  2. If using the Intel® C++ Compiler, make sure the appropriate compiler is available in your PATH - (e.g., by sourcing the appropriate iccvars script for the compiler to be used). -
  3. Explicitly specify the architecture when invoking GNU make, e.g. make arch=ia32. -
- -

The default make target will build the release and debug versions of the Intel TBB library.

-

Other targets are available in the top-level Makefile. You might find the following targets useful: -

    -
  • make test will build and run Intel TBB unit-tests; -
  • make examples will build and run Intel TBB examples. Available in the open-source version only. -For the commercial version, you can download Intel TBB Samples at the Intel® Software Product Samples and Tutorials website; -
  • make all will do all of the above. Available in the open-source version only. -
-See also the list of other targets below. -

- -

-By default, the libraries will be built in sub-directories within the build/ directory. -The sub-directories are named according to the operating system, architecture, compiler and software environment used -(the sub-directory names also distinguish release vs. debug libraries). On Linux OS, the software environment comprises -the GCC, libc and kernel version used. On macOS, the software environment comprises the GCC and OS version used. -On Windows OS, the software environment comprises the Microsoft* Visual Studio* version used. -See below for how to change the default build directory. -

- -

-To perform different build and/or test operations, use the following steps. -

-
    -
  1. Change to the top-level directory of the installed software. -
  2. If using the Intel® C++ Compiler, make sure the appropriate compiler is available in your PATH - (e.g., by sourcing the appropriate iccvars script for the compiler to be used). -
  3. Invoke GNU make by using one or more of the following commands. -
    -
    make -
    Default build. Equivalent to make tbb tbbmalloc. -
    make all -
    Equivalent to make tbb tbbmalloc test examples. Available in the open-source version only. -
    cd src;make release -
    Build and test release libraries only. -
    cd src;make debug -
    Build and test debug libraries only. -
    make tbb -
    Make Intel TBB release and debug libraries. -
    make tbbmalloc -
    Make Intel TBB scalable memory allocator libraries. -
    make test -
    Compile and run unit-tests -
    make examples -
    Build libraries and run all examples, like doing make debug clean release from the general example Makefile. - Available in the open-source version only. -
    make python -
    Build, install, and test Python* API for Intel TBB. See details here. -
    make compiler={icl, icc, gcc, clang} [(above options or targets)] -
    Build and run as above, but use specified compilers instead of default, native compilers -
      -
    1. {icl, icc} - to use Intel® compilers (icl on Windows OS, icc on Linux OS or macOS).
      2. -
    2. gcc - to use g++ (e.g. MinGW on Windows OS)
      3. -
    3. clang - to use Clang compiler
      4. -
    -
    make compiler=clang stdlib=libc++ [(above options or targets)] -
    Build and run as above, but use libc++ as a standard c++ library for clang. -
    make stdver={c++11, c++14, ...} [(above options or targets)] -
    Build and run as above, but additionally specify the version of the C++ standard or dialect to be used by - the compiler. The specified value of stdver will be used as a parameter to the appropriate - compiler option (such as -std); the behavior in case of unsupported value is compiler-specific. -
    make target_app={win8ui, uwp} [target_mode=store] [(above options or targets)] -
    Build and run as above, but use API that is compliant with Universal Windows* applications. Use win8ui option, if you want to use Intel TBB in Windows* 8 Universal application and uwp in case of Windows* 10 Universal Windows application. - target_mode=store is used to produce binaries that are compliant with Windows Store* application container. In later case they won't work with Intel TBB unit tests but work only with Windows Store* applications. -
    ndk-build target=android [(above options or targets)] -
    Build and run as above, but build libraries for Android* OS by Android NDK that should be installed. Makefiles were tested with revision 8. -
    make arch={ia32, intel64, ia64} [(above options or targets)] -
    Build and run as above, but build libraries for the selected ABI. - Might be useful for cross-compilation; ensure proper environment is set before running this command. -
    make tbb_root={(Intel TBB directory)} [(above options or targets)] -
    Build and run as above; for use when invoking make from a directory other than the top-level directory. -
    make tbb_build_dir={(build directory)} [(above options or targets)] -
    Build and run as above, but place the built libraries in the specified directory, rather than in the default sub-directory within the build/ directory. This command might have troubles with the build in case the sources installed to the directory with spaces in the path. -
    make tbb_build_prefix={(build sub-directory)} [(above options or targets)] -
    Build and run as above, but place the built libraries in the specified sub-directory within the build/ directory, rather than using the default sub-directory name. -
    make tbb_cpf=1 [(above options or targets)] -
    Build and run as above, but build and use libraries with the Community Preview Features enabled, rather than the default libraries. -
    make [(above options)] clean -
    Remove any executables or intermediate files produced by the above commands. - Includes build directories, object files, libraries and test executables. -
    -
- -

To Port

 -

-This section provides information on how to port Intel TBB to a new platform, operating system or architecture. -A subset or a superset of these steps may be required for porting to a given platform. -

- -

To port the Intel TBB source code:

-
    -
  1. If porting to a new architecture, create a file that describes the architecture-specific details for that architecture. -
      -
    • Create a <os>_<architecture>.h file in the include/tbb/machine directory - that describes these details. -
        -
      • The <os>_<architecture>.h is named after the operating system and architecture as recognized by - include/tbb/tbb_machine.h and the Makefile infrastructure. -
      • This file defines the implementations of synchronization operations, and also the - scheduler yield function, for the operating system and architecture. -
      • Several examples of <os>_<architecture>.h files can be found in the - include/tbb/machine directory. -
          -
        • A minimal implementation defines the 4-byte and 8-byte compare-and-swap operations, - and the scheduler yield function. See include/tbb/machine/mac_ppc.h - for an example of a minimal implementation. -
        • More complex implementation examples can also be found in the - include/tbb/machine directory - that implement all the individual variants of synchronization operations that Intel TBB uses. - Such implementations are more verbose but may achieve better performance on a given architecture. -
        • In a given implementation, any synchronization operation that is not defined is implemented, by default, - in terms of 4-byte or 8-byte compare-and-swap. More operations can thus be added incrementally to increase - the performance of an implementation. -
        • In most cases, synchronization operations are implemented as inline assembly code; examples also exist, - (e.g., for Intel® Itanium® processors) that use out-of-line assembly code in *.s or *.asm files - (see the assembly code sub-directories in the src/tbb directory). -
        -
      -
    • Modify include/tbb/tbb_machine.h, if needed, to invoke the appropriate - <os>_<architecture>.h file in the include/tbb/machine directory. -
    -
  2. Add an implementation of DetectNumberOfWorkers() in src/tbb/tbb_misc.h, - that returns the number of cores found on the system in case it is not supported by the current implementation. - This is used to determine the default number of threads for the Intel TBB task scheduler. -
  3. Either properly define FillDynamicLinks for use in - src/tbb/cache_aligned_allocator.cpp, - or hardcode the allocator to be used. -
  4. Additional types might be required in the union defined in - include/tbb/aligned_space.h - to ensure proper alignment on your platform. -
  5. Changes may be required in include/tbb/tick_count.h - for systems that do not provide gettimeofday. -
- -

To port the Makefile infrastructure:

-Modify the appropriate files in the Makefile infrastructure to add a new platform, operating system or architecture as needed. -See the Makefile infrastructure files for examples. -
    -
  1. The top-level Makefile includes common.inc to determine the operating system. -
      -
    • To add a new operating system, add the appropriate test to common.inc, and create the needed <os>.inc and <os>.<compiler>.inc files (see below). -
    -
  2. The <os>.inc file makes OS-specific settings for a particular operating systems. -
      -
    • For example, linux.inc makes settings specific to Linux operating systems. -
    • This file performs OS-dependent tests to determine the specific platform and/or architecture, and sets other platform-dependent values. -
    • Add a new <os>.inc file for each new operating system added. -
    -
  3. The <os>.<compiler>.inc file makes compiler-specific settings for a particular - <os> / <compiler> combination. -
      -
    • For example, linux.gcc.inc makes specific settings for using GCC on Linux OS, and linux.icc.inc makes specific settings for using the Intel® C++ compiler on Linux OS. -
    • This file sets particular compiler, assembler and linker options required when using a particular <os> / <compiler> combination. -
    • Add a new <os>.<compiler>.inc file for each new <os> / <compiler> combination added. -
    -
- -
-Up to parent directory -

-Copyright © 2005-2017 Intel Corporation. All Rights Reserved. -

-Intel, the Intel logo and Itanium are trademarks of Intel Corporation or its subsidiaries in the U.S. and/or other countries. -

-* Other names and brands may be claimed as the property of others. - - diff --git a/src/tbb/build/ios.macos.inc b/src/tbb/build/ios.macos.inc deleted file mode 100644 index ae723bd52..000000000 --- a/src/tbb/build/ios.macos.inc +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright (c) 2005-2017 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -# -# - -ifneq ($(arch),$(filter $(arch),ia32 intel64 armv7 armv7s arm64)) - $(error $(arch) is unknown architecture. Known arhitechtures are ia32 intel64 armv7 armv7s arm64) -endif - -# If target is ios but arch is ia32/intel64 then build for 32/64 simulator! -ifeq (,$(SDKROOT)) - ifeq ($(arch),$(filter $(arch),ia32 intel64)) - export SDKROOT:=$(shell xcodebuild -sdk -version | grep -o -E '/.*SDKs/iPhoneSimulator.*' 2>/dev/null) - else - export SDKROOT:=$(shell xcodebuild -sdk -version | grep -o -E '/.*SDKs/iPhoneOS.*' 2>/dev/null) - endif -endif -ifeq (,$(SDKROOT)) - $(error iOS* SDK not found) -endif - -ios_version:=$(shell echo $(SDKROOT) | sed -e "s/.*[a-z,A-Z]\(.*\).sdk/\1/") -runtime:=cc$(clang_version)_ios$(ios_version) diff --git a/src/tbb/build/linux.clang.inc b/src/tbb/build/linux.clang.inc deleted file mode 100644 index edabab26a..000000000 --- a/src/tbb/build/linux.clang.inc +++ /dev/null @@ -1,115 +0,0 @@ -# Copyright (c) 2005-2017 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -# -# - -COMPILE_ONLY = -c -MMD -PREPROC_ONLY = -E -x c++ -INCLUDE_KEY = -I -DEFINE_KEY = -D -OUTPUT_KEY = -o # -OUTPUTOBJ_KEY = -o # -PIC_KEY = -fPIC -WARNING_AS_ERROR_KEY = -Werror -WARNING_KEY = -Wall -TEST_WARNING_KEY = -Wextra -Wshadow -Wcast-qual -Woverloaded-virtual -Wnon-virtual-dtor -WARNING_SUPPRESS = -Wno-parentheses -Wno-non-virtual-dtor -Wno-dangling-else -DYLIB_KEY = -shared -EXPORT_KEY = -Wl,--version-script, -LIBDL = -ldl - -CPLUS = clang++ -CONLY = clang -LIB_LINK_FLAGS = $(DYLIB_KEY) -Wl,-soname=$(BUILDING_LIBRARY) -LIBS += -lpthread -lrt -LINK_FLAGS = -Wl,-rpath-link=. -rdynamic -C_FLAGS = $(CPLUS_FLAGS) - -ifeq ($(cfg), release) - CPLUS_FLAGS = $(ITT_NOTIFY) -g -O2 -DUSE_PTHREAD -endif -ifeq ($(cfg), debug) - CPLUS_FLAGS = -DTBB_USE_DEBUG $(ITT_NOTIFY) -g -O0 -DUSE_PTHREAD -endif - -ifneq (,$(stdlib)) - CPLUS_FLAGS += -stdlib=$(stdlib) - LIB_LINK_FLAGS += -stdlib=$(stdlib) -endif - -ifneq (,$(gcc_version)) - # TODO: do not assume that GCC minor and patchlevel versions are always single-digit. - CPLUS_FLAGS += -DTBB_USE_GLIBCXX_VERSION=$(subst .,0,$(gcc_version)) -endif - -TBB_ASM.OBJ= -MALLOC_ASM.OBJ= - -ifeq (intel64,$(arch)) - ITT_NOTIFY = -DDO_ITT_NOTIFY - CPLUS_FLAGS += -m64 - LIB_LINK_FLAGS += -m64 -endif - -ifeq (ia32,$(arch)) - ITT_NOTIFY = -DDO_ITT_NOTIFY - CPLUS_FLAGS += -m32 -march=pentium4 - LIB_LINK_FLAGS += -m32 -endif - -ifeq (ppc64,$(arch)) - CPLUS_FLAGS += -m64 - LIB_LINK_FLAGS += -m64 -endif - -ifeq (ppc32,$(arch)) - CPLUS_FLAGS += -m32 - LIB_LINK_FLAGS += -m32 -endif - -ifeq (bg,$(arch)) - CPLUS = bgclang++ - CONLY = bgclang -endif - -#------------------------------------------------------------------------------ -# Setting assembler data. -#------------------------------------------------------------------------------ -ASM = as -ifeq (intel64,$(arch)) - ASM_FLAGS += --64 -endif -ifeq (ia32,$(arch)) - ASM_FLAGS += --32 -endif -ifeq ($(cfg),debug) - ASM_FLAGS += -g -endif - -ASSEMBLY_SOURCE=$(arch)-gas -#------------------------------------------------------------------------------ -# End of setting assembler data. -#------------------------------------------------------------------------------ - -#------------------------------------------------------------------------------ -# Setting tbbmalloc data. -#------------------------------------------------------------------------------ - -M_CPLUS_FLAGS = $(CPLUS_FLAGS) -fno-rtti -fno-exceptions - -#------------------------------------------------------------------------------ -# End of setting tbbmalloc data. -#------------------------------------------------------------------------------ diff --git a/src/tbb/build/linux.gcc.inc b/src/tbb/build/linux.gcc.inc deleted file mode 100644 index 348bac0fc..000000000 --- a/src/tbb/build/linux.gcc.inc +++ /dev/null @@ -1,155 +0,0 @@ -# Copyright (c) 2005-2017 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -# -# - -COMPILE_ONLY = -c -MMD -PREPROC_ONLY = -E -x c++ -INCLUDE_KEY = -I -DEFINE_KEY = -D -OUTPUT_KEY = -o # -OUTPUTOBJ_KEY = -o # -PIC_KEY = -fPIC -WARNING_AS_ERROR_KEY = -Werror -WARNING_KEY = -Wall -TEST_WARNING_KEY = -Wshadow -Wcast-qual -Woverloaded-virtual -Wnon-virtual-dtor - -WARNING_SUPPRESS = -Wno-parentheses -Wno-non-virtual-dtor -DYLIB_KEY = -shared -EXPORT_KEY = -Wl,--version-script, -LIBDL = -ldl - -CPLUS = g++ -CONLY = gcc -LIB_LINK_FLAGS = $(DYLIB_KEY) -Wl,-soname=$(BUILDING_LIBRARY) -LIBS += -lpthread -lrt -LINK_FLAGS = -Wl,-rpath-link=. -rdynamic -C_FLAGS = $(CPLUS_FLAGS) - -# gcc 4.2 and higher support OpenMP -ifneq (,$(shell gcc -dumpversion | egrep "^(4\.[2-9]|[5-9])")) - OPENMP_FLAG = -fopenmp -endif - -# gcc 4.8 and later support RTM intrinsics, but require command line switch to enable them -ifneq (,$(shell gcc -dumpversion | egrep "^(4\.[8-9]|[5-9])")) - RTM_KEY = -mrtm -endif - -# gcc 4.0 and later have -Wextra that is used by some our customers. -ifneq (,$(shell gcc -dumpversion | egrep "^([4-9])")) - TEST_WARNING_KEY += -Wextra -endif - -# gcc 5.0 and later have -Wsuggest-override option -# enable it via a pre-included header in order to limit to C++11 and above -ifneq (,$(shell gcc -dumpversion | egrep "^([5-9])")) - INCLUDE_TEST_HEADERS = -include $(tbb_root)/src/test/harness_preload.h -endif - -# gcc 6.0 and later have -flifetime-dse option that controls -# elimination of stores done outside the object lifetime -ifneq (,$(shell gcc -dumpversion | egrep "^([6-9])")) - # keep pre-contruction stores for zero initialization - DSE_KEY = -flifetime-dse=1 -endif - -ifeq ($(cfg), release) - CPLUS_FLAGS = $(ITT_NOTIFY) -g -O2 -DUSE_PTHREAD -endif -ifeq ($(cfg), debug) - CPLUS_FLAGS = -DTBB_USE_DEBUG $(ITT_NOTIFY) -g -O0 -DUSE_PTHREAD -endif - -TBB_ASM.OBJ= -MALLOC_ASM.OBJ= - -ifeq (ia64,$(arch)) -# Position-independent code (PIC) is a must on IA-64 architecture, even for regular (not shared) executables - CPLUS_FLAGS += $(PIC_KEY) -endif - -ifeq (intel64,$(arch)) - ITT_NOTIFY = -DDO_ITT_NOTIFY - CPLUS_FLAGS += -m64 $(RTM_KEY) - LIB_LINK_FLAGS += -m64 -endif - -ifeq (ia32,$(arch)) - ITT_NOTIFY = -DDO_ITT_NOTIFY - CPLUS_FLAGS += -m32 -march=pentium4 $(RTM_KEY) - LIB_LINK_FLAGS += -m32 -endif - -ifeq (ppc64,$(arch)) - CPLUS_FLAGS += -m64 - LIB_LINK_FLAGS += -m64 -endif - -ifeq (ppc32,$(arch)) - CPLUS_FLAGS += -m32 - LIB_LINK_FLAGS += -m32 -endif - -ifeq (bg,$(arch)) - CPLUS = $(firstword $(notdir $(shell which powerpc{64,32,}-bg{z..a}-linux-g++ 2>/dev/null))) - CONLY = $(firstword $(notdir $(shell which powerpc{64,32,}-bg{z..a}-linux-gcc 2>/dev/null))) -endif - -# for some gcc versions on Solaris, -m64 may imply V9, but perhaps not everywhere (TODO: verify) -ifeq (sparc,$(arch)) - CPLUS_FLAGS += -mcpu=v9 -m64 - LIB_LINK_FLAGS += -mcpu=v9 -m64 -endif - -# automatically generate "IT" instructions when compiling for Thumb ISA -ifeq (armv7,$(arch)) - CPLUS_FLAGS += -Wa,-mimplicit-it=thumb -endif - -#------------------------------------------------------------------------------ -# Setting assembler data. -#------------------------------------------------------------------------------ -ASM = as -ifeq (intel64,$(arch)) - ASM_FLAGS += --64 -endif -ifeq (ia32,$(arch)) - ASM_FLAGS += --32 -endif -ifeq ($(cfg),debug) - ASM_FLAGS += -g -endif - -ASSEMBLY_SOURCE=$(arch)-gas -ifeq (ia64,$(arch)) - ASM_FLAGS += -xexplicit - TBB_ASM.OBJ += atomic_support.o lock_byte.o log2.o pause.o ia64_misc.o - MALLOC_ASM.OBJ += atomic_support.o lock_byte.o pause.o log2.o -endif -#------------------------------------------------------------------------------ -# End of setting assembler data. -#------------------------------------------------------------------------------ - -#------------------------------------------------------------------------------ -# Setting tbbmalloc data. -#------------------------------------------------------------------------------ - -M_CPLUS_FLAGS = $(CPLUS_FLAGS) -fno-rtti -fno-exceptions - -#------------------------------------------------------------------------------ -# End of setting tbbmalloc data. -#------------------------------------------------------------------------------ diff --git a/src/tbb/build/linux.icc.inc b/src/tbb/build/linux.icc.inc deleted file mode 100644 index c1830f579..000000000 --- a/src/tbb/build/linux.icc.inc +++ /dev/null @@ -1,120 +0,0 @@ -# Copyright (c) 2005-2017 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -# -# - -COMPILE_ONLY = -c -MMD -PREPROC_ONLY = -E -x c++ -INCLUDE_KEY = -I -DEFINE_KEY = -D -OUTPUT_KEY = -o # -OUTPUTOBJ_KEY = -o # -PIC_KEY = -fPIC -WARNING_AS_ERROR_KEY = -Werror -WARNING_KEY = -w1 -DYLIB_KEY = -shared -EXPORT_KEY = -Wl,--version-script, -NOINTRINSIC_KEY = -fno-builtin -LIBDL = -ldl -ifneq (,$(shell icc -dumpversion | egrep "1[2-9]\.")) -SDL_FLAGS = -fstack-protector -Wformat -Wformat-security -endif - -CPLUS = icpc -CONLY = icc - -ITT_NOTIFY = -DDO_ITT_NOTIFY -ifeq (release,$(cfg)) -SDL_FLAGS += -D_FORTIFY_SOURCE=2 -CPLUS_FLAGS = $(ITT_NOTIFY) -O2 -g -DUSE_PTHREAD -else -CPLUS_FLAGS = $(ITT_NOTIFY) -O0 -g -DUSE_PTHREAD -DTBB_USE_DEBUG -endif - -LIB_LINK_FLAGS = -shared -static-intel -Wl,-soname=$(BUILDING_LIBRARY) -z relro -z now -LIBS += -lpthread -lrt -LINK_FLAGS = -rdynamic -C_FLAGS = $(CPLUS_FLAGS) - -ifneq (,$(shell icc -dumpversion | egrep "^1[6-9]\.")) -OPENMP_FLAG = -qopenmp -else -OPENMP_FLAG = -openmp -endif - -# ICC 12.0 and higher provide Intel(R) Cilk(TM) Plus -ifneq (,$(shell icc -dumpversion | egrep "^1[2-9]\.")) - CILK_AVAILABLE = yes -endif - -TBB_ASM.OBJ= -MALLOC_ASM.OBJ= - -ifeq (ia32,$(arch)) - CPLUS_FLAGS += -m32 -falign-stack=maintain-16-byte - LIB_LINK_FLAGS += -m32 -endif - -ifeq (ia64,$(arch)) - ITT_NOTIFY = -# Position-independent code (PIC) is a must on IA-64 architecture, even for regular (not shared) executables -# strict-ansi does not work with on RHEL 4 AS - CPLUS_FLAGS += $(PIC_KEY) $(if $(findstring cc3.,$(runtime)),-ansi,-strict-ansi) -else -# For ICC 16 and older, in std=c++14 mode -strict-ansi does not work with GNU C++ library headers -# egrep returns 0 or 1, compare it in concatenation - CPLUS_FLAGS += $(if $(findstring c++14_1,$(stdver)_$(shell icc -dumpversion| egrep -c "^1[1-6]\.")),-ansi,-strict-ansi) -endif - -ifneq (,$(codecov)) -# no tool support for code coverage, need profile data generation - ITT_NOTIFY = -prof-gen=srcpos -endif - -#------------------------------------------------------------------------------ -# Setting assembler data. -#------------------------------------------------------------------------------ -ASM = as -ifeq (intel64,$(arch)) - ASM_FLAGS += --64 -endif -ifeq (ia32,$(arch)) - ASM_FLAGS += --32 -endif -ifeq ($(cfg),debug) - ASM_FLAGS += -g -endif - -ASSEMBLY_SOURCE=$(arch)-gas -ifeq (ia64,$(arch)) - ASM_FLAGS += -xexplicit - TBB_ASM.OBJ += atomic_support.o lock_byte.o log2.o pause.o ia64_misc.o - MALLOC_ASM.OBJ += atomic_support.o lock_byte.o pause.o log2.o -endif -#------------------------------------------------------------------------------ -# End of setting assembler data. -#------------------------------------------------------------------------------ - -#------------------------------------------------------------------------------ -# Setting tbbmalloc data. -#------------------------------------------------------------------------------ - -M_CPLUS_FLAGS = $(CPLUS_FLAGS) -fno-rtti -fno-exceptions - -#------------------------------------------------------------------------------ -# End of setting tbbmalloc data. -#------------------------------------------------------------------------------ - diff --git a/src/tbb/build/linux.inc b/src/tbb/build/linux.inc deleted file mode 100644 index daeaaf9a0..000000000 --- a/src/tbb/build/linux.inc +++ /dev/null @@ -1,142 +0,0 @@ -# Copyright (c) 2005-2017 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -# -# - -####### Detections and Commands ############################################### - -ifeq (icc,$(compiler)) - export COMPILER_VERSION := ICC: $(shell icc -V &1 | grep 'Version') - ifneq (,$(findstring running on IA-32, $(COMPILER_VERSION))) - export arch:=ia32 - else - ifneq (,$(findstring running on Intel(R) 64, $(COMPILER_VERSION))) - export arch:=intel64 - else - ifneq (,$(findstring IA-64, $(COMPILER_VERSION))) - export arch:=ia64 - endif - endif - endif - ifeq (,$(arch)) - $(warning "Unknown Intel compiler") - endif -endif - -ifndef arch - uname_m:=$(shell uname -m) - ifeq ($(uname_m),i686) - export arch:=ia32 - endif - ifeq ($(uname_m),ia64) - export arch:=ia64 - endif - ifeq ($(uname_m),x86_64) - export arch:=intel64 - endif - ifeq ($(uname_m),sparc64) - export arch:=sparc - endif - ifeq ($(uname_m),armv7l) - export arch:=armv7 - endif - ifndef arch - export arch:=$(uname_m) - endif -endif - -ifndef runtime - export gcc_version:=$(shell gcc -dumpversion) - os_version:=$(shell uname -r) - os_kernel_version:=$(shell uname -r | sed -e 's/-.*$$//') - export os_glibc_version_full:=$(shell getconf GNU_LIBC_VERSION | grep glibc | sed -e 's/^glibc //') - os_glibc_version:=$(shell echo "$(os_glibc_version_full)" | sed -e '2,$$d' -e 's/-.*$$//') - export runtime:=cc$(gcc_version)_libc$(os_glibc_version)_kernel$(os_kernel_version) -endif - -native_compiler := gcc -export compiler ?= gcc -debugger ?= gdb - -CMD=sh -c -CWD=$(shell pwd) -CP=cp -RM?=rm -f -RD?=rmdir -MD?=mkdir -p -NUL= /dev/null -SLASH=/ -MAKE_VERSIONS=sh $(tbb_root)/build/version_info_linux.sh $(VERSION_FLAGS) >version_string.ver -MAKE_TBBVARS=sh $(tbb_root)/build/generate_tbbvars.sh - -ifdef LD_LIBRARY_PATH - export LD_LIBRARY_PATH := .:$(LD_LIBRARY_PATH) -else - export LD_LIBRARY_PATH := . -endif - -####### Build settings ######################################################## - -OBJ = o -DLL = so -MALLOC_DLL?=$(DLL) -LIBEXT = so -SONAME_SUFFIX =$(shell grep TBB_COMPATIBLE_INTERFACE_VERSION $(tbb_root)/include/tbb/tbb_stddef.h | egrep -o [0-9.]+) - -ifeq ($(arch),ia64) - def_prefix = lin64ipf -endif -ifneq (,$(findstring $(arch),sparc s390x)) - def_prefix = lin64 -endif -ifeq ($(arch),armv7) - def_prefix = lin32 -endif -ifeq (,$(def_prefix)) - ifeq (64,$(findstring 64,$(arch))) - def_prefix = lin64 - else - def_prefix = lin32 - endif -endif -TBB.LST = $(tbb_root)/src/tbb/$(def_prefix)-tbb-export.lst -TBB.DEF = $(TBB.LST:.lst=.def) - -TBB.DLL = $(TBB_NO_VERSION.DLL).$(SONAME_SUFFIX) -TBB.LIB = $(TBB.DLL) -TBB_NO_VERSION.DLL=libtbb$(CPF_SUFFIX)$(DEBUG_SUFFIX).$(DLL) -LINK_TBB.LIB = $(TBB_NO_VERSION.DLL) - -MALLOC_NO_VERSION.DLL = libtbbmalloc$(DEBUG_SUFFIX).$(MALLOC_DLL) -MALLOC.DEF = $(MALLOC_ROOT)/$(def_prefix)-tbbmalloc-export.def -MALLOC.DLL = $(MALLOC_NO_VERSION.DLL).$(SONAME_SUFFIX) -MALLOC.LIB = $(MALLOC_NO_VERSION.DLL) -LINK_MALLOC.LIB = $(MALLOC_NO_VERSION.DLL) - -MALLOCPROXY_NO_VERSION.DLL = libtbbmalloc_proxy$(DEBUG_SUFFIX).$(DLL) -MALLOCPROXY.DEF = $(MALLOC_ROOT)/$(def_prefix)-proxy-export.def -MALLOCPROXY.DLL = $(MALLOCPROXY_NO_VERSION.DLL).$(SONAME_SUFFIX) -MALLOCPROXY.LIB = $(MALLOCPROXY_NO_VERSION.DLL) -LINK_MALLOCPROXY.LIB = $(MALLOCPROXY.LIB) - -RML_NO_VERSION.DLL = libirml$(DEBUG_SUFFIX).$(DLL) -RML.DEF = $(RML_SERVER_ROOT)/lin-rml-export.def -RML.DLL = $(RML_NO_VERSION.DLL).1 -RML.LIB = $(RML_NO_VERSION.DLL) - -TEST_LAUNCHER=sh $(tbb_root)/build/test_launcher.sh $(largs) - -OPENCL.LIB = -lOpenCL diff --git a/src/tbb/build/linux.pathcc.inc b/src/tbb/build/linux.pathcc.inc deleted file mode 100644 index 948f23160..000000000 --- a/src/tbb/build/linux.pathcc.inc +++ /dev/null @@ -1,93 +0,0 @@ -# Copyright (c) 2005-2017 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -# -# - -COMPILE_ONLY = -c -MMD -PREPROC_ONLY = -E -x c++ -INCLUDE_KEY = -I -DEFINE_KEY = -D -OUTPUT_KEY = -o # -OUTPUTOBJ_KEY = -o # -PIC_KEY = -fPIC -WARNING_AS_ERROR_KEY = -Werror -WARNING_KEY = -Wall -TEST_WARNING_KEY = -Wshadow -Wcast-qual -Woverloaded-virtual -Wnon-virtual-dtor -Wextra - -WARNING_SUPPRESS = -Wno-parentheses -Wno-non-virtual-dtor -DYLIB_KEY = -shared -EXPORT_KEY = -Wl,--version-script, -LIBDL = -ldl - -CPLUS = pathCC -CONLY = pathcc -LIB_LINK_FLAGS = $(DYLIB_KEY) -Wl,-soname=$(BUILDING_LIBRARY) -LIBS += -lstl -lpthread -lrt -LINK_FLAGS = -Wl,-rpath-link=. -rdynamic -C_FLAGS = $(CPLUS_FLAGS) - -OPENMP_FLAG = -openmp - -ifeq ($(cfg), release) - CPLUS_FLAGS = $(ITT_NOTIFY) -g -O2 -DUSE_PTHREAD -endif -ifeq ($(cfg), debug) - CPLUS_FLAGS = -DTBB_USE_DEBUG $(ITT_NOTIFY) -g -O0 -DUSE_PTHREAD -endif - -TBB_ASM.OBJ= -MALLOC_ASM.OBJ= - -ifeq (intel64,$(arch)) - ITT_NOTIFY = -DDO_ITT_NOTIFY - CPLUS_FLAGS += -m64 - LIB_LINK_FLAGS += -m64 -endif - -ifeq (ia32,$(arch)) - ITT_NOTIFY = -DDO_ITT_NOTIFY - CPLUS_FLAGS += -m32 -march=pentium4 - LIB_LINK_FLAGS += -m32 -endif - -#------------------------------------------------------------------------------ -# Setting assembler data. -#------------------------------------------------------------------------------ -ASM = as -ifeq (intel64,$(arch)) - ASM_FLAGS += --64 -endif -ifeq (ia32,$(arch)) - ASM_FLAGS += --32 -endif -ifeq ($(cfg),debug) - ASM_FLAGS += -g -endif - -ASSEMBLY_SOURCE=$(arch)-gas -#------------------------------------------------------------------------------ -# End of setting assembler data. -#------------------------------------------------------------------------------ - -#------------------------------------------------------------------------------ -# Setting tbbmalloc data. -#------------------------------------------------------------------------------ - -M_CPLUS_FLAGS = $(CPLUS_FLAGS) -fno-rtti -fno-exceptions - -#------------------------------------------------------------------------------ -# End of setting tbbmalloc data. -#------------------------------------------------------------------------------ diff --git a/src/tbb/build/linux.xl.inc b/src/tbb/build/linux.xl.inc deleted file mode 100644 index fccb559f3..000000000 --- a/src/tbb/build/linux.xl.inc +++ /dev/null @@ -1,104 +0,0 @@ -# Copyright (c) 2005-2017 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -# -# - -####### Detections and Commands ############################################### - -COMPILE_ONLY = -c -PREPROC_ONLY = -E -qsourcetype=c -INCLUDE_KEY = -I -DEFINE_KEY = -D -OUTPUT_KEY = -o # -OUTPUTOBJ_KEY = -o # -PIC_KEY = -qpic -WARNING_AS_ERROR_KEY = -qhalt=w -WARNING_KEY = -TEST_WARNING_KEY = - -WARNING_SUPPRESS = -DYLIB_KEY = -qmkshrobj -EXPORT_KEY = -Wl,--version-script, -LIBDL = -ldl - -CPLUS = xlc++_r -CONLY = xlc_r -LIB_LINK_FLAGS = $(DYLIB_KEY) -Wl,-soname=$(BUILDING_LIBRARY) -LIBS = -lpthread -lrt -C_FLAGS = $(CPLUS_FLAGS) - -ifeq ($(cfg), release) - CPLUS_FLAGS = $(ITT_NOTIFY) -O2 -DUSE_PTHREAD -endif -ifeq ($(cfg), debug) - CPLUS_FLAGS = -DTBB_USE_DEBUG $(ITT_NOTIFY) -g -O0 -DUSE_PTHREAD -endif - -# Adding directly to CPLUS_FLAGS instead of to WARNING_SUPPRESS because otherwise it would not be used in several tests (why not?). -# Suppress warnings like: -# - "1500-029: (W) WARNING: subprogram [...] could not be inlined into [...]." -# - "1501-201: (W) Maximum number of common component diagnostics, 10 has been exceeded." -# see http://www-01.ibm.com/support/docview.wss?uid=swg1LI72843 -# it seems that the internal compiler error that would ensue has now been avoided, making the condition harmless -# - "1540-0198 (W) The omitted keyword "private" is assumed for base class "no_copy"." -# - "1540-0822 (W) The name "__FUNCTION__" must not be defined as a macro." -CPLUS_FLAGS += -qsuppress=1500-029:1501-201:1540-0198:1540-0822 - -ASM= -ASM_FLAGS= - -TBB_ASM.OBJ= - -ifeq (intel64,$(arch)) - ITT_NOTIFY = -DDO_ITT_NOTIFY - CPLUS_FLAGS += -q64 - LIB_LINK_FLAGS += -q64 -endif - -# TODO: equivalent for -march=pentium4 in CPLUS_FLAGS -ifeq (ia32,$(arch)) - ITT_NOTIFY = -DDO_ITT_NOTIFY - CPLUS_FLAGS += -q32 -qarch=pentium4 - LIB_LINK_FLAGS += -q32 -endif - -ifeq (ppc64,$(arch)) - CPLUS_FLAGS += -q64 - LIB_LINK_FLAGS += -q64 -endif - -ifeq (ppc32,$(arch)) - CPLUS_FLAGS += -q32 - LIB_LINK_FLAGS += -q32 -endif - -ifeq (bg,$(arch)) - CPLUS = bgxlC_r - CONLY = bgxlc_r -endif - -#------------------------------------------------------------------------------ -# Setting tbbmalloc data. -#------------------------------------------------------------------------------ - -# Suppress innumerable warnings like "1540-1088 (W) The exception specification is being ignored." -# Suppress warnings like "1540-1090 (I) The destructor of "lock" might not be called." -# TODO: aren't these warnings an indication that -qnoeh might not be appropriate? -M_CPLUS_FLAGS = $(CPLUS_FLAGS) -qnortti -qnoeh -qsuppress=1540-1088:1540-1090 - -#------------------------------------------------------------------------------ -# End of setting tbbmalloc data. -#------------------------------------------------------------------------------ diff --git a/src/tbb/build/macos.clang.inc b/src/tbb/build/macos.clang.inc deleted file mode 100644 index 323770534..000000000 --- a/src/tbb/build/macos.clang.inc +++ /dev/null @@ -1,123 +0,0 @@ -# Copyright (c) 2005-2017 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -# -# - -CPLUS = clang++ -CONLY = clang -COMPILE_ONLY = -c -MMD -PREPROC_ONLY = -E -x c++ -INCLUDE_KEY = -I -DEFINE_KEY = -D -OUTPUT_KEY = -o # -OUTPUTOBJ_KEY = -o # -PIC_KEY = -fPIC -WARNING_AS_ERROR_KEY = -Werror -WARNING_KEY = -Wall -TEST_WARNING_KEY = -Wextra -Wshadow -Wcast-qual -Woverloaded-virtual -Wnon-virtual-dtor -WARNING_SUPPRESS = -Wno-non-virtual-dtor -Wno-dangling-else -DYLIB_KEY = -dynamiclib -EXPORT_KEY = -Wl,-exported_symbols_list, -LIBDL = -ldl - -LIBS = -lpthread -LINK_FLAGS = -LIB_LINK_FLAGS = -dynamiclib -install_name @rpath/$(BUILDING_LIBRARY) -C_FLAGS = $(CPLUS_FLAGS) - -ifeq ($(cfg), release) - CPLUS_FLAGS = -g -O2 -else - CPLUS_FLAGS = -g -O0 -DTBB_USE_DEBUG -endif - -CPLUS_FLAGS += -DUSE_PTHREAD - -# For Clang, we add the option to support RTM intrinsics *iff* xtest is found in -ifneq (,$(shell grep xtest `echo "\#include" | clang -E -M - 2>&1 | grep immintrin.h` 2>/dev/null)) - RTM_KEY = -mrtm -endif - -ifneq (,$(stdlib)) - CPLUS_FLAGS += -stdlib=$(stdlib) - LIB_LINK_FLAGS += -stdlib=$(stdlib) -endif - -ifeq (intel64,$(arch)) - CPLUS_FLAGS += -m64 $(RTM_KEY) - LINK_FLAGS += -m64 - LIB_LINK_FLAGS += -m64 -endif - -ifeq (ia32,$(arch)) - CPLUS_FLAGS += -m32 $(RTM_KEY) - LINK_FLAGS += -m32 - LIB_LINK_FLAGS += -m32 -endif - -ifeq (ppc64,$(arch)) - CPLUS_FLAGS += -arch ppc64 - LINK_FLAGS += -arch ppc64 - LIB_LINK_FLAGS += -arch ppc64 -endif - -ifeq (ppc32,$(arch)) - CPLUS_FLAGS += -arch ppc - LINK_FLAGS += -arch ppc - LIB_LINK_FLAGS += -arch ppc -endif - -ifeq ($(arch),$(filter $(arch),armv7 armv7s arm64)) - CPLUS_FLAGS += -arch $(arch) - LINK_FLAGS += -arch $(arch) - LIB_LINK_FLAGS += -arch $(arch) -endif - -ifdef SDKROOT - CPLUS_FLAGS += -isysroot $(SDKROOT) - LINK_FLAGS += -L$(SDKROOT)/usr/lib/system -L$(SDKROOT)/usr/lib/ - LIB_LINK_FLAGS += -L$(SDKROOT)/usr/lib/system -L$(SDKROOT)/usr/lib/ -endif - -#------------------------------------------------------------------------------ -# Setting assembler data. -#------------------------------------------------------------------------------ - -ASM = as -ifeq (intel64,$(arch)) - ASM_FLAGS += -arch x86_64 -endif -ifeq (ia32,$(arch)) - ASM_FLAGS += -arch i386 -endif -ifeq ($(cfg), debug) - ASM_FLAGS += -g -endif - -#------------------------------------------------------------------------------ -# End of setting assembler data. -#------------------------------------------------------------------------------ - -#------------------------------------------------------------------------------ -# Setting tbbmalloc data. -#------------------------------------------------------------------------------ - -M_CPLUS_FLAGS = $(CPLUS_FLAGS) -fno-rtti -fno-exceptions - -#------------------------------------------------------------------------------ -# End of setting tbbmalloc data. -#------------------------------------------------------------------------------ - diff --git a/src/tbb/build/macos.gcc.inc b/src/tbb/build/macos.gcc.inc deleted file mode 100644 index 65c7047cb..000000000 --- a/src/tbb/build/macos.gcc.inc +++ /dev/null @@ -1,112 +0,0 @@ -# Copyright (c) 2005-2017 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -# -# - -CPLUS = g++ -CONLY = gcc -COMPILE_ONLY = -c -MMD -PREPROC_ONLY = -E -x c++ -INCLUDE_KEY = -I -DEFINE_KEY = -D -OUTPUT_KEY = -o # -OUTPUTOBJ_KEY = -o # -PIC_KEY = -fPIC -WARNING_AS_ERROR_KEY = -Werror -WARNING_KEY = -Wall -TEST_WARNING_KEY = -Wextra -Wshadow -Wcast-qual -Woverloaded-virtual -Wnon-virtual-dtor -WARNING_SUPPRESS = -Wno-non-virtual-dtor -DYLIB_KEY = -dynamiclib -EXPORT_KEY = -Wl,-exported_symbols_list, -LIBDL = -ldl - -LIBS = -lpthread -LINK_FLAGS = -LIB_LINK_FLAGS = -dynamiclib -install_name @rpath/$(BUILDING_LIBRARY) -C_FLAGS = $(CPLUS_FLAGS) - -ifeq ($(cfg), release) - CPLUS_FLAGS = -g -O2 -else - CPLUS_FLAGS = -g -O0 -DTBB_USE_DEBUG -endif - -CPLUS_FLAGS += -DUSE_PTHREAD - -ifeq (intel64,$(arch)) - CPLUS_FLAGS += -m64 - LINK_FLAGS += -m64 - LIB_LINK_FLAGS += -m64 -endif - -ifeq (ia32,$(arch)) - CPLUS_FLAGS += -m32 - LINK_FLAGS += -m32 - LIB_LINK_FLAGS += -m32 -endif - -ifeq (ppc64,$(arch)) - CPLUS_FLAGS += -arch ppc64 - LINK_FLAGS += -arch ppc64 - LIB_LINK_FLAGS += -arch ppc64 -endif - -ifeq (ppc32,$(arch)) - CPLUS_FLAGS += -arch ppc - LINK_FLAGS += -arch ppc - LIB_LINK_FLAGS += -arch ppc -endif - -ifeq (armv7,$(arch)) - CPLUS_FLAGS += -arch armv7 - LINK_FLAGS += -arch armv7 - LIB_LINK_FLAGS += -arch armv7 -endif - -ifdef SDKROOT - CPLUS_FLAGS += -isysroot $(SDKROOT) - LIB_LINK_FLAGS += -L$(SDKROOT)/usr/lib/system -L$(SDKROOT)/usr/lib/ -endif - -#------------------------------------------------------------------------------ -# Setting assembler data. -#------------------------------------------------------------------------------ - -ASM = as -ifeq (intel64,$(arch)) - ASM_FLAGS += -arch x86_64 -endif -ifeq (ia32,$(arch)) - ASM_FLAGS += -arch i386 -endif -ifeq ($(cfg), debug) - ASM_FLAGS += -g -endif - -#------------------------------------------------------------------------------ -# End of setting assembler data. -#------------------------------------------------------------------------------ - -#------------------------------------------------------------------------------ -# Setting tbbmalloc data. -#------------------------------------------------------------------------------ - -M_CPLUS_FLAGS = $(CPLUS_FLAGS) -fno-rtti -fno-exceptions - -#------------------------------------------------------------------------------ -# End of setting tbbmalloc data. -#------------------------------------------------------------------------------ - diff --git a/src/tbb/build/macos.icc.inc b/src/tbb/build/macos.icc.inc deleted file mode 100644 index 92a339b29..000000000 --- a/src/tbb/build/macos.icc.inc +++ /dev/null @@ -1,106 +0,0 @@ -# Copyright (c) 2005-2017 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -# -# - -CPLUS = icpc -CONLY = icc -COMPILE_ONLY = -c -MMD -PREPROC_ONLY = -E -x c++ -INCLUDE_KEY = -I -DEFINE_KEY = -D -OUTPUT_KEY = -o # -OUTPUTOBJ_KEY = -o # -PIC_KEY = -fPIC -WARNING_AS_ERROR_KEY = -Werror -WARNING_KEY = -w1 -ifneq (,$(shell icc -dumpversion | egrep "1[2-9]\.")) -SDL_FLAGS = -fstack-protector -Wformat -Wformat-security -endif - -DYLIB_KEY = -dynamiclib -EXPORT_KEY = -Wl,-exported_symbols_list, -LIBDL = -ldl - -LIBS = -lpthread -LINK_FLAGS = -LIB_LINK_FLAGS = -dynamiclib -static-intel -install_name @rpath/$(BUILDING_LIBRARY) -C_FLAGS = $(CPLUS_FLAGS) - -ifneq (,$(shell icc -dumpversion | egrep "^1[6-9]\.")) -OPENMP_FLAG = -qopenmp -else -OPENMP_FLAG = -openmp -endif - -# ICC 12.0 and higher provide Intel(R) Cilk(TM) Plus -ifneq (,$(shell icc -dumpversion | egrep "^1[2-9]\.")) - CILK_AVAILABLE = yes -endif - -ifeq ($(cfg), release) - SDL_FLAGS += -D_FORTIFY_SOURCE=2 - CPLUS_FLAGS = -g -O2 -fno-omit-frame-pointer -else - CPLUS_FLAGS = -g -O0 -DTBB_USE_DEBUG -endif - -CPLUS_FLAGS += -DUSE_PTHREAD - -ifneq (,$(codecov)) - CPLUS_FLAGS += -prof-gen=srcpos -endif - -# ICC 14.0 and higher support usage of libc++, clang standard library -MACOSX_DEPLOYMENT_TARGET ?= 10.11 -ifneq (,$(shell icc -dumpversion | egrep "^1[4-9]\.")) -ifneq (,$(stdlib)) - CPLUS_FLAGS += -stdlib=$(stdlib) -mmacosx-version-min=$(MACOSX_DEPLOYMENT_TARGET) - LIB_LINK_FLAGS += -stdlib=$(stdlib) -mmacosx-version-min=$(MACOSX_DEPLOYMENT_TARGET) -endif -endif - -#------------------------------------------------------------------------------ -# Setting assembler data. -#------------------------------------------------------------------------------ - -ASM = as -ifeq (intel64,$(arch)) - ASM_FLAGS += -arch x86_64 -endif -ifeq (ia32,$(arch)) - CPLUS_FLAGS += -m32 - LINK_FLAGS += -m32 - LIB_LINK_FLAGS += -m32 - ASM_FLAGS += -arch i386 -endif -ifeq ($(cfg), debug) - ASM_FLAGS += -g -endif - -#------------------------------------------------------------------------------ -# End of setting assembler data. -#------------------------------------------------------------------------------ - -#------------------------------------------------------------------------------ -# Setting tbbmalloc data. -#------------------------------------------------------------------------------ - -M_CPLUS_FLAGS = $(CPLUS_FLAGS) -fno-rtti -fno-exceptions - -#------------------------------------------------------------------------------ -# End of setting tbbmalloc data. -#------------------------------------------------------------------------------ diff --git a/src/tbb/build/macos.inc b/src/tbb/build/macos.inc deleted file mode 100644 index 8df5bbdab..000000000 --- a/src/tbb/build/macos.inc +++ /dev/null @@ -1,107 +0,0 @@ -# Copyright (c) 2005-2017 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -# -# - -####### Detections and Commands ############################################### - -ifeq (icc,$(compiler)) - export COMPILER_VERSION := ICC: $(shell icc -V &1 | grep 'Version') - ifneq (,$(findstring running on IA-32, $(COMPILER_VERSION))) - export arch:=ia32 - else - ifneq (,$(findstring running on Intel(R) 64, $(COMPILER_VERSION))) - export arch:=intel64 - endif - endif - ifeq (,$(arch)) - $(warning "Unknown Intel compiler") - endif -endif - -ifndef arch - ifeq ($(shell /usr/sbin/sysctl -n hw.machine),Power Macintosh) - ifeq ($(shell /usr/sbin/sysctl -n hw.optional.64bitops),1) - export arch:=ppc64 - else - export arch:=ppc32 - endif - else - ifeq ($(shell /usr/sbin/sysctl -n hw.optional.x86_64 2>/dev/null),1) - export arch:=intel64 - else - export arch:=ia32 - endif - endif -endif - -ifndef runtime - clang_version:=$(shell clang -v 2>&1 >/dev/null | grep version | sed -e "s/.*version \(.*[0-9]\) .*/\1/") - ifndef os_version - os_version:=$(shell /usr/bin/sw_vers -productVersion) - endif - export runtime:=cc$(clang_version)_os$(os_version) -endif - -native_compiler := clang -export compiler ?= clang -debugger ?= lldb - -export stdlib ?= libc++ - -CMD=$(SHELL) -c -CWD=$(shell pwd) -RM?=rm -f -RD?=rmdir -MD?=mkdir -p -NUL= /dev/null -SLASH=/ -MAKE_VERSIONS=sh $(tbb_root)/build/version_info_macos.sh $(VERSION_FLAGS) >version_string.ver -MAKE_TBBVARS=sh $(tbb_root)/build/generate_tbbvars.sh DY - -ifdef DYLD_LIBRARY_PATH - export DYLD_LIBRARY_PATH := .:$(DYLD_LIBRARY_PATH) -else - export DYLD_LIBRARY_PATH := . -endif - -####### Build settings ######################################################## - -OBJ=o -DLL=dylib -MALLOC_DLL?=$(DLL) -LIBEXT=dylib - -def_prefix = $(if $(findstring 64,$(arch)),mac64,mac32) - -TBB.LST = $(tbb_root)/src/tbb/$(def_prefix)-tbb-export.lst -TBB.DEF = $(TBB.LST:.lst=.def) -TBB.DLL = libtbb$(CPF_SUFFIX)$(DEBUG_SUFFIX).$(DLL) -TBB.LIB = $(TBB.DLL) -LINK_TBB.LIB = $(TBB.LIB) - -MALLOC.DEF = $(MALLOC_ROOT)/$(def_prefix)-tbbmalloc-export.def -MALLOC.DLL = libtbbmalloc$(DEBUG_SUFFIX).$(MALLOC_DLL) -MALLOC.LIB = $(MALLOC.DLL) -LINK_MALLOC.LIB = $(MALLOC.LIB) - -MALLOCPROXY.DLL = libtbbmalloc_proxy$(DEBUG_SUFFIX).$(MALLOC_DLL) -MALLOCPROXY.LIB = $(MALLOCPROXY.DLL) -LINK_MALLOCPROXY.LIB = $(MALLOCPROXY.LIB) - -TEST_LAUNCHER=sh $(tbb_root)/build/test_launcher.sh $(largs) - -OPENCL.LIB = -framework OpenCL diff --git a/src/tbb/build/mic.icc.inc b/src/tbb/build/mic.icc.inc deleted file mode 100644 index 5574a8dc3..000000000 --- a/src/tbb/build/mic.icc.inc +++ /dev/null @@ -1,79 +0,0 @@ -# Copyright (c) 2005-2017 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -# -# - -COMPILE_ONLY = -c -MMD -PREPROC_ONLY = -E -x c++ -INCLUDE_KEY = -I -DEFINE_KEY = -D -OUTPUT_KEY = -o # -OUTPUTOBJ_KEY = -o # -PIC_KEY = -fPIC -WARNING_AS_ERROR_KEY = -Werror -WARNING_KEY = -w1 -DYLIB_KEY = -shared -Wl,-soname=$@ -EXPORT_KEY = -Wl,--version-script, -NOINTRINSIC_KEY = -fno-builtin -LIBDL = -ldl -SDL_FLAGS = -fstack-protector -Wformat -Wformat-security -CPLUS = icpc -CONLY = icc - -ifeq (release,$(cfg)) - SDL_FLAGS += -D_FORTIFY_SOURCE=2 - CPLUS_FLAGS = -O2 -g -DUSE_PTHREAD -else - CPLUS_FLAGS = -O0 -g -DUSE_PTHREAD -DTBB_USE_DEBUG -endif - -ifneq (,$(codecov)) - CPLUS_FLAGS += -prof-gen=srcpos -endif - -ifneq (,$(shell icc -dumpversion | egrep "^1[6-9]\.")) -OPENMP_FLAG = -qopenmp -else -OPENMP_FLAG = -openmp -endif - -LIB_LINK_FLAGS = -shared -static-intel -Wl,-soname=$(BUILDING_LIBRARY) -z relro -z now -LIBS += -lpthread -lrt -C_FLAGS = $(CPLUS_FLAGS) -CILK_AVAILABLE = yes - -TBB_ASM.OBJ= -MALLOC_ASM.OBJ= - -CPLUS_FLAGS += -DHARNESS_INCOMPLETE_SOURCES=1 -D__TBB_MIC_NATIVE -DTBB_USE_EXCEPTIONS=0 -qopt-streaming-stores never -CPLUS += -mmic -CONLY += -mmic -LINK_FLAGS = -Wl,-rpath-link=. -rdynamic -# Tell the icc to not link against libcilk*. Otherwise icc tries to link and emits a warning message. -LIB_LINK_FLAGS += -no-intel-extensions - -#------------------------------------------------------------------------------ -# Setting tbbmalloc data. -#------------------------------------------------------------------------------ - -M_CPLUS_FLAGS = $(CPLUS_FLAGS) -fno-rtti -fno-exceptions - -#------------------------------------------------------------------------------ -# End of setting tbbmalloc data. -#------------------------------------------------------------------------------ - - - diff --git a/src/tbb/build/mic.linux.inc b/src/tbb/build/mic.linux.inc deleted file mode 100644 index b739d6179..000000000 --- a/src/tbb/build/mic.linux.inc +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright (c) 2005-2017 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -# -# - -ifeq ($(tbb_os),mic) - $(error MIC supports only cross-compilation. Specify "target=mic" instead.) -endif - -ifneq ($(BUILDING_PHASE),1) - # The same build prefix should be used in offload.inc - ifeq (,$(tbb_build_prefix)) - tbb_build_prefix=mic_icc$(CPF_SUFFIX) - endif - # For examples - mic_tbb_build_prefix=$(tbb_build_prefix) -endif - -MAKE_VERSIONS=sh $(tbb_root)/build/version_info_linux.sh $(VERSION_FLAGS) >version_string.ver -MAKE_TBBVARS=sh $(tbb_root)/build/generate_tbbvars.sh MIC_ MIC_ -def_prefix=lin64 - -TEST_LAUNCHER= -run_cmd ?= bash $(tbb_root)/build/mic.linux.launcher.sh $(largs) - -# detects whether examples are being built. -ifeq ($(BUILDING_PHASE),0) - export UI = con - export x64 = 64 -endif # examples diff --git a/src/tbb/build/mic.linux.launcher.sh b/src/tbb/build/mic.linux.launcher.sh deleted file mode 100644 index ac69a43f2..000000000 --- a/src/tbb/build/mic.linux.launcher.sh +++ /dev/null @@ -1,161 +0,0 @@ -#!/bin/bash -# -# Copyright (c) 2005-2017 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -# -# - -# Usage: -# mic.linux.launcher.sh [-v] [-q] [-s] [-r ] [-u] [-l ] -# where: -v enables verbose output -# where: -q enables quiet mode -# where: -s runs the test in stress mode (until non-zero exit code or ctrl-c pressed) -# where: -r specifies number of times to repeat execution -# where: -u limits stack size -# where: -l specifies the library name to be assigned to LD_PRELOAD -# -# Libs and executable necessary for testing should be present in the current directory before running. -# Note: Do not remove the redirections to '/dev/null' in the script, otherwise the nightly test system will fail. -# -trap 'echo Error at line $LINENO while executing "$BASH_COMMAND"' ERR # -trap 'echo -e "\n*** Interrupted ***" && exit 1' SIGINT SIGQUIT # -# Process the optional arguments if present -while getopts "qvsr:ul:" flag # -do case $flag in # - s ) # Stress testing mode - echo Doing stress testing. Press Ctrl-C to terminate - run_env='stressed() { while $*; do :; done; };' # - run_prefix="stressed $run_prefix" ;; # - r ) # Repeats test n times - run_env="repeated() { for i in \$(seq 1 $OPTARG); do echo \$i of $OPTARG:; \$*; done; };" # - run_prefix="repeated $run_prefix" ;; # - l ) # Additional library - ldd_list+="$OPTARG " # - run_prefix+=" LD_PRELOAD=$OPTARG" ;; # - u ) # Set stack limit - run_prefix="ulimit -s 10240; $run_prefix" ;; # - q ) # Quiet mode, removes 'done' but prepends any other output by test name - SUPPRESS='>/dev/null' # - verbose=1 ;; # TODO: implement a better quiet mode - v ) # Verbose mode - verbose=1 ;; # -esac done # -shift `expr $OPTIND - 1` # -[ $verbose ] || SUPPRESS='>/dev/null' # -# -# Collect the executable name -fexename="$1" # -exename=`basename $1` # -shift # -# -: ${MICDEV:=mic0} # -RSH="sudo ssh $MICDEV" # -RCP="sudo scp" # -currentdir=$PWD # -# -# Prepare the target directory on the device -targetdir="`$RSH mktemp -d /tmp/tbbtestXXXXXX 2>/dev/null`" # -# Prepare the temporary directory on the host -hostdir="`mktemp -d /tmp/tbbtestXXXXXX 2>/dev/null`" # -# -function copy_files { # - [ $verbose ] && echo Going to copy $* # - eval "cp $* $hostdir/ $SUPPRESS 2>/dev/null || exit \$?" # - eval "$RCP $hostdir/* $MICDEV:$targetdir/ $SUPPRESS 2>/dev/null || exit \$?" # - eval "rm $hostdir/* $SUPPRESS 2>/dev/null || exit \$?" # -} # copy files -# -function clean_all() { # - eval "$RSH rm -fr $targetdir $SUPPRESS" ||: # - eval "rm -fr $hostdir $SUPPRESS" ||: # -} # clean all temporary files -# -function kill_interrupt() { # - echo -e "\n*** Killing remote $exename ***" && $RSH "killall $exename" # - clean_all # -} # kill target process -# -trap 'clean_all' SIGINT SIGQUIT # trap keyboard interrupt (control-c) -# -# Transfer the test executable file and its auxiliary libraries (named as {test}_dll.so) to the target device. -copy_files $fexename `ls ${exename%\.*}*.so 2>/dev/null ||:` # -# -# Collect all dependencies of the test and its auxiliary libraries to transfer them to the target device. -ldd_list+="libtbbmalloc*.so* libirml*.so* `$RSH ldd $targetdir/\* | grep = | cut -d= -f1 2>/dev/null`" # -fnamelist="" # -# -# Find the libraries and add them to the list. -# For example, go through MIC_LD_LIBRARY_PATH and add TBB libraries from the first -# directory that contains tbb files -mic_dir_list=`echo .:$MIC_LD_LIBRARY_PATH | tr : " "` # -[ $verbose ] && echo Searching libraries in $mic_dir_list -for name in $ldd_list; do # adds the first matched name in specified dirs - found="`find -L $mic_dir_list -name $name -a -readable -print -quit 2>/dev/null` "||: # - [ $verbose ] && echo File $name: $found - fnamelist+=$found -done # -# -# Remove extra spaces. -fnamelist=`echo $fnamelist` # -# Transfer collected executable and library files to the target device. -[ -n "$fnamelist" ] && copy_files $fnamelist -# -# Transfer input files used by example codes by scanning the executable argument list. -argfiles= # -args= # -for arg in "$@"; do # - if [ -r $arg ]; then # - argfiles+="$arg " # - args+="$(basename $arg) " # - else # - args+="$arg " # - fi # -done # -[ -n "$argfiles" ] && copy_files $argfiles # -# -# Get the list of transferred files -testfiles="`$RSH find $targetdir/ -type f | tr '\n' ' ' 2>/dev/null`" # -# -[ $verbose ] && echo Running $run_prefix ./$exename $args # -# Run the test on the target device -trap 'kill_interrupt' SIGINT SIGQUIT # trap keyboard interrupt (control-c) -trap - ERR # -run_env+="cd $targetdir; export LD_LIBRARY_PATH=.:\$LD_LIBRARY_PATH;" # -$RSH "$run_env $run_prefix ./$exename $args" # -# -# Delete the test files and get the list of output files -outfiles=`$RSH rm $testfiles 2>/dev/null; find $targetdir/ -type f 2>/dev/null` ||: # -if [ -n "$outfiles" ]; then # - for outfile in $outfiles; do # - filename=$(basename $outfile) # - subdir=$(dirname $outfile) # - subdir="${subdir#$targetdir}" # - [ -n $subdir ] subdir=$subdir/ # - # Create directories on host - [ ! -d "$hostdir/$subdir" ] && mkdir -p "$hostdir/$subdir" # - [ ! -d "$currentdir/$subdir" ] && mkdir -p "$currentdir/$subdir" # - # Copy the output file to the temporary directory on host - eval "$RCP -r '$MICDEV:${outfile#}' '$hostdir/$subdir$filename' $SUPPRESS 2>&1 || exit \$?" # - # Copy the output file from the temporary directory to the current directory - eval "cp '$hostdir/$subdir$filename' '$currentdir/$subdir$filename' $SUPPRESS 2>&1 || exit \$?" # - done # -fi # -# -# Clean up temporary directories -clean_all -# -# Return the exit code of the test. -exit $? # diff --git a/src/tbb/build/mic.offload.inc b/src/tbb/build/mic.offload.inc deleted file mode 100644 index 96f0800fb..000000000 --- a/src/tbb/build/mic.offload.inc +++ /dev/null @@ -1,118 +0,0 @@ -# Copyright (c) 2005-2017 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -# -# - -ifneq (mic,$(offload)) - $(error File mic.offload.inc should not be included directly. Use offload=mic instead.) -endif -ifneq (icc,$(compiler)) - $(error Only Intel(R) Compiler is supported for MIC offload compilation) -endif - -# The same build prefix should be used in mic.linux.inc -mic_tbb_build_prefix=mic_icc$(CPF_SUFFIX) -MIC_OFFLOAD_NATIVE_PATH?=../$(mic_tbb_build_prefix)_$(cfg) - -ifdef BUILDING_PHASE - ifeq ($(BUILDING_PHASE),1) - # Tests - export MIC_OFFLOAD_NATIVE_PATH - LINK_TBB_NATIVE.LIB=$(MIC_OFFLOAD_NATIVE_PATH)/$(TBB.LIB) - LINK_TBB.LIB=-qoffload-option,mic,ld,"$(LINK_TBB_NATIVE.LIB)" $(TBB.LIB) - LINK_MALLOC_NATIVE.LIB=$(MIC_OFFLOAD_NATIVE_PATH)/$(MALLOC.DLL) - LINK_MALLOC.LIB=-qoffload-option,mic,ld,"$(LINK_MALLOC_NATIVE.LIB)" $(MALLOC.LIB) - LINK_MALLOCPROXY_NATIVE.LIB=$(MIC_OFFLOAD_NATIVE_PATH)/$(MALLOCPROXY.DLL) - LINK_MALLOCPROXY.LIB=-qoffload-option,mic,ld,"$(LINK_MALLOCPROXY_NATIVE.LIB)" $(MALLOCPROXY.LIB) - - # Export extensions for test_launcher - export DLL - export TEST_EXT=offload.exe - OBJ=offload.o - - # Do not use -Werror because it is too strict for the early offload compiler. - # Need to set anything because WARNING_AS_ERROR_KEY should not be empty. - # Treat #2426 as a warning. Print errors only. - tbb_strict=0 - WARNING_AS_ERROR_KEY = Warning as error - WARNING_KEY = -diag-warning 2426 -w0 - - CXX_MIC_STUFF = -qoffload-attribute-target=mic -D__TBB_MIC_OFFLOAD=1 -qoffload-option,mic,compiler,"-D__TBB_MIC_OFFLOAD=1 $(CXX_MIC_NATIVE_STUFF)" - CXX_MIC_NATIVE_STUFF = -DHARNESS_INCOMPLETE_SOURCES=1 -D__TBB_MIC_NATIVE -DTBB_USE_EXCEPTIONS=0 - CPLUS_FLAGS += $(CXX_MIC_STUFF) - - # Some tests require that an executable exports its symbols. - LINK_FLAGS += -qoffload-option,mic,ld,"--export-dynamic" - - # libcoi_device.so is needed for COIProcessProxyFlush used in Harness. - LINK_FLAGS += -qoffload-option,mic,ld,"-lcoi_device" - - # DSO-linking semantics forces linking libpthread and librt to a test. - LINK_FLAGS += -qoffload-option,mic,ld,"-lpthread -lrt" - - .PHONY: FORCE - FORCE: - - $(MIC_OFFLOAD_NATIVE_PATH)/%_dll.$(DLL): FORCE - @$(MAKE) --no-print-directory -C "$(MIC_OFFLOAD_NATIVE_PATH)" target=mic offload= -f$(tbb_root)/build/Makefile.$(TESTFILE) $*_dll.$(DLL) - %_dll.$(DLL): $(MIC_OFFLOAD_NATIVE_PATH)/%_dll.$(DLL) FORCE - @$(MAKE) --no-print-directory offload= -f$(tbb_root)/build/Makefile.$(TESTFILE) $*_dll.$(DLL) - - .PRECIOUS: $(MIC_OFFLOAD_NATIVE_PATH)/%_dll.$(DLL) - - %.$(TEST_EXT): LINK_FILES+=-qoffload-option,mic,ld,"$(addprefix $(MIC_OFFLOAD_NATIVE_PATH)/,$(TEST_LIBS))" - - TEST_LAUNCHER=sh $(tbb_root)/build/test_launcher.sh $(largs) - - ifdef MIC_LD_LIBRARY_PATH - export MIC_LD_LIBRARY_PATH := $(MIC_OFFLOAD_NATIVE_PATH):$(MIC_LD_LIBRARY_PATH) - else - export MIC_LD_LIBRARY_PATH := $(MIC_OFFLOAD_NATIVE_PATH) - endif - else - # Examples - export UI = con - export x64 = 64 - endif -else - # Libraries - LIB_TARGETS = tbb tbbmalloc tbbproxy rml - addsuffixes = $(foreach suff,$(1),$(addsuffix $(suff),$(2))) - - .PHONY: $(call addsuffixes, _debug _release _debug_mic _release_mic,$(LIB_TARGETS)) - - # The dependence on *_debug and *_release targets unifies the offload support - # for top-level Makefile and src/Makefile - $(LIB_TARGETS): %: %_release %_debug - - # "override offload=" suppresses the "offload" variable value for nested makes - $(LIB_TARGETS) $(call addsuffixes, _debug _release,$(LIB_TARGETS)): override offload= - # Apply overriding for library builds - export offload - export tbb_build_prefix - # Add the dependency on target libraries - $(call addsuffixes, _debug _release,$(LIB_TARGETS)): %: %_mic - - # tbb_build_prefix should be overridden since we want to restart make in "clear" environment - $(call addsuffixes, _debug_mic _release_mic,$(LIB_TARGETS)): override tbb_build_prefix= - $(call addsuffixes, _debug_mic _release_mic,$(LIB_TARGETS)): %_mic: - @$(MAKE) --no-print-directory -C "$(full_tbb_root)/src" $* target=mic tbb_root=.. - - mic_clean: override tbb_build_prefix= - mic_clean: - @$(MAKE) --no-print-directory -C "$(full_tbb_root)/src" clean offload= target=mic tbb_root=.. - clean: mic_clean -endif diff --git a/src/tbb/build/suncc.map.pause b/src/tbb/build/suncc.map.pause deleted file mode 100644 index a92d08eb1..000000000 --- a/src/tbb/build/suncc.map.pause +++ /dev/null @@ -1 +0,0 @@ -hwcap_1 = OVERRIDE; \ No newline at end of file diff --git a/src/tbb/build/test_launcher.bat b/src/tbb/build/test_launcher.bat deleted file mode 100644 index 88c43dc45..000000000 --- a/src/tbb/build/test_launcher.bat +++ /dev/null @@ -1,74 +0,0 @@ -@echo off -REM -REM Copyright (c) 2005-2017 Intel Corporation -REM -REM Licensed under the Apache License, Version 2.0 (the "License"); -REM you may not use this file except in compliance with the License. -REM You may obtain a copy of the License at -REM -REM http://www.apache.org/licenses/LICENSE-2.0 -REM -REM Unless required by applicable law or agreed to in writing, software -REM distributed under the License is distributed on an "AS IS" BASIS, -REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -REM See the License for the specific language governing permissions and -REM limitations under the License. -REM -REM -REM -REM -REM - -set cmd_line= -if DEFINED run_prefix set cmd_line=%run_prefix% -:while -if NOT "%1"=="" ( - REM Verbose mode - if "%1"=="-v" ( - set verbose=yes - GOTO continue - ) - REM Silent mode of 'make' requires additional support for associating - REM of test output with the test name. Verbose mode is the simplest way - if "%1"=="-q" ( - set verbose=yes - GOTO continue - ) - REM Run in stress mode - if "%1"=="-s" ( - echo Doing stress testing. Press Ctrl-C to terminate - set stress=yes - GOTO continue - ) - REM Repeat execution specified number of times - if "%1"=="-r" ( - set repeat=%2 - SHIFT - GOTO continue - ) - REM no LD_PRELOAD under Windows - REM but run the test to check "#pragma comment" construction - if "%1"=="-l" ( - REM The command line may specify -l with empty dll name, - REM e.g. "test_launcher.bat -l app.exe". If the dll name is - REM empty then %2 contains the application name and the SHIFT - REM operation is not necessary. - if exist "%3" SHIFT - GOTO continue - ) - REM no need to setup up stack size under Windows - if "%1"=="-u" GOTO continue - set cmd_line=%cmd_line% %1 -:continue - SHIFT - GOTO while -) -set cmd_line=%cmd_line:./=.\% -if DEFINED verbose echo Running %cmd_line% -if DEFINED stress set cmd_line=%cmd_line% ^& IF NOT ERRORLEVEL 1 GOTO stress -:stress -if DEFINED repeat ( - for /L %%i in (1,1,%repeat%) do echo %%i of %repeat%: & %cmd_line% -) else ( - %cmd_line% -) diff --git a/src/tbb/build/test_launcher.sh b/src/tbb/build/test_launcher.sh deleted file mode 100644 index 831136f01..000000000 --- a/src/tbb/build/test_launcher.sh +++ /dev/null @@ -1,94 +0,0 @@ -#!/bin/sh -# -# Copyright (c) 2005-2017 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -# -# - -# Usage: -# test_launcher.sh [-v] [-q] [-s] [-r ] [-u] [-l ] -# where: -v enables verbose output -# where: -q enables quiet mode -# where: -s runs the test in stress mode (until non-zero exit code or ctrl-c pressed) -# where: -r specifies number of times to repeat execution -# where: -u limits stack size -# where: -l specifies the library name to be assigned to LD_PRELOAD - -while getopts "qvsr:ul:" flag # -do case $flag in # - s ) # Stress testing mode - run_prefix="stressed $run_prefix" ;; # - r ) # Repeats test n times - repeat=$OPTARG # - run_prefix="repeated $run_prefix" ;; # - l ) if [ `uname` = 'Linux' ] ; then # - LD_PRELOAD=$OPTARG # - elif [ `uname` = 'Darwin' ] ; then # - DYLD_INSERT_LIBRARIES=$OPTARG # - else # - echo 'skip' # - exit # - fi ;; # - u ) # Set stack limit - ulimit -s 10240 ;; # - q ) # Quiet mode, removes 'done' but prepends any other output by test name - OUTPUT='2>&1 | sed -e "s/done//;/^[[:space:]]*$/d;s!^!$1: !"' ;; # - v ) # Verbose mode - verbose=1 ;; # -esac done # -shift `expr $OPTIND - 1` # -if [ $MIC_OFFLOAD_NATIVE_PATH ] ; then # - LIB_NAME=${1/%.$TEST_EXT/_dll.$DLL} # - if [ -f "$MIC_OFFLOAD_NATIVE_PATH/$LIB_NAME" ]; then # - [ -z "$MIC_CARD" ] && MIC_CARD=mic0 # - TMPDIR_HOST=`mktemp -d /tmp/tbbtestXXXXXX` # - TMPDIR_MIC=`sudo ssh $MIC_CARD mktemp -d /tmp/tbbtestXXXXXX` # - sudo ssh $MIC_CARD "chmod +x $TMPDIR_MIC" # - # Test specific library may depend on libtbbmalloc* - cp "$MIC_OFFLOAD_NATIVE_PATH/$LIB_NAME" "$MIC_OFFLOAD_NATIVE_PATH"/libtbbmalloc* "$TMPDIR_HOST" >/dev/null 2>/dev/null # - sudo scp "$TMPDIR_HOST"/* $MIC_CARD:"$TMPDIR_MIC" >/dev/null 2>/dev/null # - - LD_LIBRARY_PATH=$TMPDIR_MIC:$LD_LIBRARY_PATH # - export LD_LIBRARY_PATH # - fi # -fi # -stressed() { echo Doing stress testing. Press Ctrl-C to terminate # - while :; do $*; done;# -} # -repeated() { # - i=0; while [ "$i" -lt $repeat ]; do i=`expr $i + 1`; echo $i of $repeat:; $*; done # -} # -# DYLD_LIBRARY_PATH can be purged on OS X 10.11, set it again -if [ `uname` = 'Darwin' -a -z "$DYLD_LIBRARY_PATH" ] ; then # - DYLD_LIBRARY_PATH=. # - export DYLD_LIBRARY_PATH # -fi # -# Run the command line passed via parameters -[ $verbose ] && echo Running $run_prefix $* # -if [ -n "$LD_PRELOAD" ] ; then # - export LD_PRELOAD # -elif [ -n "$DYLD_INSERT_LIBRARIES" ] ; then # - export DYLD_INSERT_LIBRARIES # -fi # -exec 4>&1 # extracting exit code of the first command in pipeline needs duplicated stdout -# custom redirection needs eval, otherwise shell cannot parse it -err=`eval '( $run_prefix $* || echo \$? >&3; )' ${OUTPUT} 3>&1 >&4` # -[ -z "$err" ] || echo $1: exited with error $err # -if [ $MIC_OFFLOAD_NATIVE_PATH ] ; then # - sudo ssh $MIC_CARD rm -fr "$TMPDIR_MIC" >/dev/null 2>/dev/null # - rm -fr "$TMPDIR_HOST" >/dev/null 2>/dev/null # -fi # -exit $err # diff --git a/src/tbb/build/version_info_aix.sh b/src/tbb/build/version_info_aix.sh deleted file mode 100644 index 622a22ed6..000000000 --- a/src/tbb/build/version_info_aix.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/sh -# -# Copyright (c) 2005-2017 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -# -# - -# Script used to generate version info string -echo "#define __TBB_VERSION_STRINGS(N) \\" -echo '#N": BUILD_HOST'"\t\t"`hostname -s`" ("`uname -m`")"'" ENDL \' -# find OS name in *-release and issue* files by filtering blank lines and lsb-release content out -echo '#N": BUILD_OS'"\t\t"`lsb_release -sd 2>/dev/null | grep -ih '[a-z] ' - /etc/*release /etc/issue 2>/dev/null | head -1 | sed -e 's/["\\\\]//g'`'" ENDL \' -echo '#N": BUILD_KERNEL'"\t"`uname -srv`'" ENDL \' -echo '#N": BUILD_GCC'"\t\t"`g++ --version &1 | grep 'g++'`'" ENDL \' -[ -z "$COMPILER_VERSION" ] || echo '#N": BUILD_COMPILER'"\t"$COMPILER_VERSION'" ENDL \' -echo '#N": BUILD_LIBC'"\t"`getconf GNU_LIBC_VERSION | grep glibc | sed -e 's/^glibc //'`'" ENDL \' -echo '#N": BUILD_LD'"\t\t"`ld -v 2>&1 | grep 'version'`'" ENDL \' -echo '#N": BUILD_TARGET'"\t$arch on $runtime"'" ENDL \' -echo '#N": BUILD_COMMAND'"\t"$*'" ENDL \' -echo "" -echo "#define __TBB_DATETIME \""`date -u`"\"" diff --git a/src/tbb/build/version_info_android.sh b/src/tbb/build/version_info_android.sh deleted file mode 100644 index 2439207f1..000000000 --- a/src/tbb/build/version_info_android.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/sh -# -# Copyright (c) 2005-2017 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -# -# - -# Script used to generate version info string -echo "#define __TBB_VERSION_STRINGS(N) \\" -echo '#N": BUILD_HOST'"\t\t"`hostname -s`" ("`uname -m`")"'" ENDL \' -# find OS name in *-release and issue* files by filtering blank lines and lsb-release content out -echo '#N": BUILD_OS'"\t\t"`lsb_release -sd 2>/dev/null | grep -ih '[a-z] ' - /etc/*release /etc/issue 2>/dev/null | head -1 | sed -e 's/["\\\\]//g'`'" ENDL \' -echo '#N": BUILD_TARGET_CXX'"\t"`$TARGET_CXX --version | head -n1`'" ENDL \' -[ -z "$COMPILER_VERSION" ] || echo '#N": BUILD_COMPILER'"\t"$COMPILER_VERSION'" ENDL \' -[ -z "$ndk_version" ] || echo '#N": BUILD_NDK'"\t\t$ndk_version"'" ENDL \' -echo '#N": BUILD_LD'"\t\t"`${tbb_tool_prefix}ld -v 2>&1 | grep 'ld'`'" ENDL \' -echo '#N": BUILD_TARGET'"\t$arch on $runtime"'" ENDL \' -echo '#N": BUILD_COMMAND'"\t"$*'" ENDL \' -echo "" -echo "#define __TBB_DATETIME \""`date -u`"\"" diff --git a/src/tbb/build/version_info_linux.sh b/src/tbb/build/version_info_linux.sh deleted file mode 100644 index 622a22ed6..000000000 --- a/src/tbb/build/version_info_linux.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/sh -# -# Copyright (c) 2005-2017 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -# -# - -# Script used to generate version info string -echo "#define __TBB_VERSION_STRINGS(N) \\" -echo '#N": BUILD_HOST'"\t\t"`hostname -s`" ("`uname -m`")"'" ENDL \' -# find OS name in *-release and issue* files by filtering blank lines and lsb-release content out -echo '#N": BUILD_OS'"\t\t"`lsb_release -sd 2>/dev/null | grep -ih '[a-z] ' - /etc/*release /etc/issue 2>/dev/null | head -1 | sed -e 's/["\\\\]//g'`'" ENDL \' -echo '#N": BUILD_KERNEL'"\t"`uname -srv`'" ENDL \' -echo '#N": BUILD_GCC'"\t\t"`g++ --version &1 | grep 'g++'`'" ENDL \' -[ -z "$COMPILER_VERSION" ] || echo '#N": BUILD_COMPILER'"\t"$COMPILER_VERSION'" ENDL \' -echo '#N": BUILD_LIBC'"\t"`getconf GNU_LIBC_VERSION | grep glibc | sed -e 's/^glibc //'`'" ENDL \' -echo '#N": BUILD_LD'"\t\t"`ld -v 2>&1 | grep 'version'`'" ENDL \' -echo '#N": BUILD_TARGET'"\t$arch on $runtime"'" ENDL \' -echo '#N": BUILD_COMMAND'"\t"$*'" ENDL \' -echo "" -echo "#define __TBB_DATETIME \""`date -u`"\"" diff --git a/src/tbb/build/version_info_macos.sh b/src/tbb/build/version_info_macos.sh deleted file mode 100644 index 397b10811..000000000 --- a/src/tbb/build/version_info_macos.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/sh -# -# Copyright (c) 2005-2017 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -# -# - -# Script used to generate version info string -echo "#define __TBB_VERSION_STRINGS(N) \\" -echo '#N": BUILD_HOST'"\t\t"`hostname -s`" ("`arch`")"'" ENDL \' -echo '#N": BUILD_OS'"\t\t"`sw_vers -productName`" version "`sw_vers -productVersion`'" ENDL \' -echo '#N": BUILD_KERNEL'"\t"`uname -v`'" ENDL \' -echo '#N": BUILD_CLANG'"\t"`clang --version &1 | grep 'version '`'" ENDL \' -echo '#N": BUILD_XCODE'"\t"`xcodebuild -version &1 | grep 'Xcode'`'" ENDL \' -[ -z "$COMPILER_VERSION" ] || echo '#N": BUILD_COMPILER'"\t"$COMPILER_VERSION'" ENDL \' -echo '#N": BUILD_TARGET'"\t$arch on $runtime"'" ENDL \' -echo '#N": BUILD_COMMAND'"\t"$*'" ENDL \' -echo "" -echo "#define __TBB_DATETIME \""`date -u`"\"" diff --git a/src/tbb/build/version_info_sunos.sh b/src/tbb/build/version_info_sunos.sh deleted file mode 100644 index 96cd9ba70..000000000 --- a/src/tbb/build/version_info_sunos.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/sh -# -# Copyright (c) 2005-2017 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -# -# - -# Script used to generate version info string -echo "#define __TBB_VERSION_STRINGS(N) \\" -echo '#N": BUILD_HOST'"\t"`hostname`" ("`arch`")"'" ENDL \' -echo '#N": BUILD_OS'"\t\t"`uname`'" ENDL \' -echo '#N": BUILD_KERNEL'"\t"`uname -srv`'" ENDL \' -echo '#N": BUILD_SUNCC'"\t"`CC -V &1 | grep 'C++'`'" ENDL \' -[ -z "$COMPILER_VERSION" ] || echo '#N": BUILD_COMPILER'"\t"$COMPILER_VERSION'" ENDL \' -echo '#N": BUILD_TARGET'"\t$arch on $runtime"'" ENDL \' -echo '#N": BUILD_COMMAND'"\t"$*'" ENDL \' -echo "" -echo "#define __TBB_DATETIME \""`date -u`"\"" diff --git a/src/tbb/build/version_info_windows.js b/src/tbb/build/version_info_windows.js deleted file mode 100644 index 57483b3c6..000000000 --- a/src/tbb/build/version_info_windows.js +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright (c) 2005-2017 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// -// -// - -var WshShell = WScript.CreateObject("WScript.Shell"); - -var tmpExec; - -WScript.Echo("#define __TBB_VERSION_STRINGS(N) \\"); - -//Getting BUILD_HOST -WScript.echo( "#N \": BUILD_HOST\\t\\t" + - WshShell.ExpandEnvironmentStrings("%COMPUTERNAME%") + - "\" ENDL \\" ); - -//Getting BUILD_OS -tmpExec = WshShell.Exec("cmd /c ver"); -while ( tmpExec.Status == 0 ) { - WScript.Sleep(100); -} -tmpExec.StdOut.ReadLine(); - -WScript.echo( "#N \": BUILD_OS\\t\\t" + - tmpExec.StdOut.ReadLine() + - "\" ENDL \\" ); - -if ( WScript.Arguments(0).toLowerCase().match("gcc") ) { - tmpExec = WshShell.Exec(WScript.Arguments(0) + " --version"); - WScript.echo( "#N \": BUILD_GCC\\t\\t" + - tmpExec.StdOut.ReadLine() + - "\" ENDL \\" ); - -} else if ( WScript.Arguments(0).toLowerCase().match("clang") ) { - tmpExec = WshShell.Exec(WScript.Arguments(0) + " --version"); - WScript.echo( "#N \": BUILD_CLANG\\t" + - tmpExec.StdOut.ReadLine() + - "\" ENDL \\" ); - -} else { // MS / Intel compilers - //Getting BUILD_CL - tmpExec = WshShell.Exec("cmd /c echo #define 0 0>empty.cpp"); - tmpExec = WshShell.Exec("cl -c empty.cpp "); - while ( tmpExec.Status == 0 ) { - WScript.Sleep(100); - } - var clVersion = tmpExec.StdErr.ReadLine(); - WScript.echo( "#N \": BUILD_CL\\t\\t" + - clVersion + - "\" ENDL \\" ); - - //Getting BUILD_COMPILER - if ( WScript.Arguments(0).toLowerCase().match("icl") ) { - tmpExec = WshShell.Exec("icl -c empty.cpp "); - while ( tmpExec.Status == 0 ) { - WScript.Sleep(100); - } - WScript.echo( "#N \": BUILD_COMPILER\\t" + - tmpExec.StdErr.ReadLine() + - "\" ENDL \\" ); - } else { - WScript.echo( "#N \": BUILD_COMPILER\\t\\t" + - clVersion + - "\" ENDL \\" ); - } - tmpExec = WshShell.Exec("cmd /c del /F /Q empty.obj empty.cpp"); -} - -//Getting BUILD_TARGET -WScript.echo( "#N \": BUILD_TARGET\\t" + - WScript.Arguments(1) + - "\" ENDL \\" ); - -//Getting BUILD_COMMAND -WScript.echo( "#N \": BUILD_COMMAND\\t" + WScript.Arguments(2) + "\" ENDL" ); - -//Getting __TBB_DATETIME and __TBB_VERSION_YMD -var date = new Date(); -WScript.echo( "#define __TBB_DATETIME \"" + date.toUTCString() + "\"" ); -WScript.echo( "#define __TBB_VERSION_YMD " + date.getUTCFullYear() + ", " + - (date.getUTCMonth() > 8 ? (date.getUTCMonth()+1):("0"+(date.getUTCMonth()+1))) + - (date.getUTCDate() > 9 ? date.getUTCDate():("0"+date.getUTCDate())) ); -WScript.echo( "" ) diff --git a/src/tbb/build/vs2013/index.html b/src/tbb/build/vs2013/index.html deleted file mode 100644 index 4e597b666..000000000 --- a/src/tbb/build/vs2013/index.html +++ /dev/null @@ -1,30 +0,0 @@ - - - -

Overview

-This directory contains the Visual Studio* 2013 solution to build Intel® Threading Building Blocks. - - -

Files

-
-
makefile.sln -
Solution file.
-
tbb.vcxproj -
Library project file.
-
tbbmalloc.vcxproj -
Scalable allocator library project file.
-
tbbmalloc_proxy.vcxproj -
Standard allocator replacement project file.
-
- -
-Up to parent directory -

-Copyright © 2017 Intel Corporation. All Rights Reserved. -

-Intel and the Intel logo are trademarks of Intel Corporation -or its subsidiaries in the U.S. and/or other countries. -

-* Other names and brands may be claimed as the property of others. - - diff --git a/src/tbb/build/vs2013/makefile.sln b/src/tbb/build/vs2013/makefile.sln deleted file mode 100644 index b913551e6..000000000 --- a/src/tbb/build/vs2013/makefile.sln +++ /dev/null @@ -1,80 +0,0 @@ -Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio 2013 -VisualStudioVersion = 12.0.40629.0 -MinimumVisualStudioVersion = 10.0.40219.1 -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{8898CE0B-0BFB-45AE-AA71-83735ED2510D}" - ProjectSection(SolutionItems) = preProject - index.html = index.html - EndProjectSection -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "tbb", "tbb.vcxproj", "{F62787DD-1327-448B-9818-030062BCFAA5}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "tbbmalloc", "tbbmalloc.vcxproj", "{B15F131E-328A-4D42-ADC2-9FF4CA6306D8}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "tbbmalloc_proxy", "tbbmalloc_proxy.vcxproj", "{02F61511-D5B6-46E6-B4BB-DEAA96E6BCC7}" -EndProject -Global - GlobalSection(SolutionConfigurationPlatforms) = preSolution - Debug|Win32 = Debug|Win32 - Debug|x64 = Debug|x64 - Debug-MT|Win32 = Debug-MT|Win32 - Debug-MT|x64 = Debug-MT|x64 - Release|Win32 = Release|Win32 - Release|x64 = Release|x64 - Release-MT|Win32 = Release-MT|Win32 - Release-MT|x64 = Release-MT|x64 - EndGlobalSection - GlobalSection(ProjectConfigurationPlatforms) = postSolution - {F62787DD-1327-448B-9818-030062BCFAA5}.Debug|Win32.ActiveCfg = Debug|Win32 - {F62787DD-1327-448B-9818-030062BCFAA5}.Debug|Win32.Build.0 = Debug|Win32 - {F62787DD-1327-448B-9818-030062BCFAA5}.Debug|x64.ActiveCfg = Debug|x64 - {F62787DD-1327-448B-9818-030062BCFAA5}.Debug|x64.Build.0 = Debug|x64 - {F62787DD-1327-448B-9818-030062BCFAA5}.Debug-MT|Win32.ActiveCfg = Debug-MT|Win32 - {F62787DD-1327-448B-9818-030062BCFAA5}.Debug-MT|Win32.Build.0 = Debug-MT|Win32 - {F62787DD-1327-448B-9818-030062BCFAA5}.Debug-MT|x64.ActiveCfg = Debug-MT|x64 - {F62787DD-1327-448B-9818-030062BCFAA5}.Debug-MT|x64.Build.0 = Debug-MT|x64 - {F62787DD-1327-448B-9818-030062BCFAA5}.Release|Win32.ActiveCfg = Release|Win32 - {F62787DD-1327-448B-9818-030062BCFAA5}.Release|Win32.Build.0 = Release|Win32 - {F62787DD-1327-448B-9818-030062BCFAA5}.Release|x64.ActiveCfg = Release|x64 - {F62787DD-1327-448B-9818-030062BCFAA5}.Release|x64.Build.0 = Release|x64 - {F62787DD-1327-448B-9818-030062BCFAA5}.Release-MT|Win32.ActiveCfg = Release-MT|Win32 - {F62787DD-1327-448B-9818-030062BCFAA5}.Release-MT|Win32.Build.0 = Release-MT|Win32 - {F62787DD-1327-448B-9818-030062BCFAA5}.Release-MT|x64.ActiveCfg = Release-MT|x64 - {F62787DD-1327-448B-9818-030062BCFAA5}.Release-MT|x64.Build.0 = Release-MT|x64 - {B15F131E-328A-4D42-ADC2-9FF4CA6306D8}.Debug|Win32.ActiveCfg = Debug|Win32 - {B15F131E-328A-4D42-ADC2-9FF4CA6306D8}.Debug|Win32.Build.0 = Debug|Win32 - {B15F131E-328A-4D42-ADC2-9FF4CA6306D8}.Debug|x64.ActiveCfg = Debug|x64 - {B15F131E-328A-4D42-ADC2-9FF4CA6306D8}.Debug|x64.Build.0 = Debug|x64 - {B15F131E-328A-4D42-ADC2-9FF4CA6306D8}.Debug-MT|Win32.ActiveCfg = Debug-MT|Win32 - {B15F131E-328A-4D42-ADC2-9FF4CA6306D8}.Debug-MT|Win32.Build.0 = Debug-MT|Win32 - {B15F131E-328A-4D42-ADC2-9FF4CA6306D8}.Debug-MT|x64.ActiveCfg = Debug-MT|x64 - {B15F131E-328A-4D42-ADC2-9FF4CA6306D8}.Debug-MT|x64.Build.0 = Debug-MT|x64 - {B15F131E-328A-4D42-ADC2-9FF4CA6306D8}.Release|Win32.ActiveCfg = Release|Win32 - {B15F131E-328A-4D42-ADC2-9FF4CA6306D8}.Release|Win32.Build.0 = Release|Win32 - {B15F131E-328A-4D42-ADC2-9FF4CA6306D8}.Release|x64.ActiveCfg = Release|x64 - {B15F131E-328A-4D42-ADC2-9FF4CA6306D8}.Release|x64.Build.0 = Release|x64 - {B15F131E-328A-4D42-ADC2-9FF4CA6306D8}.Release-MT|Win32.ActiveCfg = Release-MT|Win32 - {B15F131E-328A-4D42-ADC2-9FF4CA6306D8}.Release-MT|Win32.Build.0 = Release-MT|Win32 - {B15F131E-328A-4D42-ADC2-9FF4CA6306D8}.Release-MT|x64.ActiveCfg = Release-MT|x64 - {B15F131E-328A-4D42-ADC2-9FF4CA6306D8}.Release-MT|x64.Build.0 = Release-MT|x64 - {02F61511-D5B6-46E6-B4BB-DEAA96E6BCC7}.Debug|Win32.ActiveCfg = Debug|Win32 - {02F61511-D5B6-46E6-B4BB-DEAA96E6BCC7}.Debug|Win32.Build.0 = Debug|Win32 - {02F61511-D5B6-46E6-B4BB-DEAA96E6BCC7}.Debug|x64.ActiveCfg = Debug|x64 - {02F61511-D5B6-46E6-B4BB-DEAA96E6BCC7}.Debug|x64.Build.0 = Debug|x64 - {02F61511-D5B6-46E6-B4BB-DEAA96E6BCC7}.Debug-MT|Win32.ActiveCfg = Debug-MT|Win32 - {02F61511-D5B6-46E6-B4BB-DEAA96E6BCC7}.Debug-MT|Win32.Build.0 = Debug-MT|Win32 - {02F61511-D5B6-46E6-B4BB-DEAA96E6BCC7}.Debug-MT|x64.ActiveCfg = Debug-MT|x64 - {02F61511-D5B6-46E6-B4BB-DEAA96E6BCC7}.Debug-MT|x64.Build.0 = Debug-MT|x64 - {02F61511-D5B6-46E6-B4BB-DEAA96E6BCC7}.Release|Win32.ActiveCfg = Release|Win32 - {02F61511-D5B6-46E6-B4BB-DEAA96E6BCC7}.Release|Win32.Build.0 = Release|Win32 - {02F61511-D5B6-46E6-B4BB-DEAA96E6BCC7}.Release|x64.ActiveCfg = Release|x64 - {02F61511-D5B6-46E6-B4BB-DEAA96E6BCC7}.Release|x64.Build.0 = Release|x64 - {02F61511-D5B6-46E6-B4BB-DEAA96E6BCC7}.Release-MT|Win32.ActiveCfg = Release-MT|Win32 - {02F61511-D5B6-46E6-B4BB-DEAA96E6BCC7}.Release-MT|Win32.Build.0 = Release-MT|Win32 - {02F61511-D5B6-46E6-B4BB-DEAA96E6BCC7}.Release-MT|x64.ActiveCfg = Release-MT|x64 - {02F61511-D5B6-46E6-B4BB-DEAA96E6BCC7}.Release-MT|x64.Build.0 = Release-MT|x64 - EndGlobalSection - GlobalSection(SolutionProperties) = preSolution - HideSolutionNode = FALSE - EndGlobalSection -EndGlobal diff --git a/src/tbb/build/vs2013/tbb.vcxproj b/src/tbb/build/vs2013/tbb.vcxproj deleted file mode 100644 index 7329d81b0..000000000 --- a/src/tbb/build/vs2013/tbb.vcxproj +++ /dev/null @@ -1,696 +0,0 @@ - - - - - Debug-MT - Win32 - - - Debug-MT - x64 - - - Debug - Win32 - - - Debug - x64 - - - Release-MT - Win32 - - - Release-MT - x64 - - - Release - Win32 - - - Release - x64 - - - - {F62787DD-1327-448B-9818-030062BCFAA5} - tbb - Win32Proj - - - - DynamicLibrary - NotSet - true - v120 - - - DynamicLibrary - NotSet - v120 - - - DynamicLibrary - NotSet - true - v120 - - - DynamicLibrary - NotSet - v120 - - - DynamicLibrary - NotSet - true - v120 - - - DynamicLibrary - NotSet - v120 - - - DynamicLibrary - NotSet - true - v120 - - - DynamicLibrary - NotSet - v120 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - <_ProjectFileVersion>10.0.40219.1 - $(SolutionDir)$(Platform)\$(Configuration)\ - $(SolutionDir)$(Platform)\$(ProjectName)\$(Configuration)\ - false - $(SolutionDir)$(Platform)\$(Configuration)\ - $(SolutionDir)$(Platform)\$(ProjectName)\$(Configuration)\ - false - $(SolutionDir)$(Platform)\$(Configuration)\ - $(SolutionDir)$(Platform)\$(ProjectName)\$(Configuration)\ - false - $(SolutionDir)$(Platform)\$(Configuration)\ - $(SolutionDir)$(Platform)\$(ProjectName)\$(Configuration)\ - false - $(SolutionDir)$(Platform)\$(Configuration)\ - $(SolutionDir)$(Platform)\$(ProjectName)\$(Configuration)\ - false - $(SolutionDir)$(Platform)\$(Configuration)\ - $(SolutionDir)$(Platform)\$(ProjectName)\$(Configuration)\ - false - $(SolutionDir)$(Platform)\$(Configuration)\ - $(SolutionDir)$(Platform)\$(ProjectName)\$(Configuration)\ - false - $(SolutionDir)$(Platform)\$(Configuration)\ - $(SolutionDir)$(Platform)\$(ProjectName)\$(Configuration)\ - false - AllRules.ruleset - - - AllRules.ruleset - - - AllRules.ruleset - - - AllRules.ruleset - - - AllRules.ruleset - - - AllRules.ruleset - - - AllRules.ruleset - - - AllRules.ruleset - - - $(ProjectName)_debug - $(ProjectName)_debug - $(ProjectName)_debug - $(ProjectName)_debug - - - - /c /MDd /Od /Ob0 /Zi /EHsc /GR /Zc:forScope /Zc:wchar_t /DTBB_USE_DEBUG /D__TBB_LIB_NAME=tbb_debug.lib /DDO_ITT_NOTIFY /GS /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0502 /D__TBB_BUILD=1 /W4 /I../../src /I../../src/rml/include /I../../include - Disabled - .;%(AdditionalIncludeDirectories) - %(PreprocessorDefinitions) - true - EnableFastChecks - MultiThreadedDebugDLL - - - Level4 - ProgramDatabase - - - /DLL /MAP /DEBUG /fixed:no /INCREMENTAL:NO /DEF:"$(IntDir)tbb.def" %(AdditionalOptions) - $(OutDir)tbb_debug.dll - true - Windows - false - - - MachineX86 - - - - - X64 - - - /c /MDd /Od /Ob0 /Zi /EHsc /GR /Zc:forScope /Zc:wchar_t /DTBB_USE_DEBUG /D__TBB_LIB_NAME=tbb_debug.lib /DDO_ITT_NOTIFY /GS /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0502 /D__TBB_BUILD=1 /W4 /I../../src /I../../src/rml/include /I../../include - Disabled - .;%(AdditionalIncludeDirectories) - %(PreprocessorDefinitions) - true - EnableFastChecks - MultiThreadedDebugDLL - Level4 - ProgramDatabase - false - - - /nologo /DLL /MAP /DEBUG /fixed:no /INCREMENTAL:NO /DEF:"$(IntDir)tbb.def" %(AdditionalOptions) - $(OutDir)tbb_debug.dll - true - Windows - false - - - MachineX64 - false - - - - - /c /MD /O2 /Zi /EHsc /GR /Zc:forScope /Zc:wchar_t /D__TBB_LIB_NAME=tbb.lib /DDO_ITT_NOTIFY /GS /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0502 /D__TBB_BUILD=1 /W4 /I../../src /I../../src/rml/include /I../../include - .;%(AdditionalIncludeDirectories) - %(PreprocessorDefinitions) - MultiThreadedDLL - - - Level4 - ProgramDatabase - - - /nologo /DLL /MAP /DEBUG /fixed:no /INCREMENTAL:NO /DEF:"$(IntDir)tbb.def" %(AdditionalOptions) - $(OutDir)tbb.dll - true - Windows - true - true - false - - - MachineX86 - - - - - X64 - - - /c /MD /O2 /Zi /EHsc /GR /Zc:forScope /Zc:wchar_t /D__TBB_LIB_NAME=tbb.lib /DDO_ITT_NOTIFY /GS /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0502 /D__TBB_BUILD=1 /W4 /I../../src /I../../src/rml/include /I../../include - .;%(AdditionalIncludeDirectories) - %(PreprocessorDefinitions) - MultiThreadedDLL - Level4 - ProgramDatabase - - - /nologo /DLL /MAP /DEBUG /fixed:no /INCREMENTAL:NO /DEF:"$(IntDir)tbb.def" %(AdditionalOptions) - $(OutDir)tbb.dll - true - Windows - true - true - false - - - MachineX64 - false - - - - - /c /MTd /Od /Ob0 /Zi /EHsc /GR /Zc:forScope /Zc:wchar_t /DTBB_USE_DEBUG /D__TBB_LIB_NAME=tbb_debug.lib /DDO_ITT_NOTIFY /GS /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0502 /D__TBB_BUILD=1 /W4 /I../../src /I../../src/rml/include /I../../include - Disabled - .;%(AdditionalIncludeDirectories) - %(PreprocessorDefinitions) - true - EnableFastChecks - MultiThreadedDebug - - - Level4 - ProgramDatabase - - - /DLL /MAP /DEBUG /fixed:no /INCREMENTAL:NO /DEF:"$(IntDir)tbb.def" %(AdditionalOptions) - $(OutDir)tbb_debug.dll - true - Windows - false - - - MachineX86 - - - - - X64 - - - /c /MTd /Od /Ob0 /Zi /EHsc /GR /Zc:forScope /Zc:wchar_t /DTBB_USE_DEBUG /D__TBB_LIB_NAME=tbb_debug.lib /DDO_ITT_NOTIFY /GS /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0502 /D__TBB_BUILD=1 /W4 /I../../src /I../../src/rml/include /I../../include - Disabled - .;%(AdditionalIncludeDirectories) - %(PreprocessorDefinitions) - true - EnableFastChecks - MultiThreadedDebug - Level4 - ProgramDatabase - false - - - /nologo /DLL /MAP /DEBUG /fixed:no /INCREMENTAL:NO /DEF:"$(IntDir)tbb.def" %(AdditionalOptions) - $(OutDir)tbb_debug.dll - true - Windows - false - - - MachineX64 - false - - - - - /c /MT /O2 /Zi /EHsc /GR /Zc:forScope /Zc:wchar_t /D__TBB_LIB_NAME=tbb.lib /DDO_ITT_NOTIFY /GS /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0502 /D__TBB_BUILD=1 /W4 /I../../src /I../../src/rml/include /I../../include - .;%(AdditionalIncludeDirectories) - %(PreprocessorDefinitions) - MultiThreaded - - - Level4 - ProgramDatabase - - - /nologo /DLL /MAP /DEBUG /fixed:no /INCREMENTAL:NO /DEF:"$(IntDir)tbb.def" %(AdditionalOptions) - $(OutDir)tbb.dll - true - Windows - true - true - false - - - MachineX86 - - - - - X64 - - - /c /MT /O2 /Zi /EHsc /GR /Zc:forScope /Zc:wchar_t /D__TBB_LIB_NAME=tbb.lib /DDO_ITT_NOTIFY /GS /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0502 /D__TBB_BUILD=1 /W4 /I../../src /I../../src/rml/include /I../../include - .;%(AdditionalIncludeDirectories) - %(PreprocessorDefinitions) - MultiThreaded - Level4 - ProgramDatabase - - - /nologo /DLL /MAP /DEBUG /fixed:no /INCREMENTAL:NO /DEF:"$(IntDir)tbb.def" %(AdditionalOptions) - $(OutDir)tbb.dll - true - Windows - true - true - false - - - MachineX64 - false - - - - - /coff /Zi - true - true - /coff /Zi - true - true - /coff - true - true - /coff - true - true - - - true - building atomic_support.obj - ml64 /Fo"$(IntDir)%(FileName).obj" /DUSE_FRAME_POINTER /DEM64T=1 /c /Zi ../../src/tbb/intel64-masm/atomic_support.asm - $(IntDir)%(FileName).obj;%(Outputs) - true - building atomic_support.obj - ml64 /Fo"$(IntDir)%(FileName).obj" /DUSE_FRAME_POINTER /DEM64T=1 /c /Zi ../../src/tbb/intel64-masm/atomic_support.asm - $(IntDir)%(FileName).obj;%(Outputs) - true - building atomic_support.obj - ml64 /Fo"$(IntDir)%(FileName).obj" /DEM64T=1 /c /Zi ../../src/tbb/intel64-masm/atomic_support.asm - $(IntDir)%(FileName).obj;%(Outputs) - true - building atomic_support.obj - ml64 /Fo"$(IntDir)%(FileName).obj" /DEM64T=1 /c /Zi ../../src/tbb/intel64-masm/atomic_support.asm - $(IntDir)%(FileName).obj;%(Outputs) - - - true - building intel64_misc.obj - ml64 /Fo"$(IntDir)%(FileName).obj" /DUSE_FRAME_POINTER /DEM64T=1 /c /Zi ../../src/tbb/intel64-masm/intel64_misc.asm - $(IntDir)%(FileName).obj;%(Outputs) - true - building intel64_misc.obj - ml64 /Fo"$(IntDir)%(FileName).obj" /DUSE_FRAME_POINTER /DEM64T=1 /c /Zi ../../src/tbb/intel64-masm/intel64_misc.asm - $(IntDir)%(FileName).obj;%(Outputs) - true - building intel64_misc.obj - ml64 /Fo"$(IntDir)%(FileName).obj" /DEM64T=1 /c /Zi ../../src/tbb/intel64-masm/intel64_misc.asm - $(IntDir)%(FileName).obj;%(Outputs) - true - building intel64_misc.obj - ml64 /Fo"$(IntDir)%(FileName).obj" /DEM64T=1 /c /Zi ../../src/tbb/intel64-masm/intel64_misc.asm - $(IntDir)%(FileName).obj;%(Outputs) - - - /coff /Zi - true - true - /coff /Zi - true - true - /coff - true - true - /coff - true - true - - - true - building itsx.obj - ml64 /Fo"$(IntDir)%(FileName).obj" /DUSE_FRAME_POINTER /DEM64T=1 /c /Zi ../../src/tbb/intel64-masm/itsx.asm - $(IntDir)%(FileName).obj;%(Outputs) - true - building itsx.obj - ml64 /Fo"$(IntDir)%(FileName).obj" /DUSE_FRAME_POINTER /DEM64T=1 /c /Zi ../../src/tbb/intel64-masm/itsx.asm - $(IntDir)%(FileName).obj;%(Outputs) - true - building itsx.obj - ml64 /Fo"$(IntDir)%(FileName).obj" /DEM64T=1 /c /Zi ../../src/tbb/intel64-masm/itsx.asm - $(IntDir)%(FileName).obj;%(Outputs) - true - building itsx.obj - ml64 /Fo"$(IntDir)%(FileName).obj" /DEM64T=1 /c /Zi ../../src/tbb/intel64-masm/itsx.asm - $(IntDir)%(FileName).obj;%(Outputs) - - - /coff /Zi - true - true - /coff /Zi - /coff /Zi - true - true - /coff /Zi - /coff - true - true - /coff - true - true - - - - - generating tbb.def file - cl /nologo /TC /EP ../../src/tbb/win32-tbb-export.def /DTBB_USE_DEBUG /DDO_ITT_NOTIFY /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0400 /D__TBB_BUILD=1 /I../../src /I../../include >"$(IntDir)tbb.def" - - $(IntDir)tbb.def;%(Outputs) - true - generating tbb.def file - cl /nologo /TC /EP ../../src/tbb/win32-tbb-export.def /DTBB_USE_DEBUG /DDO_ITT_NOTIFY /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0400 /D__TBB_BUILD=1 >"$(IntDir)tbb.def" - - $(IntDir)tbb.def;%(Outputs) - generating tbb.def file - cl /nologo /TC /EP ../../src/tbb/win32-tbb-export.def /DTBB_USE_DEBUG /DDO_ITT_NOTIFY /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0400 /D__TBB_BUILD=1 /I../../src /I../../include >"$(IntDir)tbb.def" - - $(IntDir)tbb.def;%(Outputs) - true - generating tbb.def file - cl /nologo /TC /EP ../../src/tbb/win32-tbb-export.def /DTBB_USE_DEBUG /DDO_ITT_NOTIFY /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0400 /D__TBB_BUILD=1 >"$(IntDir)tbb.def" - - $(IntDir)tbb.def;%(Outputs) - generating tbb.def file - cl /nologo /TC /EP ../../src/tbb/win32-tbb-export.def /DTBB_USE_DEBUG /DDO_ITT_NOTIFY /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0400 /D__TBB_BUILD=1 /I../../src /I../../include >"$(IntDir)tbb.def" - - $(IntDir)tbb.def;%(Outputs) - true - generating tbb.def file - cl /nologo /TC /EP ../../src/tbb/win32-tbb-export.def /DTBB_USE_DEBUG /DDO_ITT_NOTIFY /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0400 /D__TBB_BUILD=1 >"$(IntDir)tbb.def" - - $(IntDir)tbb.def;%(Outputs) - generating tbb.def file - cl /nologo /TC /EP ../../src/tbb/win32-tbb-export.def /DTBB_USE_DEBUG /DDO_ITT_NOTIFY /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0400 /D__TBB_BUILD=1 /I../../src /I../../include >"$(IntDir)tbb.def" - - $(IntDir)tbb.def;%(Outputs) - true - generating tbb.def file - cl /nologo /TC /EP ../../src/tbb/win32-tbb-export.def /DTBB_USE_DEBUG /DDO_ITT_NOTIFY /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0400 /D__TBB_BUILD=1 >"$(IntDir)tbb.def" - - $(IntDir)tbb.def;%(Outputs) - - - true - generating tbb.def file - cl /nologo /TC /EP ../../src/tbb/win64-tbb-export.def /DTBB_USE_DEBUG /DDO_ITT_NOTIFY /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0400 /D__TBB_BUILD=1 >"$(IntDir)tbb.def" - - $(IntDir)tbb.def;%(Outputs) - generating tbb.def file - cl /nologo /TC /EP ../../src/tbb/win64-tbb-export.def /DTBB_USE_DEBUG /DDO_ITT_NOTIFY /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0400 /D__TBB_BUILD=1 /I../../src /I../../include >"$(IntDir)tbb.def" - - $(IntDir)tbb.def;%(Outputs) - true - generating tbb.def file - cl /nologo /TC /EP ../../src/tbb/win64-tbb-export.def /DTBB_USE_DEBUG /DDO_ITT_NOTIFY /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0400 /D__TBB_BUILD=1 >"$(IntDir)tbb.def" - - $(IntDir)tbb.def;%(Outputs) - generating tbb.def file - cl /nologo /TC /EP ../../src/tbb/win64-tbb-export.def /DTBB_USE_DEBUG /DDO_ITT_NOTIFY /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0400 /D__TBB_BUILD=1 /I../../src /I../../include >"$(IntDir)tbb.def" - - $(IntDir)tbb.def;%(Outputs) - true - generating tbb.def file - cl /nologo /TC /EP ../../src/tbb/win64-tbb-export.def /DTBB_USE_DEBUG /DDO_ITT_NOTIFY /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0400 /D__TBB_BUILD=1 >"$(IntDir)tbb.def" - - $(IntDir)tbb.def;%(Outputs) - generating tbb.def file - cl /nologo /TC /EP ../../src/tbb/win64-tbb-export.def /DTBB_USE_DEBUG /DDO_ITT_NOTIFY /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0400 /D__TBB_BUILD=1 /I../../src /I../../include >"$(IntDir)tbb.def" - - $(IntDir)tbb.def;%(Outputs) - true - generating tbb.def file - cl /nologo /TC /EP ../../src/tbb/win64-tbb-export.def /DTBB_USE_DEBUG /DDO_ITT_NOTIFY /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0400 /D__TBB_BUILD=1 >"$(IntDir)tbb.def" - - $(IntDir)tbb.def;%(Outputs) - generating tbb.def file - cl /nologo /TC /EP ../../src/tbb/win64-tbb-export.def /DTBB_USE_DEBUG /DDO_ITT_NOTIFY /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0400 /D__TBB_BUILD=1 /I../../src /I../../include >"$(IntDir)tbb.def" - - $(IntDir)tbb.def;%(Outputs) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - /I../../src /I../../include /DDO_ITT_NOTIFY /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0400 %(AdditionalOptions) - /I../../src /I../../include /DDO_ITT_NOTIFY /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0400 %(AdditionalOptions) - /I../../src /I../../include /DDO_ITT_NOTIFY /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0400 %(AdditionalOptions) - /I../../src /I../../include /DDO_ITT_NOTIFY /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0400 %(AdditionalOptions) - /I../../src /I../../include /DDO_ITT_NOTIFY /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0400 %(AdditionalOptions) - /I../../src /I../../include /DDO_ITT_NOTIFY /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0400 %(AdditionalOptions) - /I../../src /I../../include /DDO_ITT_NOTIFY /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0400 %(AdditionalOptions) - /I../../src /I../../include /DDO_ITT_NOTIFY /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0400 %(AdditionalOptions) - - - - - - - \ No newline at end of file diff --git a/src/tbb/build/vs2013/tbbmalloc.vcxproj b/src/tbb/build/vs2013/tbbmalloc.vcxproj deleted file mode 100644 index 90d5fe133..000000000 --- a/src/tbb/build/vs2013/tbbmalloc.vcxproj +++ /dev/null @@ -1,558 +0,0 @@ - - - - - Debug-MT - Win32 - - - Debug-MT - x64 - - - Debug - Win32 - - - Debug - x64 - - - Release-MT - Win32 - - - Release-MT - x64 - - - Release - Win32 - - - Release - x64 - - - - {B15F131E-328A-4D42-ADC2-9FF4CA6306D8} - tbbmalloc - Win32Proj - - - - DynamicLibrary - NotSet - true - v120 - - - DynamicLibrary - NotSet - v120 - - - DynamicLibrary - NotSet - true - v120 - - - DynamicLibrary - NotSet - v120 - - - DynamicLibrary - NotSet - true - v120 - - - DynamicLibrary - NotSet - v120 - - - DynamicLibrary - NotSet - true - v120 - - - DynamicLibrary - NotSet - v120 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - <_ProjectFileVersion>10.0.40219.1 - $(SolutionDir)$(Platform)\$(Configuration)\ - $(SolutionDir)$(Platform)\$(ProjectName)\$(Configuration)\ - false - $(SolutionDir)$(Platform)\$(Configuration)\ - $(SolutionDir)$(Platform)\$(ProjectName)\$(Configuration)\ - false - $(SolutionDir)$(Platform)\$(Configuration)\ - $(SolutionDir)$(Platform)\$(ProjectName)\$(Configuration)\ - false - $(SolutionDir)$(Platform)\$(Configuration)\ - $(SolutionDir)$(Platform)\$(ProjectName)\$(Configuration)\ - false - $(SolutionDir)$(Platform)\$(Configuration)\ - $(SolutionDir)$(Platform)\$(ProjectName)\$(Configuration)\ - false - $(SolutionDir)$(Platform)\$(Configuration)\ - $(SolutionDir)$(Platform)\$(ProjectName)\$(Configuration)\ - false - $(SolutionDir)$(Platform)\$(Configuration)\ - $(SolutionDir)$(Platform)\$(ProjectName)\$(Configuration)\ - false - $(SolutionDir)$(Platform)\$(Configuration)\ - $(SolutionDir)$(Platform)\$(ProjectName)\$(Configuration)\ - false - AllRules.ruleset - - - AllRules.ruleset - - - AllRules.ruleset - - - AllRules.ruleset - - - AllRules.ruleset - - - AllRules.ruleset - - - AllRules.ruleset - - - AllRules.ruleset - - - $(ProjectName)_debug - $(ProjectName)_debug - $(ProjectName)_debug - $(ProjectName)_debug - - - - /c /MDd /Od /Ob0 /Zi /EHsc /GR /Zc:forScope /Zc:wchar_t /DTBB_USE_DEBUG /D__TBB_LIB_NAME=tbb_debug.lib /DDO_ITT_NOTIFY /GS /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0502 /D__TBBMALLOC_BUILD=1 /I../../src /I../../src/rml/include /I../../include /I../../src/tbbmalloc /I../../src/tbbmalloc /I. - Disabled - .;%(AdditionalIncludeDirectories) - %(PreprocessorDefinitions) - true - Default - MultiThreadedDebugDLL - Level4 - ProgramDatabase - false - - - /DLL /MAP /DEBUG /fixed:no /INCREMENTAL:NO /DEF:"$(IntDir)tbbmalloc.def" %(AdditionalOptions) - $(OutDir)tbbmalloc_debug.dll - true - Windows - false - - - MachineX86 - - - - - X64 - - - /c /MDd /Od /Ob0 /Zi /EHsc /GR /Zc:forScope /Zc:wchar_t /DTBB_USE_DEBUG /D__TBB_LIB_NAME=tbb_debug.lib /DDO_ITT_NOTIFY /GS /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0502 /D__TBBMALLOC_BUILD=1 /I../../src /I../../src/rml/include /I../../include /I../../src/tbbmalloc /I../../src/tbbmalloc /I. - Disabled - .;%(AdditionalIncludeDirectories) - false - Default - MultiThreadedDebugDLL - true - Level4 - ProgramDatabase - false - false - - - /nologo /DLL /MAP /DEBUG /fixed:no /INCREMENTAL:NO /DEF:"$(IntDir)tbbmalloc.def" %(AdditionalOptions) - $(OutDir)tbbmalloc_debug.dll - true - Windows - false - - - MachineX64 - - - - - /c /MD /O2 /Zi /EHsc /GR /Zc:forScope /Zc:wchar_t /D__TBB_LIB_NAME=tbb.lib /DDO_ITT_NOTIFY /GS /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0502 /D__TBBMALLOC_BUILD=1 /I../../src /I../../src/rml/include /I../../include /I../../src/tbbmalloc /I../../src/tbbmalloc /I. - .;%(AdditionalIncludeDirectories) - %(PreprocessorDefinitions) - MultiThreadedDLL - Level4 - ProgramDatabase - false - - - /nologo /DLL /MAP /DEBUG /fixed:no /INCREMENTAL:NO /DEF:"$(IntDir)tbbmalloc.def" %(AdditionalOptions) - $(OutDir)tbbmalloc.dll - true - Windows - true - true - false - - - MachineX86 - - - - - X64 - - - /c /MD /O2 /Zi /EHsc /GR /Zc:forScope /Zc:wchar_t /D__TBB_LIB_NAME=tbb.lib /DDO_ITT_NOTIFY /GS /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0502 /D__TBBMALLOC_BUILD=1 /I../../src /I../../src/rml/include /I../../include /I../../src/tbbmalloc /I../../src/tbbmalloc /I. - .;%(AdditionalIncludeDirectories) - %(PreprocessorDefinitions) - MultiThreadedDLL - Level4 - ProgramDatabase - false - - - /nologo /DLL /MAP /DEBUG /fixed:no /INCREMENTAL:NO /DEF:"$(IntDir)tbbmalloc.def" %(AdditionalOptions) - $(OutDir)tbbmalloc.dll - true - Windows - true - true - false - - - MachineX64 - - - - - /c /MTd /Od /Ob0 /Zi /EHsc /GR /Zc:forScope /Zc:wchar_t /DTBB_USE_DEBUG /D__TBB_LIB_NAME=tbb_debug.lib /DDO_ITT_NOTIFY /GS /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0502 /D__TBBMALLOC_BUILD=1 /I../../src /I../../src/rml/include /I../../include /I../../src/tbbmalloc /I../../src/tbbmalloc /I. - Disabled - .;%(AdditionalIncludeDirectories) - %(PreprocessorDefinitions) - true - Default - MultiThreadedDebug - Level4 - ProgramDatabase - false - - - /DLL /MAP /DEBUG /fixed:no /INCREMENTAL:NO /DEF:"$(IntDir)tbbmalloc.def" %(AdditionalOptions) - $(OutDir)tbbmalloc_debug.dll - true - Windows - false - - - MachineX86 - - - - - X64 - - - /c /MTd /Od /Ob0 /Zi /EHsc /GR /Zc:forScope /Zc:wchar_t /DTBB_USE_DEBUG /D__TBB_LIB_NAME=tbb_debug.lib /DDO_ITT_NOTIFY /GS /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0502 /D__TBBMALLOC_BUILD=1 /I../../src /I../../src/rml/include /I../../include /I../../src/tbbmalloc /I../../src/tbbmalloc /I. - Disabled - .;%(AdditionalIncludeDirectories) - false - Default - MultiThreadedDebug - true - Level4 - ProgramDatabase - false - false - - - /nologo /DLL /MAP /DEBUG /fixed:no /INCREMENTAL:NO /DEF:"$(IntDir)tbbmalloc.def" %(AdditionalOptions) - $(OutDir)tbbmalloc_debug.dll - true - Windows - false - - - MachineX64 - - - - - /c /MT /O2 /Zi /EHsc /GR /Zc:forScope /Zc:wchar_t /D__TBB_LIB_NAME=tbb.lib /DDO_ITT_NOTIFY /GS /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0502 /D__TBBMALLOC_BUILD=1 /I../../src /I../../src/rml/include /I../../include /I../../src/tbbmalloc /I../../src/tbbmalloc /I. - .;%(AdditionalIncludeDirectories) - %(PreprocessorDefinitions) - MultiThreaded - Level4 - ProgramDatabase - false - - - /nologo /DLL /MAP /DEBUG /fixed:no /INCREMENTAL:NO /DEF:"$(IntDir)tbbmalloc.def" %(AdditionalOptions) - $(OutDir)tbbmalloc.dll - true - Windows - true - true - false - MachineX86 - - - - - X64 - - - /c /MT /O2 /Zi /EHsc /GR /Zc:forScope /Zc:wchar_t /D__TBB_LIB_NAME=tbb.lib /DDO_ITT_NOTIFY /GS /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0502 /D__TBBMALLOC_BUILD=1 /I../../src /I../../src/rml/include /I../../include /I../../src/tbbmalloc /I../../src/tbbmalloc /I. - .;%(AdditionalIncludeDirectories) - %(PreprocessorDefinitions) - MultiThreaded - Level4 - ProgramDatabase - false - - - /nologo /DLL /MAP /DEBUG /fixed:no /INCREMENTAL:NO /DEF:"$(IntDir)tbbmalloc.def" %(AdditionalOptions) - $(OutDir)tbbmalloc.dll - true - Windows - true - true - false - - - MachineX64 - - - - - true - building atomic_support.obj - ml64 /Fo"$(IntDir)%(FileName).obj" /DUSE_FRAME_POINTER /DEM64T=1 /c /Zi ../../src/tbb/intel64-masm/atomic_support.asm - $(IntDir)%(FileName).obj;%(Outputs) - true - building atomic_support.obj - ml64 /Fo"$(IntDir)%(FileName).obj" /DUSE_FRAME_POINTER /DEM64T=1 /c /Zi ../../src/tbb/intel64-masm/atomic_support.asm - $(IntDir)%(FileName).obj;%(Outputs) - true - building atomic_support.obj - ml64 /Fo"$(IntDir)%(FileName).obj" /DEM64T=1 /c /Zi ../../src/tbb/intel64-masm/atomic_support.asm - $(IntDir)%(FileName).obj;%(Outputs) - true - building atomic_support.obj - ml64 /Fo"$(IntDir)%(FileName).obj" /DEM64T=1 /c /Zi ../../src/tbb/intel64-masm/atomic_support.asm - $(IntDir)%(FileName).obj;%(Outputs) - - - - - generating tbbmalloc.def file - cl /nologo /TC /EP ../../src/tbbmalloc/win32-tbbmalloc-export.def /DTBB_USE_DEBUG /DDO_ITT_NOTIFY /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0400 /D__TBBMALLOC_BUILD=1 >"$(IntDir)tbbmalloc.def" - - $(IntDir)tbbmalloc.def;%(Outputs) - true - generating tbbmalloc.def file - cl /nologo /TC /EP ../../src/tbbmalloc/win32-tbbmalloc-export.def /DTBB_USE_DEBUG /DDO_ITT_NOTIFY /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0400 /D__TBBMALLOC_BUILD=1 >"$(IntDir)tbbmalloc.def" - - $(IntDir)tbbmalloc.def;%(Outputs) - generating tbbmalloc.def file - cl /nologo /TC /EP ../../src/tbbmalloc/win32-tbbmalloc-export.def /DTBB_USE_DEBUG /DDO_ITT_NOTIFY /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0400 /D__TBBMALLOC_BUILD=1 >"$(IntDir)tbbmalloc.def" - - $(IntDir)tbbmalloc.def;%(Outputs) - true - generating tbbmalloc.def file - cl /nologo /TC /EP ../../src/tbbmalloc/win32-tbbmalloc-export.def /DTBB_USE_DEBUG /DDO_ITT_NOTIFY /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0400 /D__TBBMALLOC_BUILD=1 >"$(IntDir)tbbmalloc.def" - - $(IntDir)tbbmalloc.def;%(Outputs) - generating tbbmalloc.def file - cl /nologo /TC /EP ../../src/tbbmalloc/win32-tbbmalloc-export.def /DTBB_USE_DEBUG /DDO_ITT_NOTIFY /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0400 /D__TBBMALLOC_BUILD=1 >"$(IntDir)tbbmalloc.def" - - $(IntDir)tbbmalloc.def;%(Outputs) - true - generating tbbmalloc.def file - cl /nologo /TC /EP ../../src/tbb/win32-tbbmalloc-export.def /DTBB_USE_DEBUG /DDO_ITT_NOTIFY /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0400 /D__TBBMALLOC_BUILD=1 >"$(IntDir)tbbmalloc.def" - - $(IntDir)tbbmalloc.def;%(Outputs) - generating tbbmalloc.def file - cl /nologo /TC /EP ../../src/tbbmalloc/win32-tbbmalloc-export.def /DTBB_USE_DEBUG /DDO_ITT_NOTIFY /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0400 /D__TBBMALLOC_BUILD=1 >"$(IntDir)tbbmalloc.def" - - $(IntDir)tbbmalloc.def;%(Outputs) - true - generating tbbmalloc.def file - cl /nologo /TC /EP ../../src/tbb/win32-tbbmalloc-export.def /DTBB_USE_DEBUG /DDO_ITT_NOTIFY /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0400 /D__TBBMALLOC_BUILD=1 >"$(IntDir)tbbmalloc.def" - - $(IntDir)tbbmalloc.def;%(Outputs) - - - true - generating tbbmalloc.def file - cl /nologo /TC /EP ../../src/tbb/win64-tbbmalloc-export.def /DTBB_USE_DEBUG /DDO_ITT_NOTIFY /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0400 /D__TBBMALLOC_BUILD=1 >"$(IntDir)tbbmalloc.def" - - $(IntDir)tbbmalloc.def;%(Outputs) - generating tbbmalloc.def file - cl /nologo /TC /EP ../../src/tbbmalloc/win64-tbbmalloc-export.def /DTBB_USE_DEBUG /DDO_ITT_NOTIFY /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0400 /D__TBBMALLOC_BUILD=1 >"$(IntDir)tbbmalloc.def" - - $(IntDir)tbbmalloc.def;%(Outputs) - true - generating tbbmalloc.def file - cl /nologo /TC /EP ../../src/tbb/win64-tbbmalloc-export.def /DTBB_USE_DEBUG /DDO_ITT_NOTIFY /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0400 /D__TBBMALLOC_BUILD=1 >"$(IntDir)tbbmalloc.def" - - $(IntDir)tbbmalloc.def;%(Outputs) - generating tbbmalloc.def file - cl /nologo /TC /EP ../../src/tbbmalloc/win64-tbbmalloc-export.def /DTBB_USE_DEBUG /DDO_ITT_NOTIFY /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0400 /D__TBBMALLOC_BUILD=1 >"$(IntDir)tbbmalloc.def" - - $(IntDir)tbbmalloc.def;%(Outputs) - true - generating tbbmalloc.def file - cl /nologo /TC /EP ../../src/tbb/win64-tbbmalloc-export.def /DTBB_USE_DEBUG /DDO_ITT_NOTIFY /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0400 /D__TBBMALLOC_BUILD=1 >"$(IntDir)tbbmalloc.def" - - $(IntDir)tbbmalloc.def;%(Outputs) - generating tbbmalloc.def file - cl /nologo /TC /EP ../../src/tbbmalloc/win64-tbbmalloc-export.def /DTBB_USE_DEBUG /DDO_ITT_NOTIFY /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0400 /D__TBBMALLOC_BUILD=1 >"$(IntDir)tbbmalloc.def" - - $(IntDir)tbbmalloc.def;%(Outputs) - true - generating tbbmalloc.def file - cl /nologo /TC /EP ../../src/tbb/win64-tbbmalloc-export.def /DTBB_USE_DEBUG /DDO_ITT_NOTIFY /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0400 /D__TBBMALLOC_BUILD=1 >"$(IntDir)tbbmalloc.def" - - $(IntDir)tbbmalloc.def;%(Outputs) - generating tbbmalloc.def file - cl /nologo /TC /EP ../../src/tbbmalloc/win64-tbbmalloc-export.def /DTBB_USE_DEBUG /DDO_ITT_NOTIFY /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0400 /D__TBBMALLOC_BUILD=1 >"$(IntDir)tbbmalloc.def" - - $(IntDir)tbbmalloc.def;%(Outputs) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - /I../../src /I../../include /DDO_ITT_NOTIFY /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0400 %(AdditionalOptions) - /I../../src /I../../include /DDO_ITT_NOTIFY /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0400 %(AdditionalOptions) - /I../../src /I../../include /DDO_ITT_NOTIFY /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0400 %(AdditionalOptions) - /I../../src /I../../include /DDO_ITT_NOTIFY /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0400 %(AdditionalOptions) - /I../../src /I../../include /DDO_ITT_NOTIFY /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0400 %(AdditionalOptions) - /I../../src /I../../include /DDO_ITT_NOTIFY /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0400 %(AdditionalOptions) - /I../../src /I../../include /DDO_ITT_NOTIFY /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0400 %(AdditionalOptions) - /I../../src /I../../include /DDO_ITT_NOTIFY /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0400 %(AdditionalOptions) - - - - - {f62787dd-1327-448b-9818-030062bcfaa5} - false - - - - - - - \ No newline at end of file diff --git a/src/tbb/build/vs2013/tbbmalloc_proxy.vcxproj b/src/tbb/build/vs2013/tbbmalloc_proxy.vcxproj deleted file mode 100644 index 758bb5174..000000000 --- a/src/tbb/build/vs2013/tbbmalloc_proxy.vcxproj +++ /dev/null @@ -1,425 +0,0 @@ - - - - - Debug-MT - Win32 - - - Debug-MT - x64 - - - Debug - Win32 - - - Debug - x64 - - - Release-MT - Win32 - - - Release-MT - x64 - - - Release - Win32 - - - Release - x64 - - - - {02F61511-D5B6-46E6-B4BB-DEAA96E6BCC7} - tbbmalloc_proxy - Win32Proj - - - - DynamicLibrary - NotSet - true - v120 - - - DynamicLibrary - NotSet - v120 - - - DynamicLibrary - NotSet - true - v120 - - - DynamicLibrary - NotSet - v120 - - - DynamicLibrary - NotSet - true - v120 - - - DynamicLibrary - NotSet - v120 - - - DynamicLibrary - NotSet - true - v120 - - - DynamicLibrary - NotSet - v120 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - <_ProjectFileVersion>10.0.40219.1 - $(SolutionDir)$(Platform)\$(Configuration)\ - $(SolutionDir)$(Platform)\$(ProjectName)\$(Configuration)\ - false - $(SolutionDir)$(Platform)\$(Configuration)\ - $(SolutionDir)$(Platform)\$(ProjectName)\$(Configuration)\ - false - $(SolutionDir)$(Platform)\$(Configuration)\ - $(SolutionDir)$(Platform)\$(ProjectName)\$(Configuration)\ - false - $(SolutionDir)$(Platform)\$(Configuration)\ - $(SolutionDir)$(Platform)\$(ProjectName)\$(Configuration)\ - false - $(SolutionDir)$(Platform)\$(Configuration)\ - $(SolutionDir)$(Platform)\$(ProjectName)\$(Configuration)\ - false - $(SolutionDir)$(Platform)\$(Configuration)\ - $(SolutionDir)$(Platform)\$(ProjectName)\$(Configuration)\ - false - $(SolutionDir)$(Platform)\$(Configuration)\ - $(SolutionDir)$(Platform)\$(ProjectName)\$(Configuration)\ - false - $(SolutionDir)$(Platform)\$(Configuration)\ - $(SolutionDir)$(Platform)\$(ProjectName)\$(Configuration)\ - false - AllRules.ruleset - - - AllRules.ruleset - - - AllRules.ruleset - - - AllRules.ruleset - - - AllRules.ruleset - - - AllRules.ruleset - - - AllRules.ruleset - - - AllRules.ruleset - - - $(ProjectName)_debug - $(ProjectName)_debug - $(ProjectName)_debug - $(ProjectName)_debug - - - - /c /MDd /Od /Ob0 /Zi /EHsc /GR /Zc:forScope /Zc:wchar_t /DTBB_USE_DEBUG /D__TBB_LIB_NAME=tbb_debug.lib /DDO_ITT_NOTIFY /GS /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0502 /W4 /D__TBBMALLOC_BUILD=1 /I../../src /I../../src/rml/include /I../../include /I../../src/tbbmalloc /I../../src/tbbmalloc - Disabled - .;%(AdditionalIncludeDirectories) - %(PreprocessorDefinitions) - true - Sync - Default - MultiThreadedDebugDLL - - - Level4 - ProgramDatabase - - - /DLL /MAP /DEBUG /fixed:no /INCREMENTAL:NO %(AdditionalOptions) - $(OutDir)tbbmalloc_proxy_debug.dll - true - Windows - false - - - MachineX86 - - - - - X64 - - - /c /MDd /Od /Ob0 /Zi /EHsc /GR /Zc:forScope /Zc:wchar_t /DTBB_USE_DEBUG /D__TBB_LIB_NAME=tbb_debug.lib /DDO_ITT_NOTIFY /GS /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0502 /W4 /D__TBBMALLOC_BUILD=1 /I../../src /I../../src/rml/include /I../../include /I../../src/tbbmalloc /I../../src/tbbmalloc - Disabled - .;%(AdditionalIncludeDirectories) - false - - - Default - MultiThreadedDebugDLL - true - - - Level4 - ProgramDatabase - false - - - /nologo /DLL /MAP /DEBUG /fixed:no /INCREMENTAL:NO %(AdditionalOptions) - $(OutDir)tbbmalloc_proxy_debug.dll - true - Windows - false - - - MachineX64 - - - - - /c /MD /O2 /Zi /EHsc /GR /Zc:forScope /Zc:wchar_t /D__TBB_LIB_NAME=tbb.lib /DDO_ITT_NOTIFY /GS /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0502 /W4 /D__TBBMALLOC_BUILD=1 /I../../src /I../../src/rml/include /I../../include /I../../src/tbbmalloc /I../../src/tbbmalloc - .;%(AdditionalIncludeDirectories) - %(PreprocessorDefinitions) - - - MultiThreadedDLL - - - Level4 - ProgramDatabase - - - /nologo /DLL /MAP /DEBUG /fixed:no /INCREMENTAL:NO %(AdditionalOptions) - $(OutDir)tbbmalloc_proxy.dll - true - Windows - true - true - false - - - MachineX86 - - - - - X64 - - - /c /MD /O2 /Zi /EHsc /GR /Zc:forScope /Zc:wchar_t /D__TBB_LIB_NAME=tbb.lib /DDO_ITT_NOTIFY /GS /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0502 /W4 /D__TBBMALLOC_BUILD=1 /I../../src /I../../src/rml/include /I../../include /I../../src/tbbmalloc /I../../src/tbbmalloc - .;%(AdditionalIncludeDirectories) - %(PreprocessorDefinitions) - - - MultiThreadedDLL - - - Level4 - ProgramDatabase - - - /nologo /DLL /MAP /DEBUG /fixed:no /INCREMENTAL:NO %(AdditionalOptions) - $(OutDir)tbbmalloc_proxy.dll - true - Windows - true - true - false - - - MachineX64 - - - - - /c /MTd /Od /Ob0 /Zi /EHsc /GR /Zc:forScope /Zc:wchar_t /DTBB_USE_DEBUG /D__TBB_LIB_NAME=tbb_debug.lib /DDO_ITT_NOTIFY /GS /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0502 /W4 /D__TBBMALLOC_BUILD=1 /I../../src /I../../src/rml/include /I../../include /I../../src/tbbmalloc /I../../src/tbbmalloc - Disabled - .;%(AdditionalIncludeDirectories) - %(PreprocessorDefinitions) - true - Sync - Default - MultiThreadedDebug - - - Level4 - ProgramDatabase - - - /DLL /MAP /DEBUG /fixed:no /INCREMENTAL:NO %(AdditionalOptions) - $(OutDir)tbbmalloc_proxy_debug.dll - true - Windows - false - - - MachineX86 - - - - - X64 - - - /c /MTd /Od /Ob0 /Zi /EHsc /GR /Zc:forScope /Zc:wchar_t /DTBB_USE_DEBUG /D__TBB_LIB_NAME=tbb_debug.lib /DDO_ITT_NOTIFY /GS /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0502 /W4 /D__TBBMALLOC_BUILD=1 /I../../src /I../../src/rml/include /I../../include /I../../src/tbbmalloc /I../../src/tbbmalloc - Disabled - .;%(AdditionalIncludeDirectories) - false - - - Default - MultiThreadedDebug - true - - - Level4 - ProgramDatabase - false - - - /nologo /DLL /MAP /DEBUG /fixed:no /INCREMENTAL:NO %(AdditionalOptions) - $(OutDir)tbbmalloc_proxy_debug.dll - true - Windows - false - - - MachineX64 - - - - - /c /MT /O2 /Zi /EHsc /GR /Zc:forScope /Zc:wchar_t /D__TBB_LIB_NAME=tbb.lib /DDO_ITT_NOTIFY /GS /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0502 /W4 /D__TBBMALLOC_BUILD=1 /I../../src /I../../src/rml/include /I../../include /I../../src/tbbmalloc /I../../src/tbbmalloc - .;%(AdditionalIncludeDirectories) - %(PreprocessorDefinitions) - - - MultiThreaded - - - Level4 - ProgramDatabase - - - /nologo /DLL /MAP /DEBUG /fixed:no /INCREMENTAL:NO %(AdditionalOptions) - $(OutDir)tbbmalloc_proxy.dll - true - Windows - true - true - false - - - MachineX86 - - - - - X64 - - - /c /MT /O2 /Zi /EHsc /GR /Zc:forScope /Zc:wchar_t /D__TBB_LIB_NAME=tbb.lib /DDO_ITT_NOTIFY /GS /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0502 /W4 /D__TBBMALLOC_BUILD=1 /I../../src /I../../src/rml/include /I../../include /I../../src/tbbmalloc /I../../src/tbbmalloc - .;%(AdditionalIncludeDirectories) - %(PreprocessorDefinitions) - - - MultiThreaded - - - Level4 - ProgramDatabase - - - /nologo /DLL /MAP /DEBUG /fixed:no /INCREMENTAL:NO %(AdditionalOptions) - $(OutDir)tbbmalloc_proxy.dll - true - Windows - true - true - false - - - MachineX64 - - - - - - - - - - - /I../../src /I../../include /DDO_ITT_NOTIFY /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0400 %(AdditionalOptions) - /I../../src /I../../include /DDO_ITT_NOTIFY /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0400 %(AdditionalOptions) - /I../../src /I../../include /DDO_ITT_NOTIFY /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0400 %(AdditionalOptions) - /I../../src /I../../include /DDO_ITT_NOTIFY /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0400 %(AdditionalOptions) - /I../../src /I../../include /DDO_ITT_NOTIFY /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0400 %(AdditionalOptions) - /I../../src /I../../include /DDO_ITT_NOTIFY /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0400 %(AdditionalOptions) - /I../../src /I../../include /DDO_ITT_NOTIFY /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0400 %(AdditionalOptions) - /I../../src /I../../include /DDO_ITT_NOTIFY /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE /D_WIN32_WINNT=0x0400 %(AdditionalOptions) - - - - - - - {b15f131e-328a-4d42-adc2-9ff4ca6306d8} - false - - - - - - - \ No newline at end of file diff --git a/src/tbb/build/vs2013/version_string.ver b/src/tbb/build/vs2013/version_string.ver deleted file mode 100644 index 5d8f04e5d..000000000 --- a/src/tbb/build/vs2013/version_string.ver +++ /dev/null @@ -1 +0,0 @@ -#define __TBB_VERSION_STRINGS(N) "Empty" diff --git a/src/tbb/build/windows.cl.inc b/src/tbb/build/windows.cl.inc deleted file mode 100644 index 3906d0867..000000000 --- a/src/tbb/build/windows.cl.inc +++ /dev/null @@ -1,158 +0,0 @@ -# Copyright (c) 2005-2017 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -# -# - -#------------------------------------------------------------------------------ -# Define compiler-specific variables. -#------------------------------------------------------------------------------ - - -#------------------------------------------------------------------------------ -# Setting compiler flags. -#------------------------------------------------------------------------------ -CPLUS = cl /nologo -LINK_FLAGS = /link /nologo -LIB_LINK_FLAGS=/link /nologo /DLL /MAP /DEBUG /fixed:no /INCREMENTAL:NO /DYNAMICBASE /NXCOMPAT - -ifneq (,$(stdver)) - CXX_STD_FLAGS = /std:$(stdver) -endif - -ifeq ($(arch), ia32) - LIB_LINK_FLAGS += /SAFESEH -endif - -ifeq ($(runtime), vc_mt) - MS_CRT_KEY = /MT$(if $(findstring debug,$(cfg)),d) -else - MS_CRT_KEY = /MD$(if $(findstring debug,$(cfg)),d) -endif -EH_FLAGS = $(if $(no_exceptions),/EHs-,/EHsc /GR) - -ifeq ($(cfg), release) - CPLUS_FLAGS = $(MS_CRT_KEY) /O2 /Zi $(EH_FLAGS) /Zc:forScope /Zc:wchar_t /D__TBB_LIB_NAME=$(TBB.LIB) - ASM_FLAGS = -endif -ifeq ($(cfg), debug) - CPLUS_FLAGS = $(MS_CRT_KEY) /Od /Ob0 /Zi $(EH_FLAGS) /Zc:forScope /Zc:wchar_t /DTBB_USE_DEBUG /D__TBB_LIB_NAME=$(TBB.LIB) - ASM_FLAGS = /DUSE_FRAME_POINTER -endif - -ZW_KEY = /ZW:nostdlib - -ifneq (,$(filter win8ui,$(target_app) $(target_ui))) - CPLUS_FLAGS += $(ZW_KEY) /D "_UNICODE" /D "UNICODE" /D "WINAPI_FAMILY=WINAPI_FAMILY_APP" - _WIN32_WINNT = 0x0602 -else ifneq (,$(filter uwp,$(target_app) $(target_ui))) - CPLUS_FLAGS += $(ZW_KEY) /D "_UNICODE" /D "UNICODE" /D "WINAPI_FAMILY=WINAPI_FAMILY_APP" - _WIN32_WINNT = 0x0A00 - LIB_LINK_FLAGS += /NODEFAULTLIB:kernel32.lib OneCore.lib -else - CPLUS_FLAGS += /DDO_ITT_NOTIFY -endif -ifneq (,$(filter store,$(target_mode) $(target_ui_mode))) -# it is necessary to source vcvars with 'store' argument in production - LIB_LINK_FLAGS += /APPCONTAINER -endif - -CPLUS_FLAGS += /GS - -COMPILE_ONLY = /c -PREPROC_ONLY = /TP /EP -INCLUDE_KEY = /I -DEFINE_KEY = /D -OUTPUT_KEY = /Fe -OUTPUTOBJ_KEY = /Fo -WARNING_AS_ERROR_KEY = /WX -WARNING_SUPPRESS = $(if $(no_exceptions),/wd4530 /wd4577) -BIGOBJ_KEY = /bigobj - -ifeq ($(runtime),vc7.1) - WARNING_KEY = /W3 -else - WARNING_KEY = /W4 - OPENMP_FLAG = /openmp -endif - -DYLIB_KEY = /DLL -EXPORT_KEY = /DEF: -NODEFAULTLIB_KEY = /Zl -NOINTRINSIC_KEY = /Oi- - -INCLUDE_TEST_HEADERS = /FI$(tbb_root)/src/test/harness_preload.h - -ifeq ($(runtime),vc8) - WARNING_KEY += /Wp64 - CPLUS_FLAGS += /D_USE_RTM_VERSION -endif - -# Since VS2012, VC++ provides /volatile option to control semantics of volatile variables. -# We want to use strict ISO semantics in the library and tests -ifeq (ok,$(call detect_js,/minversion cl 17)) - CPLUS_FLAGS += /volatile:iso -endif - -# Since VS2013, VC++ uses the same .pdb file for different sources so we need -# to add /FS (Force Synchronous PDB Writes) -ifeq (ok,$(call detect_js,/minversion cl 18)) - CPLUS_FLAGS += /FS -endif - -CPLUS_FLAGS += /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE \ - /D_WIN32_WINNT=$(_WIN32_WINNT) -C_FLAGS = $(subst $(ZW_KEY),,$(subst $(EH_FLAGS),,$(CPLUS_FLAGS))) - -#------------------------------------------------------------------------------ -# End of setting compiler flags. -#------------------------------------------------------------------------------ - - -#------------------------------------------------------------------------------ -# Setting assembler data. -#------------------------------------------------------------------------------ -ASSEMBLY_SOURCE=$(arch)-masm -ifeq (intel64,$(arch)) - ASM=ml64 /nologo - ASM_FLAGS += /DEM64T=1 /c /Zi - TBB_ASM.OBJ = atomic_support.obj intel64_misc.obj itsx.obj - MALLOC_ASM.OBJ = atomic_support.obj -else -ifeq (armv7,$(arch)) - ASM= - TBB_ASM.OBJ= -else - ASM=ml /nologo - ASM_FLAGS += /c /coff /Zi /safeseh - TBB_ASM.OBJ = atomic_support.obj lock_byte.obj itsx.obj -endif -endif -#------------------------------------------------------------------------------ -# End of setting assembler data. -#------------------------------------------------------------------------------ - - -#------------------------------------------------------------------------------ -# Setting tbbmalloc data. -#------------------------------------------------------------------------------ -M_CPLUS_FLAGS = $(CPLUS_FLAGS) -#------------------------------------------------------------------------------ -# End of setting tbbmalloc data. -#------------------------------------------------------------------------------ - -#------------------------------------------------------------------------------ -# End of define compiler-specific variables. -#------------------------------------------------------------------------------ diff --git a/src/tbb/build/windows.gcc.inc b/src/tbb/build/windows.gcc.inc deleted file mode 100644 index e4dad35cc..000000000 --- a/src/tbb/build/windows.gcc.inc +++ /dev/null @@ -1,146 +0,0 @@ -# Copyright (c) 2005-2017 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -# -# - -# Set by R -R_COMPILED_BY ?= 4.9.3 -GCC_VERSION=${subst gcc ,,${R_COMPILED_BY}} - -#------------------------------------------------------------------------------ -# Overriding settings from windows.inc -#------------------------------------------------------------------------------ - -SLASH= $(strip \) -OBJ = o -LIBEXT = dll # MinGW allows linking with DLLs directly - -TBB.RES = -MALLOC.RES = -RML.RES = -TBB.MANIFEST = -MALLOC.MANIFEST = -RML.MANIFEST = - -ifeq (ia32,$(arch)) - TBB.LST = $(tbb_root)/src/tbb/lin32-tbb-export.lst -else - TBB.LST = $(tbb_root)/src/tbb/win64-gcc-tbb-export.lst -endif -MALLOC.DEF = $(MALLOC_ROOT)/$(def_prefix)-gcc-tbbmalloc-export.def -RML.DEF = $(RML_SERVER_ROOT)/lin-rml-export.def - -LINK_TBB.LIB = $(TBB.LIB) -# no TBB proxy for the configuration -PROXY.LIB = - -#------------------------------------------------------------------------------ -# End of overridden settings -#------------------------------------------------------------------------------ -# Compiler-specific variables -#------------------------------------------------------------------------------ - -# CPLUS = g++ -COMPILE_ONLY = -c -MMD -PREPROC_ONLY = -E -x c++ -INCLUDE_KEY = -I -DEFINE_KEY = -D -OUTPUT_KEY = -o # -OUTPUTOBJ_KEY = -o # -PIC_KEY = -WARNING_AS_ERROR_KEY = -Werror -WARNING_KEY = -Wall -TEST_WARNING_KEY = -Wextra -Wshadow -Wcast-qual -Woverloaded-virtual -Wnon-virtual-dtor -Wno-uninitialized -WARNING_SUPPRESS = -Wno-parentheses -Wno-uninitialized -Wno-non-virtual-dtor -DYLIB_KEY = -shared -LIBDL = -EXPORT_KEY = -Wl,--version-script, -LIBS = -lpsapi -BIGOBJ_KEY = -Wa,-mbig-obj - -#------------------------------------------------------------------------------ -# End of compiler-specific variables -#------------------------------------------------------------------------------ -# Command lines -#------------------------------------------------------------------------------ - -LINK_FLAGS = -Wl,--enable-auto-import -LIB_LINK_FLAGS = $(DYLIB_KEY) - -# gcc 4.8 and later support RTM intrinsics, but require command line switch to enable them -ifeq ($(shell expr $(GCC_VERSION) \>= 4.8), 1) - RTM_KEY = -mrtm -endif - -# gcc 6.0 and later have -flifetime-dse option that controls -# elimination of stores done outside the object lifetime -ifeq ($(shell expr $(GCC_VERSION) \>= 6.0), 1) - # keep pre-contruction stores for zero initialization - DSE_KEY = -flifetime-dse=1 -endif - -ifeq ($(cfg), release) - CPLUS_FLAGS = -g -O2 -endif -ifeq ($(cfg), debug) - CPLUS_FLAGS = -g -O0 -DTBB_USE_DEBUG -endif - -CPLUS_FLAGS += -DUSE_WINTHREAD -CPLUS_FLAGS += -D_WIN32_WINNT=$(_WIN32_WINNT) - -# MinGW specific -CPLUS_FLAGS += -DMINGW_HAS_SECURE_API=1 -D__MSVCRT_VERSION__=0x0700 -msse -mthreads - -# CONLY = gcc -debugger = gdb -C_FLAGS = $(CPLUS_FLAGS) - -ifeq (intel64,$(arch)) - CPLUS_FLAGS += -m64 $(RTM_KEY) - LIB_LINK_FLAGS += -m64 -endif - -ifeq (ia32,$(arch)) - CPLUS_FLAGS += -m32 -march=i686 $(RTM_KEY) - LIB_LINK_FLAGS += -m32 -endif - -# For examples -export UNIXMODE = 1 - -#------------------------------------------------------------------------------ -# End of command lines -#------------------------------------------------------------------------------ -# Setting assembler data -#------------------------------------------------------------------------------ - -ASM= -ASM_FLAGS= -TBB_ASM.OBJ= -ASSEMBLY_SOURCE=$(arch)-gas - -#------------------------------------------------------------------------------ -# End of setting assembler data -#------------------------------------------------------------------------------ -# Setting tbbmalloc data -#------------------------------------------------------------------------------ - -M_CPLUS_FLAGS = $(CPLUS_FLAGS) -fno-rtti -fno-exceptions - -#------------------------------------------------------------------------------ -# End of setting tbbmalloc data -#------------------------------------------------------------------------------ diff --git a/src/tbb/build/windows.icl.inc b/src/tbb/build/windows.icl.inc deleted file mode 100644 index 7a0b1165e..000000000 --- a/src/tbb/build/windows.icl.inc +++ /dev/null @@ -1,178 +0,0 @@ -# Copyright (c) 2005-2017 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -# -# - -#------------------------------------------------------------------------------ -# Define compiler-specific variables. -#------------------------------------------------------------------------------ - - -#------------------------------------------------------------------------------ -# Setting default configuration to release. -#------------------------------------------------------------------------------ -cfg ?= release -#------------------------------------------------------------------------------ -# End of setting default configuration to release. -#------------------------------------------------------------------------------ - - -#------------------------------------------------------------------------------ -# Setting compiler flags. -#------------------------------------------------------------------------------ -CPLUS = icl /nologo $(VCCOMPAT_FLAG) -LINK_FLAGS = /link /nologo -LIB_LINK_FLAGS= /link /nologo /DLL /MAP /DEBUG /fixed:no /INCREMENTAL:NO /DYNAMICBASE /NXCOMPAT - -ifeq ($(arch), ia32) - LIB_LINK_FLAGS += /SAFESEH -endif - -ifneq (,$(stdver)) - CXX_STD_FLAGS = /Qstd=$(stdver) -endif - -# ICC 12.0 and higher provide Intel(R) Cilk(TM) Plus -ifeq (ok,$(call detect_js,/minversion icl 12)) - CILK_AVAILABLE = yes -endif - -# ICC 17.0.4 and higher provide support for VS2017 -ifeq (ok,$(call detect_js,/minversion icl 17 4)) - VS2017_SUPPORT = yes -endif - -ifeq ($(runtime), vc_mt) - MS_CRT_KEY = /MT$(if $(findstring debug,$(cfg)),d) -else - MS_CRT_KEY = /MD$(if $(findstring debug,$(cfg)),d) -endif -EH_FLAGS = $(if $(no_exceptions),/EHs-,/EHsc /GR) - -ifeq ($(cfg), release) - CPLUS_FLAGS = $(MS_CRT_KEY) /O2 /Zi $(EH_FLAGS) /Zc:forScope /Zc:wchar_t /D__TBB_LIB_NAME=$(TBB.LIB) - ASM_FLAGS = -endif -ifeq ($(cfg), debug) - CPLUS_FLAGS = $(MS_CRT_KEY) /Od /Ob0 /Zi $(EH_FLAGS) /Zc:forScope /Zc:wchar_t /DTBB_USE_DEBUG /D__TBB_LIB_NAME=$(TBB.LIB) - ASM_FLAGS = /DUSE_FRAME_POINTER -endif -CPLUS_FLAGS += /GS - -COMPILE_ONLY = /c /QMMD -# PREPROC_ONLY should really use /TP which applies to all files in the command line. -# But with /TP, ICL does not preprocess *.def files. -PREPROC_ONLY = /EP /Tp -INCLUDE_KEY = /I -DEFINE_KEY = /D -OUTPUT_KEY = /Fe -OUTPUTOBJ_KEY = /Fo -WARNING_AS_ERROR_KEY = /WX -WARNING_KEY = /W3 -WARNING_SUPPRESS = $(if $(no_exceptions),/wd583) -DYLIB_KEY = /DLL -EXPORT_KEY = /DEF: -NODEFAULTLIB_KEY = /Zl -NOINTRINSIC_KEY = /Oi- -BIGOBJ_KEY = /bigobj -INCLUDE_TEST_HEADERS = /FI$(tbb_root)/src/test/harness_preload.h - - -ifneq (,$(codecov)) - CPLUS_FLAGS += /Qprof-genx -else - CPLUS_FLAGS += /DDO_ITT_NOTIFY -endif - -OPENMP_FLAG = /Qopenmp -CPLUS_FLAGS += /DUSE_WINTHREAD /D_CRT_SECURE_NO_DEPRECATE \ - /D_WIN32_WINNT=$(_WIN32_WINNT) - -ifeq ($(runtime),vc8) - CPLUS_FLAGS += /D_USE_RTM_VERSION -endif - - -C_FLAGS = $(subst $(EH_FLAGS),,$(CPLUS_FLAGS)) - -VCVERSION:=$(runtime) -VCCOMPAT_FLAG ?= $(if $(findstring vc7.1, $(VCVERSION)),/Qvc7.1) -ifeq ($(VCCOMPAT_FLAG),) - VCCOMPAT_FLAG := $(if $(findstring vc8, $(VCVERSION)),/Qvc8) -endif -ifeq ($(VCCOMPAT_FLAG),) - VCCOMPAT_FLAG := $(if $(findstring vc_mt, $(VCVERSION)),/Qvc10) -endif -ifeq ($(VCCOMPAT_FLAG),) - VCCOMPAT_FLAG := $(if $(findstring vc9, $(VCVERSION)),/Qvc9) -endif -ifeq ($(VCCOMPAT_FLAG),) - VCCOMPAT_FLAG := $(if $(findstring vc10, $(VCVERSION)),/Qvc10) -endif -ifeq ($(VCCOMPAT_FLAG),) - VCCOMPAT_FLAG := $(if $(findstring vc11, $(VCVERSION)),/Qvc11) -endif -ifeq ($(VCCOMPAT_FLAG),) - VCCOMPAT_FLAG := $(if $(findstring vc12, $(VCVERSION)),/Qvc12) -endif -ifeq ($(VCCOMPAT_FLAG),) - VCCOMPAT_FLAG := $(if $(findstring vc14, $(VCVERSION)),/Qvc14) - ifeq ($(VS2017_SUPPORT),yes) - ifneq (,$(findstring vc14.1, $(VCVERSION))) - VCCOMPAT_FLAG := /Qvc14.1 - endif - endif -endif -ifeq ($(VCCOMPAT_FLAG),) - $(error VC version not detected correctly: $(VCVERSION) ) -endif -export VCCOMPAT_FLAG - -#------------------------------------------------------------------------------ -# End of setting compiler flags. -#------------------------------------------------------------------------------ - - -#------------------------------------------------------------------------------ -# Setting assembler data. -#------------------------------------------------------------------------------ -ASSEMBLY_SOURCE=$(arch)-masm -ifeq (intel64,$(arch)) - ASM=ml64 /nologo - ASM_FLAGS += /DEM64T=1 /c /Zi - TBB_ASM.OBJ = atomic_support.obj intel64_misc.obj itsx.obj - MALLOC_ASM.OBJ = atomic_support.obj -else - ASM=ml /nologo - ASM_FLAGS += /c /coff /Zi /safeseh - TBB_ASM.OBJ = atomic_support.obj lock_byte.obj itsx.obj -endif -#------------------------------------------------------------------------------ -# End of setting assembler data. -#------------------------------------------------------------------------------ - - -#------------------------------------------------------------------------------ -# Setting tbbmalloc data. -#------------------------------------------------------------------------------ -M_CPLUS_FLAGS = $(CPLUS_FLAGS) -#------------------------------------------------------------------------------ -# End of setting tbbmalloc data. -#------------------------------------------------------------------------------ - -#------------------------------------------------------------------------------ -# End of define compiler-specific variables. -#------------------------------------------------------------------------------ diff --git a/src/tbb/build/windows.inc b/src/tbb/build/windows.inc deleted file mode 100644 index 9f5b91344..000000000 --- a/src/tbb/build/windows.inc +++ /dev/null @@ -1,128 +0,0 @@ -# Copyright (c) 2005-2017 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -# -# - -ifdef tbb_build_dir - test_dir:=$(tbb_build_dir) -else - test_dir:=. -endif - -ifndef rtools - export SHELL = cmd - CMD:=cmd /C - SLASH=\\ - RM=cmd /C del /Q /F - RD=cmd /C rmdir - MD=cmd /c mkdir - NUL= nul -else - ifneq ($(COMSPEC), ) - # Use comspec env var if present - # Since it represents a full path, quote in case there are some spaces. - CMD:="$(subst \,/,$(COMSPEC))" /C - else - # otherwise rely on cmd being in the path - CMD:=cmd /C - endif - export SHELL = sh.exe - SLASH=/ - RD=rmdir - MD=mkdir - RM=rm - NUL= /dev/null -endif - -# A convenience wrapper for calls to detect.js. -# $(1) is the full command line for the script, e.g. /minversion icl 12 -detect_js = $(shell $(CMD) "cscript /nologo /E:jscript $(tbb_root)/build/detect.js $(1)") - -# TODO give an error if archs doesn't match -ifndef arch - export arch:=$(call detect_js, /arch $(compiler)) -endif - -ifndef runtime - export runtime:=$(call detect_js, /runtime $(compiler)) -endif - -native_compiler := cl -export compiler ?= cl -debugger ?= devenv /debugexe - -CWD=$(shell $(CMD) echo %CD%) - -AR=lib -AR_OUTPUT_KEY=/out: -AR_FLAGS=/nologo /nodefaultlib - -OBJ = obj -DLL = dll -LIBEXT = lib -ASMEXT = asm - -def_prefix = $(if $(findstring intel64,$(arch)),win64,win32) - -# Target Windows version. Do not increase beyond 0x0502 without prior discussion! -# Used as the value for macro definition option in windows.cl.inc etc. -# For tests, we need at least Windows XP SP2 for sake of enabling stack backtraces. -_WIN32_WINNT=0x0502 - -TBB.LST = $(tbb_root)/src/tbb/$(def_prefix)-tbb-export.lst -TBB.DEF = $(TBB.LST:.lst=.def) -TBB.DLL = tbb$(CPF_SUFFIX)$(DEBUG_SUFFIX).$(DLL) -TBB.LIB = tbb$(CPF_SUFFIX)$(DEBUG_SUFFIX).$(LIBEXT) -TBB.RES = tbb_resource.res -# On Windows, we use #pragma comment to set the proper TBB lib to link with. -# But for cross-configuration testing, need to link explicitly. -# Tests use this variable to detect dependency on TBB binary, so have to be non-empty. -LINK_TBB.LIB = $(if $(crosstest),$(TBB.LIB),$(DEFINE_KEY)__TBB_IMPLICITLY_LINKED) -TBB.MANIFEST = -ifneq ($(filter vc8 vc9,$(runtime)),) - TBB.MANIFEST = tbbmanifest.exe.manifest -endif - -MALLOC.DEF = $(MALLOC_ROOT)/$(def_prefix)-tbbmalloc-export.def -MALLOC.DLL = tbbmalloc$(DEBUG_SUFFIX).$(DLL) -MALLOC.LIB = tbbmalloc$(DEBUG_SUFFIX).$(LIBEXT) -MALLOC.RES = tbbmalloc.res -MALLOC.MANIFEST = -ifneq ($(filter vc8 vc9,$(runtime)),) -MALLOC.MANIFEST = tbbmanifest.exe.manifest -endif -LINK_MALLOC.LIB = $(MALLOC.LIB) - -MALLOCPROXY.DLL = tbbmalloc_proxy$(DEBUG_SUFFIX).$(DLL) -MALLOCPROXY.LIB = tbbmalloc_proxy$(DEBUG_SUFFIX).$(LIBEXT) -LINK_MALLOCPROXY.LIB = $(MALLOCPROXY.LIB) - -PROXY.LIB = tbbproxy$(DEBUG_SUFFIX).$(LIBEXT) - -RML.DEF = $(RML_SERVER_ROOT)/$(def_prefix)-rml-export.def -RML.DLL = irml$(DEBUG_SUFFIX).$(DLL) -RML.LIB = irml$(DEBUG_SUFFIX).$(LIBEXT) -RML.RES = irml.res -ifneq ($(filter vc8 vc9,$(runtime)),) -RML.MANIFEST = tbbmanifest.exe.manifest -endif - -MAKE_VERSIONS = $(CMD) "cscript /nologo /E:jscript $(subst \,/,$(tbb_root))/build/version_info_windows.js $(CONLY) $(arch) $(subst \,/,$(VERSION_FLAGS))" > version_string.ver -MAKE_TBBVARS = $(CMD) "$(subst /,\,$(tbb_root))\build\generate_tbbvars.bat" - -TEST_LAUNCHER = $(subst /,\,$(tbb_root))\build\test_launcher.bat $(largs) - -OPENCL.LIB = OpenCL.$(LIBEXT) diff --git a/src/tbb/cmake/README.md b/src/tbb/cmake/README.md new file mode 100644 index 000000000..3a357218d --- /dev/null +++ b/src/tbb/cmake/README.md @@ -0,0 +1,332 @@ +# Build System Description + +The project uses CMake* build configuration. + +The following controls are available during the configure stage: +``` +TBB_TEST:BOOL - Enable testing (ON by default) +TBB_STRICT:BOOL - Treat compiler warnings as errors (ON by default) +TBB_SANITIZE:STRING - Sanitizer parameter, passed to compiler/linker +TBB_SIGNTOOL:FILEPATH - Tool for digital signing, used in post-install step for libraries if provided. +TBB_SIGNTOOL_ARGS:STRING - Additional arguments for TBB_SIGNTOOL, used if TBB_SIGNTOOL is set. +TBB_BUILD:BOOL - Enable Intel(R) oneAPI Threading Building Blocks (oneTBB) build (ON by default) +TBB_FIND_PACKAGE - Enable search for external oneTBB using find_package instead of build from sources (OFF by default) +TBBMALLOC_BUILD:BOOL - Enable Intel(R) oneAPI Threading Building Blocks (oneTBB) memory allocator build (ON by default) +TBBMALLOC_PROXY_BUILD:BOOL - Enable Intel(R) oneAPI Threading Building Blocks (oneTBB) memory allocator proxy build (requires TBBMALLOC_BUILD. ON by default) +TBB4PY_BUILD:BOOL - Enable Intel(R) oneAPI Threading Building Blocks (oneTBB) Python module build (OFF by default) +TBB_CPF:BOOL - Enable preview features of the library (OFF by default) +TBB_INSTALL:BOOL - Enable installation (ON by default) +TBB_INSTALL_VARS:BOOL - Enable auto-generated vars installation(packages generated by `cpack` and `make install` will also include the vars script)(OFF by default) +TBB_VALGRIND_MEMCHECK:BOOL - Enable scan for memory leaks using Valgrind (OFF by default) +TBB_DISABLE_HWLOC_AUTOMATIC_SEARCH - Disable HWLOC automatic search by pkg-config tool (OFF by default) +TBB_ENABLE_IPO - Enable Interprocedural Optimization (IPO) during the compilation (ON by default) +TBB_BUILD_APPLE_FRAMEWORKS - Enable the Apple* frameworks instead of dylibs, only available on the Apple platform. (OFF by default) +``` + +## Configure, Build, and Test + +### Preparation + +To perform an out-of-source build, create a build directory and go there: + +```bash +mkdir /tmp/my-build +cd /tmp/my-build +``` + +### Configure + +```bash +cmake +``` + +Some useful options: +- `-G ` - specify particular project generator. See `cmake --help` for details. +- `-DCMAKE_BUILD_TYPE=Debug` - specify for Debug build. It is not applicable for multi-config generators, e.g., Microsoft* Visual Studio* generator. + +#### TBBBind Library Configuration + +> **_TIP:_** It is recommended to install the HWLOC* library. See [oneTBB documentation](https://oneapi-src.github.io/oneTBB/GSG/next_steps.html#hybrid-cpu-and-numa-support) for details. + +The TBBbind library has three versions: `tbbbind`, `tbbbind_2_0`, and `tbbbind_2_5`. Each of these versions is linked with the corresponding HWLOC* library version: +- `tbbbind` links with `HWLOC 1.11.x` +- `tbbbind_2_0` links with `HWLOC 2.1–2.4` +- `tbbbind_2_5` links with `HWLOC 2.5` and later + +The search for a suitable version of the HWLOC library is enabled by default. If you want to use a specific version of the library, you can specify the path to it manually using the following CMake variables: + + - `CMAKE_HWLOC__LIBRARY_PATH` - path to the corresponding HWLOC version shared library on Linux* OS or path to `.lib` file on Windows* OS + - `CMAKE_HWLOC__INCLUDE_PATH` - path to the corresponding HWLOC version including directory + + +--- +**NOTE:** Automatic HWLOC searching requires CMake version 3.6 or higher. + +--- + + +Windows* OS requires an additional variable for correct TBBBind library building: + - `CMAKE_HWLOC__DLL_PATH` - path to the corresponding HWLOC version `.dll` file. + +The `HWLOC_VER` substring used earlier can be replaced with one of the three values: +- `1_11` for the `tbbbind` library configuration +- `2` for the `tbbbind_2_0` library configuration +- `2_5` for the `tbbbind_2_5` library configuration + +If you specify variables for several TBBBind versions, the building process for all of these versions is performed during a single build session. + +--- +**TIP** + +Specify the `TBB_DISABLE_HWLOC_AUTOMATIC_SEARCH` to turn off the HWLOC library's automatic search. + +--- + + +### Build + +```bash +cmake --build . +``` + +Some useful options: +- `--target ` - specific target, "all" is the default. +- `--config ` - build configuration, applicable only for multi-config generators, e.g., Visual Studio* generator. + +The binaries are placed to `./__cxx_`. For example, `./gnu_4.8_cxx11_release`. + +#### Build For 32-bit + +* **Intel(R) Compiler**. Source Intel(R) C++ Compiler with `ia32` and build as usual. +* **MSVC**. Use switch for [generator](https://cmake.org/cmake/help/latest/manual/cmake-generators.7.html) (e.g., `-A Win32` for [VS2019](https://cmake.org/cmake/help/latest/generator/Visual%20Studio%2016%202019.html)) during the configuration stage and then build as usual. +* **GCC/Clang**. Specify `-m32` during the configuration. It can be `CXXFLAGS=-m32 cmake ..` or `cmake -DCMAKE_CXX_FLAGS=-m32 ..` +* For any other compiler, which builds for 64-bit by default, specify a 32-bit compiler key during the configuration as above. + +#### Windows* OS-Specific Builds + +--- +**NOTE** + +The following builds require CMake version 3.15 or higher. + +--- + +* **Dynamic linkage with C Runtime Library (CRT)**. The default behavior can be explicitly specified by setting `CMAKE_MSVC_RUNTIME_LIBRARY` to `MultiThreadedDLL` or `MultiThreadedDebugDLL`. +```bash +cmake .. # dynamic linkage is used by default +``` +```bash +cmake -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreadedDLL .. +``` +```bash +cmake -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreadedDebugDLL -DCMAKE_BUILD_TYPE=Debug .. +``` +* **Static linkage with CRT**. Set `CMAKE_MSVC_RUNTIME_LIBRARY` to `MultiThreaded` or `MultiThreadedDebug`. +```bash +cmake -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded .. +``` +```bash +cmake -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreadedDebug -DCMAKE_BUILD_TYPE=Debug .. +``` +* **Windows OS 10 Universal Windows application build**. Set `CMAKE_SYSTEM_NAME` to `WindowsStore` and `CMAKE_SYSTEM_VERSION` to `10.0`. + +--- +**NOTE** + +Set `TBB_NO_APPCONTAINER` to `ON` to apply the `/APPCONTAINER:NO` option during the compilation (used for testing). + +--- + +```bash +cmake -DCMAKE_SYSTEM_NAME:STRING=WindowsStore -DCMAKE_SYSTEM_VERSION:STRING=10.0 .. +``` + +* **Universal Windows OS Driver build**. Set `TBB_WINDOWS_DRIVER` to `ON` and use static linkage with CRT. + +```bash +cmake -DTBB_WINDOWS_DRIVER=ON -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded .. +``` + +#### Example + +```bash +cmake -DCMAKE_CXX_COMPILER=icpc -DCMAKE_C_COMPILER=icc -DTBB_TEST=off -DCMAKE_HWLOC_1_11_LIBRARY_PATH=/libhwloc.so.15 +-DCMAKE_HWLOC_1_11_INCLUDE_PATH= -DCMAKE_INSTALL_PREFIX=/oneTBB_install .. +make -j8 && make install +``` + +--- +**NOTE** + +The library path points to a file, while the include path points to a directory and not to ``hwloc.h``. + +--- + +### Test + +#### Build test +To build a test, use the default target ``all``: +``` +cmake --build . +``` + +Or use a specific test target: +``` +cmake --build . --target # e.g. test_version +``` + +#### Run Test + +You can run a test by using CTest: +```bash +ctest +``` + +Or by using the ``test`` target: +```bash +cmake --build . --target test # currently does not work on Windows* OS +``` + +## Installation +See [Installation from Sources](../INSTALL.md) to learn how to install oneTBB. + +To install oneTBB from the release packages, use the following commands: +```bash +tar -xvf oneapi-tbb-xxx.xx.x-*.tgz +source env/vars.sh +``` + + +## Sanitizers - Configure, Build, and Run + +```bash +mkdir build +cd build +cmake -DTBB_SANITIZE=thread .. # or -DTBB_SANITIZE=memory or any other sanitizer +make -j +ctest -V +``` + +## Valgrind Memcheck - Configure, Build, and Run + +### Prerequisites +* Valgrind tool executable +### Example +```bash +mkdir build +cd build +cmake -DTBB_VALGRIND_MEMCHECK=ON .. +make -j memcheck- # or memcheck-all to scan all tests +``` + +## Test Specification + +Use Doxygen* to generate oneTBB test specification: + +```bash +mkdir build +cd build +cmake -DTBB_TEST_SPEC=ON .. +make test_spec +``` + +## TBBConfig - Integration of Binary Packages + +It is a configuration module that is used for the integration of prebuilt oneTBB. It consists of two files (``TBBConfig.cmake`` and ``TBBConfigVersion.cmake``) and can be used via the [find_package](https://cmake.org/cmake/help/latest/command/find_package.html) function. + +To use this module in your CMake project: + 1. Let CMake know where to search for TBBConfig, e.g. specify the location of ``TBBConfig.cmake`` in `TBB_DIR` (for more details about search paths, see [find_package](https://cmake.org/cmake/help/latest/command/find_package.html)). + 2. Use [find_package](https://cmake.org/cmake/help/latest/command/find_package.html) to find oneTBB. + 3. Use provided variables and/or imported targets (described below) to work with the found oneTBB. + +Example: + +```cmake +add_executable(foo foo.cpp) +find_package(TBB) +target_link_libraries(foo TBB::tbb) +``` + +oneTBB components can be passed to [find_package](https://cmake.org/cmake/help/latest/command/find_package.html) after keyword ``COMPONENTS`` or ``REQUIRED``. +Use basic names of components (`tbb`, `tbbmalloc`, etc.). + +If components are not specified, then the default set is used: `tbb`, `tbbmalloc`, and ``tbbmalloc_proxy``. + +If `tbbmalloc_proxy` is requested, the `tbbmalloc` component is also added and set as a dependency for `tbbmalloc_proxy`. + +TBBConfig creates [imported targets](https://cmake.org/cmake/help/latest/manual/cmake-buildsystem.7.html#imported-targets>) as +shared libraries using the following format: `TBB::`. For example, `TBB::tbb` or `TBB::tbbmalloc`. + +To search only for release oneTBB version, set `TBB_FIND_RELEASE_ONLY` to `TRUE` before calling `find_package`. This variable helps to avoid simultaneous linkage of release and debug oneTBB versions when CMake configuration is `Debug,` but a third-party component depends on the release oneTBB version. + +Variables set during TBB configuration: + +Variable | Description +--- | --- +`TBB_FOUND` | oneTBB is found +`TBB__FOUND` | Specific oneTBB component is found +`TBB_VERSION` | oneTBB version (format: `...`) +`TBB_IMPORTED_TARGETS` | All created oneTBB imported targets (not supported for builds from source code) + +Starting from [oneTBB 2021.1](https://github.com/oneapi-src/oneTBB/releases/tag/v2021.1), GitHub* release TBBConfig files in the binary packages are located under `/lib/cmake/TBB`. +For example, `TBB_DIR` should be set to `/lib/cmake/TBB`. + +TBBConfig files are automatically created during the build from source code and can be installed together with the library. +Also, oneTBB provides a helper function that creates TBBConfig files from predefined templates. See `tbb_generate_config` in `cmake/config_generation.cmake`. + +## oneTBB Python Module Support +The `TBB4PY_BUILD` Cmake option provides the ability to build a Python module for oneTBB. + +### Targets: + - `irml` - build IPC RML server + - `python_build` - build oneTBB module for Python + +`python_build` target requirements: + - Python version 3.5 or newer + - SWIG version 3.0.6 or newer + +## CMake Files + +### Compile and Link Options + +Compile and link options may be specific for certain compilers. This part is handled in `cmake/compilers/*` files. + +Options in TBB CMake are handled via variables in two ways for convenience: + +* by options group +* by the specific option + +#### Options Group + +Naming convention is the following: `TBB___`, where: + +* `` can be: + * `LIB` - options applied during libraries build. + * `TEST` - options applied during test build. + * `BENCH` - options applied during benchmarks build. + * `COMMON` - options applied during all (libraries, test, benchmarks) builds. +* `` can be: + * `COMPILE` - options applied during the compilation. + * `LINK` - options applied during the linkage. +* `` can be: + * `FLAGS` - list of flags + * `LIBS` - list of libraries + +*Examples* + +Variable | Description +--- | --- +`TBB_COMMON_COMPILE_FLAGS` | Applied to libraries, tests, and benchmarks as compile options +`TBB_LIB_LINK_FLAGS` | Applied to libraries as link options +`TBB_LIB_LINK_LIBS ` | Applied to libraries as link libraries +`TBB_TEST_COMPILE_FLAGS` | Applied to tests as compile options + +Specify the `LINK` options prefixed with a dash(-) for MSVC(Visual Studio) compiler with CMake < 3.13 to avoid issues caused by `target_link_libraries` CMake command usage. + +#### Specific Options + +If the option is used only in part of the places (library, tests, benchmarks) and adding this option to the group of other options is not possible, +then the option must be named using common sense. + +Warning suppressions should be added to the `TBB_WARNING_SUPPRESS` variable, which is applied during the compilation of libraries, tests, and benchmarks. +Additional warnings should be added to the `TBB_WARNING_TEST_FLAGS` variable, which is applied during the compilation of tests. diff --git a/src/tbb/cmake/android/device_environment_cleanup.cmake b/src/tbb/cmake/android/device_environment_cleanup.cmake new file mode 100644 index 000000000..53bd9e54f --- /dev/null +++ b/src/tbb/cmake/android/device_environment_cleanup.cmake @@ -0,0 +1,17 @@ +# Copyright (c) 2020-2021 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +include(${CMAKE_CURRENT_LIST_DIR}/environment.cmake) + +execute_on_device("rm -rf ${ANDROID_DEVICE_TESTING_DIRECTORY}") diff --git a/src/tbb/cmake/android/environment.cmake b/src/tbb/cmake/android/environment.cmake new file mode 100644 index 000000000..c209dfc5c --- /dev/null +++ b/src/tbb/cmake/android/environment.cmake @@ -0,0 +1,35 @@ +# Copyright (c) 2020-2021 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set(ANDROID_DEVICE_TESTING_DIRECTORY "/data/local/tmp/tbb_testing") + +find_program(adb_executable adb) +if (NOT adb_executable) + message(FATAL_ERROR "Could not find adb") +endif() + +macro(execute_on_device cmd) + execute_process(COMMAND ${adb_executable} shell ${cmd} RESULT_VARIABLE CMD_RESULT) + if (CMD_RESULT) + message(FATAL_ERROR "Error while on device execution: ${cmd} error_code: ${CMD_RESULT}") + endif() +endmacro() + +macro(transfer_data data_path) + execute_process(COMMAND ${adb_executable} push --sync ${data_path} ${ANDROID_DEVICE_TESTING_DIRECTORY} + RESULT_VARIABLE CMD_RESULT OUTPUT_QUIET) + if (CMD_RESULT) + message(FATAL_ERROR "Error while data transferring: ${data_path} error_code: ${CMD_RESULT}") + endif() +endmacro() diff --git a/src/tbb/cmake/android/test_launcher.cmake b/src/tbb/cmake/android/test_launcher.cmake new file mode 100644 index 000000000..843d9bef1 --- /dev/null +++ b/src/tbb/cmake/android/test_launcher.cmake @@ -0,0 +1,27 @@ +# Copyright (c) 2020-2021 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +include(${CMAKE_CURRENT_LIST_DIR}/environment.cmake) + +# transfer data to device +execute_on_device("mkdir -m 755 -p ${ANDROID_DEVICE_TESTING_DIRECTORY}") + +file (GLOB_RECURSE BINARIES_LIST "${BINARIES_PATH}/*.so*" "${BINARIES_PATH}/${TEST_NAME}") +foreach(BINARY_FILE ${BINARIES_LIST}) + transfer_data(${BINARY_FILE}) +endforeach() + +# execute binary +execute_on_device("chmod -R 755 ${ANDROID_DEVICE_TESTING_DIRECTORY}") +execute_on_device("LD_LIBRARY_PATH=${ANDROID_DEVICE_TESTING_DIRECTORY} ${ANDROID_DEVICE_TESTING_DIRECTORY}/${TEST_NAME}") diff --git a/src/tbb/cmake/compilers/AppleClang.cmake b/src/tbb/cmake/compilers/AppleClang.cmake new file mode 100644 index 000000000..5ebbdbd1a --- /dev/null +++ b/src/tbb/cmake/compilers/AppleClang.cmake @@ -0,0 +1,47 @@ +# Copyright (c) 2020-2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set(TBB_LINK_DEF_FILE_FLAG -Wl,-exported_symbols_list,) +set(TBB_DEF_FILE_PREFIX mac${TBB_ARCH}) +set(TBB_WARNING_LEVEL -Wall -Wextra $<$:-Werror>) +set(TBB_TEST_WARNING_FLAGS -Wshadow -Wcast-qual -Woverloaded-virtual -Wnon-virtual-dtor) +set(TBB_WARNING_SUPPRESS -Wno-parentheses -Wno-non-virtual-dtor -Wno-dangling-else) +# For correct ucontext.h structures layout +set(TBB_COMMON_COMPILE_FLAGS -D_XOPEN_SOURCE) + +# Depfile options (e.g. -MD) are inserted automatically in some cases. +# Don't add -MMD to avoid conflicts in such cases. +if (NOT CMAKE_GENERATOR MATCHES "Ninja" AND NOT CMAKE_CXX_DEPENDS_USE_COMPILER) + set(TBB_MMD_FLAG -MMD) +endif() + +# Ignore -Werror set through add_compile_options() or added to CMAKE_CXX_FLAGS if TBB_STRICT is disabled. +if (NOT TBB_STRICT AND COMMAND tbb_remove_compile_flag) + tbb_remove_compile_flag(-Werror) +endif() + +# Enable Intel(R) Transactional Synchronization Extensions (-mrtm) and WAITPKG instructions support (-mwaitpkg) on relevant processors +if (CMAKE_OSX_ARCHITECTURES) + set(_tbb_target_architectures "${CMAKE_OSX_ARCHITECTURES}") +else() + set(_tbb_target_architectures "${CMAKE_SYSTEM_PROCESSOR}") +endif() +if ("${_tbb_target_architectures}" MATCHES "(x86_64|amd64|AMD64)") # OSX systems are 64-bit only + set(TBB_COMMON_COMPILE_FLAGS ${TBB_COMMON_COMPILE_FLAGS} -mrtm $<$>:-mwaitpkg>) +endif() +unset(_tbb_target_architectures) + +# TBB malloc settings +set(TBBMALLOC_LIB_COMPILE_FLAGS -fno-rtti -fno-exceptions) + diff --git a/src/tbb/cmake/compilers/Clang.cmake b/src/tbb/cmake/compilers/Clang.cmake new file mode 100644 index 000000000..dcd66634f --- /dev/null +++ b/src/tbb/cmake/compilers/Clang.cmake @@ -0,0 +1,87 @@ +# Copyright (c) 2020-2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if (EMSCRIPTEN) + set(TBB_EMSCRIPTEN 1) + set(TBB_COMMON_COMPILE_FLAGS ${TBB_COMMON_COMPILE_FLAGS} -fexceptions) + set(TBB_TEST_LINK_FLAGS ${TBB_COMMON_LINK_FLAGS} -fexceptions -sINITIAL_MEMORY=65536000 -sALLOW_MEMORY_GROWTH=1 -sMALLOC=mimalloc -sEXIT_RUNTIME=1) + if (NOT EMSCRIPTEN_WITHOUT_PTHREAD) + set_property(TARGET Threads::Threads PROPERTY INTERFACE_LINK_LIBRARIES "-pthread") + endif() + set(TBB_EMSCRIPTEN_STACK_SIZE 65536) + set(TBB_LIB_COMPILE_FLAGS -D__TBB_EMSCRIPTEN_STACK_SIZE=${TBB_EMSCRIPTEN_STACK_SIZE}) + set(TBB_TEST_LINK_FLAGS ${TBB_TEST_LINK_FLAGS} -sTOTAL_STACK=${TBB_EMSCRIPTEN_STACK_SIZE}) + unset(TBB_EMSCRIPTEN_STACK_SIZE) +endif() + +if (MINGW) + set(TBB_LINK_DEF_FILE_FLAG "") + set(TBB_DEF_FILE_PREFIX "") +elseif (APPLE) + set(TBB_LINK_DEF_FILE_FLAG -Wl,-exported_symbols_list,) + set(TBB_DEF_FILE_PREFIX mac${TBB_ARCH}) + + # For correct ucontext.h structures layout + set(TBB_COMMON_COMPILE_FLAGS ${TBB_COMMON_COMPILE_FLAGS} -D_XOPEN_SOURCE) +elseif (MSVC) + include(${CMAKE_CURRENT_LIST_DIR}/MSVC.cmake) + return() +else() + set(TBB_LINK_DEF_FILE_FLAG -Wl,--version-script=) + set(TBB_DEF_FILE_PREFIX lin${TBB_ARCH}) + set(TBB_TEST_COMPILE_FLAGS ${TBB_TEST_COMPILE_FLAGS} $<$>:-ffp-model=precise>) +endif() + +# Depfile options (e.g. -MD) are inserted automatically in some cases. +# Don't add -MMD to avoid conflicts in such cases. +if (NOT CMAKE_GENERATOR MATCHES "Ninja" AND NOT CMAKE_CXX_DEPENDS_USE_COMPILER) + set(TBB_MMD_FLAG -MMD) +endif() + +set(TBB_WARNING_LEVEL -Wall -Wextra $<$:-Werror>) +set(TBB_TEST_WARNING_FLAGS -Wshadow -Wcast-qual -Woverloaded-virtual -Wnon-virtual-dtor) + +# Ignore -Werror set through add_compile_options() or added to CMAKE_CXX_FLAGS if TBB_STRICT is disabled. +if (NOT TBB_STRICT AND COMMAND tbb_remove_compile_flag) + tbb_remove_compile_flag(-Werror) +endif() + +# Enable Intel(R) Transactional Synchronization Extensions (-mrtm) and WAITPKG instructions support (-mwaitpkg) on relevant processors +if (CMAKE_SYSTEM_PROCESSOR MATCHES "(AMD64|amd64|i.86|x86)" AND NOT EMSCRIPTEN) + set(TBB_COMMON_COMPILE_FLAGS ${TBB_COMMON_COMPILE_FLAGS} -mrtm $<$>:-mwaitpkg>) +endif() + +# Clang flags to prevent compiler from optimizing out security checks +set(TBB_COMMON_COMPILE_FLAGS ${TBB_COMMON_COMPILE_FLAGS} -Wformat -Wformat-security -Werror=format-security -fPIC $<$>:-fstack-protector-strong>) + +# -z switch is not supported on MacOS +if (NOT APPLE) + set(TBB_LIB_LINK_FLAGS ${TBB_LIB_LINK_FLAGS} -Wl,-z,relro,-z,now) +endif() + +set(TBB_COMMON_LINK_LIBS ${CMAKE_DL_LIBS}) + +if (NOT CMAKE_CXX_FLAGS MATCHES "_FORTIFY_SOURCE") + set(TBB_COMMON_COMPILE_FLAGS ${TBB_COMMON_COMPILE_FLAGS} $<$>:-D_FORTIFY_SOURCE=2>) +endif () + +if (MINGW) + list(APPEND TBB_COMMON_COMPILE_FLAGS -U__STRICT_ANSI__) +endif() + +set(TBB_IPO_COMPILE_FLAGS $<$>:-flto>) +set(TBB_IPO_LINK_FLAGS $<$>:-flto>) + +# TBB malloc settings +set(TBBMALLOC_LIB_COMPILE_FLAGS -fno-rtti -fno-exceptions) diff --git a/src/tbb/cmake/compilers/GNU.cmake b/src/tbb/cmake/compilers/GNU.cmake new file mode 100644 index 000000000..cf6d8bdbc --- /dev/null +++ b/src/tbb/cmake/compilers/GNU.cmake @@ -0,0 +1,118 @@ +# Copyright (c) 2020-2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if (MINGW) + set(TBB_LINK_DEF_FILE_FLAG "") + set(TBB_DEF_FILE_PREFIX "") +elseif (APPLE) + set(TBB_LINK_DEF_FILE_FLAG -Wl,-exported_symbols_list,) + set(TBB_DEF_FILE_PREFIX mac${TBB_ARCH}) + + # For correct ucontext.h structures layout + set(TBB_COMMON_COMPILE_FLAGS ${TBB_COMMON_COMPILE_FLAGS} -D_XOPEN_SOURCE) +else() + set(TBB_LINK_DEF_FILE_FLAG -Wl,--version-script=) + set(TBB_DEF_FILE_PREFIX lin${TBB_ARCH}) +endif() + +set(TBB_WARNING_LEVEL -Wall -Wextra $<$:-Werror> -Wfatal-errors) +set(TBB_TEST_WARNING_FLAGS -Wshadow -Wcast-qual -Woverloaded-virtual -Wnon-virtual-dtor) + +# Depfile options (e.g. -MD) are inserted automatically in some cases. +# Don't add -MMD to avoid conflicts in such cases. +if (NOT CMAKE_GENERATOR MATCHES "Ninja" AND NOT CMAKE_CXX_DEPENDS_USE_COMPILER) + set(TBB_MMD_FLAG -MMD) +endif() + + +# Binutils < 2.31.1 do not support the tpause instruction. When compiling with +# a modern version of GCC (supporting it) but relying on an outdated assembler, +# will result in an error reporting "no such instruction: tpause". +# The following code invokes the GNU assembler to extract the version number +# and convert it to an integer that can be used in the C++ code to compare +# against, and conditionally disable the __TBB_WAITPKG_INTRINSICS_PRESENT +# macro if the version is incompatible. Binutils only report the version in the +# MAJOR.MINOR format, therefore the version checked is >=2.32 (instead of +# >=2.31.1). Capturing the output in CMake can be done like below. The version +# information is written to either stdout or stderr. To not make any +# assumptions, both are captured. +execute_process( + COMMAND ${CMAKE_COMMAND} -E env "LANG=C" ${CMAKE_CXX_COMPILER} -xc -c /dev/null -Wa,-v -o/dev/null + OUTPUT_VARIABLE ASSEMBLER_VERSION_LINE_OUT + ERROR_VARIABLE ASSEMBLER_VERSION_LINE_ERR + OUTPUT_STRIP_TRAILING_WHITESPACE + ERROR_STRIP_TRAILING_WHITESPACE +) +set(ASSEMBLER_VERSION_LINE ${ASSEMBLER_VERSION_LINE_OUT}${ASSEMBLER_VERSION_LINE_ERR}) +string(REGEX REPLACE ".*GNU assembler version ([0-9]+)\\.([0-9]+).*" "\\1" _tbb_gnu_asm_major_version "${ASSEMBLER_VERSION_LINE}") +string(REGEX REPLACE ".*GNU assembler version ([0-9]+)\\.([0-9]+).*" "\\2" _tbb_gnu_asm_minor_version "${ASSEMBLER_VERSION_LINE}") +unset(ASSEMBLER_VERSION_LINE_OUT) +unset(ASSEMBLER_VERSION_LINE_ERR) +unset(ASSEMBLER_VERSION_LINE) +message(TRACE "Extracted GNU assembler version: major=${_tbb_gnu_asm_major_version} minor=${_tbb_gnu_asm_minor_version}") + +math(EXPR _tbb_gnu_asm_version_number "${_tbb_gnu_asm_major_version} * 1000 + ${_tbb_gnu_asm_minor_version}") +set(TBB_COMMON_COMPILE_FLAGS ${TBB_COMMON_COMPILE_FLAGS} "-D__TBB_GNU_ASM_VERSION=${_tbb_gnu_asm_version_number}") +message(STATUS "GNU Assembler version: ${_tbb_gnu_asm_major_version}.${_tbb_gnu_asm_minor_version} (${_tbb_gnu_asm_version_number})") + +# Enable Intel(R) Transactional Synchronization Extensions (-mrtm) and WAITPKG instructions support (-mwaitpkg) on relevant processors +if (CMAKE_SYSTEM_PROCESSOR MATCHES "(AMD64|amd64|i.86|x86)" AND NOT EMSCRIPTEN) + set(TBB_COMMON_COMPILE_FLAGS ${TBB_COMMON_COMPILE_FLAGS} -mrtm $<$>,$>>:-mwaitpkg>) +endif() + +set(TBB_COMMON_LINK_LIBS ${CMAKE_DL_LIBS}) + +# Ignore -Werror set through add_compile_options() or added to CMAKE_CXX_FLAGS if TBB_STRICT is disabled. +if (NOT TBB_STRICT AND COMMAND tbb_remove_compile_flag) + tbb_remove_compile_flag(-Werror) +endif() + +if (NOT ${CMAKE_CXX_COMPILER_ID} STREQUAL Intel) + # gcc 6.0 and later have -flifetime-dse option that controls elimination of stores done outside the object lifetime + set(TBB_DSE_FLAG $<$>:-flifetime-dse=1>) + set(TBB_COMMON_COMPILE_FLAGS ${TBB_COMMON_COMPILE_FLAGS} $<$>:-fstack-clash-protection>) + + # Suppress GCC 12.x-13.x warning here that to_wait_node(n)->my_is_in_list might have size 0 + set(TBB_COMMON_LINK_FLAGS ${TBB_COMMON_LINK_FLAGS} $<$>,$>:-Wno-stringop-overflow>) +endif() + +# Workaround for heavy tests and too many symbols in debug (rellocation truncated to fit: R_MIPS_CALL16) +if ("${CMAKE_SYSTEM_PROCESSOR}" MATCHES "mips") + set(TBB_TEST_COMPILE_FLAGS ${TBB_TEST_COMPILE_FLAGS} -DTBB_TEST_LOW_WORKLOAD $<$:-fPIE -mxgot>) + set(TBB_TEST_LINK_FLAGS ${TBB_TEST_LINK_FLAGS} $<$:-pie>) +endif() + +set(TBB_IPO_COMPILE_FLAGS $<$>:-flto>) +set(TBB_IPO_LINK_FLAGS $<$>:-flto>) + + +if (MINGW AND CMAKE_SYSTEM_PROCESSOR MATCHES "i.86") + list (APPEND TBB_COMMON_COMPILE_FLAGS -msse2) +endif () + +# Gnu flags to prevent compiler from optimizing out security checks +set(TBB_COMMON_COMPILE_FLAGS ${TBB_COMMON_COMPILE_FLAGS} -fno-strict-overflow -fno-delete-null-pointer-checks -fwrapv) +set(TBB_COMMON_COMPILE_FLAGS ${TBB_COMMON_COMPILE_FLAGS} -Wformat -Wformat-security -Werror=format-security + -fstack-protector-strong ) +# -z switch is not supported on MacOS and MinGW +if (NOT APPLE AND NOT MINGW) + set(TBB_LIB_LINK_FLAGS ${TBB_LIB_LINK_FLAGS} -Wl,-z,relro,-z,now,-z,noexecstack) +endif() +if (NOT CMAKE_CXX_FLAGS MATCHES "_FORTIFY_SOURCE") + set(TBB_COMMON_COMPILE_FLAGS ${TBB_COMMON_COMPILE_FLAGS} $<$>:-D_FORTIFY_SOURCE=2> ) +endif () + +# TBB malloc settings +set(TBBMALLOC_LIB_COMPILE_FLAGS -fno-rtti -fno-exceptions) +set(TBB_OPENMP_FLAG -fopenmp) diff --git a/src/tbb/cmake/compilers/Intel.cmake b/src/tbb/cmake/compilers/Intel.cmake new file mode 100644 index 000000000..531e078ec --- /dev/null +++ b/src/tbb/cmake/compilers/Intel.cmake @@ -0,0 +1,38 @@ +# Copyright (c) 2020-2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if (MSVC) + include(${CMAKE_CURRENT_LIST_DIR}/MSVC.cmake) + set(TBB_WARNING_LEVEL ${TBB_WARNING_LEVEL} /W3) + set(TBB_OPENMP_FLAG /Qopenmp) + set(TBB_IPO_COMPILE_FLAGS $<$>:/Qipo>) + set(TBB_IPO_LINK_FLAGS $<$>:/INCREMENTAL:NO>) +elseif (APPLE) + include(${CMAKE_CURRENT_LIST_DIR}/AppleClang.cmake) + set(TBB_COMMON_COMPILE_FLAGS ${TBB_COMMON_COMPILE_FLAGS} -fstack-protector -Wformat -Wformat-security + $<$>:-fno-omit-frame-pointer -qno-opt-report-embed>) + if (NOT CMAKE_CXX_FLAGS MATCHES "_FORTIFY_SOURCE") + set(TBB_COMMON_COMPILE_FLAGS ${TBB_COMMON_COMPILE_FLAGS} $<$>:-D_FORTIFY_SOURCE=2>) + endif () + + set(TBB_OPENMP_FLAG -qopenmp) + set(TBB_IPO_COMPILE_FLAGS $<$>:-ipo>) +else() + include(${CMAKE_CURRENT_LIST_DIR}/GNU.cmake) + set(TBB_COMMON_COMPILE_FLAGS ${TBB_COMMON_COMPILE_FLAGS} $<$:-falign-stack=maintain-16-byte>) + set(TBB_LIB_LINK_FLAGS ${TBB_LIB_LINK_FLAGS} -static-intel) + set(TBB_OPENMP_FLAG -qopenmp) + set(TBB_IPO_COMPILE_FLAGS $<$>:-ipo>) +endif() +set(TBB_IPO_LINK_FLAGS ${TBB_IPO_LINK_FLAGS} ${TBB_IPO_COMPILE_FLAGS}) diff --git a/src/tbb/cmake/compilers/IntelLLVM.cmake b/src/tbb/cmake/compilers/IntelLLVM.cmake new file mode 100644 index 000000000..b51437816 --- /dev/null +++ b/src/tbb/cmake/compilers/IntelLLVM.cmake @@ -0,0 +1,28 @@ +# Copyright (c) 2020-2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if (WIN32) + include(${CMAKE_CURRENT_LIST_DIR}/MSVC.cmake) + set(TBB_OPENMP_FLAG /Qopenmp) + set(TBB_IPO_COMPILE_FLAGS $<$>:/Qipo>) + set(TBB_IPO_LINK_FLAGS $<$>:/INCREMENTAL:NO>) +else() + include(${CMAKE_CURRENT_LIST_DIR}/Clang.cmake) + set(TBB_IPO_COMPILE_FLAGS $<$>:-ipo>) + # "--exclude-libs,ALL" is used to avoid accidental exporting of symbols + # from statically linked libraries + set(TBB_LIB_LINK_FLAGS ${TBB_LIB_LINK_FLAGS} -static-intel -Wl,--exclude-libs,ALL) + set(TBB_OPENMP_FLAG -qopenmp) +endif() +set(TBB_IPO_LINK_FLAGS ${TBB_IPO_LINK_FLAGS} ${TBB_IPO_COMPILE_FLAGS}) diff --git a/src/tbb/cmake/compilers/MSVC.cmake b/src/tbb/cmake/compilers/MSVC.cmake new file mode 100644 index 000000000..6568ec7eb --- /dev/null +++ b/src/tbb/cmake/compilers/MSVC.cmake @@ -0,0 +1,92 @@ +# Copyright (c) 2020-2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set(TBB_LINK_DEF_FILE_FLAG ${CMAKE_LINK_DEF_FILE_FLAG}) +set(TBB_DEF_FILE_PREFIX win${TBB_ARCH}) + +# Workaround for CMake issue https://gitlab.kitware.com/cmake/cmake/issues/18317. +# TODO: consider use of CMP0092 CMake policy. +string(REGEX REPLACE "/W[0-4]" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") + +set(TBB_WARNING_LEVEL $<$>:/W4> $<$:/WX>) + +# Warning suppression C4324: structure was padded due to alignment specifier +set(TBB_WARNING_SUPPRESS /wd4324) + +set(TBB_TEST_COMPILE_FLAGS ${TBB_TEST_COMPILE_FLAGS} /bigobj) +if (MSVC_VERSION LESS_EQUAL 1900) + # Warning suppression C4503 for VS2015 and earlier: + # decorated name length exceeded, name was truncated. + # More info can be found at + # https://docs.microsoft.com/en-us/cpp/error-messages/compiler-warnings/compiler-warning-level-1-c4503 + set(TBB_TEST_COMPILE_FLAGS ${TBB_TEST_COMPILE_FLAGS} /wd4503) +endif() +set(TBB_LIB_COMPILE_FLAGS -D_CRT_SECURE_NO_WARNINGS /GS) +set(TBB_COMMON_COMPILE_FLAGS ${TBB_COMMON_COMPILE_FLAGS} /volatile:iso /FS /EHsc) + +set(TBB_LIB_LINK_FLAGS ${TBB_LIB_LINK_FLAGS} /DEPENDENTLOADFLAG:0x2000 /DYNAMICBASE /NXCOMPAT) + +if (TBB_ARCH EQUAL 32) + set(TBB_LIB_LINK_FLAGS ${TBB_LIB_LINK_FLAGS} /SAFESEH ) +endif() + +# Ignore /WX set through add_compile_options() or added to CMAKE_CXX_FLAGS if TBB_STRICT is disabled. +if (NOT TBB_STRICT AND COMMAND tbb_remove_compile_flag) + tbb_remove_compile_flag(/WX) +endif() + +if (WINDOWS_STORE OR TBB_WINDOWS_DRIVER) + set(TBB_COMMON_COMPILE_FLAGS ${TBB_COMMON_COMPILE_FLAGS} /D_WIN32_WINNT=0x0A00) + set(TBB_COMMON_LINK_FLAGS -NODEFAULTLIB:kernel32.lib -INCREMENTAL:NO) + set(TBB_COMMON_LINK_LIBS OneCore.lib) +endif() + +if (WINDOWS_STORE) + if (NOT CMAKE_SYSTEM_VERSION EQUAL 10.0) + message(FATAL_ERROR "CMAKE_SYSTEM_VERSION must be equal to 10.0") + endif() + + set(TBB_COMMON_COMPILE_FLAGS ${TBB_COMMON_COMPILE_FLAGS} /ZW /ZW:nostdlib) + + # CMake define this extra lib, remove it for this build type + string(REGEX REPLACE "WindowsApp.lib" "" CMAKE_CXX_STANDARD_LIBRARIES "${CMAKE_CXX_STANDARD_LIBRARIES}") + + if (TBB_NO_APPCONTAINER) + set(TBB_LIB_LINK_FLAGS ${TBB_LIB_LINK_FLAGS} -APPCONTAINER:NO) + endif() +endif() + +if (TBB_WINDOWS_DRIVER) + # Since this is universal driver disable this variable + set(CMAKE_SYSTEM_PROCESSOR "") + + # CMake define list additional libs, remove it for this build type + set(CMAKE_CXX_STANDARD_LIBRARIES "") + + set(TBB_COMMON_COMPILE_FLAGS ${TBB_COMMON_COMPILE_FLAGS} /D _UNICODE /DUNICODE /DWINAPI_FAMILY=WINAPI_FAMILY_APP /D__WRL_NO_DEFAULT_LIB__) +endif() + +if (CMAKE_CXX_COMPILER_ID MATCHES "(Clang|IntelLLVM)") + if (CMAKE_SYSTEM_PROCESSOR MATCHES "(x86|AMD64|i.86)") + set(TBB_COMMON_COMPILE_FLAGS ${TBB_COMMON_COMPILE_FLAGS} -mrtm -mwaitpkg) + endif() + set(TBB_IPO_COMPILE_FLAGS $<$>:-flto>) + set(TBB_IPO_LINK_FLAGS $<$>:-flto>) +else() + set(TBB_IPO_COMPILE_FLAGS $<$>:/GL>) + set(TBB_IPO_LINK_FLAGS $<$>:-LTCG> $<$>:-INCREMENTAL:NO>) +endif() + +set(TBB_OPENMP_FLAG /openmp) +set(TBB_OPENMP_NO_LINK_FLAG TRUE) # TBB_OPENMP_FLAG will be used only on compilation but not on linkage diff --git a/src/tbb/cmake/compilers/QCC.cmake b/src/tbb/cmake/compilers/QCC.cmake new file mode 100644 index 000000000..52285a616 --- /dev/null +++ b/src/tbb/cmake/compilers/QCC.cmake @@ -0,0 +1,18 @@ +# Copyright (c) 2021 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/compilers/GNU.cmake) + +# Remove dl library not present in QNX systems +unset(TBB_COMMON_LINK_LIBS) diff --git a/src/tbb/cmake/config_generation.cmake b/src/tbb/cmake/config_generation.cmake new file mode 100644 index 000000000..e4ef7bcea --- /dev/null +++ b/src/tbb/cmake/config_generation.cmake @@ -0,0 +1,142 @@ +# Copyright (c) 2020-2023 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Save current location, +# see for details: https://cmake.org/cmake/help/latest/variable/CMAKE_CURRENT_LIST_DIR.html +set(_tbb_gen_cfg_path ${CMAKE_CURRENT_LIST_DIR}) + +include(CMakeParseArguments) + +function(tbb_generate_config) + set(options HANDLE_SUBDIRS) + set(oneValueArgs INSTALL_DIR + SYSTEM_NAME + LIB_REL_PATH INC_REL_PATH + VERSION + TBB_BINARY_VERSION + TBBMALLOC_BINARY_VERSION + TBBMALLOC_PROXY_BINARY_VERSION + TBBBIND_BINARY_VERSION) + + cmake_parse_arguments(tbb_gen_cfg "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + + get_filename_component(config_install_dir ${tbb_gen_cfg_INSTALL_DIR} ABSOLUTE) + file(MAKE_DIRECTORY ${config_install_dir}) + + file(TO_CMAKE_PATH "${tbb_gen_cfg_LIB_REL_PATH}" TBB_LIB_REL_PATH) + file(TO_CMAKE_PATH "${tbb_gen_cfg_INC_REL_PATH}" TBB_INC_REL_PATH) + + set(TBB_VERSION ${tbb_gen_cfg_VERSION}) + + set(_tbb_pc_lib_name tbb) + set(_prefix_for_pc_file "\${pcfiledir}/../../") + set(_includedir_for_pc_file "\${prefix}/include") + + set(TBB_COMPONENTS_BIN_VERSION " +set(_tbb_bin_version ${tbb_gen_cfg_TBB_BINARY_VERSION}) +set(_tbbmalloc_bin_version ${tbb_gen_cfg_TBBMALLOC_BINARY_VERSION}) +set(_tbbmalloc_proxy_bin_version ${tbb_gen_cfg_TBBMALLOC_PROXY_BINARY_VERSION}) +set(_tbbbind_bin_version ${tbb_gen_cfg_TBBBIND_BINARY_VERSION}) +") + + if (tbb_gen_cfg_SYSTEM_NAME STREQUAL "Linux") + set(TBB_LIB_PREFIX "lib") + set(TBB_LIB_EXT "so.\${_\${_tbb_component}_bin_version}") + + set (TBB_HANDLE_IMPLIB " + set (_tbb_release_dll \${_tbb_release_lib}) + set (_tbb_debug_dll \${_tbb_debug_lib}) +") + if (tbb_gen_cfg_HANDLE_SUBDIRS) + set(TBB_HANDLE_SUBDIRS "set(_tbb_subdir gcc4.8)") + + set(_libdir_for_pc_file "\${prefix}/lib/intel64/gcc4.8") + set(_tbb_pc_extra_libdir "-L\${prefix}/lib") + configure_file(${_tbb_gen_cfg_path}/../integration/pkg-config/tbb.pc.in ${config_install_dir}/tbb.pc @ONLY) + + set(_libdir_for_pc_file "\${prefix}/lib/ia32/gcc4.8") + set(_tbb_pc_extra_libdir "-L\${prefix}/lib32") + configure_file(${_tbb_gen_cfg_path}/../integration/pkg-config/tbb.pc.in ${config_install_dir}/tbb32.pc @ONLY) + endif() + elseif (tbb_gen_cfg_SYSTEM_NAME STREQUAL "Darwin") + set(TBB_LIB_PREFIX "lib") + set(TBB_LIB_EXT "\${_\${_tbb_component}_bin_version}.dylib") + + set (TBB_HANDLE_IMPLIB " + set (_tbb_release_dll \${_tbb_release_lib}) + set (_tbb_debug_dll \${_tbb_debug_lib}) +") + set(_libdir_for_pc_file "\${prefix}/lib") + configure_file(${_tbb_gen_cfg_path}/../integration/pkg-config/tbb.pc.in ${config_install_dir}/tbb.pc @ONLY) + elseif (tbb_gen_cfg_SYSTEM_NAME STREQUAL "Windows") + set(TBB_LIB_PREFIX "") + set(TBB_LIB_EXT "lib") + set(TBB_COMPILE_DEFINITIONS " + INTERFACE_COMPILE_DEFINITIONS \"__TBB_NO_IMPLICIT_LINKAGE=1\"") + + # .lib - installed to TBB_LIB_REL_PATH (e.g. /lib) and are passed as IMPORTED_IMPLIB_ property to target + # .dll - installed to /bin or /redist and are passed as IMPORTED_LOCATION_ property to target + set (TBB_HANDLE_IMPLIB " + find_file(_tbb_release_dll + NAMES \${_tbb_component}\${_bin_version}.dll + PATHS \${_tbb_root} + PATH_SUFFIXES \"redist/\${_tbb_intel_arch}/\${_tbb_subdir}\" \"bin\${_tbb_arch_suffix}/\${_tbb_subdir}\" \"bin\${_tbb_arch_suffix}/\" \"bin\" + NO_DEFAULT_PATH + ) + + if (EXISTS \"\${_tbb_debug_lib}\") + find_file(_tbb_debug_dll + NAMES \${_tbb_component}\${_bin_version}_debug.dll + PATHS \${_tbb_root} + PATH_SUFFIXES \"redist/\${_tbb_intel_arch}/\${_tbb_subdir}\" \"bin\${_tbb_arch_suffix}/\${_tbb_subdir}\" \"bin\${_tbb_arch_suffix}/\" \"bin\" + NO_DEFAULT_PATH + ) + endif() +") + set(TBB_IMPLIB_RELEASE " + IMPORTED_IMPLIB_RELEASE \"\${_tbb_release_lib}\"") + set(TBB_IMPLIB_DEBUG " + IMPORTED_IMPLIB_DEBUG \"\${_tbb_debug_lib}\"") + + if (tbb_gen_cfg_HANDLE_SUBDIRS) + set(TBB_HANDLE_SUBDIRS " +set(_tbb_subdir vc14) +if (WINDOWS_STORE) + set(_tbb_subdir \${_tbb_subdir}_uwp) +endif() +") + set(_tbb_pc_lib_name ${_tbb_pc_lib_name}${TBB_BINARY_VERSION}) + + set(_libdir_for_pc_file "\${prefix}/lib/intel64/vc14") + set(_tbb_pc_extra_libdir "-L\${prefix}/lib") + configure_file(${_tbb_gen_cfg_path}/../integration/pkg-config/tbb.pc.in ${config_install_dir}/tbb.pc @ONLY) + + set(_libdir_for_pc_file "\${prefix}/lib/ia32/vc14") + set(_tbb_pc_extra_libdir "-L\${prefix}/lib32") + configure_file(${_tbb_gen_cfg_path}/../integration/pkg-config/tbb.pc.in ${config_install_dir}/tbb32.pc @ONLY) + endif() + + set(TBB_HANDLE_BIN_VERSION " + unset(_bin_version) + if (_tbb_component STREQUAL tbb) + set(_bin_version \${_tbb_bin_version}) + endif() +") + else() + message(FATAL_ERROR "Unsupported OS name: ${tbb_system_name}") + endif() + + configure_file(${_tbb_gen_cfg_path}/templates/TBBConfig.cmake.in ${config_install_dir}/TBBConfig.cmake @ONLY) + configure_file(${_tbb_gen_cfg_path}/templates/TBBConfigVersion.cmake.in ${config_install_dir}/TBBConfigVersion.cmake @ONLY) +endfunction() diff --git a/src/tbb/cmake/hwloc_detection.cmake b/src/tbb/cmake/hwloc_detection.cmake new file mode 100644 index 000000000..aaca5a599 --- /dev/null +++ b/src/tbb/cmake/hwloc_detection.cmake @@ -0,0 +1,64 @@ +# Copyright (c) 2020-2023 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +list(APPEND HWLOC_REQUIRED_VERSIONS 1_11 2 2_5) + +foreach(hwloc_version ${HWLOC_REQUIRED_VERSIONS}) + if (NOT WIN32) + set(CMAKE_HWLOC_${hwloc_version}_DLL_PATH STUB) + endif() + set(HWLOC_TARGET_NAME HWLOC::hwloc_${hwloc_version}) + + if (NOT TARGET ${HWLOC_TARGET_NAME} AND + CMAKE_HWLOC_${hwloc_version}_LIBRARY_PATH AND + CMAKE_HWLOC_${hwloc_version}_DLL_PATH AND + CMAKE_HWLOC_${hwloc_version}_INCLUDE_PATH + ) + add_library(${HWLOC_TARGET_NAME} SHARED IMPORTED) + set_target_properties(${HWLOC_TARGET_NAME} PROPERTIES INTERFACE_INCLUDE_DIRECTORIES + "${CMAKE_HWLOC_${hwloc_version}_INCLUDE_PATH}") + if (WIN32) + set_target_properties(${HWLOC_TARGET_NAME} PROPERTIES + IMPORTED_LOCATION "${CMAKE_HWLOC_${hwloc_version}_DLL_PATH}" + IMPORTED_IMPLIB "${CMAKE_HWLOC_${hwloc_version}_LIBRARY_PATH}") + else() + set_target_properties(${HWLOC_TARGET_NAME} PROPERTIES + IMPORTED_LOCATION "${CMAKE_HWLOC_${hwloc_version}_LIBRARY_PATH}") + endif() + endif() + + if (TARGET ${HWLOC_TARGET_NAME}) + set(HWLOC_TARGET_EXPLICITLY_DEFINED TRUE) + endif() +endforeach() + +unset(HWLOC_TARGET_NAME) + +if (NOT HWLOC_TARGET_EXPLICITLY_DEFINED AND + NOT TBB_DISABLE_HWLOC_AUTOMATIC_SEARCH +) + find_package(PkgConfig QUIET) + if (PKG_CONFIG_FOUND AND NOT CMAKE_VERSION VERSION_LESS 3.6) + pkg_search_module(HWLOC hwloc IMPORTED_TARGET) + if (TARGET PkgConfig::HWLOC) + if (HWLOC_VERSION VERSION_LESS 2) + set(TBBBIND_LIBRARY_NAME tbbbind) + elseif(HWLOC_VERSION VERSION_LESS 2.5) + set(TBBBIND_LIBRARY_NAME tbbbind_2_0) + else() + set(TBBBIND_LIBRARY_NAME tbbbind_2_5) + endif() + endif() + endif() +endif() diff --git a/src/tbb/cmake/memcheck.cmake b/src/tbb/cmake/memcheck.cmake new file mode 100644 index 000000000..fd5e920c5 --- /dev/null +++ b/src/tbb/cmake/memcheck.cmake @@ -0,0 +1,112 @@ +# Copyright (c) 2020-2023 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +option(TBB_VALGRIND_MEMCHECK "Enable scan for memory leaks using Valgrind" OFF) + +if (NOT TBB_VALGRIND_MEMCHECK) + return() +endif() + +add_custom_target(memcheck-all + COMMENT "Run memcheck on all tests") + +find_program(VALGRIND_EXE valgrind) + +if (NOT VALGRIND_EXE) + message(FATAL_ERROR "Valgrind executable is not found, add tool to PATH or turn off TBB_VALGRIND_MEMCHECK") +else() + message(STATUS "Found Valgrind to run memory leak scan") +endif() + +file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/memcheck) + +function(_tbb_run_memcheck test_target subdir) + set(target_name memcheck-${test_target}) + if(${subdir} STREQUAL "tbbmalloc") + # Valgring intercepts all allocation symbols with its own by default, + # so it disables using tbbmalloc. In case of tbbmalloc tests + # intercept allocation symbols only in the default system libraries, + # but not in any other shared library or the executable + # defining public malloc or operator new related functions. + set(option "--soname-synonyms=somalloc=nouserintercepts") + endif() + add_custom_target(${target_name} + COMMAND ${VALGRIND_EXE} ${option} --leak-check=full --show-leak-kinds=all --log-file=${CMAKE_BINARY_DIR}/memcheck/${target_name}.log -v $) + add_dependencies(memcheck-all ${target_name}) +endfunction() + +add_custom_target(memcheck-short + COMMENT "Run memcheck scan on specified list") + +# List of reasonable and quick enough tests to use in automated memcheck +add_dependencies(memcheck-short + memcheck-test_allocators + memcheck-test_arena_constraints + memcheck-test_dynamic_link + memcheck-test_concurrent_lru_cache + memcheck-conformance_concurrent_unordered_map + memcheck-conformance_concurrent_unordered_set + memcheck-conformance_concurrent_map + memcheck-conformance_concurrent_set + memcheck-conformance_concurrent_priority_queue + memcheck-conformance_concurrent_vector + memcheck-conformance_concurrent_queue + memcheck-conformance_concurrent_hash_map + memcheck-test_parallel_for + memcheck-test_parallel_for_each + memcheck-test_parallel_reduce + memcheck-test_parallel_sort + memcheck-test_parallel_invoke + memcheck-test_parallel_scan + memcheck-test_parallel_pipeline + memcheck-test_eh_algorithms + memcheck-test_task_group + memcheck-test_task_arena + memcheck-test_enumerable_thread_specific + memcheck-test_resumable_tasks + memcheck-conformance_mutex + memcheck-test_function_node + memcheck-test_multifunction_node + memcheck-test_broadcast_node + memcheck-test_buffer_node + memcheck-test_composite_node + memcheck-test_continue_node + memcheck-test_eh_flow_graph + memcheck-test_flow_graph + memcheck-test_flow_graph_priorities + memcheck-test_flow_graph_whitebox + memcheck-test_indexer_node + memcheck-test_join_node + memcheck-test_join_node_key_matching + memcheck-test_join_node_msg_key_matching + memcheck-test_priority_queue_node + memcheck-test_sequencer_node + memcheck-test_split_node + memcheck-test_tagged_msg + memcheck-test_overwrite_node + memcheck-test_write_once_node + memcheck-test_async_node + memcheck-test_input_node + memcheck-test_profiling + memcheck-test_concurrent_queue_whitebox + memcheck-test_intrusive_list + memcheck-test_semaphore + memcheck-test_environment_whitebox + memcheck-test_handle_perror + memcheck-test_hw_concurrency + memcheck-test_eh_thread + memcheck-test_global_control + memcheck-test_task + memcheck-test_concurrent_monitor +) diff --git a/src/tbb/cmake/packaging.cmake b/src/tbb/cmake/packaging.cmake new file mode 100644 index 000000000..aa2acc4d4 --- /dev/null +++ b/src/tbb/cmake/packaging.cmake @@ -0,0 +1,24 @@ +# Copyright (c) 2020-2023 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Note: current implementation uses CMAKE_BUILD_TYPE, +# this parameter is not defined for multi-config generators. +set(CPACK_PACKAGE_NAME "${PROJECT_NAME}") +set(CPACK_PACKAGE_VERSION "${TBB_VERSION}") +string(TOLOWER ${CPACK_PACKAGE_NAME}-${PROJECT_VERSION}-${CMAKE_SYSTEM_NAME}_${TBB_OUTPUT_DIR_BASE}_${CMAKE_BUILD_TYPE} CPACK_PACKAGE_FILE_NAME) +set(CPACK_GENERATOR ZIP) +# Note: this is an internal non-documented variable set by CPack +if (NOT CPack_CMake_INCLUDED) + include(CPack) +endif() diff --git a/src/tbb/cmake/post_install/CMakeLists.txt b/src/tbb/cmake/post_install/CMakeLists.txt new file mode 100644 index 000000000..a7a025457 --- /dev/null +++ b/src/tbb/cmake/post_install/CMakeLists.txt @@ -0,0 +1,22 @@ +# Copyright (c) 2020-2021 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Add code signing as post-install step. +if (DEFINED TBB_SIGNTOOL) + file(TO_CMAKE_PATH "${TBB_SIGNTOOL}" TBB_SIGNTOOL) + install(CODE " + file(GLOB_RECURSE FILES_TO_SIGN \${CMAKE_INSTALL_PREFIX}/*${CMAKE_SHARED_LIBRARY_SUFFIX}) + execute_process(COMMAND ${TBB_SIGNTOOL} \${FILES_TO_SIGN} ${TBB_SIGNTOOL_ARGS}) +") +endif() diff --git a/src/tbb/cmake/python/test_launcher.cmake b/src/tbb/cmake/python/test_launcher.cmake new file mode 100644 index 000000000..9b1bdde98 --- /dev/null +++ b/src/tbb/cmake/python/test_launcher.cmake @@ -0,0 +1,38 @@ +# Copyright (c) 2020-2021 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +find_package(PythonInterp 3.5 REQUIRED) + +file(GLOB_RECURSE MODULES_LIST "${PYTHON_MODULE_BUILD_PATH}/*TBB.py*" ) +list(LENGTH MODULES_LIST MODULES_COUNT) + +if (MODULES_COUNT EQUAL 0) + message(FATAL_ERROR "Cannot find oneTBB Python module") +elseif (MODULES_COUNT GREATER 1) + message(WARNING "Found more than oneTBB Python modules, the only first found module will be tested") +endif() + +list(GET MODULES_LIST 0 PYTHON_MODULE) +get_filename_component(PYTHON_MODULE_PATH ${PYTHON_MODULE} DIRECTORY) + +execute_process( + COMMAND + ${CMAKE_COMMAND} -E env LD_LIBRARY_PATH=${TBB_BINARIES_PATH} + ${PYTHON_EXECUTABLE} -m tbb test + WORKING_DIRECTORY ${PYTHON_MODULE_PATH} + RESULT_VARIABLE CMD_RESULT +) +if (CMD_RESULT) + message(FATAL_ERROR "Error while test execution: ${cmd} error_code: ${CMD_RESULT}") +endif() diff --git a/src/tbb/cmake/resumable_tasks.cmake b/src/tbb/cmake/resumable_tasks.cmake new file mode 100644 index 000000000..d379d4ed3 --- /dev/null +++ b/src/tbb/cmake/resumable_tasks.cmake @@ -0,0 +1,31 @@ +# Copyright (c) 2023 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +include(CheckSymbolExists) + +if (UNIX) + set(CMAKE_REQUIRED_FLAGS -Wno-deprecated-declarations) + if (APPLE) + set(CMAKE_REQUIRED_DEFINITIONS -D_XOPEN_SOURCE) + endif() + + check_symbol_exists("getcontext" "ucontext.h" _tbb_have_ucontext) + if (NOT _tbb_have_ucontext) + set(TBB_RESUMABLE_TASKS_USE_THREADS "__TBB_RESUMABLE_TASKS_USE_THREADS=1") + endif() + + unset(_tbb_have_ucontext) + unset(CMAKE_REQUIRED_DEFINITIONS) + unset(CMAKE_REQUIRED_FLAGS) +endif() diff --git a/src/tbb/cmake/sanitize.cmake b/src/tbb/cmake/sanitize.cmake new file mode 100644 index 000000000..d07b32986 --- /dev/null +++ b/src/tbb/cmake/sanitize.cmake @@ -0,0 +1,43 @@ +# Copyright (c) 2020-2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set(TBB_SANITIZE ${TBB_SANITIZE} CACHE STRING "Sanitizer parameter passed to compiler/linker" FORCE) +# Possible values of sanitizer parameter for cmake-gui for convenience, user still can use any other value. +set_property(CACHE TBB_SANITIZE PROPERTY STRINGS "thread" "memory" "leak" "address -fno-omit-frame-pointer") + +if (NOT TBB_SANITIZE) + return() +endif() + +set(TBB_SANITIZE_OPTION -fsanitize=${TBB_SANITIZE}) + +# It is required to add sanitizer option to CMAKE_REQUIRED_LIBRARIES to make check_cxx_compiler_flag working properly: +# sanitizer option should be passed during the compilation phase as well as during the compilation. +set(CMAKE_REQUIRED_LIBRARIES "${TBB_SANITIZE_OPTION} ${CMAKE_REQUIRED_LIBRARIES}") + +string(MAKE_C_IDENTIFIER ${TBB_SANITIZE_OPTION} FLAG_DISPLAY_NAME) +check_cxx_compiler_flag(${TBB_SANITIZE_OPTION} ${FLAG_DISPLAY_NAME}) +if (NOT ${FLAG_DISPLAY_NAME}) + message(FATAL_ERROR + "${TBB_SANITIZE_OPTION} is not supported by compiler ${CMAKE_CXX_COMPILER_ID}:${CMAKE_CXX_COMPILER_VERSION}, " + "please try another compiler or omit TBB_SANITIZE variable") +endif() + +set(TBB_TESTS_ENVIRONMENT ${TBB_TESTS_ENVIRONMENT} + "TSAN_OPTIONS=suppressions=${CMAKE_CURRENT_SOURCE_DIR}/cmake/suppressions/tsan.suppressions" + "LSAN_OPTIONS=suppressions=${CMAKE_CURRENT_SOURCE_DIR}/cmake/suppressions/lsan.suppressions") + +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${TBB_SANITIZE_OPTION}") +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${TBB_SANITIZE_OPTION}") +set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${TBB_SANITIZE_OPTION}") diff --git a/src/tbb/cmake/scripts/cmake_gen_github_configs.cmake b/src/tbb/cmake/scripts/cmake_gen_github_configs.cmake new file mode 100644 index 000000000..4d9eb73d6 --- /dev/null +++ b/src/tbb/cmake/scripts/cmake_gen_github_configs.cmake @@ -0,0 +1,49 @@ +# Copyright (c) 2020-2023 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +include(${CMAKE_CURRENT_LIST_DIR}/../config_generation.cmake) + +# TBBConfig in TBB provided packages are expected to be placed into: /lib/cmake/tbb* +set(TBB_ROOT_REL_PATH "../../..") + +# Paths relative to TBB root directory +set(INC_REL_PATH "include") +set(LIB_REL_PATH "lib") + +# Parse version info +file(READ ${CMAKE_CURRENT_LIST_DIR}/../../include/oneapi/tbb/version.h _tbb_version_info) +string(REGEX REPLACE ".*#define TBB_VERSION_MAJOR ([0-9]+).*" "\\1" _tbb_ver_major "${_tbb_version_info}") +string(REGEX REPLACE ".*#define TBB_VERSION_MINOR ([0-9]+).*" "\\1" _tbb_ver_minor "${_tbb_version_info}") +string(REGEX REPLACE ".*#define TBB_VERSION_PATCH ([0-9]+).*" "\\1" _tbb_ver_patch "${_tbb_version_info}") +string(REGEX REPLACE ".*#define __TBB_BINARY_VERSION ([0-9]+).*" "\\1" TBB_BINARY_VERSION "${_tbb_version_info}") +file(READ ${CMAKE_CURRENT_LIST_DIR}/../../CMakeLists.txt _tbb_cmakelist) +string(REGEX REPLACE ".*TBBMALLOC_BINARY_VERSION ([0-9]+).*" "\\1" TBBMALLOC_BINARY_VERSION "${_tbb_cmakelist}") +set(TBBMALLOC_PROXY_BINARY_VERSION ${TBBMALLOC_BINARY_VERSION}) +string(REGEX REPLACE ".*TBBBIND_BINARY_VERSION ([0-9]+).*" "\\1" TBBBIND_BINARY_VERSION "${_tbb_cmakelist}") + +set(COMMON_ARGS + TBB_ROOT_REL_PATH ${TBB_ROOT_REL_PATH} + INC_REL_PATH ${INC_REL_PATH} + LIB_REL_PATH ${LIB_REL_PATH} + VERSION ${_tbb_ver_major}.${_tbb_ver_minor}.${_tbb_ver_patch} + TBB_BINARY_VERSION ${TBB_BINARY_VERSION} + TBBMALLOC_BINARY_VERSION ${TBBMALLOC_BINARY_VERSION} + TBBMALLOC_PROXY_BINARY_VERSION ${TBBMALLOC_PROXY_BINARY_VERSION} + TBBBIND_BINARY_VERSION ${TBBBIND_BINARY_VERSION} +) + +tbb_generate_config(INSTALL_DIR ${INSTALL_DIR}/linux SYSTEM_NAME Linux HANDLE_SUBDIRS ${COMMON_ARGS}) +tbb_generate_config(INSTALL_DIR ${INSTALL_DIR}/windows SYSTEM_NAME Windows HANDLE_SUBDIRS ${COMMON_ARGS}) +tbb_generate_config(INSTALL_DIR ${INSTALL_DIR}/darwin SYSTEM_NAME Darwin ${COMMON_ARGS}) +message(STATUS "TBBConfig files were created in ${INSTALL_DIR}") diff --git a/src/tbb/cmake/suppressions/lsan.suppressions b/src/tbb/cmake/suppressions/lsan.suppressions new file mode 100644 index 000000000..4a2dea29b --- /dev/null +++ b/src/tbb/cmake/suppressions/lsan.suppressions @@ -0,0 +1,2 @@ +# LSAN suppression for ltdl library known issue. +leak:libltdl.so diff --git a/src/tbb/cmake/suppressions/tsan.suppressions b/src/tbb/cmake/suppressions/tsan.suppressions new file mode 100644 index 000000000..1bbb833b6 --- /dev/null +++ b/src/tbb/cmake/suppressions/tsan.suppressions @@ -0,0 +1,4 @@ +# TSAN suppression for known issues. +# Possible data race during ittnotify initialization. Low impact. +race:__itt_nullify_all_pointers +race:__itt_init_ittlib diff --git a/src/tbb/cmake/templates/TBBConfig.cmake.in b/src/tbb/cmake/templates/TBBConfig.cmake.in new file mode 100644 index 000000000..3131e3dd7 --- /dev/null +++ b/src/tbb/cmake/templates/TBBConfig.cmake.in @@ -0,0 +1,128 @@ +# Copyright (c) 2017-2023 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# It defines the following variables: +# TBB__FOUND +# TBB_IMPORTED_TARGETS +# +# TBBConfigVersion.cmake defines TBB_VERSION +# +# Initialize to default values +if (NOT TBB_IMPORTED_TARGETS) + set(TBB_IMPORTED_TARGETS "") +endif() + +if (NOT TBB_FIND_COMPONENTS) + set(TBB_FIND_COMPONENTS "tbb;tbbmalloc;tbbmalloc_proxy") + foreach (_tbb_component ${TBB_FIND_COMPONENTS}) + set(TBB_FIND_REQUIRED_${_tbb_component} 1) + endforeach() +endif() + +get_filename_component(_tbb_root "${CMAKE_CURRENT_LIST_DIR}" REALPATH) +get_filename_component(_tbb_root "${_tbb_root}/@TBB_ROOT_REL_PATH@" ABSOLUTE) + +set(TBB_INTERFACE_VERSION @TBB_INTERFACE_VERSION@) +@TBB_COMPONENTS_BIN_VERSION@ +# Add components with internal dependencies: tbbmalloc_proxy -> tbbmalloc +list(FIND TBB_FIND_COMPONENTS tbbmalloc_proxy _tbbmalloc_proxy_ix) +if (NOT _tbbmalloc_proxy_ix EQUAL -1) + list(APPEND TBB_FIND_COMPONENTS tbbmalloc) + list(REMOVE_DUPLICATES TBB_FIND_COMPONENTS) + set(TBB_FIND_REQUIRED_tbbmalloc ${TBB_FIND_REQUIRED_tbbmalloc_proxy}) +endif() +unset(_tbbmalloc_proxy_ix) + +if (CMAKE_SIZEOF_VOID_P STREQUAL "8") + set(_tbb_intel_arch intel64) +else () + set(_tbb_intel_arch ia32) + set(_tbb_arch_suffix 32) +endif() + +@TBB_HANDLE_SUBDIRS@ +foreach (_tbb_component ${TBB_FIND_COMPONENTS}) + unset(_tbb_release_dll CACHE) + unset(_tbb_debug_dll CACHE) + unset(_tbb_release_lib CACHE) + unset(_tbb_debug_lib CACHE) + + set(TBB_${_tbb_component}_FOUND 0) + @TBB_HANDLE_BIN_VERSION@ + + find_library(_tbb_release_lib + NAMES @TBB_LIB_PREFIX@${_tbb_component}${_bin_version}.@TBB_LIB_EXT@ + PATHS ${_tbb_root} + PATH_SUFFIXES "@TBB_LIB_REL_PATH@/${_tbb_intel_arch}/${_tbb_subdir}" "@TBB_LIB_REL_PATH@${_tbb_arch_suffix}/${_tbb_subdir}" "@TBB_LIB_REL_PATH@${_tbb_arch_suffix}" "@TBB_LIB_REL_PATH@" + NO_DEFAULT_PATH + ) + + if (NOT TBB_FIND_RELEASE_ONLY) + find_library(_tbb_debug_lib + NAMES @TBB_LIB_PREFIX@${_tbb_component}${_bin_version}_debug.@TBB_LIB_EXT@ + PATHS ${_tbb_root} + PATH_SUFFIXES "@TBB_LIB_REL_PATH@/${_tbb_intel_arch}/${_tbb_subdir}" "@TBB_LIB_REL_PATH@${_tbb_arch_suffix}/${_tbb_subdir}" "@TBB_LIB_REL_PATH@${_tbb_arch_suffix}" "@TBB_LIB_REL_PATH@" + NO_DEFAULT_PATH + ) + endif() + + if (EXISTS "${_tbb_release_lib}" OR EXISTS "${_tbb_debug_lib}") + if (NOT TARGET TBB::${_tbb_component}) + add_library(TBB::${_tbb_component} SHARED IMPORTED) + + get_filename_component(_tbb_include_dir "${_tbb_root}/@TBB_INC_REL_PATH@" ABSOLUTE) + set_target_properties(TBB::${_tbb_component} PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${_tbb_include_dir}"@TBB_COMPILE_DEFINITIONS@) + unset(_tbb_current_realpath) + unset(_tbb_include_dir) + + @TBB_HANDLE_IMPLIB@ + + if (EXISTS "${_tbb_release_dll}") + set_target_properties(TBB::${_tbb_component} PROPERTIES + IMPORTED_LOCATION_RELEASE "${_tbb_release_dll}"@TBB_IMPLIB_RELEASE@) + set_property(TARGET TBB::${_tbb_component} APPEND PROPERTY IMPORTED_CONFIGURATIONS RELEASE) + endif() + + if (EXISTS "${_tbb_debug_dll}") + set_target_properties(TBB::${_tbb_component} PROPERTIES + IMPORTED_LOCATION_DEBUG "${_tbb_debug_dll}"@TBB_IMPLIB_DEBUG@) + set_property(TARGET TBB::${_tbb_component} APPEND PROPERTY IMPORTED_CONFIGURATIONS DEBUG) + endif() + + # Add internal dependencies for imported targets: TBB::tbbmalloc_proxy -> TBB::tbbmalloc + if (_tbb_component STREQUAL tbbmalloc_proxy) + set_target_properties(TBB::tbbmalloc_proxy PROPERTIES INTERFACE_LINK_LIBRARIES TBB::tbbmalloc) + endif() + endif() + list(APPEND TBB_IMPORTED_TARGETS TBB::${_tbb_component}) + set(TBB_${_tbb_component}_FOUND 1) + elseif (TBB_FIND_REQUIRED AND TBB_FIND_REQUIRED_${_tbb_component}) + message(STATUS "Missed required oneTBB component: ${_tbb_component}") + if (TBB_FIND_RELEASE_ONLY) + message(STATUS " ${_tbb_release_lib} must exist.") + else() + message(STATUS " one or both of:\n ${_tbb_release_lib}\n ${_tbb_debug_lib}\n files must exist.") + endif() + set(TBB_FOUND FALSE) + endif() +endforeach() +list(REMOVE_DUPLICATES TBB_IMPORTED_TARGETS) +unset(_tbb_release_dll) +unset(_tbb_debug_dll) +unset(_tbb_release_lib) +unset(_tbb_debug_lib) +unset(_tbb_root) +unset(_tbb_intel_arch) +unset(_tbb_arch_suffix) diff --git a/src/tbb/cmake/templates/TBBConfigVersion.cmake.in b/src/tbb/cmake/templates/TBBConfigVersion.cmake.in new file mode 100644 index 000000000..d97bc3fea --- /dev/null +++ b/src/tbb/cmake/templates/TBBConfigVersion.cmake.in @@ -0,0 +1,24 @@ +# Copyright (c) 2017-2021 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set(PACKAGE_VERSION @TBB_VERSION@) + +if ("${PACKAGE_VERSION}" VERSION_LESS "${PACKAGE_FIND_VERSION}") + set(PACKAGE_VERSION_COMPATIBLE FALSE) +else() + set(PACKAGE_VERSION_COMPATIBLE TRUE) + if ("${PACKAGE_VERSION}" VERSION_EQUAL "${PACKAGE_FIND_VERSION}") + set(PACKAGE_VERSION_EXACT TRUE) + endif() +endif() diff --git a/src/tbb/cmake/test_spec.cmake b/src/tbb/cmake/test_spec.cmake new file mode 100644 index 000000000..9ac811f50 --- /dev/null +++ b/src/tbb/cmake/test_spec.cmake @@ -0,0 +1,35 @@ +# Copyright (c) 2020-2021 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +option(TBB_TEST_SPEC "Generate test specification (Doxygen)" OFF) + +if (TBB_TEST_SPEC) + find_package(Doxygen REQUIRED) + + set(DOXYGEN_PREDEFINED_MACROS + "TBB_USE_EXCEPTIONS \ + __TBB_RESUMABLE_TASKS \ + __TBB_HWLOC_PRESENT \ + __TBB_CPP17_DEDUCTION_GUIDES_PRESENT \ + __TBB_CPP17_MEMORY_RESOURCE_PRESENT \ + __TBB_CPP14_GENERIC_LAMBDAS_PRESENT" + ) + + add_custom_target( + test_spec ALL + COMMAND ${DOXYGEN_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile + COMMENT "Generating test specification with Doxygen" + VERBATIM) + configure_file(${CMAKE_CURRENT_SOURCE_DIR}/doc/Doxyfile.in ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile @ONLY) +endif() diff --git a/src/tbb/cmake/toolchains/mips.cmake b/src/tbb/cmake/toolchains/mips.cmake new file mode 100644 index 000000000..13af9519b --- /dev/null +++ b/src/tbb/cmake/toolchains/mips.cmake @@ -0,0 +1,40 @@ +# Copyright (c) 2020-2021 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Prevent double invocation. +if (MIPS_TOOLCHAIN_INCLUDED) + return() +endif() +set(MIPS_TOOLCHAIN_INCLUDED TRUE) + +set(CMAKE_SYSTEM_NAME Linux) +set(CMAKE_SYSTEM_VERSION 1) +set(CMAKE_SYSTEM_PROCESSOR mips) + +set(CMAKE_C_COMPILER ${CMAKE_FIND_ROOT_PATH}/bin/mips-img-linux-gnu-gcc) +set(CMAKE_CXX_COMPILER ${CMAKE_FIND_ROOT_PATH}/bin/mips-img-linux-gnu-g++) +set(CMAKE_LINKER ${CMAKE_FIND_ROOT_PATH}/bin/mips-img-linux-gnu-ld) + +# Define result for try_run used in find_package(Threads). +# In old CMake versions (checked on 3.5) there is invocation of try_run command in FindThreads.cmake module. +# It can't be executed on host system in case of cross-compilation for MIPS architecture. +# Define return code for this try_run as 0 since threads are expected to be available on target machine. +set(THREADS_PTHREAD_ARG "0" CACHE STRING "Result from TRY_RUN" FORCE) + +set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) + +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -EL -mabi=64 -march=mips64r6 -mcrc -mfp64 -mmt -mtune=mips64r6 -ggdb -ffp-contract=off -mhard-float" CACHE INTERNAL "") +set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -mvirt -mxpa" CACHE INTERNAL "") +set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -mvirt -mxpa" CACHE INTERNAL "") # for tests diff --git a/src/tbb/cmake/toolchains/riscv64.cmake b/src/tbb/cmake/toolchains/riscv64.cmake new file mode 100644 index 000000000..96c0014b6 --- /dev/null +++ b/src/tbb/cmake/toolchains/riscv64.cmake @@ -0,0 +1,34 @@ +# Copyright (c) 2023 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Prevent double invocation. +if (RISCV_TOOLCHAIN_INCLUDED) + return() +endif() +set(RISCV_TOOLCHAIN_INCLUDED TRUE) + +set(CMAKE_SYSTEM_NAME Linux) +set(CMAKE_SYSTEM_VERSION 1) +set(CMAKE_SYSTEM_PROCESSOR riscv) + +# User can use -DCMAKE_FIND_ROOT_PATH to specific toolchain path +set(CMAKE_C_COMPILER ${CMAKE_FIND_ROOT_PATH}/bin/riscv64-unknown-linux-gnu-clang) +set(CMAKE_CXX_COMPILER ${CMAKE_FIND_ROOT_PATH}/bin/riscv64-unknown-linux-gnu-clang++) +set(CMAKE_LINKER ${CMAKE_FIND_ROOT_PATH}/bin/riscv64-unknown-linux-gnu-ld) + +set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) + +# Most linux on riscv64 support rv64imafd_zba_zbb extensions +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=rv64imafd_zba_zbb -mabi=lp64d " CACHE INTERNAL "") diff --git a/src/tbb/cmake/utils.cmake b/src/tbb/cmake/utils.cmake new file mode 100644 index 000000000..211019896 --- /dev/null +++ b/src/tbb/cmake/utils.cmake @@ -0,0 +1,74 @@ +# Copyright (c) 2020-2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +macro(tbb_remove_compile_flag flag) + get_property(_tbb_compile_options DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY COMPILE_OPTIONS) + list(REMOVE_ITEM _tbb_compile_options ${flag}) + set_property(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY COMPILE_OPTIONS ${_tbb_compile_options}) + unset(_tbb_compile_options) + if (CMAKE_CXX_FLAGS) + string(REGEX REPLACE "(^|[ \t\r\n]+)${flag}($|[ \t\r\n]+)" " " CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS}) + endif() +endmacro() + +macro(tbb_install_target target) + if (TBB_INSTALL) + install(TARGETS ${target} + EXPORT TBBTargets + LIBRARY + DESTINATION ${CMAKE_INSTALL_LIBDIR} + NAMELINK_SKIP + COMPONENT runtime + RUNTIME + DESTINATION ${CMAKE_INSTALL_BINDIR} + COMPONENT runtime + ARCHIVE + DESTINATION ${CMAKE_INSTALL_LIBDIR} + COMPONENT devel + FRAMEWORK + DESTINATION ${CMAKE_INSTALL_LIBDIR} + COMPONENT runtime + OPTIONAL) + + if (BUILD_SHARED_LIBS) + install(TARGETS ${target} + LIBRARY + DESTINATION ${CMAKE_INSTALL_LIBDIR} + NAMELINK_ONLY + COMPONENT devel) + endif() + if (MSVC AND BUILD_SHARED_LIBS) + install(FILES $ + DESTINATION ${CMAKE_INSTALL_BINDIR} + COMPONENT devel + OPTIONAL) + endif() + endif() +endmacro() + +macro(tbb_handle_ipo target) + if (TBB_IPO_PROPERTY) + set_target_properties(${target} PROPERTIES + INTERPROCEDURAL_OPTIMIZATION TRUE + INTERPROCEDURAL_OPTIMIZATION_DEBUG FALSE + ) + elseif (TBB_IPO_FLAGS) + target_compile_options(${target} PRIVATE ${TBB_IPO_COMPILE_FLAGS}) + if (COMMAND target_link_options) + target_link_options(${target} PRIVATE ${TBB_IPO_LINK_FLAGS}) + else() + target_link_libraries(${target} PRIVATE ${TBB_IPO_LINK_FLAGS}) + endif() + endif() +endmacro() diff --git a/src/tbb/cmake/vars_utils.cmake b/src/tbb/cmake/vars_utils.cmake new file mode 100644 index 000000000..54a9fda18 --- /dev/null +++ b/src/tbb/cmake/vars_utils.cmake @@ -0,0 +1,56 @@ +# Copyright (c) 2020-2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +option(TBB_INSTALL_VARS "Enable auto-generated vars installation" OFF) + +if (WIN32) + set(TBB_VARS_TEMPLATE "windows/env/vars.bat.in") +elseif (APPLE) + set(TBB_VARS_TEMPLATE "mac/env/vars.sh.in") +else() + set(TBB_VARS_TEMPLATE "linux/env/vars.sh.in") +endif() + +get_filename_component(TBB_VARS_TEMPLATE_NAME ${PROJECT_SOURCE_DIR}/integration/${TBB_VARS_TEMPLATE} NAME) +string(REPLACE ".in" "" TBB_VARS_NAME ${TBB_VARS_TEMPLATE_NAME}) + +macro(tbb_gen_vars target) + if (NOT TBB_BUILD_APPLE_FRAMEWORKS) + set(BIN_PATH $) + else() + # For Apple* frameworks, the binaries are placed in a framework bundle. + # When using an Apple* framework, you refer to the bundle, not the binary inside, so we take the bundle's path and go up one level. + # This path will then be used to generate the vars file, and the contents of the vars file will use the bundle's parent directory. + set(BIN_PATH $/..) + endif() + if (${CMAKE_PROJECT_NAME} STREQUAL ${PROJECT_NAME}) + add_custom_command(TARGET ${target} POST_BUILD COMMAND + ${CMAKE_COMMAND} + -DBINARY_DIR=${CMAKE_BINARY_DIR} + -DSOURCE_DIR=${PROJECT_SOURCE_DIR} + -DBIN_PATH=${BIN_PATH} + -DVARS_TEMPLATE=${TBB_VARS_TEMPLATE} + -DVARS_NAME=${TBB_VARS_NAME} + -DTBB_INSTALL_VARS=${TBB_INSTALL_VARS} + -DTBB_CMAKE_INSTALL_LIBDIR=${CMAKE_INSTALL_LIBDIR} + -P ${PROJECT_SOURCE_DIR}/integration/cmake/generate_vars.cmake + ) + endif() +endmacro(tbb_gen_vars) + +if (TBB_INSTALL_VARS) + install(PROGRAMS "${CMAKE_BINARY_DIR}/internal_install_vars" + DESTINATION env + RENAME ${TBB_VARS_NAME}) +endif() diff --git a/src/tbb/include/index.html b/src/tbb/include/index.html deleted file mode 100644 index b0962e012..000000000 --- a/src/tbb/include/index.html +++ /dev/null @@ -1,25 +0,0 @@ - - - -

Overview

-Include files for Intel® Threading Building Blocks (Intel® TBB). - -

Directories

-
-
tbb -
Include files for Intel TBB classes and functions. -
serial/tbb -
Include files for a sequential implementation of the parallel_for algorithm. -
- -
-Up to parent directory -

-Copyright © 2005-2017 Intel Corporation. All Rights Reserved. -

-Intel is a registered trademark or trademark of Intel Corporation -or its subsidiaries in the United States and other countries. -

-* Other names and brands may be claimed as the property of others. - - diff --git a/src/tbb/include/oneapi/tbb.h b/src/tbb/include/oneapi/tbb.h new file mode 100644 index 000000000..ad9601137 --- /dev/null +++ b/src/tbb/include/oneapi/tbb.h @@ -0,0 +1,76 @@ +/* + Copyright (c) 2005-2023 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_tbb_H +#define __TBB_tbb_H + +/** + This header bulk-includes declarations or definitions of all the functionality + provided by TBB (save for tbbmalloc and 3rd party dependent headers). + + If you use only a few TBB constructs, consider including specific headers only. + Any header listed below can be included independently of others. +**/ + +#include "oneapi/tbb/blocked_range.h" +#include "oneapi/tbb/blocked_range2d.h" +#include "oneapi/tbb/blocked_range3d.h" +#if TBB_PREVIEW_BLOCKED_RANGE_ND +#include "tbb/blocked_rangeNd.h" +#endif +#include "oneapi/tbb/cache_aligned_allocator.h" +#include "oneapi/tbb/combinable.h" +#include "oneapi/tbb/concurrent_hash_map.h" +#if TBB_PREVIEW_CONCURRENT_LRU_CACHE +#include "tbb/concurrent_lru_cache.h" +#endif +#include "oneapi/tbb/collaborative_call_once.h" +#include "oneapi/tbb/concurrent_priority_queue.h" +#include "oneapi/tbb/concurrent_queue.h" +#include "oneapi/tbb/concurrent_unordered_map.h" +#include "oneapi/tbb/concurrent_unordered_set.h" +#include "oneapi/tbb/concurrent_map.h" +#include "oneapi/tbb/concurrent_set.h" +#include "oneapi/tbb/concurrent_vector.h" +#include "oneapi/tbb/enumerable_thread_specific.h" +#include "oneapi/tbb/flow_graph.h" +#include "oneapi/tbb/global_control.h" +#include "oneapi/tbb/info.h" +#include "oneapi/tbb/null_mutex.h" +#include "oneapi/tbb/null_rw_mutex.h" +#include "oneapi/tbb/parallel_for.h" +#include "oneapi/tbb/parallel_for_each.h" +#include "oneapi/tbb/parallel_invoke.h" +#include "oneapi/tbb/parallel_pipeline.h" +#include "oneapi/tbb/parallel_reduce.h" +#include "oneapi/tbb/parallel_scan.h" +#include "oneapi/tbb/parallel_sort.h" +#include "oneapi/tbb/partitioner.h" +#include "oneapi/tbb/queuing_mutex.h" +#include "oneapi/tbb/queuing_rw_mutex.h" +#include "oneapi/tbb/spin_mutex.h" +#include "oneapi/tbb/spin_rw_mutex.h" +#include "oneapi/tbb/mutex.h" +#include "oneapi/tbb/rw_mutex.h" +#include "oneapi/tbb/task.h" +#include "oneapi/tbb/task_arena.h" +#include "oneapi/tbb/task_group.h" +#include "oneapi/tbb/task_scheduler_observer.h" +#include "oneapi/tbb/tbb_allocator.h" +#include "oneapi/tbb/tick_count.h" +#include "oneapi/tbb/version.h" + +#endif /* __TBB_tbb_H */ diff --git a/inst/include/tbb/blocked_range.h b/src/tbb/include/oneapi/tbb/blocked_range.h similarity index 79% rename from inst/include/tbb/blocked_range.h rename to src/tbb/include/oneapi/tbb/blocked_range.h index 9f24cd2bd..12862fa2a 100644 --- a/inst/include/tbb/blocked_range.h +++ b/src/tbb/include/oneapi/tbb/blocked_range.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,18 +12,21 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_blocked_range_H #define __TBB_blocked_range_H -#include "tbb_stddef.h" +#include + +#include "detail/_range_common.h" +#include "detail/_namespace_injection.h" + +#include "version.h" namespace tbb { +namespace detail { +namespace d1 { /** \page range_req Requirements on range concept Class \c R implementing the concept of range must define: @@ -37,19 +40,16 @@ namespace tbb { //! A range over which to iterate. /** @ingroup algorithms */ template + __TBB_requires(blocked_range_value) class blocked_range { public: //! Type of a value /** Called a const_iterator for sake of algorithms that need to treat a blocked_range as an STL container. */ - typedef Value const_iterator; + using const_iterator = Value; //! Type for size of a range - typedef std::size_t size_type; - - //! Construct range with default-constructed values for begin and end. - /** Requires that Value have a default constructor. */ - blocked_range() : my_end(), my_begin() {} + using size_type = std::size_t; //! Construct range over half-open interval [begin,end), with the given grainsize. blocked_range( Value begin_, Value end_, size_type grainsize_=1 ) : @@ -59,10 +59,10 @@ class blocked_range { } //! Beginning of range. - const_iterator begin() const {return my_begin;} + const_iterator begin() const { return my_begin; } //! One past last value in range. - const_iterator end() const {return my_end;} + const_iterator end() const { return my_end; } //! Size of the range /** Unspecified if end() + __TBB_requires(blocked_range_value && + blocked_range_value) friend class blocked_range2d; template + __TBB_requires(blocked_range_value && + blocked_range_value && + blocked_range_value) friend class blocked_range3d; + + template + __TBB_requires(blocked_range_value) + friend class blocked_rangeNd_impl; }; +} // namespace d1 +} // namespace detail + +inline namespace v1 { +using detail::d1::blocked_range; +// Split types +using detail::split; +using detail::proportional_split; +} // namespace v1 + } // namespace tbb #endif /* __TBB_blocked_range_H */ diff --git a/inst/include/tbb/blocked_range2d.h b/src/tbb/include/oneapi/tbb/blocked_range2d.h similarity index 71% rename from inst/include/tbb/blocked_range2d.h rename to src/tbb/include/oneapi/tbb/blocked_range2d.h index f1b9f35d6..41385db98 100644 --- a/inst/include/tbb/blocked_range2d.h +++ b/src/tbb/include/oneapi/tbb/blocked_range2d.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,52 +12,54 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_blocked_range2d_H #define __TBB_blocked_range2d_H -#include "tbb_stddef.h" +#include + +#include "detail/_config.h" +#include "detail/_namespace_injection.h" +#include "detail/_range_common.h" + #include "blocked_range.h" namespace tbb { +namespace detail { +namespace d1 { //! A 2-dimensional range that models the Range concept. /** @ingroup algorithms */ -template +template + __TBB_requires(blocked_range_value && + blocked_range_value) class blocked_range2d { public: //! Type for size of an iteration range - typedef blocked_range row_range_type; - typedef blocked_range col_range_type; + using row_range_type = blocked_range; + using col_range_type = blocked_range; private: row_range_type my_rows; col_range_type my_cols; public: - blocked_range2d( RowValue row_begin, RowValue row_end, typename row_range_type::size_type row_grainsize, ColValue col_begin, ColValue col_end, typename col_range_type::size_type col_grainsize ) : my_rows(row_begin,row_end,row_grainsize), my_cols(col_begin,col_end,col_grainsize) - { - } + {} blocked_range2d( RowValue row_begin, RowValue row_end, ColValue col_begin, ColValue col_end ) : my_rows(row_begin,row_end), my_cols(col_begin,col_end) - { - } + {} //! True if range is empty bool empty() const { - // Yes, it is a logical OR here, not AND. + // Range is empty if at least one dimension is empty. return my_rows.empty() || my_cols.empty(); } @@ -74,35 +76,36 @@ class blocked_range2d { do_split(r, split_obj); } -#if __TBB_USE_PROPORTIONAL_SPLIT_IN_BLOCKED_RANGES - //! Static field to support proportional split - static const bool is_splittable_in_proportion = true; - blocked_range2d( blocked_range2d& r, proportional_split& proportion ) : my_rows(r.my_rows), my_cols(r.my_cols) { do_split(r, proportion); } -#endif /* __TBB_USE_PROPORTIONAL_SPLIT_IN_BLOCKED_RANGES */ + //! The rows of the iteration space + const row_range_type& rows() const { return my_rows; } + + //! The columns of the iteration space + const col_range_type& cols() const { return my_cols; } + +private: template - void do_split( blocked_range2d& r, Split& split_obj ) - { - if( my_rows.size()*double(my_cols.grainsize()) < my_cols.size()*double(my_rows.grainsize()) ) { + void do_split( blocked_range2d& r, Split& split_obj ) { + if ( my_rows.size()*double(my_cols.grainsize()) < my_cols.size()*double(my_rows.grainsize()) ) { my_cols.my_begin = col_range_type::do_split(r.my_cols, split_obj); } else { my_rows.my_begin = row_range_type::do_split(r.my_rows, split_obj); } } - - //! The rows of the iteration space - const row_range_type& rows() const {return my_rows;} - - //! The columns of the iteration space - const col_range_type& cols() const {return my_cols;} }; +} // namespace d1 +} // namespace detail + +inline namespace v1 { +using detail::d1::blocked_range2d; +} // namespace v1 } // namespace tbb #endif /* __TBB_blocked_range2d_H */ diff --git a/inst/include/tbb/blocked_range3d.h b/src/tbb/include/oneapi/tbb/blocked_range3d.h similarity index 75% rename from inst/include/tbb/blocked_range3d.h rename to src/tbb/include/oneapi/tbb/blocked_range3d.h index c62565eeb..d8932192b 100644 --- a/inst/include/tbb/blocked_range3d.h +++ b/src/tbb/include/oneapi/tbb/blocked_range3d.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,29 +12,34 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_blocked_range3d_H #define __TBB_blocked_range3d_H -#include "tbb_stddef.h" +#include + +#include "detail/_config.h" +#include "detail/_namespace_injection.h" + #include "blocked_range.h" namespace tbb { +namespace detail { +namespace d1 { //! A 3-dimensional range that models the Range concept. /** @ingroup algorithms */ -template +template + __TBB_requires(blocked_range_value && + blocked_range_value && + blocked_range_value) class blocked_range3d { public: //! Type for size of an iteration range - typedef blocked_range page_range_type; - typedef blocked_range row_range_type; - typedef blocked_range col_range_type; + using page_range_type = blocked_range; + using row_range_type = blocked_range; + using col_range_type = blocked_range; private: page_range_type my_pages; @@ -49,8 +54,7 @@ class blocked_range3d { my_pages(page_begin,page_end), my_rows(row_begin,row_end), my_cols(col_begin,col_end) - { - } + {} blocked_range3d( PageValue page_begin, PageValue page_end, typename page_range_type::size_type page_grainsize, RowValue row_begin, RowValue row_end, typename row_range_type::size_type row_grainsize, @@ -58,12 +62,11 @@ class blocked_range3d { my_pages(page_begin,page_end,page_grainsize), my_rows(row_begin,row_end,row_grainsize), my_cols(col_begin,col_end,col_grainsize) - { - } + {} //! True if range is empty bool empty() const { - // Yes, it is a logical OR here, not AND. + // Range is empty if at least one dimension is empty. return my_pages.empty() || my_rows.empty() || my_cols.empty(); } @@ -72,19 +75,14 @@ class blocked_range3d { return my_pages.is_divisible() || my_rows.is_divisible() || my_cols.is_divisible(); } - blocked_range3d( blocked_range3d& r, split ) : + blocked_range3d( blocked_range3d& r, split split_obj ) : my_pages(r.my_pages), my_rows(r.my_rows), my_cols(r.my_cols) { - split split_obj; do_split(r, split_obj); } -#if __TBB_USE_PROPORTIONAL_SPLIT_IN_BLOCKED_RANGES - //! Static field to support proportional split - static const bool is_splittable_in_proportion = true; - blocked_range3d( blocked_range3d& r, proportional_split& proportion ) : my_pages(r.my_pages), my_rows(r.my_rows), @@ -92,18 +90,26 @@ class blocked_range3d { { do_split(r, proportion); } -#endif /* __TBB_USE_PROPORTIONAL_SPLIT_IN_BLOCKED_RANGES */ + //! The pages of the iteration space + const page_range_type& pages() const { return my_pages; } + + //! The rows of the iteration space + const row_range_type& rows() const { return my_rows; } + + //! The columns of the iteration space + const col_range_type& cols() const { return my_cols; } + +private: template - void do_split( blocked_range3d& r, Split& split_obj) - { + void do_split( blocked_range3d& r, Split& split_obj) { if ( my_pages.size()*double(my_rows.grainsize()) < my_rows.size()*double(my_pages.grainsize()) ) { if ( my_rows.size()*double(my_cols.grainsize()) < my_cols.size()*double(my_rows.grainsize()) ) { my_cols.my_begin = col_range_type::do_split(r.my_cols, split_obj); } else { my_rows.my_begin = row_range_type::do_split(r.my_rows, split_obj); } - } else { + } else { if ( my_pages.size()*double(my_cols.grainsize()) < my_cols.size()*double(my_pages.grainsize()) ) { my_cols.my_begin = col_range_type::do_split(r.my_cols, split_obj); } else { @@ -111,18 +117,14 @@ class blocked_range3d { } } } - - //! The pages of the iteration space - const page_range_type& pages() const {return my_pages;} - - //! The rows of the iteration space - const row_range_type& rows() const {return my_rows;} - - //! The columns of the iteration space - const col_range_type& cols() const {return my_cols;} - }; +} // namespace d1 +} // namespace detail + +inline namespace v1 { +using detail::d1::blocked_range3d; +} // namespace v1 } // namespace tbb #endif /* __TBB_blocked_range3d_H */ diff --git a/src/tbb/include/oneapi/tbb/blocked_rangeNd.h b/src/tbb/include/oneapi/tbb/blocked_rangeNd.h new file mode 100644 index 000000000..a7ba13750 --- /dev/null +++ b/src/tbb/include/oneapi/tbb/blocked_rangeNd.h @@ -0,0 +1,147 @@ +/* + Copyright (c) 2017-2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_blocked_rangeNd_H +#define __TBB_blocked_rangeNd_H + +#if !TBB_PREVIEW_BLOCKED_RANGE_ND + #error Set TBB_PREVIEW_BLOCKED_RANGE_ND to include blocked_rangeNd.h +#endif + +#include // std::any_of +#include +#include +#include // std::is_same, std::enable_if + +#include "detail/_config.h" +#include "detail/_template_helpers.h" // index_sequence, make_index_sequence +#include "detail/_range_common.h" + +#include "blocked_range.h" + +namespace tbb { +namespace detail { +namespace d1 { + +/* + The blocked_rangeNd_impl uses make_index_sequence to automatically generate a ctor with + exactly N arguments of the type tbb::blocked_range. Such ctor provides an opportunity + to use braced-init-list parameters to initialize each dimension. + Use of parameters, whose representation is a braced-init-list, but they're not + std::initializer_list or a reference to one, produces a non-deduced context + within template argument deduction. + + NOTE: blocked_rangeNd must be exactly a templated alias to the blocked_rangeNd_impl + (and not e.g. a derived class), otherwise it would need to declare its own ctor + facing the same problem that the impl class solves. +*/ + +template> + __TBB_requires(blocked_range_value) +class blocked_rangeNd_impl; + +template + __TBB_requires(blocked_range_value) +class blocked_rangeNd_impl> { +public: + //! Type of a value. + using value_type = Value; + +private: + //! Helper type to construct range with N tbb::blocked_range objects. + template + using dim_type_helper = tbb::blocked_range; + +public: + blocked_rangeNd_impl() = delete; + + //! Constructs N-dimensional range over N half-open intervals each represented as tbb::blocked_range. + blocked_rangeNd_impl(const dim_type_helper&... args) : my_dims{ {args...} } {} + + //! Dimensionality of a range. + static constexpr unsigned int ndims() { return N; } + + //! Range in certain dimension. + const tbb::blocked_range& dim(unsigned int dimension) const { + __TBB_ASSERT(dimension < N, "out of bound"); + return my_dims[dimension]; + } + + //------------------------------------------------------------------------ + // Methods that implement Range concept + //------------------------------------------------------------------------ + + //! True if at least one dimension is empty. + bool empty() const { + return std::any_of(my_dims.begin(), my_dims.end(), [](const tbb::blocked_range& d) { + return d.empty(); + }); + } + + //! True if at least one dimension is divisible. + bool is_divisible() const { + return std::any_of(my_dims.begin(), my_dims.end(), [](const tbb::blocked_range& d) { + return d.is_divisible(); + }); + } + + blocked_rangeNd_impl(blocked_rangeNd_impl& r, proportional_split proportion) : my_dims(r.my_dims) { + do_split(r, proportion); + } + + blocked_rangeNd_impl(blocked_rangeNd_impl& r, split proportion) : my_dims(r.my_dims) { + do_split(r, proportion); + } + +private: + static_assert(N != 0, "zero dimensional blocked_rangeNd can't be constructed"); + + //! Ranges in each dimension. + std::array, N> my_dims; + + template + void do_split(blocked_rangeNd_impl& r, split_type proportion) { + static_assert((std::is_same::value || std::is_same::value), "type of split object is incorrect"); + __TBB_ASSERT(r.is_divisible(), "can't split not divisible range"); + + auto my_it = std::max_element(my_dims.begin(), my_dims.end(), [](const tbb::blocked_range& first, const tbb::blocked_range& second) { + return (first.size() * second.grainsize() < second.size() * first.grainsize()); + }); + + auto r_it = r.my_dims.begin() + (my_it - my_dims.begin()); + + my_it->my_begin = tbb::blocked_range::do_split(*r_it, proportion); + + // (!(my_it->my_begin < r_it->my_end) && !(r_it->my_end < my_it->my_begin)) equals to + // (my_it->my_begin == r_it->my_end), but we can't use operator== due to Value concept + __TBB_ASSERT(!(my_it->my_begin < r_it->my_end) && !(r_it->my_end < my_it->my_begin), + "blocked_range has been split incorrectly"); + } +}; + +template +using blocked_rangeNd = blocked_rangeNd_impl; + +} // namespace d1 +} // namespace detail + +inline namespace v1 { +using detail::d1::blocked_rangeNd; +} // namespace v1 +} // namespace tbb + +#endif /* __TBB_blocked_rangeNd_H */ + diff --git a/src/tbb/include/oneapi/tbb/cache_aligned_allocator.h b/src/tbb/include/oneapi/tbb/cache_aligned_allocator.h new file mode 100644 index 000000000..0ff3972d2 --- /dev/null +++ b/src/tbb/include/oneapi/tbb/cache_aligned_allocator.h @@ -0,0 +1,189 @@ +/* + Copyright (c) 2005-2022 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_cache_aligned_allocator_H +#define __TBB_cache_aligned_allocator_H + +#include "detail/_utils.h" +#include "detail/_namespace_injection.h" +#include +#include + +#if __TBB_CPP17_MEMORY_RESOURCE_PRESENT +#include +#endif + +namespace tbb { +namespace detail { + +namespace r1 { +TBB_EXPORT void* __TBB_EXPORTED_FUNC cache_aligned_allocate(std::size_t size); +TBB_EXPORT void __TBB_EXPORTED_FUNC cache_aligned_deallocate(void* p); +TBB_EXPORT std::size_t __TBB_EXPORTED_FUNC cache_line_size(); +} + +namespace d1 { + +template +class cache_aligned_allocator { +public: + using value_type = T; + using propagate_on_container_move_assignment = std::true_type; + + //! Always defined for TBB containers (supported since C++17 for std containers) + using is_always_equal = std::true_type; + + cache_aligned_allocator() = default; + template cache_aligned_allocator(const cache_aligned_allocator&) noexcept {} + + //! Allocate space for n objects, starting on a cache/sector line. + __TBB_nodiscard T* allocate(std::size_t n) { + return static_cast(r1::cache_aligned_allocate(n * sizeof(value_type))); + } + + //! Free block of memory that starts on a cache line + void deallocate(T* p, std::size_t) { + r1::cache_aligned_deallocate(p); + } + + //! Largest value for which method allocate might succeed. + std::size_t max_size() const noexcept { + return (~std::size_t(0) - r1::cache_line_size()) / sizeof(value_type); + } + +#if TBB_ALLOCATOR_TRAITS_BROKEN + using pointer = value_type*; + using const_pointer = const value_type*; + using reference = value_type&; + using const_reference = const value_type&; + using difference_type = std::ptrdiff_t; + using size_type = std::size_t; + template struct rebind { + using other = cache_aligned_allocator; + }; + template + void construct(U *p, Args&&... args) + { ::new (p) U(std::forward(args)...); } + void destroy(pointer p) { p->~value_type(); } + pointer address(reference x) const { return &x; } + const_pointer address(const_reference x) const { return &x; } +#endif // TBB_ALLOCATOR_TRAITS_BROKEN +}; + +#if TBB_ALLOCATOR_TRAITS_BROKEN + template<> + class cache_aligned_allocator { + public: + using pointer = void*; + using const_pointer = const void*; + using value_type = void; + template struct rebind { + using other = cache_aligned_allocator; + }; + }; +#endif + +template +bool operator==(const cache_aligned_allocator&, const cache_aligned_allocator&) noexcept { return true; } + +#if !__TBB_CPP20_COMPARISONS_PRESENT +template +bool operator!=(const cache_aligned_allocator&, const cache_aligned_allocator&) noexcept { return false; } +#endif + +#if __TBB_CPP17_MEMORY_RESOURCE_PRESENT + +//! C++17 memory resource wrapper to ensure cache line size alignment +class cache_aligned_resource : public std::pmr::memory_resource { +public: + cache_aligned_resource() : cache_aligned_resource(std::pmr::get_default_resource()) {} + explicit cache_aligned_resource(std::pmr::memory_resource* upstream) : m_upstream(upstream) {} + + std::pmr::memory_resource* upstream_resource() const { + return m_upstream; + } + +private: + //! We don't know what memory resource set. Use padding to guarantee alignment + void* do_allocate(std::size_t bytes, std::size_t alignment) override { + // TODO: make it common with tbb_allocator.cpp + std::size_t cache_line_alignment = correct_alignment(alignment); + std::size_t space = correct_size(bytes) + cache_line_alignment; + std::uintptr_t base = reinterpret_cast(m_upstream->allocate(space)); + __TBB_ASSERT(base != 0, "Upstream resource returned nullptr."); + + // Round up to the next cache line (align the base address) + std::uintptr_t result = (base + cache_line_alignment) & ~(cache_line_alignment - 1); + __TBB_ASSERT((result - base) >= sizeof(std::uintptr_t), "Can`t store a base pointer to the header"); + __TBB_ASSERT(space - (result - base) >= bytes, "Not enough space for the storage"); + + // Record where block actually starts. + (reinterpret_cast(result))[-1] = base; + return reinterpret_cast(result); + } + + void do_deallocate(void* ptr, std::size_t bytes, std::size_t alignment) override { + if (ptr) { + // Recover where block actually starts + std::uintptr_t base = (reinterpret_cast(ptr))[-1]; + m_upstream->deallocate(reinterpret_cast(base), correct_size(bytes) + correct_alignment(alignment)); + } + } + + bool do_is_equal(const std::pmr::memory_resource& other) const noexcept override { + if (this == &other) { return true; } +#if __TBB_USE_OPTIONAL_RTTI + const cache_aligned_resource* other_res = dynamic_cast(&other); + return other_res && (upstream_resource() == other_res->upstream_resource()); +#else + return false; +#endif + } + + std::size_t correct_alignment(std::size_t alignment) { + __TBB_ASSERT(tbb::detail::is_power_of_two(alignment), "Alignment is not a power of 2"); +#if __TBB_CPP17_HW_INTERFERENCE_SIZE_PRESENT + std::size_t cache_line_size = std::hardware_destructive_interference_size; +#else + std::size_t cache_line_size = r1::cache_line_size(); +#endif + return alignment < cache_line_size ? cache_line_size : alignment; + } + + std::size_t correct_size(std::size_t bytes) { + // To handle the case, when small size requested. There could be not + // enough space to store the original pointer. + return bytes < sizeof(std::uintptr_t) ? sizeof(std::uintptr_t) : bytes; + } + + std::pmr::memory_resource* m_upstream; +}; + +#endif // __TBB_CPP17_MEMORY_RESOURCE_PRESENT + +} // namespace d1 +} // namespace detail + +inline namespace v1 { +using detail::d1::cache_aligned_allocator; +#if __TBB_CPP17_MEMORY_RESOURCE_PRESENT +using detail::d1::cache_aligned_resource; +#endif +} // namespace v1 +} // namespace tbb + +#endif /* __TBB_cache_aligned_allocator_H */ + diff --git a/src/tbb/include/oneapi/tbb/collaborative_call_once.h b/src/tbb/include/oneapi/tbb/collaborative_call_once.h new file mode 100644 index 000000000..0a24f17d2 --- /dev/null +++ b/src/tbb/include/oneapi/tbb/collaborative_call_once.h @@ -0,0 +1,256 @@ +/* + Copyright (c) 2021-2024 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_collaborative_call_once_H +#define __TBB_collaborative_call_once_H + +#include "task_arena.h" +#include "task_group.h" + +#include + +namespace tbb { +namespace detail { +namespace d1 { + +#if _MSC_VER && !defined(__INTEL_COMPILER) + // Suppress warning: structure was padded due to alignment specifier + // #pragma warning (push) + // #pragma warning (disable: 4324) +#endif + +template +class collaborative_call_stack_task : public task { + const F& m_func; + wait_context& m_wait_ctx; + + void finalize() { + m_wait_ctx.release(); + } + task* execute(d1::execution_data&) override { + task* res = d2::task_ptr_or_nullptr(m_func); + finalize(); + return res; + } + task* cancel(d1::execution_data&) override { + finalize(); + return nullptr; + } +public: + collaborative_call_stack_task(const F& f, wait_context& wctx) : m_func(f), m_wait_ctx(wctx) {} +}; + +constexpr std::uintptr_t collaborative_once_max_references = max_nfs_size; +constexpr std::uintptr_t collaborative_once_references_mask = collaborative_once_max_references-1; + +class alignas(max_nfs_size) collaborative_once_runner : no_copy { + + struct storage_t { + task_arena m_arena{ task_arena::attach{} }; + wait_context m_wait_context{1}; + }; + + std::atomic m_ref_count{0}; + std::atomic m_is_ready{false}; + + // Storage with task_arena and wait_context must be initialized only by winner thread + union { + storage_t m_storage; + }; + + template + void isolated_execute(Fn f) { + auto func = [f] { + f(); + // delegate_base requires bool returning functor while isolate_within_arena ignores the result + return true; + }; + + delegated_function delegate(func); + + r1::isolate_within_arena(delegate, reinterpret_cast(this)); + } + +public: + class lifetime_guard : no_copy { + collaborative_once_runner& m_runner; + public: + lifetime_guard(collaborative_once_runner& r) : m_runner(r) { + m_runner.m_ref_count++; + } + ~lifetime_guard() { + m_runner.m_ref_count--; + } + }; + + collaborative_once_runner() {} + + ~collaborative_once_runner() { + spin_wait_until_eq(m_ref_count, 0, std::memory_order_acquire); + if (m_is_ready.load(std::memory_order_relaxed)) { + m_storage.~storage_t(); + } + } + + std::uintptr_t to_bits() { + return reinterpret_cast(this); + } + + static collaborative_once_runner* from_bits(std::uintptr_t bits) { + __TBB_ASSERT( (bits & collaborative_once_references_mask) == 0, "invalid pointer, last log2(max_nfs_size) bits must be zero" ); + return reinterpret_cast(bits); + } + + template + void run_once(F&& f) { + __TBB_ASSERT(!m_is_ready.load(std::memory_order_relaxed), "storage with task_arena and wait_context is already initialized"); + // Initialize internal state + new(&m_storage) storage_t(); + m_storage.m_arena.execute([&] { + isolated_execute([&] { + task_group_context context{ task_group_context::bound, + task_group_context::default_traits | task_group_context::concurrent_wait }; + + collaborative_call_stack_task t{ std::forward(f), m_storage.m_wait_context }; + + // Set the ready flag after entering the execute body to prevent + // moonlighting threads from occupying all slots inside the arena. + m_is_ready.store(true, std::memory_order_release); + execute_and_wait(t, context, m_storage.m_wait_context, context); + }); + }); + } + + void assist() noexcept { + // Do not join the arena until the winner thread takes the slot + spin_wait_while_eq(m_is_ready, false); + m_storage.m_arena.execute([&] { + isolated_execute([&] { + // We do not want to get an exception from user functor on moonlighting threads. + // The exception is handled with the winner thread + task_group_context stub_context; + wait(m_storage.m_wait_context, stub_context); + }); + }); + } + +}; + +class collaborative_once_flag : no_copy { + enum state : std::uintptr_t { + uninitialized, + done, +#if TBB_USE_ASSERT + dead +#endif + }; + std::atomic m_state{ state::uninitialized }; + + template + friend void collaborative_call_once(collaborative_once_flag& flag, Fn&& f, Args&&... args); + + void set_completion_state(std::uintptr_t runner_bits, std::uintptr_t desired) { + std::uintptr_t expected = runner_bits; + do { + expected = runner_bits; + // Possible inefficiency: when we start waiting, + // some moonlighting threads might continue coming that will prolong our waiting. + // Fortunately, there are limited number of threads on the system so wait time is limited. + spin_wait_until_eq(m_state, expected); + } while (!m_state.compare_exchange_strong(expected, desired)); + } + + template + void do_collaborative_call_once(Fn&& f) { + std::uintptr_t expected = m_state.load(std::memory_order_acquire); + collaborative_once_runner runner; + + do { + if (expected == state::uninitialized && m_state.compare_exchange_strong(expected, runner.to_bits())) { + // Winner thread + runner.run_once([&] { + try_call([&] { + std::forward(f)(); + }).on_exception([&] { + // Reset the state to uninitialized to allow other threads to try initialization again + set_completion_state(runner.to_bits(), state::uninitialized); + }); + // We successfully executed functor + set_completion_state(runner.to_bits(), state::done); + }); + break; + } else { + // Moonlighting thread: we need to add a reference to the state to prolong runner lifetime. + // However, the maximum number of references are limited with runner alignment. + // So, we use CAS loop and spin_wait to guarantee that references never exceed "max_value". + do { + auto max_value = expected | collaborative_once_references_mask; + expected = spin_wait_while_eq(m_state, max_value); + // "expected > state::done" prevents storing values, when state is uninitialized or done + } while (expected > state::done && !m_state.compare_exchange_strong(expected, expected + 1)); + + if (auto shared_runner = collaborative_once_runner::from_bits(expected & ~collaborative_once_references_mask)) { + collaborative_once_runner::lifetime_guard guard{*shared_runner}; + m_state.fetch_sub(1); + + // The moonlighting threads are not expected to handle exceptions from user functor. + // Therefore, no exception is expected from assist(). + shared_runner->assist(); + } + } + __TBB_ASSERT(m_state.load(std::memory_order_relaxed) != state::dead, + "collaborative_once_flag has been prematurely destroyed"); + } while (expected != state::done); + } + +#if TBB_USE_ASSERT +public: + ~collaborative_once_flag() { + m_state.store(state::dead, std::memory_order_relaxed); + } +#endif +}; + + +template +void collaborative_call_once(collaborative_once_flag& flag, Fn&& fn, Args&&... args) { + __TBB_ASSERT(flag.m_state.load(std::memory_order_relaxed) != collaborative_once_flag::dead, + "collaborative_once_flag has been prematurely destroyed"); + if (flag.m_state.load(std::memory_order_acquire) != collaborative_once_flag::done) { + #if __TBB_GCC_PARAMETER_PACK_IN_LAMBDAS_BROKEN + // Using stored_pack to suppress bug in GCC 4.8 + // with parameter pack expansion in lambda + auto stored_pack = save_pack(std::forward(args)...); + auto func = [&] { call(std::forward(fn), std::move(stored_pack)); }; + #else + auto func = [&] { fn(std::forward(args)...); }; + #endif + flag.do_collaborative_call_once(func); + } +} + +#if _MSC_VER && !defined(__INTEL_COMPILER) + // #pragma warning (pop) // 4324 warning +#endif + +} // namespace d1 +} // namespace detail + +using detail::d1::collaborative_call_once; +using detail::d1::collaborative_once_flag; +} // namespace tbb + +#endif // __TBB_collaborative_call_once_H diff --git a/src/tbb/include/oneapi/tbb/combinable.h b/src/tbb/include/oneapi/tbb/combinable.h new file mode 100644 index 000000000..b676a30cc --- /dev/null +++ b/src/tbb/include/oneapi/tbb/combinable.h @@ -0,0 +1,69 @@ +/* + Copyright (c) 2005-2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_combinable_H +#define __TBB_combinable_H + +#include "detail/_namespace_injection.h" + +#include "enumerable_thread_specific.h" +#include "cache_aligned_allocator.h" + +namespace tbb { +namespace detail { +namespace d1 { +/** \name combinable **/ +//@{ +//! Thread-local storage with optional reduction +/** @ingroup containers */ +template +class combinable { + using my_alloc = typename tbb::cache_aligned_allocator; + using my_ets_type = typename tbb::enumerable_thread_specific; + my_ets_type my_ets; + +public: + combinable() = default; + + template + explicit combinable(Finit _finit) : my_ets(_finit) { } + + void clear() { my_ets.clear(); } + + T& local() { return my_ets.local(); } + + T& local(bool& exists) { return my_ets.local(exists); } + + // combine_func_t has signature T(T,T) or T(const T&, const T&) + template + T combine(CombineFunc f_combine) { return my_ets.combine(f_combine); } + + // combine_func_t has signature void(T) or void(const T&) + template + void combine_each(CombineFunc f_combine) { my_ets.combine_each(f_combine); } +}; + +} // namespace d1 +} // namespace detail + +inline namespace v1 { +using detail::d1::combinable; +} // inline namespace v1 + +} // namespace tbb + +#endif /* __TBB_combinable_H */ + diff --git a/src/tbb/include/oneapi/tbb/concurrent_hash_map.h b/src/tbb/include/oneapi/tbb/concurrent_hash_map.h new file mode 100644 index 000000000..b30033742 --- /dev/null +++ b/src/tbb/include/oneapi/tbb/concurrent_hash_map.h @@ -0,0 +1,1664 @@ +/* + Copyright (c) 2005-2022 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_concurrent_hash_map_H +#define __TBB_concurrent_hash_map_H + +#include "detail/_namespace_injection.h" +#include "detail/_utils.h" +#include "detail/_assert.h" +#include "detail/_allocator_traits.h" +#include "detail/_containers_helpers.h" +#include "detail/_template_helpers.h" +#include "detail/_hash_compare.h" +#include "detail/_range_common.h" +#include "tbb_allocator.h" +#include "spin_rw_mutex.h" + +#include +#include +#include +#include +#include // Need std::pair +#include // Need std::memset + +namespace tbb { +namespace detail { +namespace d2 { + +#if __TBB_PREVIEW_CONCURRENT_HASH_MAP_EXTENSIONS && __TBB_CPP20_CONCEPTS_PRESENT +template +concept ch_map_rw_scoped_lockable = rw_scoped_lockable && + requires(const typename Mutex::scoped_lock& sl) { + { sl.is_writer() } -> std::convertible_to; +}; +#endif + +template +struct hash_map_node_base : no_copy { + using mutex_type = MutexType; + // Scoped lock type for mutex + using scoped_type = typename MutexType::scoped_lock; + // Next node in chain + hash_map_node_base* next; + mutex_type mutex; +}; + +// Incompleteness flag value +static void* const rehash_req_flag = reinterpret_cast(std::size_t(3)); +// Rehashed empty bucket flag +static void* const empty_rehashed_flag = reinterpret_cast(std::size_t(0)); + +template +bool rehash_required( hash_map_node_base* node_ptr ) { + return reinterpret_cast(node_ptr) == rehash_req_flag; +} + +#if TBB_USE_ASSERT +template +bool empty_rehashed( hash_map_node_base* node_ptr ) { + return reinterpret_cast(node_ptr) == empty_rehashed_flag; +} +#endif + +// base class of concurrent_hash_map + +template +class hash_map_base { +public: + using size_type = std::size_t; + using hashcode_type = std::size_t; + using segment_index_type = std::size_t; + using node_base = hash_map_node_base; + + struct bucket : no_copy { + using mutex_type = MutexType; + using scoped_type = typename mutex_type::scoped_lock; + + bucket() : node_list(nullptr) {} + bucket( node_base* ptr ) : node_list(ptr) {} + + mutex_type mutex; + std::atomic node_list; + }; + + using allocator_type = Allocator; + using allocator_traits_type = tbb::detail::allocator_traits; + using bucket_allocator_type = typename allocator_traits_type::template rebind_alloc; + using bucket_allocator_traits = tbb::detail::allocator_traits; + + // Count of segments in the first block + static constexpr size_type embedded_block = 1; + // Count of segments in the first block + static constexpr size_type embedded_buckets = 1 << embedded_block; + // Count of segments in the first block + static constexpr size_type first_block = 8; //including embedded_block. perfect with bucket size 16, so the allocations are power of 4096 + // Size of a pointer / table size + static constexpr size_type pointers_per_table = sizeof(segment_index_type) * 8; // one segment per bit + + using segment_ptr_type = bucket*; + using atomic_segment_type = std::atomic; + using segments_table_type = atomic_segment_type[pointers_per_table]; + + hash_map_base( const allocator_type& alloc ) : my_allocator(alloc), my_mask(embedded_buckets - 1), my_size(0) { + for (size_type i = 0; i != embedded_buckets; ++i) { + my_embedded_segment[i].node_list.store(nullptr, std::memory_order_relaxed); + } + + for (size_type segment_index = 0; segment_index < pointers_per_table; ++segment_index) { + auto argument = segment_index < embedded_block ? my_embedded_segment + segment_base(segment_index) : nullptr; + my_table[segment_index].store(argument, std::memory_order_relaxed); + } + + __TBB_ASSERT( embedded_block <= first_block, "The first block number must include embedded blocks"); + } + + // segment index of given index in the array + static segment_index_type segment_index_of( size_type index ) { + return segment_index_type(tbb::detail::log2( index|1 )); + } + + // the first array index of given segment + static segment_index_type segment_base( segment_index_type k ) { + return (segment_index_type(1) << k & ~segment_index_type(1)); + } + + // segment size except for k == 0 + static size_type segment_size( segment_index_type k ) { + return size_type(1) << k; // fake value for k==0 + } + + // true if ptr is valid pointer + static bool is_valid( void* ptr ) { + return reinterpret_cast(ptr) > uintptr_t(63); + } + + template + void init_buckets_impl( segment_ptr_type ptr, size_type sz, const Args&... args ) { + for (size_type i = 0; i < sz; ++i) { + bucket_allocator_traits::construct(my_allocator, ptr + i, args...); + } + } + + // Initialize buckets + void init_buckets( segment_ptr_type ptr, size_type sz, bool is_initial ) { + if (is_initial) { + init_buckets_impl(ptr, sz); + } else { + init_buckets_impl(ptr, sz, reinterpret_cast(rehash_req_flag)); + } + } + + // Add node n to bucket b + static void add_to_bucket( bucket* b, node_base* n ) { + __TBB_ASSERT(!rehash_required(b->node_list.load(std::memory_order_relaxed)), nullptr); + n->next = b->node_list.load(std::memory_order_relaxed); + b->node_list.store(n, std::memory_order_relaxed); // its under lock and flag is set + } + + const bucket_allocator_type& get_allocator() const { + return my_allocator; + } + + bucket_allocator_type& get_allocator() { + return my_allocator; + } + + // Enable segment + void enable_segment( segment_index_type k, bool is_initial = false ) { + __TBB_ASSERT( k, "Zero segment must be embedded" ); + size_type sz; + __TBB_ASSERT( !is_valid(my_table[k].load(std::memory_order_relaxed)), "Wrong concurrent assignment"); + if (k >= first_block) { + sz = segment_size(k); + segment_ptr_type ptr = nullptr; + try_call( [&] { + ptr = bucket_allocator_traits::allocate(my_allocator, sz); + } ).on_exception( [&] { + my_table[k].store(nullptr, std::memory_order_relaxed); + }); + + __TBB_ASSERT(ptr, nullptr); + init_buckets(ptr, sz, is_initial); + my_table[k].store(ptr, std::memory_order_release); + sz <<= 1;// double it to get entire capacity of the container + } else { // the first block + __TBB_ASSERT( k == embedded_block, "Wrong segment index" ); + sz = segment_size(first_block); + segment_ptr_type ptr = nullptr; + try_call( [&] { + ptr = bucket_allocator_traits::allocate(my_allocator, sz - embedded_buckets); + } ).on_exception( [&] { + my_table[k].store(nullptr, std::memory_order_relaxed); + }); + + __TBB_ASSERT(ptr, nullptr); + init_buckets(ptr, sz - embedded_buckets, is_initial); + ptr -= segment_base(embedded_block); + for(segment_index_type i = embedded_block; i < first_block; i++) // calc the offsets + my_table[i].store(ptr + segment_base(i), std::memory_order_release); + } + my_mask.store(sz-1, std::memory_order_release); + } + + void delete_segment( segment_index_type s ) { + segment_ptr_type buckets_ptr = my_table[s].load(std::memory_order_relaxed); + size_type sz = segment_size( s ? s : 1 ); + + size_type deallocate_size = 0; + + if (s >= first_block) { // the first segment or the next + deallocate_size = sz; + } else if (s == embedded_block && embedded_block != first_block) { + deallocate_size = segment_size(first_block) - embedded_buckets; + } + + for (size_type i = 0; i < deallocate_size; ++i) { + bucket_allocator_traits::destroy(my_allocator, buckets_ptr + i); + } + if (deallocate_size != 0) { + bucket_allocator_traits::deallocate(my_allocator, buckets_ptr, deallocate_size); + } + + if (s >= embedded_block) my_table[s].store(nullptr, std::memory_order_relaxed); + } + + // Get bucket by (masked) hashcode + bucket *get_bucket( hashcode_type h ) const noexcept { + segment_index_type s = segment_index_of( h ); + h -= segment_base(s); + segment_ptr_type seg = my_table[s].load(std::memory_order_acquire); + __TBB_ASSERT( is_valid(seg), "hashcode must be cut by valid mask for allocated segments" ); + return &seg[h]; + } + + // detail serial rehashing helper + void mark_rehashed_levels( hashcode_type h ) noexcept { + segment_index_type s = segment_index_of( h ); + while (segment_ptr_type seg = my_table[++s].load(std::memory_order_relaxed)) + if (rehash_required(seg[h].node_list.load(std::memory_order_relaxed))) { + seg[h].node_list.store(reinterpret_cast(empty_rehashed_flag), std::memory_order_relaxed); + mark_rehashed_levels( h + ((hashcode_type)1<node_list.load(std::memory_order_acquire))) { + return true; + } + } + return false; + } + + // Insert a node and check for load factor. @return segment index to enable. + segment_index_type insert_new_node( bucket *b, node_base *n, hashcode_type mask ) { + size_type sz = ++my_size; // prefix form is to enforce allocation after the first item inserted + add_to_bucket( b, n ); + // check load factor + if( sz >= mask ) { // TODO: add custom load_factor + segment_index_type new_seg = tbb::detail::log2( mask+1 ); //optimized segment_index_of + __TBB_ASSERT( is_valid(my_table[new_seg-1].load(std::memory_order_relaxed)), "new allocations must not publish new mask until segment has allocated"); + static const segment_ptr_type is_allocating = segment_ptr_type(2); + segment_ptr_type disabled = nullptr; + if (!(my_table[new_seg].load(std::memory_order_acquire)) + && my_table[new_seg].compare_exchange_strong(disabled, is_allocating)) + return new_seg; // The value must be processed + } + return 0; + } + + // Prepare enough segments for number of buckets + void reserve(size_type buckets) { + if( !buckets-- ) return; + bool is_initial = !my_size.load(std::memory_order_relaxed); + for (size_type m = my_mask.load(std::memory_order_relaxed); buckets > m; + m = my_mask.load(std::memory_order_relaxed)) + { + enable_segment( segment_index_of( m+1 ), is_initial ); + } + } + + // Swap hash_map_bases + void internal_swap_content(hash_map_base &table) { + using std::swap; + swap_atomics_relaxed(my_mask, table.my_mask); + swap_atomics_relaxed(my_size, table.my_size); + + for(size_type i = 0; i < embedded_buckets; i++) { + auto temp = my_embedded_segment[i].node_list.load(std::memory_order_relaxed); + my_embedded_segment[i].node_list.store(table.my_embedded_segment[i].node_list.load(std::memory_order_relaxed), + std::memory_order_relaxed); + table.my_embedded_segment[i].node_list.store(temp, std::memory_order_relaxed); + } + for(size_type i = embedded_block; i < pointers_per_table; i++) { + auto temp = my_table[i].load(std::memory_order_relaxed); + my_table[i].store(table.my_table[i].load(std::memory_order_relaxed), + std::memory_order_relaxed); + table.my_table[i].store(temp, std::memory_order_relaxed); + } + } + + void internal_move(hash_map_base&& other) { + my_mask.store(other.my_mask.load(std::memory_order_relaxed), std::memory_order_relaxed); + other.my_mask.store(embedded_buckets - 1, std::memory_order_relaxed); + + my_size.store(other.my_size.load(std::memory_order_relaxed), std::memory_order_relaxed); + other.my_size.store(0, std::memory_order_relaxed); + + for (size_type i = 0; i < embedded_buckets; ++i) { + my_embedded_segment[i].node_list.store(other.my_embedded_segment[i].node_list, std::memory_order_relaxed); + other.my_embedded_segment[i].node_list.store(nullptr, std::memory_order_relaxed); + } + + for (size_type i = embedded_block; i < pointers_per_table; ++i) { + my_table[i].store(other.my_table[i].load(std::memory_order_relaxed), + std::memory_order_relaxed); + other.my_table[i].store(nullptr, std::memory_order_relaxed); + } + } + +protected: + bucket_allocator_type my_allocator; + // Hash mask = sum of allocated segment sizes - 1 + std::atomic my_mask; + // Size of container in stored items + std::atomic my_size; // It must be in separate cache line from my_mask due to performance effects + // Zero segment + bucket my_embedded_segment[embedded_buckets]; + // Segment pointers table. Also prevents false sharing between my_mask and my_size + segments_table_type my_table; +}; + +template +class hash_map_range; + +// Meets requirements of a forward iterator for STL +// Value is either the T or const T type of the container. +template +class hash_map_iterator { + using map_type = Container; + using node = typename Container::node; + using map_base = typename Container::base_type; + using node_base = typename map_base::node_base; + using bucket = typename map_base::bucket; +public: + using value_type = Value; + using size_type = typename Container::size_type; + using difference_type = typename Container::difference_type; + using pointer = value_type*; + using reference = value_type&; + using iterator_category = std::forward_iterator_tag; + + // Construct undefined iterator + hash_map_iterator(): my_map(), my_index(), my_bucket(), my_node() {} + hash_map_iterator( const hash_map_iterator& other ) : + my_map(other.my_map), + my_index(other.my_index), + my_bucket(other.my_bucket), + my_node(other.my_node) + {} + + hash_map_iterator& operator=( const hash_map_iterator& other ) { + my_map = other.my_map; + my_index = other.my_index; + my_bucket = other.my_bucket; + my_node = other.my_node; + return *this; + } + + Value& operator*() const { + __TBB_ASSERT( map_base::is_valid(my_node), "iterator uninitialized or at end of container?" ); + return my_node->value(); + } + + Value* operator->() const {return &operator*();} + + hash_map_iterator& operator++() { + my_node = static_cast( my_node->next ); + if( !my_node ) advance_to_next_bucket(); + return *this; + } + + // Post increment + hash_map_iterator operator++(int) { + hash_map_iterator old(*this); + operator++(); + return old; + } +private: + template + friend bool operator==( const hash_map_iterator& i, const hash_map_iterator& j ); + + template + friend bool operator!=( const hash_map_iterator& i, const hash_map_iterator& j ); + + template + friend ptrdiff_t operator-( const hash_map_iterator& i, const hash_map_iterator& j ); + + template + friend class hash_map_iterator; + + template + friend class hash_map_range; + + void advance_to_next_bucket() { // TODO?: refactor to iterator_base class + size_t k = my_index+1; + __TBB_ASSERT( my_bucket, "advancing an invalid iterator?"); + while (k <= my_map->my_mask.load(std::memory_order_relaxed)) { + // Following test uses 2's-complement wizardry + if( k&(k-2) ) // not the beginning of a segment + ++my_bucket; + else my_bucket = my_map->get_bucket( k ); + node_base *n = my_bucket->node_list.load(std::memory_order_relaxed); + if( map_base::is_valid(n) ) { + my_node = static_cast(n); + my_index = k; + return; + } + ++k; + } + my_bucket = nullptr; my_node = nullptr; my_index = k; // the end + } + + template + __TBB_requires(tbb::detail::hash_compare && + ch_map_rw_scoped_lockable) +#else + > + __TBB_requires(tbb::detail::hash_compare) +#endif + friend class concurrent_hash_map; + + hash_map_iterator( const Container &map, std::size_t index, const bucket *b, node_base *n ) : + my_map(&map), my_index(index), my_bucket(b), my_node(static_cast(n)) + { + if( b && !map_base::is_valid(n) ) + advance_to_next_bucket(); + } + + // concurrent_hash_map over which we are iterating. + const Container *my_map; + // Index in hash table for current item + size_t my_index; + // Pointer to bucket + const bucket* my_bucket; + // Pointer to node that has current item + node* my_node; +}; + +template +bool operator==( const hash_map_iterator& i, const hash_map_iterator& j ) { + return i.my_node == j.my_node && i.my_map == j.my_map; +} + +template +bool operator!=( const hash_map_iterator& i, const hash_map_iterator& j ) { + return i.my_node != j.my_node || i.my_map != j.my_map; +} + +// Range class used with concurrent_hash_map +template +class hash_map_range { + using map_type = typename Iterator::map_type; +public: + // Type for size of a range + using size_type = std::size_t; + using value_type = typename Iterator::value_type; + using reference = typename Iterator::reference; + using difference_type = typename Iterator::difference_type; + using iterator = Iterator; + + // True if range is empty. + bool empty() const { return my_begin == my_end; } + + // True if range can be partitioned into two subranges. + bool is_divisible() const { + return my_midpoint != my_end; + } + + // Split range. + hash_map_range( hash_map_range& r, split ) : + my_end(r.my_end), + my_grainsize(r.my_grainsize) + { + r.my_end = my_begin = r.my_midpoint; + __TBB_ASSERT( !empty(), "Splitting despite the range is not divisible" ); + __TBB_ASSERT( !r.empty(), "Splitting despite the range is not divisible" ); + set_midpoint(); + r.set_midpoint(); + } + + // Init range with container and grainsize specified + hash_map_range( const map_type &map, size_type grainsize_ = 1 ) : + my_begin( Iterator( map, 0, map.my_embedded_segment, map.my_embedded_segment->node_list.load(std::memory_order_relaxed) ) ), + my_end( Iterator( map, map.my_mask.load(std::memory_order_relaxed) + 1, nullptr, nullptr ) ), + my_grainsize( grainsize_ ) + { + __TBB_ASSERT( grainsize_>0, "grainsize must be positive" ); + set_midpoint(); + } + + Iterator begin() const { return my_begin; } + Iterator end() const { return my_end; } + // The grain size for this range. + size_type grainsize() const { return my_grainsize; } + +private: + Iterator my_begin; + Iterator my_end; + mutable Iterator my_midpoint; + size_t my_grainsize; + // Set my_midpoint to point approximately half way between my_begin and my_end. + void set_midpoint() const; + template friend class hash_map_range; +}; + +template +void hash_map_range::set_midpoint() const { + // Split by groups of nodes + size_t m = my_end.my_index-my_begin.my_index; + if( m > my_grainsize ) { + m = my_begin.my_index + m/2u; + auto b = my_begin.my_map->get_bucket(m); + my_midpoint = Iterator(*my_begin.my_map,m,b,b->node_list.load(std::memory_order_relaxed)); + } else { + my_midpoint = my_end; + } + __TBB_ASSERT( my_begin.my_index <= my_midpoint.my_index, + "my_begin is after my_midpoint" ); + __TBB_ASSERT( my_midpoint.my_index <= my_end.my_index, + "my_midpoint is after my_end" ); + __TBB_ASSERT( my_begin != my_midpoint || my_begin == my_end, + "[my_begin, my_midpoint) range should not be empty" ); +} + +template , + typename Allocator = tbb_allocator> +#if __TBB_PREVIEW_CONCURRENT_HASH_MAP_EXTENSIONS + , typename MutexType = spin_rw_mutex + > + __TBB_requires(tbb::detail::hash_compare && + ch_map_rw_scoped_lockable) +#else + > + __TBB_requires(tbb::detail::hash_compare) +#endif +class concurrent_hash_map +#if __TBB_PREVIEW_CONCURRENT_HASH_MAP_EXTENSIONS + : protected hash_map_base +#else + : protected hash_map_base +#endif +{ + template + friend class hash_map_iterator; + + template + friend class hash_map_range; + using allocator_traits_type = tbb::detail::allocator_traits; + +#if __TBB_PREVIEW_CONCURRENT_HASH_MAP_EXTENSIONS + using base_type = hash_map_base; +#else + using base_type = hash_map_base; +#endif +public: + using key_type = Key; + using mapped_type = T; + // type_identity is needed to disable implicit deduction guides for std::initializer_list constructors + // and copy/move constructor with explicit allocator argument + using allocator_type = tbb::detail::type_identity_t; + using hash_compare_type = tbb::detail::type_identity_t; + using value_type = std::pair; + using size_type = typename base_type::size_type; + using difference_type = std::ptrdiff_t; +#if __TBB_PREVIEW_CONCURRENT_HASH_MAP_EXTENSIONS + using mutex_type = MutexType; +#endif + using pointer = typename allocator_traits_type::pointer; + using const_pointer = typename allocator_traits_type::const_pointer; + + using reference = value_type&; + using const_reference = const value_type&; + using iterator = hash_map_iterator; + using const_iterator = hash_map_iterator; + using range_type = hash_map_range; + using const_range_type = hash_map_range; + +protected: + static_assert(std::is_same::value, + "value_type of the container must be the same as its allocator's"); + + friend class const_accessor; + class node; + using segment_index_type = typename base_type::segment_index_type; + using segment_ptr_type = typename base_type::segment_ptr_type; + using node_base = typename base_type::node_base; + using bucket = typename base_type::bucket; + using hashcode_type = typename base_type::hashcode_type; + using bucket_allocator_type = typename base_type::bucket_allocator_type; + using node_allocator_type = typename base_type::allocator_traits_type::template rebind_alloc; + using node_allocator_traits = tbb::detail::allocator_traits; + hash_compare_type my_hash_compare; + + class node : public node_base { + public: + node() {} + ~node() {} + pointer storage() { return &my_value; } + value_type& value() { return *storage(); } + private: + union { + value_type my_value; + }; + }; + + void delete_node( node_base *n ) { + node_allocator_type node_allocator(this->get_allocator()); + node_allocator_traits::destroy(node_allocator, static_cast(n)->storage()); + node_allocator_traits::destroy(node_allocator, static_cast(n)); + node_allocator_traits::deallocate(node_allocator, static_cast(n), 1); + } + + template + static node* create_node(bucket_allocator_type& allocator, Args&&... args) { + node_allocator_type node_allocator(allocator); + node* node_ptr = node_allocator_traits::allocate(node_allocator, 1); + auto guard = make_raii_guard([&] { + node_allocator_traits::destroy(node_allocator, node_ptr); + node_allocator_traits::deallocate(node_allocator, node_ptr, 1); + }); + + node_allocator_traits::construct(node_allocator, node_ptr); + node_allocator_traits::construct(node_allocator, node_ptr->storage(), std::forward(args)...); + guard.dismiss(); + return node_ptr; + } + + static node* allocate_node_copy_construct(bucket_allocator_type& allocator, const Key &key, const T * t){ + return create_node(allocator, key, *t); + } + + static node* allocate_node_move_construct(bucket_allocator_type& allocator, const Key &key, const T * t){ + return create_node(allocator, key, std::move(*const_cast(t))); + } + + template + static node* allocate_node_default_construct(bucket_allocator_type& allocator, const K &key, const T * ){ + // Emplace construct an empty T object inside the pair + return create_node(allocator, std::piecewise_construct, + std::forward_as_tuple(key), std::forward_as_tuple()); + } + + static node* do_not_allocate_node(bucket_allocator_type& , const Key &, const T * ){ + __TBB_ASSERT(false,"this dummy function should not be called"); + return nullptr; + } + + template + node *search_bucket( const K &key, bucket *b ) const { + node *n = static_cast( b->node_list.load(std::memory_order_relaxed) ); + while (this->is_valid(n) && !my_hash_compare.equal(key, n->value().first)) + n = static_cast( n->next ); + __TBB_ASSERT(!rehash_required(n), "Search can be executed only for rehashed bucket"); + return n; + } + + // bucket accessor is to find, rehash, acquire a lock, and access a bucket + class bucket_accessor : public bucket::scoped_type { + bucket *my_b; + public: + bucket_accessor( concurrent_hash_map *base, const hashcode_type h, bool writer = false ) { acquire( base, h, writer ); } + // find a bucket by masked hashcode, optionally rehash, and acquire the lock + inline void acquire( concurrent_hash_map *base, const hashcode_type h, bool writer = false ) { + my_b = base->get_bucket( h ); + // TODO: actually, notification is unnecessary here, just hiding double-check + if (rehash_required(my_b->node_list.load(std::memory_order_acquire)) + && bucket::scoped_type::try_acquire( my_b->mutex, /*write=*/true ) ) + { + if (rehash_required(my_b->node_list.load(std::memory_order_relaxed))) base->rehash_bucket(my_b, h); // recursive rehashing + } + else bucket::scoped_type::acquire( my_b->mutex, writer ); + __TBB_ASSERT(!rehash_required(my_b->node_list.load(std::memory_order_relaxed)), nullptr); + } + + // get bucket pointer + bucket *operator() () { return my_b; } + }; + + // TODO refactor to hash_base + void rehash_bucket( bucket *b_new, const hashcode_type hash ) { + __TBB_ASSERT( hash > 1, "The lowermost buckets can't be rehashed" ); + b_new->node_list.store(reinterpret_cast(empty_rehashed_flag), std::memory_order_release); // mark rehashed + hashcode_type mask = (hashcode_type(1) << tbb::detail::log2(hash)) - 1; // get parent mask from the topmost bit + bucket_accessor b_old( this, hash & mask ); + + mask = (mask<<1) | 1; // get full mask for new bucket + __TBB_ASSERT( (mask&(mask+1))==0 && (hash & mask) == hash, nullptr ); + restart: + node_base* prev = nullptr; + node_base* curr = b_old()->node_list.load(std::memory_order_acquire); + while (this->is_valid(curr)) { + hashcode_type curr_node_hash = my_hash_compare.hash(static_cast(curr)->value().first); + + if ((curr_node_hash & mask) == hash) { + if (!b_old.is_writer()) { + if (!b_old.upgrade_to_writer()) { + goto restart; // node ptr can be invalid due to concurrent erase + } + } + node_base* next = curr->next; + // exclude from b_old + if (prev == nullptr) { + b_old()->node_list.store(curr->next, std::memory_order_relaxed); + } else { + prev->next = curr->next; + } + this->add_to_bucket(b_new, curr); + curr = next; + } else { + prev = curr; + curr = curr->next; + } + } + } + + template + using hash_compare_is_transparent = dependent_bool, U>; + +public: + + class accessor; + // Combines data access, locking, and garbage collection. + class const_accessor : private node::scoped_type /*which derived from no_copy*/ { +#if __TBB_PREVIEW_CONCURRENT_HASH_MAP_EXTENSIONS + friend class concurrent_hash_map; +#else + friend class concurrent_hash_map; +#endif + friend class accessor; + public: + // Type of value + using value_type = const typename concurrent_hash_map::value_type; + + // True if result is empty. + bool empty() const { return !my_node; } + + // Set to null + void release() { + if( my_node ) { + node::scoped_type::release(); + my_node = nullptr; + } + } + + // Return reference to associated value in hash table. + const_reference operator*() const { + __TBB_ASSERT( my_node, "attempt to dereference empty accessor" ); + return my_node->value(); + } + + // Return pointer to associated value in hash table. + const_pointer operator->() const { + return &operator*(); + } + + // Create empty result + const_accessor() : my_node(nullptr), my_hash() {} + + // Destroy result after releasing the underlying reference. + ~const_accessor() { + my_node = nullptr; // scoped lock's release() is called in its destructor + } + protected: + bool is_writer() { return node::scoped_type::is_writer(); } + node *my_node; + hashcode_type my_hash; + }; + + // Allows write access to elements and combines data access, locking, and garbage collection. + class accessor: public const_accessor { + public: + // Type of value + using value_type = typename concurrent_hash_map::value_type; + + // Return reference to associated value in hash table. + reference operator*() const { + __TBB_ASSERT( this->my_node, "attempt to dereference empty accessor" ); + return this->my_node->value(); + } + + // Return pointer to associated value in hash table. + pointer operator->() const { + return &operator*(); + } + }; + + explicit concurrent_hash_map( const hash_compare_type& compare, const allocator_type& a = allocator_type() ) + : base_type(a) + , my_hash_compare(compare) + {} + + concurrent_hash_map() : concurrent_hash_map(hash_compare_type()) {} + + explicit concurrent_hash_map( const allocator_type& a ) + : concurrent_hash_map(hash_compare_type(), a) + {} + + // Construct empty table with n preallocated buckets. This number serves also as initial concurrency level. + concurrent_hash_map( size_type n, const allocator_type &a = allocator_type() ) + : concurrent_hash_map(a) + { + this->reserve(n); + } + + concurrent_hash_map( size_type n, const hash_compare_type& compare, const allocator_type& a = allocator_type() ) + : concurrent_hash_map(compare, a) + { + this->reserve(n); + } + + // Copy constructor + concurrent_hash_map( const concurrent_hash_map &table ) + : concurrent_hash_map(node_allocator_traits::select_on_container_copy_construction(table.get_allocator())) + { + try_call( [&] { + internal_copy(table); + }).on_exception( [&] { + this->clear(); + }); + } + + concurrent_hash_map( const concurrent_hash_map &table, const allocator_type &a) + : concurrent_hash_map(a) + { + try_call( [&] { + internal_copy(table); + }).on_exception( [&] { + this->clear(); + }); + } + + // Move constructor + concurrent_hash_map( concurrent_hash_map &&table ) + : concurrent_hash_map(std::move(table.get_allocator())) + { + this->internal_move(std::move(table)); + } + + // Move constructor + concurrent_hash_map( concurrent_hash_map &&table, const allocator_type &a ) + : concurrent_hash_map(a) + { + using is_equal_type = typename node_allocator_traits::is_always_equal; + internal_move_construct_with_allocator(std::move(table), a, is_equal_type()); + } + + // Construction with copying iteration range and given allocator instance + template + concurrent_hash_map( I first, I last, const allocator_type &a = allocator_type() ) + : concurrent_hash_map(a) + { + try_call( [&] { + internal_copy(first, last, std::distance(first, last)); + }).on_exception( [&] { + this->clear(); + }); + } + + template + concurrent_hash_map( I first, I last, const hash_compare_type& compare, const allocator_type& a = allocator_type() ) + : concurrent_hash_map(compare, a) + { + try_call( [&] { + internal_copy(first, last, std::distance(first, last)); + }).on_exception( [&] { + this->clear(); + }); + } + + concurrent_hash_map( std::initializer_list il, const hash_compare_type& compare = hash_compare_type(), const allocator_type& a = allocator_type() ) + : concurrent_hash_map(compare, a) + { + try_call( [&] { + internal_copy(il.begin(), il.end(), il.size()); + }).on_exception( [&] { + this->clear(); + }); + } + + concurrent_hash_map( std::initializer_list il, const allocator_type& a ) + : concurrent_hash_map(il, hash_compare_type(), a) {} + + // Assignment + concurrent_hash_map& operator=( const concurrent_hash_map &table ) { + if( this != &table ) { + clear(); + copy_assign_allocators(this->my_allocator, table.my_allocator); + internal_copy(table); + } + return *this; + } + + // Move Assignment + concurrent_hash_map& operator=( concurrent_hash_map &&table ) { + if( this != &table ) { + using pocma_type = typename node_allocator_traits::propagate_on_container_move_assignment; + using is_equal_type = typename node_allocator_traits::is_always_equal; + move_assign_allocators(this->my_allocator, table.my_allocator); + internal_move_assign(std::move(table), tbb::detail::disjunction()); + } + return *this; + } + + // Assignment + concurrent_hash_map& operator=( std::initializer_list il ) { + clear(); + internal_copy(il.begin(), il.end(), il.size()); + return *this; + } + + // Rehashes and optionally resizes the whole table. + /** Useful to optimize performance before or after concurrent operations. + Also enables using of find() and count() concurrent methods in serial context. */ + void rehash(size_type sz = 0) { + this->reserve(sz); // TODO: add reduction of number of buckets as well + hashcode_type mask = this->my_mask.load(std::memory_order_relaxed); + hashcode_type b = (mask+1)>>1; // size or first index of the last segment + __TBB_ASSERT((b&(b-1))==0, nullptr); // zero or power of 2 + bucket *bp = this->get_bucket( b ); // only the last segment should be scanned for rehashing + for(; b <= mask; b++, bp++ ) { + node_base *n = bp->node_list.load(std::memory_order_relaxed); + __TBB_ASSERT( this->is_valid(n) || empty_rehashed(n) || rehash_required(n), "Broken internal structure" ); + __TBB_ASSERT( *reinterpret_cast(&bp->mutex) == 0, "concurrent or unexpectedly terminated operation during rehash() execution" ); + if (rehash_required(n)) { // rehash bucket, conditional because rehashing of a previous bucket may affect this one + hashcode_type h = b; bucket *b_old = bp; + do { + __TBB_ASSERT( h > 1, "The lowermost buckets can't be rehashed" ); + hashcode_type m = ( hashcode_type(1) << tbb::detail::log2( h ) ) - 1; // get parent mask from the topmost bit + b_old = this->get_bucket( h &= m ); + } while( rehash_required(b_old->node_list.load(std::memory_order_relaxed)) ); + // now h - is index of the root rehashed bucket b_old + this->mark_rehashed_levels( h ); // mark all non-rehashed children recursively across all segments + node_base* prev = nullptr; + node_base* curr = b_old->node_list.load(std::memory_order_relaxed); + while (this->is_valid(curr)) { + hashcode_type curr_node_hash = my_hash_compare.hash(static_cast(curr)->value().first); + + if ((curr_node_hash & mask) != h) { // should be rehashed + node_base* next = curr->next; + // exclude from b_old + if (prev == nullptr) { + b_old->node_list.store(curr->next, std::memory_order_relaxed); + } else { + prev->next = curr->next; + } + bucket *b_new = this->get_bucket(curr_node_hash & mask); + __TBB_ASSERT(!rehash_required(b_new->node_list.load(std::memory_order_relaxed)), "hash() function changed for key in table or internal error"); + this->add_to_bucket(b_new, curr); + curr = next; + } else { + prev = curr; + curr = curr->next; + } + } + } + } + } + + // Clear table + void clear() { + hashcode_type m = this->my_mask.load(std::memory_order_relaxed); + __TBB_ASSERT((m&(m+1))==0, "data structure is invalid"); + this->my_size.store(0, std::memory_order_relaxed); + segment_index_type s = this->segment_index_of( m ); + __TBB_ASSERT( s+1 == this->pointers_per_table || !this->my_table[s+1].load(std::memory_order_relaxed), "wrong mask or concurrent grow" ); + do { + __TBB_ASSERT(this->is_valid(this->my_table[s].load(std::memory_order_relaxed)), "wrong mask or concurrent grow" ); + segment_ptr_type buckets_ptr = this->my_table[s].load(std::memory_order_relaxed); + size_type sz = this->segment_size( s ? s : 1 ); + for( segment_index_type i = 0; i < sz; i++ ) + for( node_base *n = buckets_ptr[i].node_list.load(std::memory_order_relaxed); + this->is_valid(n); n = buckets_ptr[i].node_list.load(std::memory_order_relaxed) ) + { + buckets_ptr[i].node_list.store(n->next, std::memory_order_relaxed); + delete_node( n ); + } + this->delete_segment(s); + } while(s-- > 0); + this->my_mask.store(this->embedded_buckets - 1, std::memory_order_relaxed); + } + + // Clear table and destroy it. + ~concurrent_hash_map() { clear(); } + + //------------------------------------------------------------------------ + // Parallel algorithm support + //------------------------------------------------------------------------ + range_type range( size_type grainsize=1 ) { + return range_type( *this, grainsize ); + } + const_range_type range( size_type grainsize=1 ) const { + return const_range_type( *this, grainsize ); + } + + //------------------------------------------------------------------------ + // STL support - not thread-safe methods + //------------------------------------------------------------------------ + iterator begin() { return iterator( *this, 0, this->my_embedded_segment, this->my_embedded_segment->node_list.load(std::memory_order_relaxed) ); } + const_iterator begin() const { return const_iterator( *this, 0, this->my_embedded_segment, this->my_embedded_segment->node_list.load(std::memory_order_relaxed) ); } + const_iterator cbegin() const { return const_iterator( *this, 0, this->my_embedded_segment, this->my_embedded_segment->node_list.load(std::memory_order_relaxed) ); } + iterator end() { return iterator( *this, 0, nullptr, nullptr ); } + const_iterator end() const { return const_iterator( *this, 0, nullptr, nullptr ); } + const_iterator cend() const { return const_iterator( *this, 0, nullptr, nullptr ); } + std::pair equal_range( const Key& key ) { return internal_equal_range( key, end() ); } + std::pair equal_range( const Key& key ) const { return internal_equal_range( key, end() ); } + + template + typename std::enable_if::value, + std::pair>::type equal_range( const K& key ) { + return internal_equal_range(key, end()); + } + + template + typename std::enable_if::value, + std::pair>::type equal_range( const K& key ) const { + return internal_equal_range(key, end()); + } + + // Number of items in table. + size_type size() const { return this->my_size.load(std::memory_order_acquire); } + + // True if size()==0. + __TBB_nodiscard bool empty() const { return size() == 0; } + + // Upper bound on size. + size_type max_size() const { + return allocator_traits_type::max_size(base_type::get_allocator()); + } + + // Returns the current number of buckets + size_type bucket_count() const { return this->my_mask.load(std::memory_order_relaxed) + 1; } + + // return allocator object + allocator_type get_allocator() const { return base_type::get_allocator(); } + + // swap two instances. Iterators are invalidated + void swap(concurrent_hash_map& table) { + using pocs_type = typename node_allocator_traits::propagate_on_container_swap; + using is_equal_type = typename node_allocator_traits::is_always_equal; + swap_allocators(this->my_allocator, table.my_allocator); + internal_swap(table, tbb::detail::disjunction()); + } + + //------------------------------------------------------------------------ + // concurrent map operations + //------------------------------------------------------------------------ + + // Return count of items (0 or 1) + size_type count( const Key &key ) const { + return const_cast(this)->lookup(key, nullptr, nullptr, /*write=*/false, &do_not_allocate_node); + } + + template + typename std::enable_if::value, + size_type>::type count( const K& key ) const { + return const_cast(this)->lookup(key, nullptr, nullptr, /*write=*/false, &do_not_allocate_node); + } + + // Find item and acquire a read lock on the item. + /** Return true if item is found, false otherwise. */ + bool find( const_accessor &result, const Key &key ) const { + result.release(); + return const_cast(this)->lookup(key, nullptr, &result, /*write=*/false, &do_not_allocate_node ); + } + + // Find item and acquire a write lock on the item. + /** Return true if item is found, false otherwise. */ + bool find( accessor &result, const Key &key ) { + result.release(); + return lookup(key, nullptr, &result, /*write=*/true, &do_not_allocate_node); + } + + template + typename std::enable_if::value, + bool>::type find( const_accessor& result, const K& key ) { + result.release(); + return lookup(key, nullptr, &result, /*write=*/false, &do_not_allocate_node); + } + + template + typename std::enable_if::value, + bool>::type find( accessor& result, const K& key ) { + result.release(); + return lookup(key, nullptr, &result, /*write=*/true, &do_not_allocate_node); + } + + // Insert item (if not already present) and acquire a read lock on the item. + /** Returns true if item is new. */ + bool insert( const_accessor &result, const Key &key ) { + result.release(); + return lookup(key, nullptr, &result, /*write=*/false, &allocate_node_default_construct<>); + } + + // Insert item (if not already present) and acquire a write lock on the item. + /** Returns true if item is new. */ + bool insert( accessor &result, const Key &key ) { + result.release(); + return lookup(key, nullptr, &result, /*write=*/true, &allocate_node_default_construct<>); + } + + template + typename std::enable_if::value && + std::is_constructible::value, + bool>::type insert( const_accessor& result, const K& key ) { + result.release(); + return lookup(key, nullptr, &result, /*write=*/false, &allocate_node_default_construct); + } + + template + typename std::enable_if::value && + std::is_constructible::value, + bool>::type insert( accessor& result, const K& key ) { + result.release(); + return lookup(key, nullptr, &result, /*write=*/true, &allocate_node_default_construct); + } + + // Insert item by copying if there is no such key present already and acquire a read lock on the item. + /** Returns true if item is new. */ + bool insert( const_accessor &result, const value_type &value ) { + result.release(); + return lookup(value.first, &value.second, &result, /*write=*/false, &allocate_node_copy_construct); + } + + // Insert item by copying if there is no such key present already and acquire a write lock on the item. + /** Returns true if item is new. */ + bool insert( accessor &result, const value_type &value ) { + result.release(); + return lookup(value.first, &value.second, &result, /*write=*/true, &allocate_node_copy_construct); + } + + // Insert item by copying if there is no such key present already + /** Returns true if item is inserted. */ + bool insert( const value_type &value ) { + return lookup(value.first, &value.second, nullptr, /*write=*/false, &allocate_node_copy_construct); + } + + // Insert item by copying if there is no such key present already and acquire a read lock on the item. + /** Returns true if item is new. */ + bool insert( const_accessor &result, value_type && value ) { + return generic_move_insert(result, std::move(value)); + } + + // Insert item by copying if there is no such key present already and acquire a write lock on the item. + /** Returns true if item is new. */ + bool insert( accessor &result, value_type && value ) { + return generic_move_insert(result, std::move(value)); + } + + // Insert item by copying if there is no such key present already + /** Returns true if item is inserted. */ + bool insert( value_type && value ) { + return generic_move_insert(accessor_not_used(), std::move(value)); + } + + // Insert item by copying if there is no such key present already and acquire a read lock on the item. + /** Returns true if item is new. */ + template + bool emplace( const_accessor &result, Args&&... args ) { + return generic_emplace(result, std::forward(args)...); + } + + // Insert item by copying if there is no such key present already and acquire a write lock on the item. + /** Returns true if item is new. */ + template + bool emplace( accessor &result, Args&&... args ) { + return generic_emplace(result, std::forward(args)...); + } + + // Insert item by copying if there is no such key present already + /** Returns true if item is inserted. */ + template + bool emplace( Args&&... args ) { + return generic_emplace(accessor_not_used(), std::forward(args)...); + } + + // Insert range [first, last) + template + void insert( I first, I last ) { + for ( ; first != last; ++first ) + insert( *first ); + } + + // Insert initializer list + void insert( std::initializer_list il ) { + insert( il.begin(), il.end() ); + } + + // Erase item. + /** Return true if item was erased by particularly this call. */ + bool erase( const Key &key ) { + return internal_erase(key); + } + + template + typename std::enable_if::value, + bool>::type erase( const K& key ) { + return internal_erase(key); + } + + // Erase item by const_accessor. + /** Return true if item was erased by particularly this call. */ + bool erase( const_accessor& item_accessor ) { + return exclude( item_accessor ); + } + + // Erase item by accessor. + /** Return true if item was erased by particularly this call. */ + bool erase( accessor& item_accessor ) { + return exclude( item_accessor ); + } + +protected: + template + node* allocate_node_helper( const K& key, const T* t, AllocateNodeType allocate_node, std::true_type ) { + return allocate_node(base_type::get_allocator(), key, t); + } + + template + node* allocate_node_helper( const K&, const T*, AllocateNodeType, std::false_type ) { + __TBB_ASSERT(false, "allocate_node_helper with std::false_type should never been called"); + return nullptr; + } + + // Insert or find item and optionally acquire a lock on the item. + template + bool lookup( const K &key, const T *t, const_accessor *result, bool write, AllocateNodeType allocate_node, node *tmp_n = nullptr) + { + __TBB_ASSERT( !result || !result->my_node, nullptr ); + bool return_value; + hashcode_type const h = my_hash_compare.hash( key ); + hashcode_type m = this->my_mask.load(std::memory_order_acquire); + segment_index_type grow_segment = 0; + node *n; + restart: + {//lock scope + __TBB_ASSERT((m&(m+1))==0, "data structure is invalid"); + return_value = false; + // get bucket + bucket_accessor b( this, h & m ); + // find a node + n = search_bucket( key, b() ); + if( OpInsert ) { + // [opt] insert a key + if( !n ) { + if( !tmp_n ) { + tmp_n = allocate_node_helper(key, t, allocate_node, std::integral_constant{}); + } + while ( !b.is_writer() && !b.upgrade_to_writer() ) { // TODO: improved insertion + // Rerun search list, in case another thread inserted the intem during the upgrade + n = search_bucket(key, b()); + if (this->is_valid(n)) { // unfortunately, it did + if (!b.downgrade_to_reader()) { + // If the lock was downgraded with reacquiring the mutex + // Rerun search list in case another thread removed the item during the downgrade + n = search_bucket(key, b()); + if (!this->is_valid(n)) { + // Unfortunately, it did + // We need to try upgrading to writer again + continue; + } + } + goto exists; + } + } + + if( this->check_mask_race(h, m) ) + goto restart; // b.release() is done in ~b(). + // insert and set flag to grow the container + grow_segment = this->insert_new_node( b(), n = tmp_n, m ); + tmp_n = nullptr; + return_value = true; + } + } else { // find or count + if( !n ) { + if( this->check_mask_race( h, m ) ) + goto restart; // b.release() is done in ~b(). TODO: replace by continue + return false; + } + return_value = true; + } + exists: + if( !result ) goto check_growth; + // TODO: the following seems as generic/regular operation + // acquire the item + if( !result->try_acquire( n->mutex, write ) ) { + for( tbb::detail::atomic_backoff backoff(true);; ) { + if( result->try_acquire( n->mutex, write ) ) break; + if( !backoff.bounded_pause() ) { + // the wait takes really long, restart the operation + b.release(); + __TBB_ASSERT( !OpInsert || !return_value, "Can't acquire new item in locked bucket?" ); + yield(); + m = this->my_mask.load(std::memory_order_acquire); + goto restart; + } + } + } + }//lock scope + result->my_node = n; + result->my_hash = h; + check_growth: + // [opt] grow the container + if( grow_segment ) { + this->enable_segment( grow_segment ); + } + if( tmp_n ) // if OpInsert only + delete_node( tmp_n ); + return return_value; + } + + struct accessor_not_used { void release(){}}; + friend const_accessor* accessor_location( accessor_not_used const& ){ return nullptr;} + friend const_accessor* accessor_location( const_accessor & a ) { return &a;} + + friend bool is_write_access_needed( accessor const& ) { return true;} + friend bool is_write_access_needed( const_accessor const& ) { return false;} + friend bool is_write_access_needed( accessor_not_used const& ) { return false;} + + template + bool generic_move_insert( Accessor && result, value_type && value ) { + result.release(); + return lookup(value.first, &value.second, accessor_location(result), is_write_access_needed(result), &allocate_node_move_construct); + } + + template + bool generic_emplace( Accessor && result, Args &&... args ) { + result.release(); + node * node_ptr = create_node(base_type::get_allocator(), std::forward(args)...); + return lookup(node_ptr->value().first, nullptr, accessor_location(result), is_write_access_needed(result), &do_not_allocate_node, node_ptr); + } + + // delete item by accessor + bool exclude( const_accessor &item_accessor ) { + __TBB_ASSERT( item_accessor.my_node, nullptr ); + node_base *const exclude_node = item_accessor.my_node; + hashcode_type const hash = item_accessor.my_hash; + hashcode_type mask = this->my_mask.load(std::memory_order_acquire); + do { + // get bucket + bucket_accessor b( this, hash & mask, /*writer=*/true ); + node_base* prev = nullptr; + node_base* curr = b()->node_list.load(std::memory_order_relaxed); + + while (curr && curr != exclude_node) { + prev = curr; + curr = curr->next; + } + + if (curr == nullptr) { // someone else was first + if (this->check_mask_race(hash, mask)) + continue; + item_accessor.release(); + return false; + } + __TBB_ASSERT( curr == exclude_node, nullptr ); + // remove from container + if (prev == nullptr) { + b()->node_list.store(curr->next, std::memory_order_relaxed); + } else { + prev->next = curr->next; + } + + this->my_size--; + break; + } while(true); + if (!item_accessor.is_writer()) { // need to get exclusive lock + item_accessor.upgrade_to_writer(); // return value means nothing here + } + + item_accessor.release(); + delete_node(exclude_node); // Only one thread can delete it + return true; + } + + template + bool internal_erase( const K& key ) { + node_base *erase_node; + hashcode_type const hash = my_hash_compare.hash(key); + hashcode_type mask = this->my_mask.load(std::memory_order_acquire); + restart: + {//lock scope + // get bucket + bucket_accessor b( this, hash & mask ); + search: + node_base* prev = nullptr; + erase_node = b()->node_list.load(std::memory_order_relaxed); + while (this->is_valid(erase_node) && !my_hash_compare.equal(key, static_cast(erase_node)->value().first ) ) { + prev = erase_node; + erase_node = erase_node->next; + } + + if (erase_node == nullptr) { // not found, but mask could be changed + if (this->check_mask_race(hash, mask)) + goto restart; + return false; + } else if (!b.is_writer() && !b.upgrade_to_writer()) { + if (this->check_mask_race(hash, mask)) // contended upgrade, check mask + goto restart; + goto search; + } + + // remove from container + if (prev == nullptr) { + b()->node_list.store(erase_node->next, std::memory_order_relaxed); + } else { + prev->next = erase_node->next; + } + this->my_size--; + } + { + typename node::scoped_type item_locker( erase_node->mutex, /*write=*/true ); + } + // note: there should be no threads pretending to acquire this mutex again, do not try to upgrade const_accessor! + delete_node(erase_node); // Only one thread can delete it due to write lock on the bucket + return true; + } + + // Returns an iterator for an item defined by the key, or for the next item after it (if upper==true) + template + std::pair internal_equal_range( const K& key, I end_ ) const { + hashcode_type h = my_hash_compare.hash( key ); + hashcode_type m = this->my_mask.load(std::memory_order_relaxed); + __TBB_ASSERT((m&(m+1))==0, "data structure is invalid"); + h &= m; + bucket *b = this->get_bucket( h ); + while (rehash_required(b->node_list.load(std::memory_order_relaxed))) { + m = ( hashcode_type(1) << tbb::detail::log2( h ) ) - 1; // get parent mask from the topmost bit + b = this->get_bucket( h &= m ); + } + node *n = search_bucket( key, b ); + if( !n ) + return std::make_pair(end_, end_); + iterator lower(*this, h, b, n), upper(lower); + return std::make_pair(lower, ++upper); + } + + // Copy "source" to *this, where *this must start out empty. + void internal_copy( const concurrent_hash_map& source ) { + hashcode_type mask = source.my_mask.load(std::memory_order_relaxed); + if( this->my_mask.load(std::memory_order_relaxed) == mask ) { // optimized version + this->reserve(source.my_size.load(std::memory_order_relaxed)); // TODO: load_factor? + bucket *dst = nullptr, *src = nullptr; + bool rehashing_required = false; + for( hashcode_type k = 0; k <= mask; k++ ) { + if( k & (k-2) ) ++dst,src++; // not the beginning of a segment + else { dst = this->get_bucket( k ); src = source.get_bucket( k ); } + __TBB_ASSERT(!rehash_required(dst->node_list.load(std::memory_order_relaxed)), "Invalid bucket in destination table"); + node *n = static_cast( src->node_list.load(std::memory_order_relaxed) ); + if (rehash_required(n)) { // source is not rehashed, items are in previous buckets + rehashing_required = true; + dst->node_list.store(reinterpret_cast(rehash_req_flag), std::memory_order_relaxed); + } else for(; n; n = static_cast( n->next ) ) { + node* node_ptr = create_node(base_type::get_allocator(), n->value().first, n->value().second); + this->add_to_bucket( dst, node_ptr); + this->my_size.fetch_add(1, std::memory_order_relaxed); + } + } + if( rehashing_required ) rehash(); + } else internal_copy(source.begin(), source.end(), source.my_size.load(std::memory_order_relaxed)); + } + + template + void internal_copy( I first, I last, size_type reserve_size ) { + this->reserve(reserve_size); // TODO: load_factor? + hashcode_type m = this->my_mask.load(std::memory_order_relaxed); + for(; first != last; ++first) { + hashcode_type h = my_hash_compare.hash( (*first).first ); + bucket *b = this->get_bucket( h & m ); + __TBB_ASSERT(!rehash_required(b->node_list.load(std::memory_order_relaxed)), "Invalid bucket in destination table"); + node* node_ptr = create_node(base_type::get_allocator(), (*first).first, (*first).second); + this->add_to_bucket( b, node_ptr ); + ++this->my_size; // TODO: replace by non-atomic op + } + } + + void internal_move_construct_with_allocator( concurrent_hash_map&& other, const allocator_type&, + /*is_always_equal=*/std::true_type ) + { + this->internal_move(std::move(other)); + } + + void internal_move_construct_with_allocator( concurrent_hash_map&& other, const allocator_type& a, + /*is_always_equal=*/std::false_type ) + { + if (a == other.get_allocator()){ + this->internal_move(std::move(other)); + } else { + try_call( [&] { + internal_copy(std::make_move_iterator(other.begin()), std::make_move_iterator(other.end()), + other.size()); + }).on_exception( [&] { + this->clear(); + }); + } + } + + void internal_move_assign( concurrent_hash_map&& other, + /*is_always_equal || POCMA = */std::true_type) + { + this->internal_move(std::move(other)); + } + + void internal_move_assign(concurrent_hash_map&& other, /*is_always_equal=*/ std::false_type) { + if (this->my_allocator == other.my_allocator) { + this->internal_move(std::move(other)); + } else { + //do per element move + internal_copy(std::make_move_iterator(other.begin()), std::make_move_iterator(other.end()), + other.size()); + } + } + + void internal_swap(concurrent_hash_map& other, /*is_always_equal || POCS = */ std::true_type) { + this->internal_swap_content(other); + } + + void internal_swap(concurrent_hash_map& other, /*is_always_equal || POCS = */ std::false_type) { + __TBB_ASSERT(this->my_allocator == other.my_allocator, nullptr); + this->internal_swap_content(other); + } + + // Fast find when no concurrent erasure is used. For internal use inside TBB only! + /** Return pointer to item with given key, or nullptr if no such item exists. + Must not be called concurrently with erasure operations. */ + const_pointer internal_fast_find( const Key& key ) const { + hashcode_type h = my_hash_compare.hash( key ); + hashcode_type m = this->my_mask.load(std::memory_order_acquire); + node *n; + restart: + __TBB_ASSERT((m&(m+1))==0, "data structure is invalid"); + bucket *b = this->get_bucket( h & m ); + // TODO: actually, notification is unnecessary here, just hiding double-check + if (rehash_required(b->node_list.load(std::memory_order_acquire))) + { + typename bucket::scoped_type lock; + if( lock.try_acquire( b->mutex, /*write=*/true ) ) { + if (rehash_required(b->node_list.load(std::memory_order_relaxed))) + const_cast(this)->rehash_bucket( b, h & m ); //recursive rehashing + } + else lock.acquire( b->mutex, /*write=*/false ); + __TBB_ASSERT(!rehash_required(b->node_list.load(std::memory_order_relaxed)), nullptr); + } + n = search_bucket( key, b ); + if( n ) + return n->storage(); + else if( this->check_mask_race( h, m ) ) + goto restart; + return nullptr; + } +}; + +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT +template >, + typename Alloc = tbb_allocator>, + typename = std::enable_if_t>, + typename = std::enable_if_t>, + typename = std::enable_if_t>> +concurrent_hash_map( It, It, HashCompare = HashCompare(), Alloc = Alloc() ) +-> concurrent_hash_map, iterator_mapped_t, HashCompare, Alloc>; + +template >, + typename = std::enable_if_t>> +concurrent_hash_map( It, It, Alloc ) +-> concurrent_hash_map, iterator_mapped_t, d1::tbb_hash_compare>, Alloc>; + +template >, + typename Alloc = tbb_allocator>, + typename = std::enable_if_t>, + typename = std::enable_if_t>> +concurrent_hash_map( std::initializer_list>, HashCompare = HashCompare(), Alloc = Alloc() ) +-> concurrent_hash_map, T, HashCompare, Alloc>; + +template >> +concurrent_hash_map( std::initializer_list>, Alloc ) +-> concurrent_hash_map, T, d1::tbb_hash_compare>, Alloc>; + +#endif /* __TBB_CPP17_DEDUCTION_GUIDES_PRESENT */ + +template +inline bool operator==(const concurrent_hash_map &a, const concurrent_hash_map &b) { + if(a.size() != b.size()) return false; + typename concurrent_hash_map::const_iterator i(a.begin()), i_end(a.end()); + typename concurrent_hash_map::const_iterator j, j_end(b.end()); + for(; i != i_end; ++i) { + j = b.equal_range(i->first).first; + if( j == j_end || !(i->second == j->second) ) return false; + } + return true; +} + +#if !__TBB_CPP20_COMPARISONS_PRESENT +template +inline bool operator!=(const concurrent_hash_map &a, const concurrent_hash_map &b) +{ return !(a == b); } +#endif // !__TBB_CPP20_COMPARISONS_PRESENT + +template +inline void swap(concurrent_hash_map &a, concurrent_hash_map &b) +{ a.swap( b ); } + +} // namespace d2 +} // namespace detail + +inline namespace v1 { + using detail::split; + using detail::d2::concurrent_hash_map; + using detail::d1::tbb_hash_compare; +} // namespace v1 + +} // namespace tbb + +#endif /* __TBB_concurrent_hash_map_H */ diff --git a/src/tbb/include/oneapi/tbb/concurrent_lru_cache.h b/src/tbb/include/oneapi/tbb/concurrent_lru_cache.h new file mode 100644 index 000000000..83d0576ea --- /dev/null +++ b/src/tbb/include/oneapi/tbb/concurrent_lru_cache.h @@ -0,0 +1,374 @@ +/* + Copyright (c) 2005-2022 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_concurrent_lru_cache_H +#define __TBB_concurrent_lru_cache_H + +#if ! TBB_PREVIEW_CONCURRENT_LRU_CACHE + #error Set TBB_PREVIEW_CONCURRENT_LRU_CACHE to include concurrent_lru_cache.h +#endif + +#include "detail/_assert.h" +#include "detail/_aggregator.h" + +#include // for std::map +#include // for std::list +#include // for std::make_pair +#include // for std::find +#include // for std::atomic + +namespace tbb { + +namespace detail { +namespace d1 { + +//----------------------------------------------------------------------------- +// Concurrent LRU cache +//----------------------------------------------------------------------------- + +template +class concurrent_lru_cache : no_assign { +// incapsulated helper classes +private: + struct handle_object; + struct storage_map_value_type; + + struct aggregator_operation; + struct retrieve_aggregator_operation; + struct signal_end_of_usage_aggregator_operation; + +// typedefs +public: + using key_type = KeyT; + using value_type = ValT; + using pointer = ValT*; + using reference = ValT&; + using const_pointer = const ValT*; + using const_reference = const ValT&; + + using value_function_type = KeyToValFunctorT; + using handle = handle_object; +private: + using lru_cache_type = concurrent_lru_cache; + + using storage_map_type = std::map; + using storage_map_iterator_type = typename storage_map_type::iterator; + using storage_map_pointer_type = typename storage_map_type::pointer; + using storage_map_reference_type = typename storage_map_type::reference; + + using history_list_type = std::list; + using history_list_iterator_type = typename history_list_type::iterator; + + using aggregator_operation_type = aggregator_operation; + using aggregator_function_type = aggregating_functor; + using aggregator_type = aggregator; + + friend class aggregating_functor; + +// fields +private: + value_function_type my_value_function; + aggregator_type my_aggregator; + + storage_map_type my_storage_map; // storage map for used objects + history_list_type my_history_list; // history list for unused objects + const std::size_t my_history_list_capacity; // history list's allowed capacity + +// interface +public: + + concurrent_lru_cache(value_function_type value_function, std::size_t cache_capacity) + : my_value_function(value_function), my_history_list_capacity(cache_capacity) { + my_aggregator.initialize_handler(aggregator_function_type(this)); + } + + handle operator[](key_type key) { + retrieve_aggregator_operation op(key); + my_aggregator.execute(&op); + + if (op.is_new_value_needed()) { + op.result().second.my_value = my_value_function(key); + op.result().second.my_is_ready.store(true, std::memory_order_release); + } else { + spin_wait_while_eq(op.result().second.my_is_ready, false); + } + + return handle(*this, op.result()); + } + +private: + + void handle_operations(aggregator_operation* op_list) { + while (op_list) { + op_list->cast_and_handle(*this); + aggregator_operation* prev_op = op_list; + op_list = op_list->next; + + (prev_op->status).store(1, std::memory_order_release); + } + } + + void signal_end_of_usage(storage_map_reference_type map_record_ref) { + signal_end_of_usage_aggregator_operation op(map_record_ref); + my_aggregator.execute(&op); + } + + void signal_end_of_usage_serial(storage_map_reference_type map_record_ref) { + storage_map_iterator_type map_it = my_storage_map.find(map_record_ref.first); + + __TBB_ASSERT(map_it != my_storage_map.end(), + "cache should not return past-end iterators to outer world"); + __TBB_ASSERT(&(*map_it) == &map_record_ref, + "dangling reference has been returned to outside world: data race?"); + __TBB_ASSERT(std::find(my_history_list.begin(), my_history_list.end(), map_it) == my_history_list.end(), + "object in use should not be in list of unused objects "); + + // if it was the last reference, put it to the LRU history + if (! --(map_it->second.my_ref_counter)) { + // if the LRU history is full, evict the oldest items to get space + if (my_history_list.size() >= my_history_list_capacity) { + if (my_history_list_capacity == 0) { + // Since LRU history capacity is zero, there is no need to keep the element in history + my_storage_map.erase(map_it); + return; + } + std::size_t number_of_elements_to_evict = 1 + my_history_list.size() - my_history_list_capacity; + + for (std::size_t i = 0; i < number_of_elements_to_evict; ++i) { + storage_map_iterator_type map_it_to_evict = my_history_list.back(); + + __TBB_ASSERT(map_it_to_evict->second.my_ref_counter == 0, + "item to be evicted should not have a live references"); + + // TODO: can we use forward_list instead of list? pop_front / insert_after last + my_history_list.pop_back(); + my_storage_map.erase(map_it_to_evict); + } + } + + // TODO: can we use forward_list instead of list? pop_front / insert_after last + my_history_list.push_front(map_it); + map_it->second.my_history_list_iterator = my_history_list.begin(); + } + } + + storage_map_reference_type retrieve_serial(key_type key, bool& is_new_value_needed) { + storage_map_iterator_type map_it = my_storage_map.find(key); + + if (map_it == my_storage_map.end()) { + map_it = my_storage_map.emplace_hint( + map_it, std::piecewise_construct, std::make_tuple(key), std::make_tuple(value_type(), 0, my_history_list.end(), false)); + is_new_value_needed = true; + } else { + history_list_iterator_type list_it = map_it->second.my_history_list_iterator; + if (list_it != my_history_list.end()) { + __TBB_ASSERT(map_it->second.my_ref_counter == 0, + "item to be evicted should not have a live references"); + + // Item is going to be used. Therefore it is not a subject for eviction, + // so we remove it from LRU history. + my_history_list.erase(list_it); + map_it->second.my_history_list_iterator = my_history_list.end(); + } + } + + ++(map_it->second.my_ref_counter); + return *map_it; + } +}; + +//----------------------------------------------------------------------------- +// Value type for storage map in concurrent LRU cache +//----------------------------------------------------------------------------- + +template +struct concurrent_lru_cache::storage_map_value_type { +//typedefs +public: + using ref_counter_type = std::size_t; + +// fields +public: + value_type my_value; + ref_counter_type my_ref_counter; + history_list_iterator_type my_history_list_iterator; + std::atomic my_is_ready; + +// interface +public: + storage_map_value_type( + value_type const& value, ref_counter_type ref_counter, + history_list_iterator_type history_list_iterator, bool is_ready) + : my_value(value), my_ref_counter(ref_counter), + my_history_list_iterator(history_list_iterator), my_is_ready(is_ready) {} +}; + +//----------------------------------------------------------------------------- +// Handle object for operator[] in concurrent LRU cache +//----------------------------------------------------------------------------- + +template +struct concurrent_lru_cache::handle_object { +// fields +private: + lru_cache_type* my_lru_cache_ptr; + storage_map_pointer_type my_map_record_ptr; + +// interface +public: + handle_object() + : my_lru_cache_ptr(nullptr), my_map_record_ptr(nullptr) {} + handle_object(lru_cache_type& lru_cache_ref, storage_map_reference_type map_record_ref) + : my_lru_cache_ptr(&lru_cache_ref), my_map_record_ptr(&map_record_ref) {} + + handle_object(handle_object&) = delete; + void operator=(handle_object&) = delete; + + handle_object(handle_object&& other) + : my_lru_cache_ptr(other.my_lru_cache_ptr), my_map_record_ptr(other.my_map_record_ptr) { + + __TBB_ASSERT( + (other.my_lru_cache_ptr != nullptr && other.my_map_record_ptr != nullptr) || + (other.my_lru_cache_ptr == nullptr && other.my_map_record_ptr == nullptr), + "invalid state of moving object?"); + + other.my_lru_cache_ptr = nullptr; + other.my_map_record_ptr = nullptr; + } + + handle_object& operator=(handle_object&& other) { + __TBB_ASSERT( + (other.my_lru_cache_ptr != nullptr && other.my_map_record_ptr != nullptr) || + (other.my_lru_cache_ptr == nullptr && other.my_map_record_ptr == nullptr), + "invalid state of moving object?"); + + if (my_lru_cache_ptr) + my_lru_cache_ptr->signal_end_of_usage(*my_map_record_ptr); + + my_lru_cache_ptr = other.my_lru_cache_ptr; + my_map_record_ptr = other.my_map_record_ptr; + other.my_lru_cache_ptr = nullptr; + other.my_map_record_ptr = nullptr; + + return *this; + } + + ~handle_object() { + if (my_lru_cache_ptr) + my_lru_cache_ptr->signal_end_of_usage(*my_map_record_ptr); + } + + operator bool() const { + return (my_lru_cache_ptr && my_map_record_ptr); + } + + value_type& value() { + __TBB_ASSERT(my_lru_cache_ptr, "get value from already moved object?"); + __TBB_ASSERT(my_map_record_ptr, "get value from an invalid or already moved object?"); + + return my_map_record_ptr->second.my_value; + } +}; + +//----------------------------------------------------------------------------- +// Aggregator operation for aggregator type in concurrent LRU cache +//----------------------------------------------------------------------------- + +template +struct concurrent_lru_cache::aggregator_operation + : aggregated_operation { +// incapsulated helper classes +public: + enum class op_type { retrieve, signal_end_of_usage }; + +// fields +private: + op_type my_op; + +// interface +public: + aggregator_operation(op_type op) : my_op(op) {} + + // TODO: aggregator_operation can be implemented + // - as a statically typed variant type or CRTP? (static, dependent on the use case) + // - or use pointer to function and apply_visitor (dynamic) + // - or use virtual functions (dynamic) + void cast_and_handle(lru_cache_type& lru_cache_ref) { + if (my_op == op_type::retrieve) + static_cast(this)->handle(lru_cache_ref); + else + static_cast(this)->handle(lru_cache_ref); + } +}; + +template +struct concurrent_lru_cache::retrieve_aggregator_operation + : aggregator_operation, private no_assign { +public: + key_type my_key; + storage_map_pointer_type my_map_record_ptr; + bool my_is_new_value_needed; + +public: + retrieve_aggregator_operation(key_type key) + : aggregator_operation(aggregator_operation::op_type::retrieve), + my_key(key), my_map_record_ptr(nullptr), my_is_new_value_needed(false) {} + + void handle(lru_cache_type& lru_cache_ref) { + my_map_record_ptr = &lru_cache_ref.retrieve_serial(my_key, my_is_new_value_needed); + } + + storage_map_reference_type result() { + __TBB_ASSERT(my_map_record_ptr, "Attempt to call result() before calling handle()"); + return *my_map_record_ptr; + } + + bool is_new_value_needed() { return my_is_new_value_needed; } +}; + +template +struct concurrent_lru_cache::signal_end_of_usage_aggregator_operation + : aggregator_operation, private no_assign { + +private: + storage_map_reference_type my_map_record_ref; + +public: + signal_end_of_usage_aggregator_operation(storage_map_reference_type map_record_ref) + : aggregator_operation(aggregator_operation::op_type::signal_end_of_usage), + my_map_record_ref(map_record_ref) {} + + void handle(lru_cache_type& lru_cache_ref) { + lru_cache_ref.signal_end_of_usage_serial(my_map_record_ref); + } +}; + +// TODO: if we have guarantees that KeyToValFunctorT always have +// ValT as a return type and KeyT as an argument type +// we can deduce template parameters of concurrent_lru_cache +// by pattern matching on KeyToValFunctorT + +} // namespace d1 +} // namespace detail + +inline namespace v1 { + +using detail::d1::concurrent_lru_cache; + +} // inline namespace v1 +} // namespace tbb + +#endif // __TBB_concurrent_lru_cache_H diff --git a/src/tbb/include/oneapi/tbb/concurrent_map.h b/src/tbb/include/oneapi/tbb/concurrent_map.h new file mode 100644 index 000000000..c1817a18c --- /dev/null +++ b/src/tbb/include/oneapi/tbb/concurrent_map.h @@ -0,0 +1,350 @@ +/* + Copyright (c) 2019-2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_concurrent_map_H +#define __TBB_concurrent_map_H + +#include "detail/_namespace_injection.h" +#include "detail/_concurrent_skip_list.h" +#include "tbb_allocator.h" +#include +#include +#include + +namespace tbb { +namespace detail { +namespace d2 { + +template +struct map_traits { + static constexpr std::size_t max_level = RandomGenerator::max_level; + using random_level_generator_type = RandomGenerator; + using key_type = Key; + using mapped_type = Value; + using compare_type = KeyCompare; + using value_type = std::pair; + using reference = value_type&; + using const_reference = const value_type&; + using allocator_type = Allocator; + + static constexpr bool allow_multimapping = AllowMultimapping; + + class value_compare { + public: + bool operator()(const value_type& lhs, const value_type& rhs) const { + return comp(lhs.first, rhs.first); + } + + protected: + value_compare(compare_type c) : comp(c) {} + + friend struct map_traits; + + compare_type comp; + }; + + static value_compare value_comp(compare_type comp) { return value_compare(comp); } + + static const key_type& get_key(const_reference val) { + return val.first; + } +}; // struct map_traits + +template +class concurrent_multimap; + +template , typename Allocator = tbb::tbb_allocator>> +class concurrent_map : public concurrent_skip_list, Allocator, false>> { + using base_type = concurrent_skip_list, Allocator, false>>; +public: + using key_type = Key; + using mapped_type = Value; + using value_type = typename base_type::value_type; + using size_type = typename base_type::size_type; + using difference_type = typename base_type::difference_type; + using key_compare = Compare; + using value_compare = typename base_type::value_compare; + using allocator_type = Allocator; + + using reference = typename base_type::reference; + using const_reference = typename base_type::const_reference; + using pointer = typename base_type::pointer; + using const_pointer = typename base_type::const_pointer; + + using iterator = typename base_type::iterator; + using const_iterator = typename base_type::const_iterator; + + using node_type = typename base_type::node_type; + + // Include constructors of base type + using base_type::base_type; + + // Required for implicit deduction guides + concurrent_map() = default; + concurrent_map( const concurrent_map& ) = default; + concurrent_map( const concurrent_map& other, const allocator_type& alloc ) : base_type(other, alloc) {} + concurrent_map( concurrent_map&& ) = default; + concurrent_map( concurrent_map&& other, const allocator_type& alloc ) : base_type(std::move(other), alloc) {} + // Required to respect the rule of 5 + concurrent_map& operator=( const concurrent_map& ) = default; + concurrent_map& operator=( concurrent_map&& ) = default; + + concurrent_map& operator=( std::initializer_list il ) { + base_type::operator= (il); + return *this; + } + + // Observers + mapped_type& at(const key_type& key) { + iterator it = this->find(key); + + if (it == this->end()) { + throw_exception(exception_id::invalid_key); + } + return it->second; + } + + const mapped_type& at(const key_type& key) const { + return const_cast(this)->at(key); + } + + mapped_type& operator[](const key_type& key) { + iterator it = this->find(key); + + if (it == this->end()) { + it = this->emplace(std::piecewise_construct, std::forward_as_tuple(key), std::tuple<>()).first; + } + return it->second; + } + + mapped_type& operator[](key_type&& key) { + iterator it = this->find(key); + + if (it == this->end()) { + it = this->emplace(std::piecewise_construct, std::forward_as_tuple(std::move(key)), std::tuple<>()).first; + } + return it->second; + } + + using base_type::insert; + + template + typename std::enable_if::value, + std::pair>::type insert( P&& value ) + { + return this->emplace(std::forward

(value)); + } + + template + typename std::enable_if::value, + iterator>::type insert( const_iterator hint, P&& value ) + { + return this->emplace_hint(hint, std::forward

(value)); + } + + template + void merge(concurrent_map& source) { + this->internal_merge(source); + } + + template + void merge(concurrent_map&& source) { + this->internal_merge(std::move(source)); + } + + template + void merge(concurrent_multimap& source) { + this->internal_merge(source); + } + + template + void merge(concurrent_multimap&& source) { + this->internal_merge(std::move(source)); + } +}; // class concurrent_map + +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +template >, + typename Alloc = tbb::tbb_allocator>, + typename = std::enable_if_t>, + typename = std::enable_if_t>, + typename = std::enable_if_t>> +concurrent_map( It, It, Comp = Comp(), Alloc = Alloc() ) +-> concurrent_map, iterator_mapped_t, Comp, Alloc>; + +template >, + typename Alloc = tbb::tbb_allocator>, + typename = std::enable_if_t>, + typename = std::enable_if_t>> +concurrent_map( std::initializer_list>, Comp = Comp(), Alloc = Alloc() ) +-> concurrent_map, T, Comp, Alloc>; + +template >, + typename = std::enable_if_t>> +concurrent_map( It, It, Alloc ) +-> concurrent_map, iterator_mapped_t, + std::less>, Alloc>; + +template >> +concurrent_map( std::initializer_list>, Alloc ) +-> concurrent_map, T, std::less>, Alloc>; + +#endif // __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +template +void swap( concurrent_map& lhs, + concurrent_map& rhs ) +{ + lhs.swap(rhs); +} + +template , typename Allocator = tbb::tbb_allocator>> +class concurrent_multimap : public concurrent_skip_list, Allocator, true>> { + using base_type = concurrent_skip_list, Allocator, true>>; +public: + using key_type = Key; + using mapped_type = Value; + using value_type = typename base_type::value_type; + using size_type = typename base_type::size_type; + using difference_type = typename base_type::difference_type; + using key_compare = Compare; + using value_compare = typename base_type::value_compare; + using allocator_type = Allocator; + + using reference = typename base_type::reference; + using const_reference = typename base_type::const_reference; + using pointer = typename base_type::pointer; + using const_pointer = typename base_type::const_pointer; + + using iterator = typename base_type::iterator; + using const_iterator = typename base_type::const_iterator; + + using node_type = typename base_type::node_type; + + // Include constructors of base_type + using base_type::base_type; + using base_type::insert; + + // Required for implicit deduction guides + concurrent_multimap() = default; + concurrent_multimap( const concurrent_multimap& ) = default; + concurrent_multimap( const concurrent_multimap& other, const allocator_type& alloc ) : base_type(other, alloc) {} + concurrent_multimap( concurrent_multimap&& ) = default; + concurrent_multimap( concurrent_multimap&& other, const allocator_type& alloc ) : base_type(std::move(other), alloc) {} + // Required to respect the rule of 5 + concurrent_multimap& operator=( const concurrent_multimap& ) = default; + concurrent_multimap& operator=( concurrent_multimap&& ) = default; + + concurrent_multimap& operator=( std::initializer_list il ) { + base_type::operator= (il); + return *this; + } + + template + typename std::enable_if::value, + std::pair>::type insert( P&& value ) + { + return this->emplace(std::forward

(value)); + } + + template + typename std::enable_if::value, + iterator>::type insert( const_iterator hint, P&& value ) + { + return this->emplace_hint(hint, std::forward

(value)); + } + + template + void merge(concurrent_multimap& source) { + this->internal_merge(source); + } + + template + void merge(concurrent_multimap&& source) { + this->internal_merge(std::move(source)); + } + + template + void merge(concurrent_map& source) { + this->internal_merge(source); + } + + template + void merge(concurrent_map&& source) { + this->internal_merge(std::move(source)); + } +}; // class concurrent_multimap + +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +template >, + typename Alloc = tbb::tbb_allocator>, + typename = std::enable_if_t>, + typename = std::enable_if_t>, + typename = std::enable_if_t>> +concurrent_multimap( It, It, Comp = Comp(), Alloc = Alloc() ) +-> concurrent_multimap, iterator_mapped_t, Comp, Alloc>; + +template >, + typename Alloc = tbb::tbb_allocator>, + typename = std::enable_if_t>, + typename = std::enable_if_t>> +concurrent_multimap( std::initializer_list>, Comp = Comp(), Alloc = Alloc() ) +-> concurrent_multimap, T, Comp, Alloc>; + +template >, + typename = std::enable_if_t>> +concurrent_multimap( It, It, Alloc ) +-> concurrent_multimap, iterator_mapped_t, + std::less>, Alloc>; + +template >> +concurrent_multimap( std::initializer_list>, Alloc ) +-> concurrent_multimap, T, std::less>, Alloc>; + + +#endif // __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +template +void swap( concurrent_multimap& lhs, + concurrent_multimap& rhs ) +{ + lhs.swap(rhs); +} + +} // namespace d2 +} // namespace detail + +inline namespace v1 { + +using detail::d2::concurrent_map; +using detail::d2::concurrent_multimap; +using detail::split; + +} // inline namespace v1 +} // namespace tbb + +#endif // __TBB_concurrent_map_H diff --git a/src/tbb/include/oneapi/tbb/concurrent_priority_queue.h b/src/tbb/include/oneapi/tbb/concurrent_priority_queue.h new file mode 100644 index 000000000..6d7eb3eb9 --- /dev/null +++ b/src/tbb/include/oneapi/tbb/concurrent_priority_queue.h @@ -0,0 +1,490 @@ +/* + Copyright (c) 2005-2022 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_concurrent_priority_queue_H +#define __TBB_concurrent_priority_queue_H + +#include "detail/_namespace_injection.h" +#include "detail/_aggregator.h" +#include "detail/_template_helpers.h" +#include "detail/_allocator_traits.h" +#include "detail/_range_common.h" +#include "detail/_exception.h" +#include "detail/_utils.h" +#include "detail/_containers_helpers.h" +#include "cache_aligned_allocator.h" +#include +#include +#include +#include +#include +#include + +namespace tbb { +namespace detail { +namespace d1 { + +template , typename Allocator = cache_aligned_allocator> +class concurrent_priority_queue { +public: + using value_type = T; + using reference = T&; + using const_reference = const T&; + + using size_type = std::size_t; + using difference_type = std::ptrdiff_t; + + using allocator_type = Allocator; + + concurrent_priority_queue() : concurrent_priority_queue(allocator_type{}) {} + + explicit concurrent_priority_queue( const allocator_type& alloc ) + : mark(0), my_size(0), my_compare(), data(alloc) + { + my_aggregator.initialize_handler(functor{this}); + } + + explicit concurrent_priority_queue( const Compare& compare, const allocator_type& alloc = allocator_type() ) + : mark(0), my_size(0), my_compare(compare), data(alloc) + { + my_aggregator.initialize_handler(functor{this}); + } + + explicit concurrent_priority_queue( size_type init_capacity, const allocator_type& alloc = allocator_type() ) + : mark(0), my_size(0), my_compare(), data(alloc) + { + data.reserve(init_capacity); + my_aggregator.initialize_handler(functor{this}); + } + + explicit concurrent_priority_queue( size_type init_capacity, const Compare& compare, const allocator_type& alloc = allocator_type() ) + : mark(0), my_size(0), my_compare(compare), data(alloc) + { + data.reserve(init_capacity); + my_aggregator.initialize_handler(functor{this}); + } + + template + concurrent_priority_queue( InputIterator begin, InputIterator end, const Compare& compare, const allocator_type& alloc = allocator_type() ) + : mark(0), my_compare(compare), data(begin, end, alloc) + { + my_aggregator.initialize_handler(functor{this}); + heapify(); + my_size.store(data.size(), std::memory_order_relaxed); + } + + template + concurrent_priority_queue( InputIterator begin, InputIterator end, const allocator_type& alloc = allocator_type() ) + : concurrent_priority_queue(begin, end, Compare(), alloc) {} + + concurrent_priority_queue( std::initializer_list init, const Compare& compare, const allocator_type& alloc = allocator_type() ) + : concurrent_priority_queue(init.begin(), init.end(), compare, alloc) {} + + concurrent_priority_queue( std::initializer_list init, const allocator_type& alloc = allocator_type() ) + : concurrent_priority_queue(init, Compare(), alloc) {} + + concurrent_priority_queue( const concurrent_priority_queue& other ) + : mark(other.mark), my_size(other.my_size.load(std::memory_order_relaxed)), my_compare(other.my_compare), + data(other.data) + { + my_aggregator.initialize_handler(functor{this}); + } + + concurrent_priority_queue( const concurrent_priority_queue& other, const allocator_type& alloc ) + : mark(other.mark), my_size(other.my_size.load(std::memory_order_relaxed)), my_compare(other.my_compare), + data(other.data, alloc) + { + my_aggregator.initialize_handler(functor{this}); + } + + concurrent_priority_queue( concurrent_priority_queue&& other ) + : mark(other.mark), my_size(other.my_size.load(std::memory_order_relaxed)), my_compare(other.my_compare), + data(std::move(other.data)) + { + my_aggregator.initialize_handler(functor{this}); + } + + concurrent_priority_queue( concurrent_priority_queue&& other, const allocator_type& alloc ) + : mark(other.mark), my_size(other.my_size.load(std::memory_order_relaxed)), my_compare(other.my_compare), + data(std::move(other.data), alloc) + { + my_aggregator.initialize_handler(functor{this}); + } + + concurrent_priority_queue& operator=( const concurrent_priority_queue& other ) { + if (this != &other) { + data = other.data; + mark = other.mark; + my_size.store(other.my_size.load(std::memory_order_relaxed), std::memory_order_relaxed); + } + return *this; + } + + concurrent_priority_queue& operator=( concurrent_priority_queue&& other ) { + if (this != &other) { + // TODO: check if exceptions from std::vector::operator=(vector&&) should be handled separately + data = std::move(other.data); + mark = other.mark; + my_size.store(other.my_size.load(std::memory_order_relaxed), std::memory_order_relaxed); + } + return *this; + } + + concurrent_priority_queue& operator=( std::initializer_list init ) { + assign(init.begin(), init.end()); + return *this; + } + + template + void assign( InputIterator begin, InputIterator end ) { + data.assign(begin, end); + mark = 0; + my_size.store(data.size(), std::memory_order_relaxed); + heapify(); + } + + void assign( std::initializer_list init ) { + assign(init.begin(), init.end()); + } + + /* Returned value may not reflect results of pending operations. + This operation reads shared data and will trigger a race condition. */ + __TBB_nodiscard bool empty() const { return size() == 0; } + + // Returns the current number of elements contained in the queue + /* Returned value may not reflect results of pending operations. + This operation reads shared data and will trigger a race condition. */ + size_type size() const { return my_size.load(std::memory_order_relaxed); } + + /* This operation can be safely used concurrently with other push, try_pop or emplace operations. */ + void push( const value_type& value ) { + cpq_operation op_data(value, PUSH_OP); + my_aggregator.execute(&op_data); + if (op_data.status == FAILED) + throw_exception(exception_id::bad_alloc); + } + + /* This operation can be safely used concurrently with other push, try_pop or emplace operations. */ + void push( value_type&& value ) { + cpq_operation op_data(value, PUSH_RVALUE_OP); + my_aggregator.execute(&op_data); + if (op_data.status == FAILED) + throw_exception(exception_id::bad_alloc); + } + + /* This operation can be safely used concurrently with other push, try_pop or emplace operations. */ + template + void emplace( Args&&... args ) { + // TODO: support uses allocator construction in this place + push(value_type(std::forward(args)...)); + } + + // Gets a reference to and removes highest priority element + /* If a highest priority element was found, sets elem and returns true, + otherwise returns false. + This operation can be safely used concurrently with other push, try_pop or emplace operations. */ + bool try_pop( value_type& value ) { + cpq_operation op_data(value, POP_OP); + my_aggregator.execute(&op_data); + return op_data.status == SUCCEEDED; + } + + // This operation affects the whole container => it is not thread-safe + void clear() { + data.clear(); + mark = 0; + my_size.store(0, std::memory_order_relaxed); + } + + // This operation affects the whole container => it is not thread-safe + void swap( concurrent_priority_queue& other ) { + if (this != &other) { + using std::swap; + swap(data, other.data); + swap(mark, other.mark); + + size_type sz = my_size.load(std::memory_order_relaxed); + my_size.store(other.my_size.load(std::memory_order_relaxed), std::memory_order_relaxed); + other.my_size.store(sz, std::memory_order_relaxed); + } + } + + allocator_type get_allocator() const { return data.get_allocator(); } +private: + enum operation_type {INVALID_OP, PUSH_OP, POP_OP, PUSH_RVALUE_OP}; + enum operation_status {WAIT = 0, SUCCEEDED, FAILED}; + + class cpq_operation : public aggregated_operation { + public: + operation_type type; + union { + value_type* elem; + size_type sz; + }; + cpq_operation( const value_type& value, operation_type t ) + : type(t), elem(const_cast(&value)) {} + }; // class cpq_operation + + class functor { + concurrent_priority_queue* my_cpq; + public: + functor() : my_cpq(nullptr) {} + functor( concurrent_priority_queue* cpq ) : my_cpq(cpq) {} + + void operator()(cpq_operation* op_list) { + __TBB_ASSERT(my_cpq != nullptr, "Invalid functor"); + my_cpq->handle_operations(op_list); + } + }; // class functor + + void handle_operations( cpq_operation* op_list ) { + call_itt_notify(acquired, this); + cpq_operation* tmp, *pop_list = nullptr; + __TBB_ASSERT(mark == data.size(), nullptr); + + // First pass processes all constant (amortized; reallocation may happen) time pushes and pops. + while(op_list) { + // ITT note: &(op_list->status) tag is used to cover accesses to op_list + // node. This thread is going to handle the operation, and so will acquire it + // and perform the associated operation w/o triggering a race condition; the + // thread that created the operation is waiting on the status field, so when + // this thread is done with the operation, it will perform a + // store_with_release to give control back to the waiting thread in + // aggregator::insert_operation. + // TODO: enable + call_itt_notify(acquired, &(op_list->status)); + __TBB_ASSERT(op_list->type != INVALID_OP, nullptr); + + tmp = op_list; + op_list = op_list->next.load(std::memory_order_relaxed); + if (tmp->type == POP_OP) { + if (mark < data.size() && + my_compare(data[0], data.back())) + { + // there are newly pushed elems and the last one is higher than top + *(tmp->elem) = std::move(data.back()); + my_size.store(my_size.load(std::memory_order_relaxed) - 1, std::memory_order_relaxed); + tmp->status.store(uintptr_t(SUCCEEDED), std::memory_order_release); + + data.pop_back(); + __TBB_ASSERT(mark <= data.size(), nullptr); + } else { // no convenient item to pop; postpone + tmp->next.store(pop_list, std::memory_order_relaxed); + pop_list = tmp; + } + } else { // PUSH_OP or PUSH_RVALUE_OP + __TBB_ASSERT(tmp->type == PUSH_OP || tmp->type == PUSH_RVALUE_OP, "Unknown operation"); +#if TBB_USE_EXCEPTIONS + try +#endif + { + if (tmp->type == PUSH_OP) { + push_back_helper(*(tmp->elem)); + } else { + data.push_back(std::move(*(tmp->elem))); + } + my_size.store(my_size.load(std::memory_order_relaxed) + 1, std::memory_order_relaxed); + tmp->status.store(uintptr_t(SUCCEEDED), std::memory_order_release); + } +#if TBB_USE_EXCEPTIONS + catch(...) { + tmp->status.store(uintptr_t(FAILED), std::memory_order_release); + } +#endif + } + } + + // Second pass processes pop operations + while(pop_list) { + tmp = pop_list; + pop_list = pop_list->next.load(std::memory_order_relaxed); + __TBB_ASSERT(tmp->type == POP_OP, nullptr); + if (data.empty()) { + tmp->status.store(uintptr_t(FAILED), std::memory_order_release); + } else { + __TBB_ASSERT(mark <= data.size(), nullptr); + if (mark < data.size() && + my_compare(data[0], data.back())) + { + // there are newly pushed elems and the last one is higher than top + *(tmp->elem) = std::move(data.back()); + my_size.store(my_size.load(std::memory_order_relaxed) - 1, std::memory_order_relaxed); + tmp->status.store(uintptr_t(SUCCEEDED), std::memory_order_release); + data.pop_back(); + } else { // extract top and push last element down heap + *(tmp->elem) = std::move(data[0]); + my_size.store(my_size.load(std::memory_order_relaxed) - 1, std::memory_order_relaxed); + tmp->status.store(uintptr_t(SUCCEEDED), std::memory_order_release); + reheap(); + } + } + } + + // heapify any leftover pushed elements before doing the next + // batch of operations + if (mark < data.size()) heapify(); + __TBB_ASSERT(mark == data.size(), nullptr); + call_itt_notify(releasing, this); + } + + // Merge unsorted elements into heap + void heapify() { + if (!mark && data.size() > 0) mark = 1; + for (; mark < data.size(); ++mark) { + // for each unheapified element under size + size_type cur_pos = mark; + value_type to_place = std::move(data[mark]); + do { // push to_place up the heap + size_type parent = (cur_pos - 1) >> 1; + if (!my_compare(data[parent], to_place)) + break; + data[cur_pos] = std::move(data[parent]); + cur_pos = parent; + } while(cur_pos); + data[cur_pos] = std::move(to_place); + } + } + + // Re-heapify after an extraction + // Re-heapify by pushing last element down the heap from the root. + void reheap() { + size_type cur_pos = 0, child = 1; + + while(child < mark) { + size_type target = child; + if (child + 1 < mark && my_compare(data[child], data[child + 1])) + ++target; + // target now has the higher priority child + if (my_compare(data[target], data.back())) + break; + data[cur_pos] = std::move(data[target]); + cur_pos = target; + child = (cur_pos << 1) + 1; + } + if (cur_pos != data.size() - 1) + data[cur_pos] = std::move(data.back()); + data.pop_back(); + if (mark > data.size()) mark = data.size(); + } + + void push_back_helper( const T& value ) { + push_back_helper_impl(value, std::is_copy_constructible{}); + } + + void push_back_helper_impl( const T& value, /*is_copy_constructible = */std::true_type ) { + data.push_back(value); + } + + void push_back_helper_impl( const T&, /*is_copy_constructible = */std::false_type ) { + __TBB_ASSERT(false, "error: calling tbb::concurrent_priority_queue.push(const value_type&) for move-only type"); + } + + using aggregator_type = aggregator; + + aggregator_type my_aggregator; + // Padding added to avoid false sharing + char padding1[max_nfs_size - sizeof(aggregator_type)]; + // The point at which unsorted elements begin + size_type mark; + std::atomic my_size; + Compare my_compare; + + // Padding added to avoid false sharing + char padding2[max_nfs_size - (2*sizeof(size_type)) - sizeof(Compare)]; + //! Storage for the heap of elements in queue, plus unheapified elements + /** data has the following structure: + + binary unheapified + heap elements + ____|_______|____ + | | | + v v v + [_|...|_|_|...|_| |...| ] + 0 ^ ^ ^ + | | |__capacity + | |__my_size + |__mark + + Thus, data stores the binary heap starting at position 0 through + mark-1 (it may be empty). Then there are 0 or more elements + that have not yet been inserted into the heap, in positions + mark through my_size-1. */ + + using vector_type = std::vector; + vector_type data; + + friend bool operator==( const concurrent_priority_queue& lhs, + const concurrent_priority_queue& rhs ) + { + return lhs.data == rhs.data; + } + +#if !__TBB_CPP20_COMPARISONS_PRESENT + friend bool operator!=( const concurrent_priority_queue& lhs, + const concurrent_priority_queue& rhs ) + { + return !(lhs == rhs); + } +#endif +}; // class concurrent_priority_queue + +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT +template >, + typename Alloc = tbb::cache_aligned_allocator>, + typename = std::enable_if_t>, + typename = std::enable_if_t>, + typename = std::enable_if_t>> +concurrent_priority_queue( It, It, Comp = Comp(), Alloc = Alloc() ) +-> concurrent_priority_queue, Comp, Alloc>; + +template >, + typename = std::enable_if_t>> +concurrent_priority_queue( It, It, Alloc ) +-> concurrent_priority_queue, std::less>, Alloc>; + +template , + typename Alloc = tbb::cache_aligned_allocator, + typename = std::enable_if_t>, + typename = std::enable_if_t>> +concurrent_priority_queue( std::initializer_list, Comp = Comp(), Alloc = Alloc() ) +-> concurrent_priority_queue; + +template >> +concurrent_priority_queue( std::initializer_list, Alloc ) +-> concurrent_priority_queue, Alloc>; + +#endif // __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +template +void swap( concurrent_priority_queue& lhs, + concurrent_priority_queue& rhs ) +{ + lhs.swap(rhs); +} + +} // namespace d1 +} // namespace detail +inline namespace v1 { +using detail::d1::concurrent_priority_queue; + +} // inline namespace v1 +} // namespace tbb + +#endif // __TBB_concurrent_priority_queue_H diff --git a/src/tbb/include/oneapi/tbb/concurrent_queue.h b/src/tbb/include/oneapi/tbb/concurrent_queue.h new file mode 100644 index 000000000..cfd5db6a5 --- /dev/null +++ b/src/tbb/include/oneapi/tbb/concurrent_queue.h @@ -0,0 +1,700 @@ +/* + Copyright (c) 2005-2023 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_concurrent_queue_H +#define __TBB_concurrent_queue_H + +#include "detail/_namespace_injection.h" +#include "detail/_concurrent_queue_base.h" +#include "detail/_allocator_traits.h" +#include "detail/_exception.h" +#include "detail/_containers_helpers.h" +#include "cache_aligned_allocator.h" + +namespace tbb { +namespace detail { +namespace d2 { + +template +std::pair internal_try_pop_impl(void* dst, QueueRep& queue, Allocator& alloc ) { + ticket_type ticket{}; + do { + // Basically, we need to read `head_counter` before `tail_counter`. To achieve it we build happens-before on `head_counter` + ticket = queue.head_counter.load(std::memory_order_acquire); + do { + if (static_cast(queue.tail_counter.load(std::memory_order_relaxed) - ticket) <= 0) { // queue is empty + // Queue is empty + return { false, ticket }; + } + // Queue had item with ticket k when we looked. Attempt to get that item. + // Another thread snatched the item, retry. + } while (!queue.head_counter.compare_exchange_strong(ticket, ticket + 1)); + } while (!queue.choose(ticket).pop(dst, ticket, queue, alloc)); + return { true, ticket }; +} + +// A high-performance thread-safe non-blocking concurrent queue. +// Multiple threads may each push and pop concurrently. +// Assignment construction is not allowed. +template > +class concurrent_queue { + using allocator_traits_type = tbb::detail::allocator_traits; + using queue_representation_type = concurrent_queue_rep; + using queue_allocator_type = typename allocator_traits_type::template rebind_alloc; + using queue_allocator_traits = tbb::detail::allocator_traits; +public: + using size_type = std::size_t; + using value_type = T; + using reference = T&; + using const_reference = const T&; + using difference_type = std::ptrdiff_t; + + using allocator_type = Allocator; + using pointer = typename allocator_traits_type::pointer; + using const_pointer = typename allocator_traits_type::const_pointer; + + using iterator = concurrent_queue_iterator; + using const_iterator = concurrent_queue_iterator; + + concurrent_queue() : concurrent_queue(allocator_type()) {} + + explicit concurrent_queue(const allocator_type& a) : + my_allocator(a), my_queue_representation(nullptr) + { + my_queue_representation = static_cast(r1::cache_aligned_allocate(sizeof(queue_representation_type))); + queue_allocator_traits::construct(my_allocator, my_queue_representation); + + __TBB_ASSERT(is_aligned(my_queue_representation, max_nfs_size), "alignment error" ); + __TBB_ASSERT(is_aligned(&my_queue_representation->head_counter, max_nfs_size), "alignment error" ); + __TBB_ASSERT(is_aligned(&my_queue_representation->tail_counter, max_nfs_size), "alignment error" ); + __TBB_ASSERT(is_aligned(&my_queue_representation->array, max_nfs_size), "alignment error" ); + } + + template + concurrent_queue(InputIterator begin, InputIterator end, const allocator_type& a = allocator_type()) : + concurrent_queue(a) + { + for (; begin != end; ++begin) + push(*begin); + } + + concurrent_queue( std::initializer_list init, const allocator_type& alloc = allocator_type() ) : + concurrent_queue(init.begin(), init.end(), alloc) + {} + + concurrent_queue(const concurrent_queue& src, const allocator_type& a) : + concurrent_queue(a) + { + my_queue_representation->assign(*src.my_queue_representation, my_allocator, copy_construct_item); + } + + concurrent_queue(const concurrent_queue& src) : + concurrent_queue(queue_allocator_traits::select_on_container_copy_construction(src.get_allocator())) + { + my_queue_representation->assign(*src.my_queue_representation, my_allocator, copy_construct_item); + } + + // Move constructors + concurrent_queue(concurrent_queue&& src) : + concurrent_queue(std::move(src.my_allocator)) + { + internal_swap(src); + } + + concurrent_queue(concurrent_queue&& src, const allocator_type& a) : + concurrent_queue(a) + { + // checking that memory allocated by one instance of allocator can be deallocated + // with another + if (my_allocator == src.my_allocator) { + internal_swap(src); + } else { + // allocators are different => performing per-element move + my_queue_representation->assign(*src.my_queue_representation, my_allocator, move_construct_item); + src.clear(); + } + } + + // Destroy queue + ~concurrent_queue() { + clear(); + my_queue_representation->clear(my_allocator); + queue_allocator_traits::destroy(my_allocator, my_queue_representation); + r1::cache_aligned_deallocate(my_queue_representation); + } + + concurrent_queue& operator=( const concurrent_queue& other ) { + //TODO: implement support for std::allocator_traits::propagate_on_container_copy_assignment + if (my_queue_representation != other.my_queue_representation) { + clear(); + my_allocator = other.my_allocator; + my_queue_representation->assign(*other.my_queue_representation, my_allocator, copy_construct_item); + } + return *this; + } + + concurrent_queue& operator=( concurrent_queue&& other ) { + //TODO: implement support for std::allocator_traits::propagate_on_container_move_assignment + if (my_queue_representation != other.my_queue_representation) { + clear(); + if (my_allocator == other.my_allocator) { + internal_swap(other); + } else { + my_queue_representation->assign(*other.my_queue_representation, other.my_allocator, move_construct_item); + other.clear(); + my_allocator = std::move(other.my_allocator); + } + } + return *this; + } + + concurrent_queue& operator=( std::initializer_list init ) { + assign(init); + return *this; + } + + template + void assign( InputIterator first, InputIterator last ) { + concurrent_queue src(first, last); + clear(); + my_queue_representation->assign(*src.my_queue_representation, my_allocator, move_construct_item); + } + + void assign( std::initializer_list init ) { + assign(init.begin(), init.end()); + } + + void swap ( concurrent_queue& other ) { + //TODO: implement support for std::allocator_traits::propagate_on_container_swap + __TBB_ASSERT(my_allocator == other.my_allocator, "unequal allocators"); + internal_swap(other); + } + + // Enqueue an item at tail of queue. + void push(const T& value) { + internal_push(value); + } + + void push(T&& value) { + internal_push(std::move(value)); + } + + template + void emplace( Args&&... args ) { + internal_push(std::forward(args)...); + } + + // Attempt to dequeue an item from head of queue. + /** Does not wait for item to become available. + Returns true if successful; false otherwise. */ + bool try_pop( T& result ) { + return internal_try_pop(&result); + } + + // Return the number of items in the queue; thread unsafe + size_type unsafe_size() const { + std::ptrdiff_t size = my_queue_representation->size(); + return size < 0 ? 0 : size_type(size); + } + + // Equivalent to size()==0. + __TBB_nodiscard bool empty() const { + return my_queue_representation->empty(); + } + + // Clear the queue. not thread-safe. + void clear() { + my_queue_representation->clear(my_allocator); + } + + // Return allocator object + allocator_type get_allocator() const { return my_allocator; } + + //------------------------------------------------------------------------ + // The iterators are intended only for debugging. They are slow and not thread safe. + //------------------------------------------------------------------------ + + iterator unsafe_begin() { return concurrent_queue_iterator_provider::get(*this); } + iterator unsafe_end() { return iterator(); } + const_iterator unsafe_begin() const { return concurrent_queue_iterator_provider::get(*this); } + const_iterator unsafe_end() const { return const_iterator(); } + const_iterator unsafe_cbegin() const { return concurrent_queue_iterator_provider::get(*this); } + const_iterator unsafe_cend() const { return const_iterator(); } + +private: + void internal_swap(concurrent_queue& src) { + using std::swap; + swap(my_queue_representation, src.my_queue_representation); + } + + template + void internal_push( Args&&... args ) { + ticket_type k = my_queue_representation->tail_counter++; + my_queue_representation->choose(k).push(k, *my_queue_representation, my_allocator, std::forward(args)...); + } + + bool internal_try_pop( void* dst ) { + return internal_try_pop_impl(dst, *my_queue_representation, my_allocator).first; + } + + template + friend class concurrent_queue_iterator; + + static void copy_construct_item(T* location, const void* src) { + // TODO: use allocator_traits for copy construction + new (location) value_type(*static_cast(src)); + // queue_allocator_traits::construct(my_allocator, location, *static_cast(src)); + } + + static void move_construct_item(T* location, const void* src) { + // TODO: use allocator_traits for move construction + new (location) value_type(std::move(*static_cast(const_cast(src)))); + } + + queue_allocator_type my_allocator; + queue_representation_type* my_queue_representation; + + friend void swap( concurrent_queue& lhs, concurrent_queue& rhs ) { + lhs.swap(rhs); + } + + friend bool operator==( const concurrent_queue& lhs, const concurrent_queue& rhs ) { + return lhs.unsafe_size() == rhs.unsafe_size() && std::equal(lhs.unsafe_begin(), lhs.unsafe_end(), rhs.unsafe_begin()); + } + +#if !__TBB_CPP20_COMPARISONS_PRESENT + friend bool operator!=( const concurrent_queue& lhs, const concurrent_queue& rhs ) { + return !(lhs == rhs); + } +#endif // __TBB_CPP20_COMPARISONS_PRESENT +}; // class concurrent_queue + +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT +// Deduction guide for the constructor from two iterators +template >, + typename = std::enable_if_t>, + typename = std::enable_if_t>> +concurrent_queue( It, It, Alloc = Alloc() ) +-> concurrent_queue, Alloc>; + +#endif /* __TBB_CPP17_DEDUCTION_GUIDES_PRESENT */ + +class concurrent_monitor; + +// The concurrent monitor tags for concurrent_bounded_queue. +static constexpr std::size_t cbq_slots_avail_tag = 0; +static constexpr std::size_t cbq_items_avail_tag = 1; +} // namespace d2 + + +namespace r1 { + class concurrent_monitor; + + TBB_EXPORT std::uint8_t* __TBB_EXPORTED_FUNC allocate_bounded_queue_rep( std::size_t queue_rep_size ); + TBB_EXPORT void __TBB_EXPORTED_FUNC deallocate_bounded_queue_rep( std::uint8_t* mem, std::size_t queue_rep_size ); + TBB_EXPORT void __TBB_EXPORTED_FUNC abort_bounded_queue_monitors( concurrent_monitor* monitors ); + TBB_EXPORT void __TBB_EXPORTED_FUNC notify_bounded_queue_monitor( concurrent_monitor* monitors, std::size_t monitor_tag + , std::size_t ticket ); + TBB_EXPORT void __TBB_EXPORTED_FUNC wait_bounded_queue_monitor( concurrent_monitor* monitors, std::size_t monitor_tag, + std::ptrdiff_t target, d1::delegate_base& predicate ); +} // namespace r1 + + +namespace d2 { +// A high-performance thread-safe blocking concurrent bounded queue. +// Supports boundedness and blocking semantics. +// Multiple threads may each push and pop concurrently. +// Assignment construction is not allowed. +template > +class concurrent_bounded_queue { + using allocator_traits_type = tbb::detail::allocator_traits; + using queue_representation_type = concurrent_queue_rep; + using queue_allocator_type = typename allocator_traits_type::template rebind_alloc; + using queue_allocator_traits = tbb::detail::allocator_traits; + + template + void internal_wait(r1::concurrent_monitor* monitors, std::size_t monitor_tag, std::ptrdiff_t target, FuncType pred) { + d1::delegated_function func(pred); + r1::wait_bounded_queue_monitor(monitors, monitor_tag, target, func); + } +public: + using size_type = std::ptrdiff_t; + using value_type = T; + using reference = T&; + using const_reference = const T&; + using difference_type = std::ptrdiff_t; + + using allocator_type = Allocator; + using pointer = typename allocator_traits_type::pointer; + using const_pointer = typename allocator_traits_type::const_pointer; + + using iterator = concurrent_queue_iterator; + using const_iterator = concurrent_queue_iterator ; + + concurrent_bounded_queue() : concurrent_bounded_queue(allocator_type()) {} + + explicit concurrent_bounded_queue( const allocator_type& a ) : + my_allocator(a), my_capacity(0), my_abort_counter(0), my_queue_representation(nullptr) + { + my_queue_representation = reinterpret_cast( + r1::allocate_bounded_queue_rep(sizeof(queue_representation_type))); + my_monitors = reinterpret_cast(my_queue_representation + 1); + queue_allocator_traits::construct(my_allocator, my_queue_representation); + my_capacity = std::size_t(-1) / (queue_representation_type::item_size > 1 ? queue_representation_type::item_size : 2); + + __TBB_ASSERT(is_aligned(my_queue_representation, max_nfs_size), "alignment error" ); + __TBB_ASSERT(is_aligned(&my_queue_representation->head_counter, max_nfs_size), "alignment error" ); + __TBB_ASSERT(is_aligned(&my_queue_representation->tail_counter, max_nfs_size), "alignment error" ); + __TBB_ASSERT(is_aligned(&my_queue_representation->array, max_nfs_size), "alignment error" ); + } + + template + concurrent_bounded_queue( InputIterator begin, InputIterator end, const allocator_type& a = allocator_type() ) : + concurrent_bounded_queue(a) + { + for (; begin != end; ++begin) + push(*begin); + } + + concurrent_bounded_queue( std::initializer_list init, const allocator_type& alloc = allocator_type() ): + concurrent_bounded_queue(init.begin(), init.end(), alloc) + {} + + concurrent_bounded_queue( const concurrent_bounded_queue& src, const allocator_type& a ) : + concurrent_bounded_queue(a) + { + my_queue_representation->assign(*src.my_queue_representation, my_allocator, copy_construct_item); + } + + concurrent_bounded_queue( const concurrent_bounded_queue& src ) : + concurrent_bounded_queue(queue_allocator_traits::select_on_container_copy_construction(src.get_allocator())) + { + my_queue_representation->assign(*src.my_queue_representation, my_allocator, copy_construct_item); + } + + // Move constructors + concurrent_bounded_queue( concurrent_bounded_queue&& src ) : + concurrent_bounded_queue(std::move(src.my_allocator)) + { + internal_swap(src); + } + + concurrent_bounded_queue( concurrent_bounded_queue&& src, const allocator_type& a ) : + concurrent_bounded_queue(a) + { + // checking that memory allocated by one instance of allocator can be deallocated + // with another + if (my_allocator == src.my_allocator) { + internal_swap(src); + } else { + // allocators are different => performing per-element move + my_queue_representation->assign(*src.my_queue_representation, my_allocator, move_construct_item); + src.clear(); + } + } + + // Destroy queue + ~concurrent_bounded_queue() { + clear(); + my_queue_representation->clear(my_allocator); + queue_allocator_traits::destroy(my_allocator, my_queue_representation); + r1::deallocate_bounded_queue_rep(reinterpret_cast(my_queue_representation), + sizeof(queue_representation_type)); + } + + concurrent_bounded_queue& operator=( const concurrent_bounded_queue& other ) { + //TODO: implement support for std::allocator_traits::propagate_on_container_copy_assignment + if (my_queue_representation != other.my_queue_representation) { + clear(); + my_allocator = other.my_allocator; + my_queue_representation->assign(*other.my_queue_representation, my_allocator, copy_construct_item); + } + return *this; + } + + concurrent_bounded_queue& operator=( concurrent_bounded_queue&& other ) { + //TODO: implement support for std::allocator_traits::propagate_on_container_move_assignment + if (my_queue_representation != other.my_queue_representation) { + clear(); + if (my_allocator == other.my_allocator) { + internal_swap(other); + } else { + my_queue_representation->assign(*other.my_queue_representation, other.my_allocator, move_construct_item); + other.clear(); + my_allocator = std::move(other.my_allocator); + } + } + return *this; + } + + concurrent_bounded_queue& operator=( std::initializer_list init ) { + assign(init); + return *this; + } + + template + void assign( InputIterator first, InputIterator last ) { + concurrent_bounded_queue src(first, last); + clear(); + my_queue_representation->assign(*src.my_queue_representation, my_allocator, move_construct_item); + } + + void assign( std::initializer_list init ) { + assign(init.begin(), init.end()); + } + + void swap ( concurrent_bounded_queue& other ) { + //TODO: implement support for std::allocator_traits::propagate_on_container_swap + __TBB_ASSERT(my_allocator == other.my_allocator, "unequal allocators"); + internal_swap(other); + } + + // Enqueue an item at tail of queue. + void push( const T& value ) { + internal_push(value); + } + + void push( T&& value ) { + internal_push(std::move(value)); + } + + // Enqueue an item at tail of queue if queue is not already full. + // Does not wait for queue to become not full. + // Returns true if item is pushed; false if queue was already full. + bool try_push( const T& value ) { + return internal_push_if_not_full(value); + } + + bool try_push( T&& value ) { + return internal_push_if_not_full(std::move(value)); + } + + template + void emplace( Args&&... args ) { + internal_push(std::forward(args)...); + } + + template + bool try_emplace( Args&&... args ) { + return internal_push_if_not_full(std::forward(args)...); + } + + // Attempt to dequeue an item from head of queue. + void pop( T& result ) { + internal_pop(&result); + } + + /** Does not wait for item to become available. + Returns true if successful; false otherwise. */ + bool try_pop( T& result ) { + return internal_pop_if_present(&result); + } + + void abort() { + internal_abort(); + } + + // Return the number of items in the queue; thread unsafe + std::ptrdiff_t size() const { + return my_queue_representation->size(); + } + + void set_capacity( size_type new_capacity ) { + std::ptrdiff_t c = new_capacity < 0 ? infinite_capacity : new_capacity; + my_capacity = c; + } + + size_type capacity() const { + return my_capacity; + } + + // Equivalent to size()==0. + __TBB_nodiscard bool empty() const { + return my_queue_representation->empty(); + } + + // Clear the queue. not thread-safe. + void clear() { + my_queue_representation->clear(my_allocator); + } + + // Return allocator object + allocator_type get_allocator() const { return my_allocator; } + + //------------------------------------------------------------------------ + // The iterators are intended only for debugging. They are slow and not thread safe. + //------------------------------------------------------------------------ + + iterator unsafe_begin() { return concurrent_queue_iterator_provider::get(*this); } + iterator unsafe_end() { return iterator(); } + const_iterator unsafe_begin() const { return concurrent_queue_iterator_provider::get(*this); } + const_iterator unsafe_end() const { return const_iterator(); } + const_iterator unsafe_cbegin() const { return concurrent_queue_iterator_provider::get(*this); } + const_iterator unsafe_cend() const { return const_iterator(); } + +private: + void internal_swap( concurrent_bounded_queue& src ) { + std::swap(my_queue_representation, src.my_queue_representation); + std::swap(my_monitors, src.my_monitors); + } + + static constexpr std::ptrdiff_t infinite_capacity = std::ptrdiff_t(~size_type(0) / 2); + + template + void internal_push( Args&&... args ) { + unsigned old_abort_counter = my_abort_counter.load(std::memory_order_relaxed); + ticket_type ticket = my_queue_representation->tail_counter++; + std::ptrdiff_t target = ticket - my_capacity; + + if (static_cast(my_queue_representation->head_counter.load(std::memory_order_relaxed)) <= target) { // queue is full + auto pred = [&] { + if (my_abort_counter.load(std::memory_order_relaxed) != old_abort_counter) { + throw_exception(exception_id::user_abort); + } + + return static_cast(my_queue_representation->head_counter.load(std::memory_order_relaxed)) <= target; + }; + + try_call( [&] { + internal_wait(my_monitors, cbq_slots_avail_tag, target, pred); + }).on_exception( [&] { + my_queue_representation->choose(ticket).abort_push(ticket, *my_queue_representation, my_allocator); + }); + + } + __TBB_ASSERT((static_cast(my_queue_representation->head_counter.load(std::memory_order_relaxed)) > target), nullptr); + my_queue_representation->choose(ticket).push(ticket, *my_queue_representation, my_allocator, std::forward(args)...); + r1::notify_bounded_queue_monitor(my_monitors, cbq_items_avail_tag, ticket); + } + + template + bool internal_push_if_not_full( Args&&... args ) { + ticket_type ticket = my_queue_representation->tail_counter.load(std::memory_order_relaxed); + do { + if (static_cast(ticket - my_queue_representation->head_counter.load(std::memory_order_relaxed)) >= my_capacity) { + // Queue is full + return false; + } + // Queue had empty slot with ticket k when we looked. Attempt to claim that slot. + // Another thread claimed the slot, so retry. + } while (!my_queue_representation->tail_counter.compare_exchange_strong(ticket, ticket + 1)); + + my_queue_representation->choose(ticket).push(ticket, *my_queue_representation, my_allocator, std::forward(args)...); + r1::notify_bounded_queue_monitor(my_monitors, cbq_items_avail_tag, ticket); + return true; + } + + void internal_pop( void* dst ) { + std::ptrdiff_t target; + // This loop is a single pop operation; abort_counter should not be re-read inside + unsigned old_abort_counter = my_abort_counter.load(std::memory_order_relaxed); + + do { + target = my_queue_representation->head_counter++; + if (static_cast(my_queue_representation->tail_counter.load(std::memory_order_relaxed)) <= target) { + auto pred = [&] { + if (my_abort_counter.load(std::memory_order_relaxed) != old_abort_counter) { + throw_exception(exception_id::user_abort); + } + + return static_cast(my_queue_representation->tail_counter.load(std::memory_order_relaxed)) <= target; + }; + + try_call( [&] { + internal_wait(my_monitors, cbq_items_avail_tag, target, pred); + }).on_exception( [&] { + my_queue_representation->head_counter--; + }); + } + __TBB_ASSERT(static_cast(my_queue_representation->tail_counter.load(std::memory_order_relaxed)) > target, nullptr); + } while (!my_queue_representation->choose(target).pop(dst, target, *my_queue_representation, my_allocator)); + + r1::notify_bounded_queue_monitor(my_monitors, cbq_slots_avail_tag, target); + } + + bool internal_pop_if_present( void* dst ) { + bool present{}; + ticket_type ticket{}; + std::tie(present, ticket) = internal_try_pop_impl(dst, *my_queue_representation, my_allocator); + + if (present) { + r1::notify_bounded_queue_monitor(my_monitors, cbq_slots_avail_tag, ticket); + } + return present; + } + + void internal_abort() { + ++my_abort_counter; + r1::abort_bounded_queue_monitors(my_monitors); + } + + static void copy_construct_item(T* location, const void* src) { + // TODO: use allocator_traits for copy construction + new (location) value_type(*static_cast(src)); + } + + static void move_construct_item(T* location, const void* src) { + // TODO: use allocator_traits for move construction + new (location) value_type(std::move(*static_cast(const_cast(src)))); + } + + template + friend class concurrent_queue_iterator; + + queue_allocator_type my_allocator; + std::ptrdiff_t my_capacity; + std::atomic my_abort_counter; + queue_representation_type* my_queue_representation; + + r1::concurrent_monitor* my_monitors; + + friend void swap( concurrent_bounded_queue& lhs, concurrent_bounded_queue& rhs ) { + lhs.swap(rhs); + } + + friend bool operator==( const concurrent_bounded_queue& lhs, const concurrent_bounded_queue& rhs ) { + return lhs.size() == rhs.size() && std::equal(lhs.unsafe_begin(), lhs.unsafe_end(), rhs.unsafe_begin()); + } + +#if !__TBB_CPP20_COMPARISONS_PRESENT + friend bool operator!=( const concurrent_bounded_queue& lhs, const concurrent_bounded_queue& rhs ) { + return !(lhs == rhs); + } +#endif // __TBB_CPP20_COMPARISONS_PRESENT +}; // class concurrent_bounded_queue + +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT +// Deduction guide for the constructor from two iterators +template >> +concurrent_bounded_queue( It, It, Alloc = Alloc() ) +-> concurrent_bounded_queue, Alloc>; + +#endif /* __TBB_CPP17_DEDUCTION_GUIDES_PRESENT */ + +} //namespace d2 +} // namespace detail + +inline namespace v1 { + +using detail::d2::concurrent_queue; +using detail::d2::concurrent_bounded_queue; +using detail::r1::user_abort; +using detail::r1::bad_last_alloc; + +} // inline namespace v1 +} // namespace tbb + +#endif // __TBB_concurrent_queue_H diff --git a/src/tbb/include/oneapi/tbb/concurrent_set.h b/src/tbb/include/oneapi/tbb/concurrent_set.h new file mode 100644 index 000000000..dd143311b --- /dev/null +++ b/src/tbb/include/oneapi/tbb/concurrent_set.h @@ -0,0 +1,267 @@ +/* + Copyright (c) 2019-2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_concurrent_set_H +#define __TBB_concurrent_set_H + +#include "detail/_namespace_injection.h" +#include "detail/_concurrent_skip_list.h" +#include "tbb_allocator.h" +#include +#include + +namespace tbb { +namespace detail { +namespace d2 { + +template +struct set_traits { + static constexpr std::size_t max_level = RandomGenerator::max_level; + using random_level_generator_type = RandomGenerator; + using key_type = Key; + using value_type = key_type; + using compare_type = KeyCompare; + using value_compare = compare_type; + using reference = value_type&; + using const_reference = const value_type&; + using allocator_type = Allocator; + + static constexpr bool allow_multimapping = AllowMultimapping; + + static const key_type& get_key(const_reference val) { + return val; + } + + static value_compare value_comp(compare_type comp) { return comp; } +}; // struct set_traits + +template +class concurrent_multiset; + +template , typename Allocator = tbb::tbb_allocator> +class concurrent_set : public concurrent_skip_list, Allocator, false>> { + using base_type = concurrent_skip_list, Allocator, false>>; +public: + using key_type = Key; + using value_type = typename base_type::value_type; + using size_type = typename base_type::size_type; + using difference_type = typename base_type::difference_type; + using key_compare = Compare; + using value_compare = typename base_type::value_compare; + using allocator_type = Allocator; + + using reference = typename base_type::reference; + using const_reference = typename base_type::const_reference; + using pointer = typename base_type::pointer; + using const_pointer = typename base_type::const_pointer; + + using iterator = typename base_type::iterator; + using const_iterator = typename base_type::const_iterator; + + using node_type = typename base_type::node_type; + + // Include constructors of base_type + using base_type::base_type; + + // Required for implicit deduction guides + concurrent_set() = default; + concurrent_set( const concurrent_set& ) = default; + concurrent_set( const concurrent_set& other, const allocator_type& alloc ) : base_type(other, alloc) {} + concurrent_set( concurrent_set&& ) = default; + concurrent_set( concurrent_set&& other, const allocator_type& alloc ) : base_type(std::move(other), alloc) {} + // Required to respect the rule of 5 + concurrent_set& operator=( const concurrent_set& ) = default; + concurrent_set& operator=( concurrent_set&& ) = default; + + concurrent_set& operator=( std::initializer_list il ) { + base_type::operator= (il); + return *this; + } + + template + void merge(concurrent_set& source) { + this->internal_merge(source); + } + + template + void merge(concurrent_set&& source) { + this->internal_merge(std::move(source)); + } + + template + void merge(concurrent_multiset& source) { + this->internal_merge(source); + } + + template + void merge(concurrent_multiset&& source) { + this->internal_merge(std::move(source)); + } +}; // class concurrent_set + +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +template >, + typename Alloc = tbb::tbb_allocator>, + typename = std::enable_if_t>, + typename = std::enable_if_t>, + typename = std::enable_if_t>> +concurrent_set( It, It, Comp = Comp(), Alloc = Alloc() ) +-> concurrent_set, Comp, Alloc>; + +template , + typename Alloc = tbb::tbb_allocator, + typename = std::enable_if_t>, + typename = std::enable_if_t>> +concurrent_set( std::initializer_list, Comp = Comp(), Alloc = Alloc() ) +-> concurrent_set; + +template >, + typename = std::enable_if_t>> +concurrent_set( It, It, Alloc ) +-> concurrent_set, + std::less>, Alloc>; + +template >> +concurrent_set( std::initializer_list, Alloc ) +-> concurrent_set, Alloc>; + +#endif // __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +template +void swap( concurrent_set& lhs, + concurrent_set& rhs ) +{ + lhs.swap(rhs); +} + +template , typename Allocator = tbb::tbb_allocator> +class concurrent_multiset : public concurrent_skip_list, Allocator, true>> { + using base_type = concurrent_skip_list, Allocator, true>>; +public: + using key_type = Key; + using value_type = typename base_type::value_type; + using size_type = typename base_type::size_type; + using difference_type = typename base_type::difference_type; + using key_compare = Compare; + using value_compare = typename base_type::value_compare; + using allocator_type = Allocator; + + using reference = typename base_type::reference; + using const_reference = typename base_type::const_reference; + using pointer = typename base_type::pointer; + using const_pointer = typename base_type::const_pointer; + + using iterator = typename base_type::iterator; + using const_iterator = typename base_type::const_iterator; + + using node_type = typename base_type::node_type; + + // Include constructors of base_type; + using base_type::base_type; + + // Required for implicit deduction guides + concurrent_multiset() = default; + concurrent_multiset( const concurrent_multiset& ) = default; + concurrent_multiset( const concurrent_multiset& other, const allocator_type& alloc ) : base_type(other, alloc) {} + concurrent_multiset( concurrent_multiset&& ) = default; + concurrent_multiset( concurrent_multiset&& other, const allocator_type& alloc ) : base_type(std::move(other), alloc) {} + // Required to respect the rule of 5 + concurrent_multiset& operator=( const concurrent_multiset& ) = default; + concurrent_multiset& operator=( concurrent_multiset&& ) = default; + + concurrent_multiset& operator=( std::initializer_list il ) { + base_type::operator= (il); + return *this; + } + + template + void merge(concurrent_set& source) { + this->internal_merge(source); + } + + template + void merge(concurrent_set&& source) { + this->internal_merge(std::move(source)); + } + + template + void merge(concurrent_multiset& source) { + this->internal_merge(source); + } + + template + void merge(concurrent_multiset&& source) { + this->internal_merge(std::move(source)); + } +}; // class concurrent_multiset + +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +template >, + typename Alloc = tbb::tbb_allocator>, + typename = std::enable_if_t>, + typename = std::enable_if_t>, + typename = std::enable_if_t>> +concurrent_multiset( It, It, Comp = Comp(), Alloc = Alloc() ) +-> concurrent_multiset, Comp, Alloc>; + +template , + typename Alloc = tbb::tbb_allocator, + typename = std::enable_if_t>, + typename = std::enable_if_t>> +concurrent_multiset( std::initializer_list, Comp = Comp(), Alloc = Alloc() ) +-> concurrent_multiset; + +template >, + typename = std::enable_if_t>> +concurrent_multiset( It, It, Alloc ) +-> concurrent_multiset, std::less>, Alloc>; + +template >> +concurrent_multiset( std::initializer_list, Alloc ) +-> concurrent_multiset, Alloc>; + +#endif // __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +template +void swap( concurrent_multiset& lhs, + concurrent_multiset& rhs ) +{ + lhs.swap(rhs); +} + +} // namespace d2 +} // namespace detail + +inline namespace v1 { + +using detail::d2::concurrent_set; +using detail::d2::concurrent_multiset; +using detail::split; + +} // inline namespace v1 +} // namespace tbb + +#endif // __TBB_concurrent_set_H diff --git a/src/tbb/include/oneapi/tbb/concurrent_unordered_map.h b/src/tbb/include/oneapi/tbb/concurrent_unordered_map.h new file mode 100644 index 000000000..9cade0a94 --- /dev/null +++ b/src/tbb/include/oneapi/tbb/concurrent_unordered_map.h @@ -0,0 +1,414 @@ +/* + Copyright (c) 2005-2024 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_concurrent_unordered_map_H +#define __TBB_concurrent_unordered_map_H + +#include "detail/_namespace_injection.h" +#include "detail/_concurrent_unordered_base.h" +#include "tbb_allocator.h" +#include + +namespace tbb { +namespace detail { +namespace d2 { + +template +struct concurrent_unordered_map_traits { + using value_type = std::pair; + using key_type = Key; + using allocator_type = Allocator; + using hash_compare_type = d1::hash_compare; + static constexpr bool allow_multimapping = AllowMultimapping; + + static constexpr const key_type& get_key( const value_type& value ) { + return value.first; + } +}; // struct concurrent_unordered_map_traits + +template +class concurrent_unordered_multimap; + +template , typename KeyEqual = std::equal_to, + typename Allocator = tbb::tbb_allocator> > +class concurrent_unordered_map + : public concurrent_unordered_base> +{ + using traits_type = concurrent_unordered_map_traits; + using base_type = concurrent_unordered_base; +public: + using key_type = typename base_type::key_type; + using mapped_type = T; + using value_type = typename base_type::value_type; + using size_type = typename base_type::size_type; + using difference_type = typename base_type::difference_type; + using hasher = typename base_type::hasher; + using key_equal = typename base_type::key_equal; + using allocator_type = typename base_type::allocator_type; + using reference = typename base_type::reference; + using const_reference = typename base_type::const_reference; + using pointer = typename base_type::pointer; + using const_pointer = typename base_type::const_pointer; + using iterator = typename base_type::iterator; + using const_iterator = typename base_type::const_iterator; + using local_iterator = typename base_type::local_iterator; + using const_local_iterator = typename base_type::const_local_iterator; + using node_type = typename base_type::node_type; + + // Include constructors of base type + using base_type::base_type; + + // Required for implicit deduction guides + concurrent_unordered_map() = default; + concurrent_unordered_map( const concurrent_unordered_map& ) = default; + concurrent_unordered_map( const concurrent_unordered_map& other, const allocator_type& alloc ) : base_type(other, alloc) {} + concurrent_unordered_map( concurrent_unordered_map&& ) = default; + concurrent_unordered_map( concurrent_unordered_map&& other, const allocator_type& alloc ) : base_type(std::move(other), alloc) {} + // Required to respect the rule of 5 + concurrent_unordered_map& operator=( const concurrent_unordered_map& ) = default; + concurrent_unordered_map& operator=( concurrent_unordered_map&& ) = default; + + concurrent_unordered_map& operator=( std::initializer_list il ) { + base_type::operator= (il); + return *this; + } + + // Observers + mapped_type& operator[]( const key_type& key ) { + iterator where = this->find(key); + + if (where == this->end()) { + where = this->emplace(std::piecewise_construct, std::forward_as_tuple(key), std::tuple<>()).first; + } + return where->second; + } + + mapped_type& operator[]( key_type&& key ) { + iterator where = this->find(key); + + if (where == this->end()) { + where = this->emplace(std::piecewise_construct, std::forward_as_tuple(std::move(key)), std::tuple<>()).first; + } + return where->second; + } + + mapped_type& at( const key_type& key ) { + iterator where = this->find(key); + + if (where == this->end()) { + throw_exception(exception_id::invalid_key); + } + return where->second; + } + + const mapped_type& at( const key_type& key ) const { + const_iterator where = this->find(key); + + if (where == this->end()) { + throw_exception(exception_id::out_of_range); + } + return where->second; + } + + using base_type::insert; + + template + typename std::enable_if::value, + std::pair>::type insert( P&& value ) { + return this->emplace(std::forward

(value)); + } + + template + typename std::enable_if::value, + iterator>::type insert( const_iterator hint, P&& value ) { + return this->emplace_hint(hint, std::forward

(value)); + } + + template + void merge( concurrent_unordered_map& source ) { + this->internal_merge(source); + } + + template + void merge( concurrent_unordered_map&& source ) { + this->internal_merge(std::move(source)); + } + + template + void merge( concurrent_unordered_multimap& source ) { + this->internal_merge(source); + } + + template + void merge( concurrent_unordered_multimap&& source ) { + this->internal_merge(std::move(source)); + } +}; // class concurrent_unordered_map + +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT +template >, + typename KeyEq = std::equal_to>, + typename Alloc = tbb::tbb_allocator>, + typename = std::enable_if_t>, + typename = std::enable_if_t>, + typename = std::enable_if_t>, + typename = std::enable_if_t>, + typename = std::enable_if_t>> +concurrent_unordered_map( It, It, std::size_t = {}, + Hash = Hash(), KeyEq = KeyEq(), Alloc = Alloc() ) +-> concurrent_unordered_map, iterator_mapped_t, Hash, KeyEq, Alloc>; + +template >, + typename KeyEq = std::equal_to>, + typename Alloc = tbb::tbb_allocator>, + typename = std::enable_if_t>, + typename = std::enable_if_t>, + typename = std::enable_if_t>, + typename = std::enable_if_t>> +concurrent_unordered_map( std::initializer_list>, std::size_t = {}, + Hash = Hash(), KeyEq = KeyEq(), Alloc = Alloc() ) +-> concurrent_unordered_map, T, Hash, KeyEq, Alloc>; + +template >, + typename = std::enable_if_t>> +concurrent_unordered_map( It, It, std::size_t, Alloc ) +-> concurrent_unordered_map, iterator_mapped_t, + std::hash>, + std::equal_to>, Alloc>; + +// TODO: investigate if a deduction guide for concurrent_unordered_map(It, It, Alloc) is needed + +template >, + typename = std::enable_if_t>, + typename = std::enable_if_t>, + typename = std::enable_if_t>> +concurrent_unordered_map( It, It, std::size_t, Hash, Alloc ) +-> concurrent_unordered_map, iterator_mapped_t, + Hash, std::equal_to>, Alloc>; + +template >> +concurrent_unordered_map( std::initializer_list>, std::size_t, Alloc ) +-> concurrent_unordered_map, T, std::hash>, + std::equal_to>, Alloc>; + +template >> +concurrent_unordered_map( std::initializer_list>, Alloc ) +-> concurrent_unordered_map, T, std::hash>, + std::equal_to>, Alloc>; + +template >, + typename = std::enable_if_t>, + typename = std::enable_if_t>> +concurrent_unordered_map( std::initializer_list>, std::size_t, Hash, Alloc ) +-> concurrent_unordered_map, T, Hash, + std::equal_to>, Alloc>; + +#if __APPLE__ && __TBB_CLANG_VERSION == 100000 +// An explicit deduction guide is required for copy/move constructor with allocator for APPLE LLVM 10.0.0 +// due to an issue with generating an implicit deduction guide for these constructors under several strange surcumstances. +// Currently the issue takes place because the last template parameter for Traits is boolean, it should not affect the deduction guides +// The issue reproduces only on this version of the compiler +template +concurrent_unordered_map( concurrent_unordered_map, Alloc ) +-> concurrent_unordered_map; +#endif + +#endif // __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +template +void swap( concurrent_unordered_map& lhs, + concurrent_unordered_map& rhs ) { + lhs.swap(rhs); +} + +template , typename KeyEqual = std::equal_to, + typename Allocator = tbb::tbb_allocator> > +class concurrent_unordered_multimap + : public concurrent_unordered_base> +{ + using traits_type = concurrent_unordered_map_traits; + using base_type = concurrent_unordered_base; +public: + using key_type = typename base_type::key_type; + using mapped_type = T; + using value_type = typename base_type::value_type; + using size_type = typename base_type::size_type; + using difference_type = typename base_type::difference_type; + using hasher = typename base_type::hasher; + using key_equal = typename base_type::key_equal; + using allocator_type = typename base_type::allocator_type; + using reference = typename base_type::reference; + using const_reference = typename base_type::const_reference; + using pointer = typename base_type::pointer; + using const_pointer = typename base_type::const_pointer; + using iterator = typename base_type::iterator; + using const_iterator = typename base_type::const_iterator; + using local_iterator = typename base_type::local_iterator; + using const_local_iterator = typename base_type::const_local_iterator; + using node_type = typename base_type::node_type; + + // Include constructors of base type + using base_type::base_type; + using base_type::insert; + + // Required for implicit deduction guides + concurrent_unordered_multimap() = default; + concurrent_unordered_multimap( const concurrent_unordered_multimap& ) = default; + concurrent_unordered_multimap( const concurrent_unordered_multimap& other, const allocator_type& alloc ) : base_type(other, alloc) {} + concurrent_unordered_multimap( concurrent_unordered_multimap&& ) = default; + concurrent_unordered_multimap( concurrent_unordered_multimap&& other, const allocator_type& alloc ) : base_type(std::move(other), alloc) {} + // Required to respect the rule of 5 + concurrent_unordered_multimap& operator=( const concurrent_unordered_multimap& ) = default; + concurrent_unordered_multimap& operator=( concurrent_unordered_multimap&& ) = default; + + concurrent_unordered_multimap& operator=( std::initializer_list il ) { + base_type::operator= (il); + return *this; + } + + template + typename std::enable_if::value, + std::pair>::type insert( P&& value ) { + return this->emplace(std::forward

(value)); + } + + template + typename std::enable_if::value, + iterator>::type insert( const_iterator hint, P&& value ) { + return this->emplace_hint(hint, std::forward(value)); + } + + template + void merge( concurrent_unordered_map& source ) { + this->internal_merge(source); + } + + template + void merge( concurrent_unordered_map&& source ) { + this->internal_merge(std::move(source)); + } + + template + void merge( concurrent_unordered_multimap& source ) { + this->internal_merge(source); + } + + template + void merge( concurrent_unordered_multimap&& source ) { + this->internal_merge(std::move(source)); + } +}; // class concurrent_unordered_multimap + +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +template >, + typename KeyEq = std::equal_to>, + typename Alloc = tbb::tbb_allocator>, + typename = std::enable_if_t>, + typename = std::enable_if_t>, + typename = std::enable_if_t>, + typename = std::enable_if_t>, + typename = std::enable_if_t>> +concurrent_unordered_multimap( It, It, std::size_t = {}, Hash = Hash(), KeyEq = KeyEq(), Alloc = Alloc() ) +-> concurrent_unordered_multimap, iterator_mapped_t, Hash, KeyEq, Alloc>; + +template >, + typename KeyEq = std::equal_to>, + typename Alloc = tbb::tbb_allocator>, + typename = std::enable_if_t>, + typename = std::enable_if_t>, + typename = std::enable_if_t>, + typename = std::enable_if_t>> +concurrent_unordered_multimap( std::initializer_list>, std::size_t = {}, + Hash = Hash(), KeyEq = KeyEq(), Alloc = Alloc() ) +-> concurrent_unordered_multimap, T, Hash, KeyEq, Alloc>; + +template >, + typename = std::enable_if_t>> +concurrent_unordered_multimap( It, It, std::size_t, Alloc ) +-> concurrent_unordered_multimap, iterator_mapped_t, + std::hash>, + std::equal_to>, Alloc>; + +template >, + typename = std::enable_if_t>, + typename = std::enable_if_t>, + typename = std::enable_if_t>> +concurrent_unordered_multimap( It, It, std::size_t, Hash, Alloc ) +-> concurrent_unordered_multimap, iterator_mapped_t, Hash, + std::equal_to>, Alloc>; + +template >> +concurrent_unordered_multimap( std::initializer_list>, std::size_t, Alloc ) +-> concurrent_unordered_multimap, T, std::hash>, + std::equal_to>, Alloc>; + +template >> +concurrent_unordered_multimap( std::initializer_list>, Alloc ) +-> concurrent_unordered_multimap, T, std::hash>, + std::equal_to>, Alloc>; + +template >, + typename = std::enable_if_t>, + typename = std::enable_if_t>> +concurrent_unordered_multimap( std::initializer_list>, std::size_t, Hash, Alloc ) +-> concurrent_unordered_multimap, T, Hash, + std::equal_to>, Alloc>; + +#if __APPLE__ && __TBB_CLANG_VERSION == 100000 +// An explicit deduction guide is required for copy/move constructor with allocator for APPLE LLVM 10.0.0 +// due to an issue with generating an implicit deduction guide for these constructors under several strange surcumstances. +// Currently the issue takes place because the last template parameter for Traits is boolean, it should not affect the deduction guides +// The issue reproduces only on this version of the compiler +template +concurrent_unordered_multimap( concurrent_unordered_multimap, Alloc ) +-> concurrent_unordered_multimap; +#endif +#endif // __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +template +void swap( concurrent_unordered_multimap& lhs, + concurrent_unordered_multimap& rhs ) { + lhs.swap(rhs); +} + +} // namespace d2 +} // namespace detail + +inline namespace v1 { + +using detail::d2::concurrent_unordered_map; +using detail::d2::concurrent_unordered_multimap; +using detail::split; + +} // inline namespace v1 +} // namespace tbb + +#endif // __TBB_concurrent_unordered_map_H diff --git a/src/tbb/include/oneapi/tbb/concurrent_unordered_set.h b/src/tbb/include/oneapi/tbb/concurrent_unordered_set.h new file mode 100644 index 000000000..b7e4b4caf --- /dev/null +++ b/src/tbb/include/oneapi/tbb/concurrent_unordered_set.h @@ -0,0 +1,333 @@ +/* + Copyright (c) 2005-2024 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_concurrent_unordered_set_H +#define __TBB_concurrent_unordered_set_H + +#include "detail/_namespace_injection.h" +#include "detail/_concurrent_unordered_base.h" +#include "tbb_allocator.h" + +namespace tbb { +namespace detail { +namespace d2 { + +template +struct concurrent_unordered_set_traits { + using key_type = Key; + using value_type = key_type; + using allocator_type = Allocator; + using hash_compare_type = d1::hash_compare; + static constexpr bool allow_multimapping = AllowMultimapping; + + static constexpr const key_type& get_key( const value_type& value ) { + return value; + } +}; // class concurrent_unordered_set_traits + +template +class concurrent_unordered_multiset; + +template , typename KeyEqual = std::equal_to, + typename Allocator = tbb::tbb_allocator> +class concurrent_unordered_set + : public concurrent_unordered_base> +{ + using traits_type = concurrent_unordered_set_traits; + using base_type = concurrent_unordered_base; +public: + using key_type = typename base_type::key_type; + using value_type = typename base_type::value_type; + using size_type = typename base_type::size_type; + using difference_type = typename base_type::difference_type; + using hasher = typename base_type::hasher; + using key_equal = typename base_type::key_equal; + using allocator_type = typename base_type::allocator_type; + using reference = typename base_type::reference; + using const_reference = typename base_type::const_reference; + using pointer = typename base_type::pointer; + using const_pointer = typename base_type::const_pointer; + using iterator = typename base_type::iterator; + using const_iterator = typename base_type::const_iterator; + using local_iterator = typename base_type::local_iterator; + using const_local_iterator = typename base_type::const_local_iterator; + using node_type = typename base_type::node_type; + + // Include constructors of base_type; + using base_type::base_type; + + // Required for implicit deduction guides + concurrent_unordered_set() = default; + concurrent_unordered_set( const concurrent_unordered_set& ) = default; + concurrent_unordered_set( const concurrent_unordered_set& other, const allocator_type& alloc ) : base_type(other, alloc) {} + concurrent_unordered_set( concurrent_unordered_set&& ) = default; + concurrent_unordered_set( concurrent_unordered_set&& other, const allocator_type& alloc ) : base_type(std::move(other), alloc) {} + // Required to respect the rule of 5 + concurrent_unordered_set& operator=( const concurrent_unordered_set& ) = default; + concurrent_unordered_set& operator=( concurrent_unordered_set&& ) = default; + + concurrent_unordered_set& operator=( std::initializer_list il ) { + base_type::operator= (il); + return *this; + } + + template + void merge( concurrent_unordered_set& source ) { + this->internal_merge(source); + } + + template + void merge( concurrent_unordered_set&& source ) { + this->internal_merge(std::move(source)); + } + + template + void merge( concurrent_unordered_multiset& source ) { + this->internal_merge(source); + } + + template + void merge( concurrent_unordered_multiset&& source ) { + this->internal_merge(std::move(source)); + } +}; // class concurrent_unordered_set + +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +template >, + typename KeyEq = std::equal_to>, + typename Alloc = tbb::tbb_allocator>, + typename = std::enable_if_t>, + typename = std::enable_if_t>, + typename = std::enable_if_t>, + typename = std::enable_if_t>, + typename = std::enable_if_t>> +concurrent_unordered_set( It, It, std::size_t = {}, Hash = Hash(), KeyEq = KeyEq(), Alloc = Alloc() ) +-> concurrent_unordered_set, Hash, KeyEq, Alloc>; + +template , + typename KeyEq = std::equal_to, + typename Alloc = tbb::tbb_allocator, + typename = std::enable_if_t>, + typename = std::enable_if_t>, + typename = std::enable_if_t>, + typename = std::enable_if_t>> +concurrent_unordered_set( std::initializer_list, std::size_t = {}, + Hash = Hash(), KeyEq = KeyEq(), Alloc = Alloc() ) +-> concurrent_unordered_set; + +template >, + typename = std::enable_if_t>> +concurrent_unordered_set( It, It, std::size_t, Alloc ) +-> concurrent_unordered_set, std::hash>, + std::equal_to>, Alloc>; + +template >, + typename = std::enable_if_t>, + typename = std::enable_if_t>, + typename = std::enable_if_t>> +concurrent_unordered_set( It, It, std::size_t, Hash, Alloc ) +-> concurrent_unordered_set, Hash, std::equal_to>, Alloc>; + +template >> +concurrent_unordered_set( std::initializer_list, std::size_t, Alloc ) +-> concurrent_unordered_set, std::equal_to, Alloc>; + +template >> +concurrent_unordered_set( std::initializer_list, Alloc ) +-> concurrent_unordered_set, std::equal_to, Alloc>; + +template >, + typename = std::enable_if_t>, + typename = std::enable_if_t>> +concurrent_unordered_set( std::initializer_list, std::size_t, Hash, Alloc ) +-> concurrent_unordered_set, Alloc>; + +#if __APPLE__ && __TBB_CLANG_VERSION == 100000 +// An explicit deduction guide is required for copy/move constructor with allocator for APPLE LLVM 10.0.0 +// due to an issue with generating an implicit deduction guide for these constructors under several strange surcumstances. +// Currently the issue takes place because the last template parameter for Traits is boolean, it should not affect the deduction guides +// The issue reproduces only on this version of the compiler +template +concurrent_unordered_set( concurrent_unordered_set, Alloc ) +-> concurrent_unordered_set; +#endif +#endif // __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +template +void swap( concurrent_unordered_set& lhs, + concurrent_unordered_set& rhs ) { + lhs.swap(rhs); +} + +template , typename KeyEqual = std::equal_to, + typename Allocator = tbb::tbb_allocator> +class concurrent_unordered_multiset + : public concurrent_unordered_base> +{ + using traits_type = concurrent_unordered_set_traits; + using base_type = concurrent_unordered_base; +public: + using key_type = typename base_type::key_type; + using value_type = typename base_type::value_type; + using size_type = typename base_type::size_type; + using difference_type = typename base_type::difference_type; + using hasher = typename base_type::hasher; + using key_equal = typename base_type::key_equal; + using allocator_type = typename base_type::allocator_type; + using reference = typename base_type::reference; + using const_reference = typename base_type::const_reference; + using pointer = typename base_type::pointer; + using const_pointer = typename base_type::const_pointer; + using iterator = typename base_type::iterator; + using const_iterator = typename base_type::const_iterator; + using local_iterator = typename base_type::local_iterator; + using const_local_iterator = typename base_type::const_local_iterator; + using node_type = typename base_type::node_type; + + // Include constructors of base_type; + using base_type::base_type; + + // Required for implicit deduction guides + concurrent_unordered_multiset() = default; + concurrent_unordered_multiset( const concurrent_unordered_multiset& ) = default; + concurrent_unordered_multiset( const concurrent_unordered_multiset& other, const allocator_type& alloc ) : base_type(other, alloc) {} + concurrent_unordered_multiset( concurrent_unordered_multiset&& ) = default; + concurrent_unordered_multiset( concurrent_unordered_multiset&& other, const allocator_type& alloc ) : base_type(std::move(other), alloc) {} + // Required to respect the rule of 5 + concurrent_unordered_multiset& operator=( const concurrent_unordered_multiset& ) = default; + concurrent_unordered_multiset& operator=( concurrent_unordered_multiset&& ) = default; + + concurrent_unordered_multiset& operator=( std::initializer_list il ) { + base_type::operator= (il); + return *this; + } + + template + void merge( concurrent_unordered_set& source ) { + this->internal_merge(source); + } + + template + void merge( concurrent_unordered_set&& source ) { + this->internal_merge(std::move(source)); + } + + template + void merge( concurrent_unordered_multiset& source ) { + this->internal_merge(source); + } + + template + void merge( concurrent_unordered_multiset&& source ) { + this->internal_merge(std::move(source)); + } +}; // class concurrent_unordered_multiset + +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT +template >, + typename KeyEq = std::equal_to>, + typename Alloc = tbb::tbb_allocator>, + typename = std::enable_if_t>, + typename = std::enable_if_t>, + typename = std::enable_if_t>, + typename = std::enable_if_t>, + typename = std::enable_if_t>> +concurrent_unordered_multiset( It, It, std::size_t = {}, Hash = Hash(), KeyEq = KeyEq(), Alloc = Alloc() ) +-> concurrent_unordered_multiset, Hash, KeyEq, Alloc>; + +template , + typename KeyEq = std::equal_to, + typename Alloc = tbb::tbb_allocator, + typename = std::enable_if_t>, + typename = std::enable_if_t>, + typename = std::enable_if_t>, + typename = std::enable_if_t>> +concurrent_unordered_multiset( std::initializer_list, std::size_t = {}, + Hash = Hash(), KeyEq = KeyEq(), Alloc = Alloc() ) +-> concurrent_unordered_multiset; + +template >, + typename = std::enable_if_t>> +concurrent_unordered_multiset( It, It, std::size_t, Alloc ) +-> concurrent_unordered_multiset, std::hash>, + std::equal_to>, Alloc>; + +template >, + typename = std::enable_if_t>, + typename = std::enable_if_t>, + typename = std::enable_if_t>> +concurrent_unordered_multiset( It, It, std::size_t, Hash, Alloc ) +-> concurrent_unordered_multiset, Hash, std::equal_to>, Alloc>; + +template >> +concurrent_unordered_multiset( std::initializer_list, std::size_t, Alloc ) +-> concurrent_unordered_multiset, std::equal_to, Alloc>; + +template >> +concurrent_unordered_multiset( std::initializer_list, Alloc ) +-> concurrent_unordered_multiset, std::equal_to, Alloc>; + +template >, + typename = std::enable_if_t>, + typename = std::enable_if_t>> +concurrent_unordered_multiset( std::initializer_list, std::size_t, Hash, Alloc ) +-> concurrent_unordered_multiset, Alloc>; + +#if __APPLE__ && __TBB_CLANG_VERSION == 100000 +// An explicit deduction guide is required for copy/move constructor with allocator for APPLE LLVM 10.0.0 +// due to an issue with generating an implicit deduction guide for these constructors under several strange surcumstances. +// Currently the issue takes place because the last template parameter for Traits is boolean, it should not affect the deduction guides +// The issue reproduces only on this version of the compiler +template +concurrent_unordered_multiset( concurrent_unordered_multiset, Alloc ) +-> concurrent_unordered_multiset; +#endif +#endif // __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +template +void swap( concurrent_unordered_multiset& lhs, + concurrent_unordered_multiset& rhs ) { + lhs.swap(rhs); +} + +} // namespace d2 +} // namespace detail + +inline namespace v1 { + +using detail::d2::concurrent_unordered_set; +using detail::d2::concurrent_unordered_multiset; +using detail::split; + +} // inline namespace v1 +} // namespace tbb + +#endif // __TBB_concurrent_unordered_set_H diff --git a/src/tbb/include/oneapi/tbb/concurrent_vector.h b/src/tbb/include/oneapi/tbb/concurrent_vector.h new file mode 100644 index 000000000..2a2cb1e4b --- /dev/null +++ b/src/tbb/include/oneapi/tbb/concurrent_vector.h @@ -0,0 +1,1129 @@ +/* + Copyright (c) 2005-2022 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_concurrent_vector_H +#define __TBB_concurrent_vector_H + +#include "detail/_namespace_injection.h" +#include "detail/_utils.h" +#include "detail/_assert.h" +#include "detail/_allocator_traits.h" +#include "detail/_segment_table.h" +#include "detail/_containers_helpers.h" +#include "blocked_range.h" +#include "cache_aligned_allocator.h" + +#include +#include // std::move_if_noexcept +#include +#if __TBB_CPP20_COMPARISONS_PRESENT +#include +#endif + +namespace tbb { +namespace detail { +namespace d1 { + +template +class vector_iterator { + using vector_type = Vector; + +public: + using value_type = Value; + using size_type = typename vector_type::size_type; + using difference_type = typename vector_type::difference_type; + using pointer = value_type*; + using reference = value_type&; + using iterator_category = std::random_access_iterator_tag; + + template + friend vector_iterator operator+( typename vector_iterator::difference_type, const vector_iterator& ); + + template + friend typename vector_iterator::difference_type operator-( const vector_iterator&, const vector_iterator& ); + + template + friend bool operator==( const vector_iterator&, const vector_iterator& ); + + template + friend bool operator<( const vector_iterator&, const vector_iterator& ); + + template + friend class vector_iterator; + + template + friend class concurrent_vector; + +private: + vector_iterator( const vector_type& vector, size_type index, value_type* item = nullptr ) + : my_vector(const_cast(&vector)), my_index(index), my_item(item) + {} + +public: + vector_iterator() : my_vector(nullptr), my_index(~size_type(0)), my_item(nullptr) + {} + + vector_iterator( const vector_iterator& other ) + : my_vector(other.my_vector), my_index(other.my_index), my_item(other.my_item) + {} + + vector_iterator& operator=( const vector_iterator& other ) { + my_vector = other.my_vector; + my_index = other.my_index; + my_item = other.my_item; + return *this; + } + + vector_iterator operator+( difference_type offset ) const { + return vector_iterator(*my_vector, my_index + offset); + } + + vector_iterator& operator+=( difference_type offset ) { + my_index += offset; + my_item = nullptr; + return *this; + } + + vector_iterator operator-( difference_type offset ) const { + return vector_iterator(*my_vector, my_index - offset); + } + + vector_iterator& operator-=( difference_type offset ) { + my_index -= offset; + my_item = nullptr; + return *this; + } + + reference operator*() const { + value_type *item = my_item; + if (item == nullptr) { + item = &my_vector->internal_subscript(my_index); + } else { + __TBB_ASSERT(item == &my_vector->internal_subscript(my_index), "corrupt cache"); + } + return *item; + } + + pointer operator->() const { return &(operator*()); } + + reference operator[]( difference_type k ) const { + return my_vector->internal_subscript(my_index + k); + } + + vector_iterator& operator++() { + ++my_index; + if (my_item != nullptr) { + if (vector_type::is_first_element_in_segment(my_index)) { + // If the iterator crosses a segment boundary, the pointer become invalid + // as possibly next segment is in another memory location + my_item = nullptr; + } else { + ++my_item; + } + } + return *this; + } + + vector_iterator operator++(int) { + vector_iterator result = *this; + ++(*this); + return result; + } + + vector_iterator& operator--() { + __TBB_ASSERT(my_index > 0, "operator--() applied to iterator already at beginning of concurrent_vector"); + --my_index; + if (my_item != nullptr) { + if (vector_type::is_first_element_in_segment(my_index)) { + // If the iterator crosses a segment boundary, the pointer become invalid + // as possibly next segment is in another memory location + my_item = nullptr; + } else { + --my_item; + } + } + return *this; + } + + vector_iterator operator--(int) { + vector_iterator result = *this; + --(*this); + return result; + } + +private: + // concurrent_vector over which we are iterating. + vector_type* my_vector; + + // Index into the vector + size_type my_index; + + // Caches my_vector *it; + // If my_item == nullptr cached value is not available use internal_subscript(my_index) + mutable value_type* my_item; +}; // class vector_iterator + +template +vector_iterator operator+( typename vector_iterator::difference_type offset, + const vector_iterator& v ) +{ + return vector_iterator(*v.my_vector, v.my_index + offset); +} + +template +typename vector_iterator::difference_type operator-( const vector_iterator& i, + const vector_iterator& j ) +{ + using difference_type = typename vector_iterator::difference_type; + return static_cast(i.my_index) - static_cast(j.my_index); +} + +template +bool operator==( const vector_iterator& i, const vector_iterator& j ) { + return i.my_vector == j.my_vector && i.my_index == j.my_index; +} + +template +bool operator!=( const vector_iterator& i, const vector_iterator& j ) { + return !(i == j); +} + +template +bool operator<( const vector_iterator& i, const vector_iterator& j ) { + return i.my_index < j.my_index; +} + +template +bool operator>( const vector_iterator& i, const vector_iterator& j ) { + return j < i; +} + +template +bool operator>=( const vector_iterator& i, const vector_iterator& j ) { + return !(i < j); +} + +template +bool operator<=( const vector_iterator& i, const vector_iterator& j ) { + return !(j < i); +} + +static constexpr std::size_t embedded_table_num_segments = 3; + +template > +class concurrent_vector + : private segment_table, embedded_table_num_segments> +{ + using self_type = concurrent_vector; + using base_type = segment_table; + + friend class segment_table; + + template + class generic_range_type : public tbb::blocked_range { + using base_type = tbb::blocked_range; + public: + using value_type = T; + using reference = T&; + using const_reference = const T&; + using iterator = Iterator; + using difference_type = std::ptrdiff_t; + + using base_type::base_type; + + template + generic_range_type( const generic_range_type& r) : blocked_range(r.begin(), r.end(), r.grainsize()) {} + generic_range_type( generic_range_type& r, split ) : blocked_range(r, split()) {} + }; // class generic_range_type + + static_assert(std::is_same::value, + "value_type of the container must be the same as its allocator's"); + using allocator_traits_type = tbb::detail::allocator_traits; + // Segment table for concurrent_vector can be extended + static constexpr bool allow_table_extending = true; + static constexpr bool is_noexcept_assignment = allocator_traits_type::propagate_on_container_move_assignment::value || + allocator_traits_type::is_always_equal::value; + static constexpr bool is_noexcept_swap = allocator_traits_type::propagate_on_container_swap::value || + allocator_traits_type::is_always_equal::value; + +public: + using value_type = T; + using allocator_type = Allocator; + using size_type = std::size_t; + using difference_type = std::ptrdiff_t; + using reference = value_type&; + using const_reference = const value_type&; + + using pointer = typename allocator_traits_type::pointer; + using const_pointer = typename allocator_traits_type::const_pointer; + + using iterator = vector_iterator; + using const_iterator = vector_iterator; + using reverse_iterator = std::reverse_iterator; + using const_reverse_iterator = std::reverse_iterator; + + using range_type = generic_range_type; + using const_range_type = generic_range_type; + + concurrent_vector() : concurrent_vector(allocator_type()) {} + + explicit concurrent_vector( const allocator_type& alloc ) noexcept + : base_type(alloc) + {} + + explicit concurrent_vector( size_type count, const value_type& value, + const allocator_type& alloc = allocator_type() ) + : concurrent_vector(alloc) + { + try_call( [&] { + grow_by(count, value); + } ).on_exception( [&] { + base_type::clear(); + }); + } + + explicit concurrent_vector( size_type count, const allocator_type& alloc = allocator_type() ) + : concurrent_vector(alloc) + { + try_call( [&] { + grow_by(count); + } ).on_exception( [&] { + base_type::clear(); + }); + } + + template + concurrent_vector( InputIterator first, InputIterator last, const allocator_type& alloc = allocator_type() ) + : concurrent_vector(alloc) + { + try_call( [&] { + grow_by(first, last); + } ).on_exception( [&] { + base_type::clear(); + }); + } + + concurrent_vector( const concurrent_vector& other ) + : base_type(segment_table_allocator_traits::select_on_container_copy_construction(other.get_allocator())) + { + try_call( [&] { + grow_by(other.begin(), other.end()); + } ).on_exception( [&] { + base_type::clear(); + }); + } + + concurrent_vector( const concurrent_vector& other, const allocator_type& alloc ) + : base_type(other, alloc) {} + + concurrent_vector(concurrent_vector&& other) noexcept + : base_type(std::move(other)) + {} + + concurrent_vector( concurrent_vector&& other, const allocator_type& alloc ) + : base_type(std::move(other), alloc) + {} + + concurrent_vector( std::initializer_list init, + const allocator_type& alloc = allocator_type() ) + : concurrent_vector(init.begin(), init.end(), alloc) + {} + + ~concurrent_vector() {} + + // Assignment + concurrent_vector& operator=( const concurrent_vector& other ) { + base_type::operator=(other); + return *this; + } + + concurrent_vector& operator=( concurrent_vector&& other ) noexcept(is_noexcept_assignment) { + base_type::operator=(std::move(other)); + return *this; + } + + concurrent_vector& operator=( std::initializer_list init ) { + assign(init); + return *this; + } + + void assign( size_type count, const value_type& value ) { + destroy_elements(); + grow_by(count, value); + } + + template + typename std::enable_if::value, void>::type + assign( InputIterator first, InputIterator last ) { + destroy_elements(); + grow_by(first, last); + } + + void assign( std::initializer_list init ) { + destroy_elements(); + assign(init.begin(), init.end()); + } + + // Concurrent growth + iterator grow_by( size_type delta ) { + return internal_grow_by_delta(delta); + } + + iterator grow_by( size_type delta, const value_type& value ) { + return internal_grow_by_delta(delta, value); + } + + template + typename std::enable_if::value, iterator>::type + grow_by( ForwardIterator first, ForwardIterator last ) { + auto delta = std::distance(first, last); + return internal_grow_by_delta(delta, first, last); + } + + iterator grow_by( std::initializer_list init ) { + return grow_by(init.begin(), init.end()); + } + + iterator grow_to_at_least( size_type n ) { + return internal_grow_to_at_least(n); + } + iterator grow_to_at_least( size_type n, const value_type& value ) { + return internal_grow_to_at_least(n, value); + } + + iterator push_back( const value_type& item ) { + return internal_emplace_back(item); + } + + iterator push_back( value_type&& item ) { + return internal_emplace_back(std::move(item)); + } + + template + iterator emplace_back( Args&&... args ) { + return internal_emplace_back(std::forward(args)...); + } + + // Items access + reference operator[]( size_type index ) { + return internal_subscript(index); + } + const_reference operator[]( size_type index ) const { + return internal_subscript(index); + } + + reference at( size_type index ) { + return internal_subscript_with_exceptions(index); + } + const_reference at( size_type index ) const { + return internal_subscript_with_exceptions(index); + } + + // Get range for iterating with parallel algorithms + range_type range( size_t grainsize = 1 ) { + return range_type(begin(), end(), grainsize); + } + + // Get const range for iterating with parallel algorithms + const_range_type range( size_t grainsize = 1 ) const { + return const_range_type(begin(), end(), grainsize); + } + + reference front() { + return internal_subscript(0); + } + + const_reference front() const { + return internal_subscript(0); + } + + reference back() { + return internal_subscript(size() - 1); + } + + const_reference back() const { + return internal_subscript(size() - 1); + } + + // Iterators + iterator begin() { return iterator(*this, 0); } + const_iterator begin() const { return const_iterator(*this, 0); } + const_iterator cbegin() const { return const_iterator(*this, 0); } + + iterator end() { return iterator(*this, size()); } + const_iterator end() const { return const_iterator(*this, size()); } + const_iterator cend() const { return const_iterator(*this, size()); } + + reverse_iterator rbegin() { return reverse_iterator(end()); } + const_reverse_iterator rbegin() const { return const_reverse_iterator(end()); } + const_reverse_iterator crbegin() const { return const_reverse_iterator(cend()); } + + reverse_iterator rend() { return reverse_iterator(begin()); } + const_reverse_iterator rend() const { return const_reverse_iterator(begin()); } + const_reverse_iterator crend() const { return const_reverse_iterator(cbegin()); } + + allocator_type get_allocator() const { + return base_type::get_allocator(); + } + + // Storage + bool empty() const noexcept { + return 0 == size(); + } + + size_type size() const noexcept { + return std::min(this->my_size.load(std::memory_order_acquire), capacity()); + } + + size_type max_size() const noexcept { + return allocator_traits_type::max_size(base_type::get_allocator()); + } + + size_type capacity() const noexcept { + return base_type::capacity(); + } + + void reserve( size_type n ) { + if (n == 0) return; + + if (n > max_size()) { + tbb::detail::throw_exception(exception_id::reservation_length_error); + } + + this->assign_first_block_if_necessary(this->segment_index_of(n - 1) + 1); + base_type::reserve(n); + } + + void resize( size_type n ) { + internal_resize(n); + } + + void resize( size_type n, const value_type& val ) { + internal_resize(n, val); + } + + void shrink_to_fit() { + internal_compact(); + } + + void swap(concurrent_vector& other) noexcept(is_noexcept_swap) { + base_type::swap(other); + } + + void clear() { + destroy_elements(); + } + +private: + using segment_type = typename base_type::segment_type; + using segment_table_type = typename base_type::segment_table_type; + using segment_table_allocator_traits = typename base_type::segment_table_allocator_traits; + using segment_index_type = typename base_type::segment_index_type; + + using segment_element_type = typename base_type::value_type; + using segment_element_allocator_type = typename allocator_traits_type::template rebind_alloc; + using segment_element_allocator_traits = tbb::detail::allocator_traits; + + segment_table_type allocate_long_table( const typename base_type::atomic_segment* embedded_table, size_type start_index ) { + __TBB_ASSERT(start_index <= this->embedded_table_size, "Start index out of embedded table"); + + // If other threads are trying to set pointers in the short segment, wait for them to finish their + // assignments before we copy the short segment to the long segment. Note: grow_to_at_least depends on it + for (segment_index_type i = 0; this->segment_base(i) < start_index; ++i) { + spin_wait_while_eq(embedded_table[i], segment_type(nullptr)); + } + + // It is possible that the table was extend by a thread allocating first_block, need to check this. + if (this->get_table() != embedded_table) { + return nullptr; + } + + // Allocate long segment table and fill with null pointers + segment_table_type new_segment_table = segment_table_allocator_traits::allocate(base_type::get_allocator(), this->pointers_per_long_table); + // Copy segment pointers from the embedded table + for (size_type segment_index = 0; segment_index < this->pointers_per_embedded_table; ++segment_index) { + segment_table_allocator_traits::construct(base_type::get_allocator(), &new_segment_table[segment_index], + embedded_table[segment_index].load(std::memory_order_relaxed)); + } + for (size_type segment_index = this->pointers_per_embedded_table; segment_index < this->pointers_per_long_table; ++segment_index) { + segment_table_allocator_traits::construct(base_type::get_allocator(), &new_segment_table[segment_index], nullptr); + } + + return new_segment_table; + } + + // create_segment function is required by the segment_table base class + segment_type create_segment( segment_table_type table, segment_index_type seg_index, size_type index ) { + size_type first_block = this->my_first_block.load(std::memory_order_relaxed); + // First block allocation + if (seg_index < first_block) { + // If 0 segment is already allocated, then it remains to wait until the segments are filled to requested + if (table[0].load(std::memory_order_acquire) != nullptr) { + spin_wait_while_eq(table[seg_index], segment_type(nullptr)); + return nullptr; + } + + segment_element_allocator_type segment_allocator(base_type::get_allocator()); + segment_type new_segment = nullptr; + size_type first_block_size = this->segment_size(first_block); + try_call( [&] { + new_segment = segment_element_allocator_traits::allocate(segment_allocator, first_block_size); + } ).on_exception( [&] { + segment_type disabled_segment = nullptr; + if (table[0].compare_exchange_strong(disabled_segment, this->segment_allocation_failure_tag)) { + size_type end_segment = table == this->my_embedded_table ? this->pointers_per_embedded_table : first_block; + for (size_type i = 1; i < end_segment; ++i) { + table[i].store(this->segment_allocation_failure_tag, std::memory_order_release); + } + } + }); + + segment_type disabled_segment = nullptr; + if (table[0].compare_exchange_strong(disabled_segment, new_segment)) { + this->extend_table_if_necessary(table, 0, first_block_size); + for (size_type i = 1; i < first_block; ++i) { + table[i].store(new_segment, std::memory_order_release); + } + + // Other threads can wait on a snapshot of an embedded table, need to fill it. + for (size_type i = 1; i < first_block && i < this->pointers_per_embedded_table; ++i) { + this->my_embedded_table[i].store(new_segment, std::memory_order_release); + } + } else if (new_segment != this->segment_allocation_failure_tag) { + // Deallocate the memory + segment_element_allocator_traits::deallocate(segment_allocator, new_segment, first_block_size); + // 0 segment is already allocated, then it remains to wait until the segments are filled to requested + spin_wait_while_eq(table[seg_index], segment_type(nullptr)); + } + } else { + size_type offset = this->segment_base(seg_index); + if (index == offset) { + __TBB_ASSERT(table[seg_index].load(std::memory_order_relaxed) == nullptr, "Only this thread can enable this segment"); + segment_element_allocator_type segment_allocator(base_type::get_allocator()); + segment_type new_segment = this->segment_allocation_failure_tag; + try_call( [&] { + new_segment = segment_element_allocator_traits::allocate(segment_allocator,this->segment_size(seg_index)); + // Shift base address to simplify access by index + new_segment -= this->segment_base(seg_index); + } ).on_completion( [&] { + table[seg_index].store(new_segment, std::memory_order_release); + }); + } else { + spin_wait_while_eq(table[seg_index], segment_type(nullptr)); + } + } + return nullptr; + } + + // Returns the number of elements in the segment to be destroy + size_type number_of_elements_in_segment( segment_index_type seg_index ) { + size_type curr_vector_size = this->my_size.load(std::memory_order_relaxed); + size_type curr_segment_base = this->segment_base(seg_index); + + if (seg_index == 0) { + return std::min(curr_vector_size, this->segment_size(seg_index)); + } else { + // Perhaps the segment is allocated, but there are no elements in it. + if (curr_vector_size < curr_segment_base) { + return 0; + } + return curr_segment_base * 2 > curr_vector_size ? curr_vector_size - curr_segment_base : curr_segment_base; + } + } + + segment_type nullify_segment( segment_table_type table, size_type segment_index ) { + segment_type target_segment = table[segment_index].load(std::memory_order_relaxed); + if (segment_index >= this->my_first_block) { + table[segment_index].store(nullptr, std::memory_order_relaxed); + } else { + if (segment_index == 0) { + for (size_type i = 0; i < this->my_first_block; ++i) { + table[i].store(nullptr, std::memory_order_relaxed); + } + } + } + + return target_segment; + } + + void deallocate_segment( segment_type address, segment_index_type seg_index ) { + segment_element_allocator_type segment_allocator(base_type::get_allocator()); + size_type first_block = this->my_first_block.load(std::memory_order_relaxed); + if (seg_index >= first_block) { + segment_element_allocator_traits::deallocate(segment_allocator, address, this->segment_size(seg_index)); + } + else if (seg_index == 0) { + size_type elements_to_deallocate = first_block > 0 ? this->segment_size(first_block) : this->segment_size(0); + segment_element_allocator_traits::deallocate(segment_allocator, address, elements_to_deallocate); + } + } + + // destroy_segment function is required by the segment_table base class + void destroy_segment( segment_type address, segment_index_type seg_index ) { + size_type elements_to_destroy = number_of_elements_in_segment(seg_index); + segment_element_allocator_type segment_allocator(base_type::get_allocator()); + + for (size_type i = 0; i < elements_to_destroy; ++i) { + segment_element_allocator_traits::destroy(segment_allocator, address + i); + } + + deallocate_segment(address, seg_index); + } + + // copy_segment function is required by the segment_table base class + void copy_segment( segment_index_type seg_index, segment_type from, segment_type to ) { + size_type i = 0; + try_call( [&] { + for (; i != number_of_elements_in_segment(seg_index); ++i) { + segment_table_allocator_traits::construct(base_type::get_allocator(), to + i, from[i]); + } + } ).on_exception( [&] { + // Zero-initialize items left not constructed after the exception + zero_unconstructed_elements(this->get_segment(seg_index) + i, this->segment_size(seg_index) - i); + + segment_index_type last_segment = this->segment_index_of(this->my_size.load(std::memory_order_relaxed)); + auto table = this->get_table(); + for (segment_index_type j = seg_index + 1; j != last_segment; ++j) { + auto curr_segment = table[j].load(std::memory_order_relaxed); + if (curr_segment) { + zero_unconstructed_elements(curr_segment + this->segment_base(j), this->segment_size(j)); + } + } + this->my_size.store(this->segment_size(seg_index) + i, std::memory_order_relaxed); + }); + } + + // move_segment function is required by the segment_table base class + void move_segment( segment_index_type seg_index, segment_type from, segment_type to ) { + size_type i = 0; + try_call( [&] { + for (; i != number_of_elements_in_segment(seg_index); ++i) { + segment_table_allocator_traits::construct(base_type::get_allocator(), to + i, std::move(from[i])); + } + } ).on_exception( [&] { + // Zero-initialize items left not constructed after the exception + zero_unconstructed_elements(this->get_segment(seg_index) + i, this->segment_size(seg_index) - i); + + segment_index_type last_segment = this->segment_index_of(this->my_size.load(std::memory_order_relaxed)); + auto table = this->get_table(); + for (segment_index_type j = seg_index + 1; j != last_segment; ++j) { + auto curr_segment = table[j].load(std::memory_order_relaxed); + if (curr_segment) { + zero_unconstructed_elements(curr_segment + this->segment_base(j), this->segment_size(j)); + } + } + this->my_size.store(this->segment_size(seg_index) + i, std::memory_order_relaxed); + }); + } + + static constexpr bool is_first_element_in_segment( size_type index ) { + // An element is the first in a segment if its index is equal to a power of two + return is_power_of_two_at_least(index, 2); + } + + const_reference internal_subscript( size_type index ) const { + return const_cast(this)->internal_subscript(index); + } + + reference internal_subscript( size_type index ) { + __TBB_ASSERT(index < this->my_size.load(std::memory_order_relaxed), "Invalid subscript index"); + return base_type::template internal_subscript(index); + } + + const_reference internal_subscript_with_exceptions( size_type index ) const { + return const_cast(this)->internal_subscript_with_exceptions(index); + } + + reference internal_subscript_with_exceptions( size_type index ) { + if (index >= this->my_size.load(std::memory_order_acquire)) { + tbb::detail::throw_exception(exception_id::out_of_range); + } + + segment_table_type table = this->my_segment_table.load(std::memory_order_acquire); + + size_type seg_index = this->segment_index_of(index); + if (base_type::number_of_segments(table) < seg_index) { + tbb::detail::throw_exception(exception_id::out_of_range); + } + + if (table[seg_index] <= this->segment_allocation_failure_tag) { + tbb::detail::throw_exception(exception_id::out_of_range); + } + + return base_type::template internal_subscript(index); + } + + static void zero_unconstructed_elements( pointer start, size_type count ) { + std::memset(static_cast(start), 0, count * sizeof(value_type)); + } + + template + iterator internal_emplace_back( Args&&... args ) { + size_type old_size = this->my_size++; + this->assign_first_block_if_necessary(default_first_block_size); + auto element_address = &base_type::template internal_subscript(old_size); + + // try_call API is not convenient here due to broken + // variadic capture on GCC 4.8.5 + auto value_guard = make_raii_guard([&] { + zero_unconstructed_elements(element_address, /*count =*/1); + }); + + segment_table_allocator_traits::construct(base_type::get_allocator(), element_address, std::forward(args)...); + value_guard.dismiss(); + return iterator(*this, old_size, element_address); + } + + template + void internal_loop_construct( segment_table_type table, size_type start_idx, size_type end_idx, const Args&... args ) { + static_assert(sizeof...(Args) < 2, "Too many parameters"); + for (size_type idx = start_idx; idx < end_idx; ++idx) { + auto element_address = &base_type::template internal_subscript(idx); + // try_call API is not convenient here due to broken + // variadic capture on GCC 4.8.5 + auto value_guard = make_raii_guard( [&] { + segment_index_type last_allocated_segment = this->find_last_allocated_segment(table); + size_type segment_size = this->segment_size(last_allocated_segment); + end_idx = end_idx < segment_size ? end_idx : segment_size; + for (size_type i = idx; i < end_idx; ++i) { + zero_unconstructed_elements(&this->internal_subscript(i), /*count =*/1); + } + }); + segment_table_allocator_traits::construct(base_type::get_allocator(), element_address, args...); + value_guard.dismiss(); + } + } + + template + void internal_loop_construct( segment_table_type table, size_type start_idx, size_type end_idx, ForwardIterator first, ForwardIterator ) { + for (size_type idx = start_idx; idx < end_idx; ++idx) { + auto element_address = &base_type::template internal_subscript(idx); + try_call( [&] { + segment_table_allocator_traits::construct(base_type::get_allocator(), element_address, *first++); + } ).on_exception( [&] { + segment_index_type last_allocated_segment = this->find_last_allocated_segment(table); + size_type segment_size = this->segment_size(last_allocated_segment); + end_idx = end_idx < segment_size ? end_idx : segment_size; + for (size_type i = idx; i < end_idx; ++i) { + zero_unconstructed_elements(&this->internal_subscript(i), /*count =*/1); + } + }); + } + } + + template + iterator internal_grow( size_type start_idx, size_type end_idx, const Args&... args ) { + this->assign_first_block_if_necessary(this->segment_index_of(end_idx - 1) + 1); + size_type seg_index = this->segment_index_of(end_idx - 1); + segment_table_type table = this->get_table(); + this->extend_table_if_necessary(table, start_idx, end_idx); + + if (seg_index > this->my_first_block.load(std::memory_order_relaxed)) { + // So that other threads be able to work with the last segment of grow_by, allocate it immediately. + // If the last segment is not less than the first block + if (table[seg_index].load(std::memory_order_relaxed) == nullptr) { + size_type first_element = this->segment_base(seg_index); + if (first_element >= start_idx && first_element < end_idx) { + segment_type segment = table[seg_index].load(std::memory_order_relaxed); + base_type::enable_segment(segment, table, seg_index, first_element); + } + } + } + + internal_loop_construct(table, start_idx, end_idx, args...); + + return iterator(*this, start_idx, &base_type::template internal_subscript(start_idx)); + } + + + template + iterator internal_grow_by_delta( size_type delta, const Args&... args ) { + if (delta == size_type(0)) { + return end(); + } + size_type start_idx = this->my_size.fetch_add(delta); + size_type end_idx = start_idx + delta; + + return internal_grow(start_idx, end_idx, args...); + } + + template + iterator internal_grow_to_at_least( size_type new_size, const Args&... args ) { + size_type old_size = this->my_size.load(std::memory_order_relaxed); + if (new_size == size_type(0)) return iterator(*this, 0); + while (old_size < new_size && !this->my_size.compare_exchange_weak(old_size, new_size)) + {} + + int delta = static_cast(new_size) - static_cast(old_size); + if (delta > 0) { + return internal_grow(old_size, new_size, args...); + } + + size_type end_segment = this->segment_index_of(new_size - 1); + + // Check/wait for segments allocation completes + if (end_segment >= this->pointers_per_embedded_table && + this->get_table() == this->my_embedded_table) + { + spin_wait_while_eq(this->my_segment_table, this->my_embedded_table); + } + + for (segment_index_type seg_idx = 0; seg_idx <= end_segment; ++seg_idx) { + if (this->get_table()[seg_idx].load(std::memory_order_relaxed) == nullptr) { + atomic_backoff backoff(true); + while (this->get_table()[seg_idx].load(std::memory_order_relaxed) == nullptr) { + backoff.pause(); + } + } + } + + #if TBB_USE_DEBUG + size_type cap = capacity(); + __TBB_ASSERT( cap >= new_size, nullptr); + #endif + return iterator(*this, size()); + } + + template + void internal_resize( size_type n, const Args&... args ) { + if (n == 0) { + clear(); + return; + } + + size_type old_size = this->my_size.load(std::memory_order_acquire); + if (n > old_size) { + reserve(n); + grow_to_at_least(n, args...); + } else { + if (old_size == n) { + return; + } + size_type last_segment = this->segment_index_of(old_size - 1); + // Delete segments + for (size_type seg_idx = this->segment_index_of(n - 1) + 1; seg_idx <= last_segment; ++seg_idx) { + this->delete_segment(seg_idx); + } + + // If n > segment_size(n) => we need to destroy all of the items in the first segment + // Otherwise, we need to destroy only items with the index < n + size_type n_segment = this->segment_index_of(n - 1); + size_type last_index_to_destroy = std::min(this->segment_base(n_segment) + this->segment_size(n_segment), old_size); + // Destroy elements in curr segment + for (size_type idx = n; idx < last_index_to_destroy; ++idx) { + segment_table_allocator_traits::destroy(base_type::get_allocator(), &base_type::template internal_subscript(idx)); + } + this->my_size.store(n, std::memory_order_release); + } + } + + void destroy_elements() { + allocator_type alloc(base_type::get_allocator()); + for (size_type i = 0; i < this->my_size.load(std::memory_order_relaxed); ++i) { + allocator_traits_type::destroy(alloc, &base_type::template internal_subscript(i)); + } + this->my_size.store(0, std::memory_order_relaxed); + } + + static bool incompact_predicate( size_type size ) { + // memory page size + const size_type page_size = 4096; + return size < page_size || ((size - 1) % page_size < page_size / 2 && size < page_size * 128); + } + + void internal_compact() { + const size_type curr_size = this->my_size.load(std::memory_order_relaxed); + segment_table_type table = this->get_table(); + const segment_index_type k_end = this->find_last_allocated_segment(table); // allocated segments + const segment_index_type k_stop = curr_size ? this->segment_index_of(curr_size - 1) + 1 : 0; // number of segments to store existing items: 0=>0; 1,2=>1; 3,4=>2; [5-8]=>3;.. + const segment_index_type first_block = this->my_first_block; // number of merged segments, getting values from atomics + + segment_index_type k = first_block; + if (k_stop < first_block) { + k = k_stop; + } + else { + while (k < k_stop && incompact_predicate(this->segment_size(k) * sizeof(value_type))) k++; + } + + if (k_stop == k_end && k == first_block) { + return; + } + + // First segment optimization + if (k != first_block && k) { + size_type max_block = std::max(first_block, k); + + auto buffer_table = segment_table_allocator_traits::allocate(base_type::get_allocator(), max_block); + + for (size_type seg_idx = 0; seg_idx < max_block; ++seg_idx) { + segment_table_allocator_traits::construct(base_type::get_allocator(), &buffer_table[seg_idx], + table[seg_idx].load(std::memory_order_relaxed)); + table[seg_idx].store(nullptr, std::memory_order_relaxed); + } + + this->my_first_block.store(k, std::memory_order_relaxed); + size_type index = 0; + try_call( [&] { + for (; index < std::min(this->segment_size(max_block), curr_size); ++index) { + auto element_address = &static_cast(this)->operator[](index); + segment_index_type seg_idx = this->segment_index_of(index); + segment_table_allocator_traits::construct(base_type::get_allocator(), element_address, + std::move_if_noexcept(buffer_table[seg_idx].load(std::memory_order_relaxed)[index])); + } + } ).on_exception( [&] { + segment_element_allocator_type allocator(base_type::get_allocator()); + for (size_type i = 0; i < index; ++i) { + auto element_adress = &this->operator[](i); + segment_element_allocator_traits::destroy(allocator, element_adress); + } + segment_element_allocator_traits::deallocate(allocator, + table[0].load(std::memory_order_relaxed), this->segment_size(max_block)); + + for (size_type seg_idx = 0; seg_idx < max_block; ++seg_idx) { + table[seg_idx].store(buffer_table[seg_idx].load(std::memory_order_relaxed), + std::memory_order_relaxed); + buffer_table[seg_idx].store(nullptr, std::memory_order_relaxed); + } + segment_table_allocator_traits::deallocate(base_type::get_allocator(), + buffer_table, max_block); + this->my_first_block.store(first_block, std::memory_order_relaxed); + }); + + // Need to correct deallocate old segments + // Method destroy_segment respect active first_block, therefore, + // in order for the segment deletion to work correctly, set the first_block size that was earlier, + // destroy the unnecessary segments. + this->my_first_block.store(first_block, std::memory_order_relaxed); + for (size_type seg_idx = max_block; seg_idx > 0 ; --seg_idx) { + auto curr_segment = buffer_table[seg_idx - 1].load(std::memory_order_relaxed); + if (curr_segment != nullptr) { + destroy_segment(buffer_table[seg_idx - 1].load(std::memory_order_relaxed) + this->segment_base(seg_idx - 1), + seg_idx - 1); + } + } + + this->my_first_block.store(k, std::memory_order_relaxed); + + for (size_type seg_idx = 0; seg_idx < max_block; ++seg_idx) { + segment_table_allocator_traits::destroy(base_type::get_allocator(), &buffer_table[seg_idx]); + } + + segment_table_allocator_traits::deallocate(base_type::get_allocator(), buffer_table, max_block); + } + // free unnecessary segments allocated by reserve() call + if (k_stop < k_end) { + for (size_type seg_idx = k_end; seg_idx != k_stop; --seg_idx) { + if (table[seg_idx - 1].load(std::memory_order_relaxed) != nullptr) { + this->delete_segment(seg_idx - 1); + } + } + if (!k) this->my_first_block.store(0, std::memory_order_relaxed); + } + } + + // Lever for adjusting the size of first_block at the very first insertion. + // TODO: consider >1 value, check performance + static constexpr size_type default_first_block_size = 1; + + template + friend class vector_iterator; +}; // class concurrent_vector + +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT +// Deduction guide for the constructor from two iterators +template >, + typename = std::enable_if_t>, + typename = std::enable_if_t>> +concurrent_vector( It, It, Alloc = Alloc() ) +-> concurrent_vector, Alloc>; +#endif + +template +void swap(concurrent_vector &lhs, + concurrent_vector &rhs) +{ + lhs.swap(rhs); +} + +template +bool operator==(const concurrent_vector &lhs, + const concurrent_vector &rhs) +{ + return lhs.size() == rhs.size() && std::equal(lhs.begin(), lhs.end(), rhs.begin()); +} + +#if !__TBB_CPP20_COMPARISONS_PRESENT +template +bool operator!=(const concurrent_vector &lhs, + const concurrent_vector &rhs) +{ + return !(lhs == rhs); +} +#endif // !__TBB_CPP20_COMPARISONS_PRESENT + +#if __TBB_CPP20_COMPARISONS_PRESENT && __TBB_CPP20_CONCEPTS_PRESENT +template +tbb::detail::synthesized_three_way_result::value_type> +operator<=>(const concurrent_vector &lhs, + const concurrent_vector &rhs) +{ + return std::lexicographical_compare_three_way(lhs.begin(), lhs.end(), + rhs.begin(), rhs.end(), + tbb::detail::synthesized_three_way_comparator{}); +} + +#else + +template +bool operator<(const concurrent_vector &lhs, + const concurrent_vector &rhs) +{ + return std::lexicographical_compare(lhs.begin(), lhs.end(), rhs.begin(), rhs.end()); +} + +template +bool operator<=(const concurrent_vector &lhs, + const concurrent_vector &rhs) +{ + return !(rhs < lhs); +} + +template +bool operator>(const concurrent_vector &lhs, + const concurrent_vector &rhs) +{ + return rhs < lhs; +} + +template +bool operator>=(const concurrent_vector &lhs, + const concurrent_vector &rhs) +{ + return !(lhs < rhs); +} +#endif // __TBB_CPP20_COMPARISONS_PRESENT && __TBB_CPP20_CONCEPTS_PRESENT + +} // namespace d1 +} // namespace detail + +inline namespace v1 { + using detail::d1::concurrent_vector; +} // namespace v1 + +} // namespace tbb + +#endif // __TBB_concurrent_vector_H diff --git a/src/tbb/include/oneapi/tbb/detail/_aggregator.h b/src/tbb/include/oneapi/tbb/detail/_aggregator.h new file mode 100644 index 000000000..2e3e5cc3d --- /dev/null +++ b/src/tbb/include/oneapi/tbb/detail/_aggregator.h @@ -0,0 +1,176 @@ +/* + Copyright (c) 2005-2022 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + + +#ifndef __TBB_detail__aggregator_H +#define __TBB_detail__aggregator_H + +#include "_assert.h" +#include "_utils.h" +#include +#if !__TBBMALLOC_BUILD // TODO: check this macro with TBB Malloc +#include "../profiling.h" +#endif + +namespace tbb { +namespace detail { +namespace d1 { + +// Base class for aggregated operation +template +class aggregated_operation { +public: + // Zero value means "wait" status, all other values are "user" specified values and + // are defined into the scope of a class which uses "status" + std::atomic status; + + std::atomic next; + aggregated_operation() : status{}, next(nullptr) {} +}; // class aggregated_operation + +// Aggregator base class +/* An aggregator for collecting operations coming from multiple sources and executing + them serially on a single thread. OperationType must be derived from + aggregated_operation. The parameter HandlerType is a functor that will be passed the + list of operations and is expected to handle each operation appropriately, setting the + status of each operation to non-zero. */ +template +class aggregator_generic { +public: + aggregator_generic() : pending_operations(nullptr), handler_busy(false) {} + + // Execute an operation + /* Places an operation into the waitlist (pending_operations), and either handles the list, + or waits for the operation to complete, or returns. + The long_life_time parameter specifies the life time of the given operation object. + Operations with long_life_time == true may be accessed after execution. + A "short" life time operation (long_life_time == false) can be destroyed + during execution, and so any access to it after it was put into the waitlist, + including status check, is invalid. As a consequence, waiting for completion + of such operation causes undefined behavior. */ + template + void execute( OperationType* op, HandlerType& handle_operations, bool long_life_time = true ) { + // op->status should be read before inserting the operation into the + // aggregator waitlist since it can become invalid after executing a + // handler (if the operation has 'short' life time.) + const uintptr_t status = op->status.load(std::memory_order_relaxed); + + // ITT note: &(op->status) tag is used to cover accesses to this op node. This + // thread has created the operation, and now releases it so that the handler + // thread may handle the associated operation w/o triggering a race condition; + // thus this tag will be acquired just before the operation is handled in the + // handle_operations functor. + call_itt_notify(releasing, &(op->status)); + // insert the operation in the queue. + OperationType* res = pending_operations.load(std::memory_order_relaxed); + do { + op->next.store(res, std::memory_order_relaxed); + } while (!pending_operations.compare_exchange_strong(res, op)); + if (!res) { // first in the list; handle the operations + // ITT note: &pending_operations tag covers access to the handler_busy flag, + // which this waiting handler thread will try to set before entering + // handle_operations. + call_itt_notify(acquired, &pending_operations); + start_handle_operations(handle_operations); + // The operation with 'short' life time can already be destroyed + if (long_life_time) + __TBB_ASSERT(op->status.load(std::memory_order_relaxed), nullptr); + } + // Not first; wait for op to be ready + else if (!status) { // operation is blocking here. + __TBB_ASSERT(long_life_time, "Waiting for an operation object that might be destroyed during processing"); + call_itt_notify(prepare, &(op->status)); + spin_wait_while_eq(op->status, uintptr_t(0)); + } + } + +private: + // Trigger the handling of operations when the handler is free + template + void start_handle_operations( HandlerType& handle_operations ) { + OperationType* op_list; + + // ITT note: &handler_busy tag covers access to pending_operations as it is passed + // between active and waiting handlers. Below, the waiting handler waits until + // the active handler releases, and the waiting handler acquires &handler_busy as + // it becomes the active_handler. The release point is at the end of this + // function, when all operations in pending_operations have been handled by the + // owner of this aggregator. + call_itt_notify(prepare, &handler_busy); + // get the handler_busy: + // only one thread can possibly spin here at a time + spin_wait_until_eq(handler_busy, uintptr_t(0)); + call_itt_notify(acquired, &handler_busy); + // acquire fence not necessary here due to causality rule and surrounding atomics + handler_busy.store(1, std::memory_order_relaxed); + + // ITT note: &pending_operations tag covers access to the handler_busy flag + // itself. Capturing the state of the pending_operations signifies that + // handler_busy has been set and a new active handler will now process that list's + // operations. + call_itt_notify(releasing, &pending_operations); + // grab pending_operations + op_list = pending_operations.exchange(nullptr); + + // handle all the operations + handle_operations(op_list); + + // release the handler + handler_busy.store(0, std::memory_order_release); + } + + // An atomically updated list (aka mailbox) of pending operations + std::atomic pending_operations; + // Controls threads access to handle_operations + std::atomic handler_busy; +}; // class aggregator_generic + +template +class aggregator : public aggregator_generic { + HandlerType handle_operations; +public: + aggregator() = default; + + void initialize_handler( HandlerType h ) { handle_operations = h; } + + void execute(OperationType* op) { + aggregator_generic::execute(op, handle_operations); + } +}; // class aggregator + +// the most-compatible friend declaration (vs, gcc, icc) is +// template friend class aggregating_functor; +template +class aggregating_functor { + AggregatingClass* my_object{nullptr}; +public: + aggregating_functor() = default; + aggregating_functor( AggregatingClass* object ) : my_object(object) { + __TBB_ASSERT(my_object, nullptr); + } + + void operator()( OperationList* op_list ) { + __TBB_ASSERT(my_object, nullptr); + my_object->handle_operations(op_list); + } +}; // class aggregating_functor + + +} // namespace d1 +} // namespace detail +} // namespace tbb + +#endif // __TBB_detail__aggregator_H diff --git a/src/tbb/include/tbb/aligned_space.h b/src/tbb/include/oneapi/tbb/detail/_aligned_space.h similarity index 72% rename from src/tbb/include/tbb/aligned_space.h rename to src/tbb/include/oneapi/tbb/detail/_aligned_space.h index ee4d9e727..13857c47c 100644 --- a/src/tbb/include/tbb/aligned_space.h +++ b/src/tbb/include/oneapi/tbb/detail/_aligned_space.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,36 +12,35 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ - #ifndef __TBB_aligned_space_H #define __TBB_aligned_space_H -#include "tbb_stddef.h" -#include "tbb_machine.h" +#include + +#include "_template_helpers.h" namespace tbb { +namespace detail { +inline namespace d0 { //! Block of space aligned sufficiently to construct an array T with N elements. /** The elements are not constructed or destroyed by this class. @ingroup memory_allocation */ -template +template class aligned_space { -private: - typedef __TBB_TypeWithAlignmentAtLeastAsStrict(T) element_type; - element_type array[(sizeof(T)*N+sizeof(element_type)-1)/sizeof(element_type)]; + alignas(alignof(T)) std::uint8_t aligned_array[N * sizeof(T)]; + public: //! Pointer to beginning of array - T* begin() const {return internal::punned_cast(this);} + T* begin() const { return punned_cast(&aligned_array); } //! Pointer to one past last element in array. - T* end() const {return begin()+N;} + T* end() const { return begin() + N; } }; +} // namespace d0 +} // namespace detail } // namespace tbb #endif /* __TBB_aligned_space_H */ diff --git a/src/tbb/include/oneapi/tbb/detail/_allocator_traits.h b/src/tbb/include/oneapi/tbb/detail/_allocator_traits.h new file mode 100644 index 000000000..8c60e25e7 --- /dev/null +++ b/src/tbb/include/oneapi/tbb/detail/_allocator_traits.h @@ -0,0 +1,107 @@ +/* + Copyright (c) 2005-2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_detail__allocator_traits_H +#define __TBB_detail__allocator_traits_H + +#include "_config.h" +#include "_template_helpers.h" +#include +#include + +namespace tbb { +namespace detail { +inline namespace d0 { + +#if !__TBB_CPP17_ALLOCATOR_IS_ALWAYS_EQUAL_PRESENT +// Struct is_always_equal_detector provides the member type "type" which is +// Allocator::is_always_equal if it is present, std::false_type otherwise +template +struct is_always_equal_detector { + using type = std::false_type; +}; + +template +struct is_always_equal_detector> +{ + using type = typename Allocator::is_always_equal; +}; +#endif // !__TBB_CPP17_ALLOCATOR_IS_ALWAYS_EQUAL_PRESENT + +template +class allocator_traits : public std::allocator_traits +{ + using base_type = std::allocator_traits; +public: +#if !__TBB_CPP17_ALLOCATOR_IS_ALWAYS_EQUAL_PRESENT + using is_always_equal = typename is_always_equal_detector::type; +#endif + + template + using rebind_traits = typename tbb::detail::allocator_traits>; +}; // struct allocator_traits + +template +void copy_assign_allocators_impl( Allocator& lhs, const Allocator& rhs, /*pocca = */std::true_type ) { + lhs = rhs; +} + +template +void copy_assign_allocators_impl( Allocator&, const Allocator&, /*pocca = */ std::false_type ) {} + +// Copy assigns allocators only if propagate_on_container_copy_assignment is true +template +void copy_assign_allocators( Allocator& lhs, const Allocator& rhs ) { + using pocca_type = typename allocator_traits::propagate_on_container_copy_assignment; + copy_assign_allocators_impl(lhs, rhs, pocca_type()); +} + +template +void move_assign_allocators_impl( Allocator& lhs, Allocator& rhs, /*pocma = */ std::true_type ) { + lhs = std::move(rhs); +} + +template +void move_assign_allocators_impl( Allocator&, Allocator&, /*pocma = */ std::false_type ) {} + +// Move assigns allocators only if propagate_on_container_move_assignment is true +template +void move_assign_allocators( Allocator& lhs, Allocator& rhs ) { + using pocma_type = typename allocator_traits::propagate_on_container_move_assignment; + move_assign_allocators_impl(lhs, rhs, pocma_type()); +} + +template +void swap_allocators_impl( Allocator& lhs, Allocator& rhs, /*pocs = */ std::true_type ) { + using std::swap; + swap(lhs, rhs); +} + +template +void swap_allocators_impl( Allocator&, Allocator&, /*pocs = */ std::false_type ) {} + +// Swaps allocators only if propagate_on_container_swap is true +template +void swap_allocators( Allocator& lhs, Allocator& rhs ) { + using pocs_type = typename allocator_traits::propagate_on_container_swap; + swap_allocators_impl(lhs, rhs, pocs_type()); +} + +} // inline namespace d0 +} // namespace detail +} // namespace tbb + +#endif // __TBB_detail__allocator_traits_H diff --git a/src/tbb/include/oneapi/tbb/detail/_assert.h b/src/tbb/include/oneapi/tbb/detail/_assert.h new file mode 100644 index 000000000..fce714ffe --- /dev/null +++ b/src/tbb/include/oneapi/tbb/detail/_assert.h @@ -0,0 +1,64 @@ +/* + Copyright (c) 2005-2022 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_detail__assert_H +#define __TBB_detail__assert_H + +#include "_config.h" + +#if __TBBMALLOC_BUILD +namespace rml { namespace internal { +#else +namespace tbb { +namespace detail { +namespace r1 { +#endif +//! Process an assertion failure. +/** Normally called from __TBB_ASSERT macro. + If assertion handler is null, print message for assertion failure and abort. + Otherwise call the assertion handler. */ +TBB_EXPORT void __TBB_EXPORTED_FUNC assertion_failure(const char* location, int line, const char* expression, const char* comment); +#if __TBBMALLOC_BUILD +}} // namespaces rml::internal +#else +} // namespace r1 +} // namespace detail +} // namespace tbb +#endif + +#if __TBBMALLOC_BUILD +//! Release version of assertions +#define __TBB_ASSERT_RELEASE(predicate,message) ((predicate)?((void)0) : rml::internal::assertion_failure(__func__,__LINE__,#predicate,message)) +#else +#define __TBB_ASSERT_RELEASE(predicate,message) ((predicate)?((void)0) : tbb::detail::r1::assertion_failure(__func__,__LINE__,#predicate,message)) +#endif + +#if TBB_USE_ASSERT + //! Assert that predicate is true. + /** If predicate is false, print assertion failure message. + If the comment argument is not nullptr, it is printed as part of the failure message. + The comment argument has no other effect. */ + #define __TBB_ASSERT(predicate,message) __TBB_ASSERT_RELEASE(predicate,message) + //! "Extended" version + #define __TBB_ASSERT_EX __TBB_ASSERT +#else + //! No-op version of __TBB_ASSERT. + #define __TBB_ASSERT(predicate,comment) ((void)0) + //! "Extended" version is useful to suppress warnings if a variable is only used with an assert + #define __TBB_ASSERT_EX(predicate,comment) ((void)(1 && (predicate))) +#endif // TBB_USE_ASSERT + +#endif // __TBB_detail__assert_H diff --git a/src/tbb/src/tbb/critical_section.cpp b/src/tbb/include/oneapi/tbb/detail/_attach.h similarity index 66% rename from src/tbb/src/tbb/critical_section.cpp rename to src/tbb/include/oneapi/tbb/detail/_attach.h index 4b32fb17f..45f29727a 100644 --- a/src/tbb/src/tbb/critical_section.cpp +++ b/src/tbb/include/oneapi/tbb/detail/_attach.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,20 +12,21 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. +*/ +#ifndef __TBB_detail__attach_H +#define __TBB_detail__attach_H +#include "_config.h" +namespace tbb { +namespace detail { +namespace d1 { -*/ - -#include "tbb/critical_section.h" -#include "itt_notify.h" + struct attach {}; -namespace tbb { - namespace internal { +} // namespace d1 +} // namespace detail +} // namespace tbb -void critical_section_v4::internal_construct() { - ITT_SYNC_CREATE(&my_impl, _T("ppl::critical_section"), _T("")); -} -} // namespace internal -} // namespace tbb +#endif // __TBB_detail__attach_H diff --git a/src/tbb/include/oneapi/tbb/detail/_concurrent_queue_base.h b/src/tbb/include/oneapi/tbb/detail/_concurrent_queue_base.h new file mode 100644 index 000000000..b195cc4ab --- /dev/null +++ b/src/tbb/include/oneapi/tbb/detail/_concurrent_queue_base.h @@ -0,0 +1,650 @@ +/* + Copyright (c) 2005-2022 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_detail__concurrent_queue_base_H +#define __TBB_detail__concurrent_queue_base_H + +#include "_utils.h" +#include "_exception.h" +#include "_machine.h" +#include "_allocator_traits.h" + +#include "../profiling.h" +#include "../spin_mutex.h" +#include "../cache_aligned_allocator.h" + +#include + +namespace tbb { +namespace detail { +namespace d2 { + +using ticket_type = std::size_t; + +template +inline bool is_valid_page(const Page p) { + return reinterpret_cast(p) > 1; +} + +template +struct concurrent_queue_rep; + +template +class micro_queue_pop_finalizer; + +#if _MSC_VER && !defined(__INTEL_COMPILER) +// unary minus operator applied to unsigned type, result still unsigned +// #pragma warning( push ) +// #pragma warning( disable: 4146 ) +#endif + +// A queue using simple locking. +// For efficiency, this class has no constructor. +// The caller is expected to zero-initialize it. +template +class micro_queue { +private: + using queue_rep_type = concurrent_queue_rep; + using self_type = micro_queue; +public: + using size_type = std::size_t; + using value_type = T; + using reference = value_type&; + using const_reference = const value_type&; + + using allocator_type = Allocator; + using allocator_traits_type = tbb::detail::allocator_traits; + using queue_allocator_type = typename allocator_traits_type::template rebind_alloc; + + static constexpr size_type item_size = sizeof(T); + static constexpr size_type items_per_page = item_size <= 8 ? 32 : + item_size <= 16 ? 16 : + item_size <= 32 ? 8 : + item_size <= 64 ? 4 : + item_size <= 128 ? 2 : 1; + + struct padded_page { + padded_page() {} + ~padded_page() {} + + reference operator[] (std::size_t index) { + __TBB_ASSERT(index < items_per_page, "Index out of range"); + return items[index]; + } + + const_reference operator[] (std::size_t index) const { + __TBB_ASSERT(index < items_per_page, "Index out of range"); + return items[index]; + } + + padded_page* next{ nullptr }; + std::atomic mask{}; + + union { + value_type items[items_per_page]; + }; + }; // struct padded_page + + using page_allocator_type = typename allocator_traits_type::template rebind_alloc; +protected: + using page_allocator_traits = tbb::detail::allocator_traits; + +public: + using item_constructor_type = void (*)(value_type* location, const void* src); + micro_queue() = default; + micro_queue( const micro_queue& ) = delete; + micro_queue& operator=( const micro_queue& ) = delete; + + size_type prepare_page( ticket_type k, queue_rep_type& base, page_allocator_type page_allocator, + padded_page*& p ) { + __TBB_ASSERT(p == nullptr, "Invalid page argument for prepare_page"); + k &= -queue_rep_type::n_queue; + size_type index = modulo_power_of_two(k / queue_rep_type::n_queue, items_per_page); + if (!index) { + try_call( [&] { + p = page_allocator_traits::allocate(page_allocator, 1); + }).on_exception( [&] { + ++base.n_invalid_entries; + invalidate_page( k ); + }); + page_allocator_traits::construct(page_allocator, p); + } + + spin_wait_until_my_turn(tail_counter, k, base); + d1::call_itt_notify(d1::acquired, &tail_counter); + + if (p) { + spin_mutex::scoped_lock lock( page_mutex ); + padded_page* q = tail_page.load(std::memory_order_relaxed); + if (is_valid_page(q)) { + q->next = p; + } else { + head_page.store(p, std::memory_order_relaxed); + } + tail_page.store(p, std::memory_order_relaxed); + } else { + p = tail_page.load(std::memory_order_relaxed); + } + return index; + } + + template + void push( ticket_type k, queue_rep_type& base, queue_allocator_type& allocator, Args&&... args ) + { + padded_page* p = nullptr; + page_allocator_type page_allocator(allocator); + size_type index = prepare_page(k, base, page_allocator, p); + __TBB_ASSERT(p != nullptr, "Page was not prepared"); + + // try_call API is not convenient here due to broken + // variadic capture on GCC 4.8.5 + auto value_guard = make_raii_guard([&] { + ++base.n_invalid_entries; + d1::call_itt_notify(d1::releasing, &tail_counter); + tail_counter.fetch_add(queue_rep_type::n_queue); + }); + + page_allocator_traits::construct(page_allocator, &(*p)[index], std::forward(args)...); + // If no exception was thrown, mark item as present. + p->mask.store(p->mask.load(std::memory_order_relaxed) | uintptr_t(1) << index, std::memory_order_relaxed); + d1::call_itt_notify(d1::releasing, &tail_counter); + + value_guard.dismiss(); + tail_counter.fetch_add(queue_rep_type::n_queue); + } + + void abort_push( ticket_type k, queue_rep_type& base, queue_allocator_type& allocator ) { + padded_page* p = nullptr; + prepare_page(k, base, allocator, p); + ++base.n_invalid_entries; + tail_counter.fetch_add(queue_rep_type::n_queue); + } + + bool pop( void* dst, ticket_type k, queue_rep_type& base, queue_allocator_type& allocator ) { + k &= -queue_rep_type::n_queue; + spin_wait_until_eq(head_counter, k); + d1::call_itt_notify(d1::acquired, &head_counter); + spin_wait_while_eq(tail_counter, k); + d1::call_itt_notify(d1::acquired, &tail_counter); + padded_page *p = head_page.load(std::memory_order_relaxed); + __TBB_ASSERT( p, nullptr ); + size_type index = modulo_power_of_two( k/queue_rep_type::n_queue, items_per_page ); + bool success = false; + { + page_allocator_type page_allocator(allocator); + micro_queue_pop_finalizer finalizer(*this, page_allocator, + k + queue_rep_type::n_queue, index == items_per_page - 1 ? p : nullptr ); + if (p->mask.load(std::memory_order_relaxed) & (std::uintptr_t(1) << index)) { + success = true; + assign_and_destroy_item(dst, *p, index); + } else { + --base.n_invalid_entries; + } + } + return success; + } + + micro_queue& assign( const micro_queue& src, queue_allocator_type& allocator, + item_constructor_type construct_item ) + { + head_counter.store(src.head_counter.load(std::memory_order_relaxed), std::memory_order_relaxed); + tail_counter.store(src.tail_counter.load(std::memory_order_relaxed), std::memory_order_relaxed); + + const padded_page* srcp = src.head_page.load(std::memory_order_relaxed); + if( is_valid_page(srcp) ) { + ticket_type g_index = head_counter.load(std::memory_order_relaxed); + size_type n_items = (tail_counter.load(std::memory_order_relaxed) - head_counter.load(std::memory_order_relaxed)) + / queue_rep_type::n_queue; + size_type index = modulo_power_of_two(head_counter.load(std::memory_order_relaxed) / queue_rep_type::n_queue, items_per_page); + size_type end_in_first_page = (index+n_items < items_per_page) ? (index + n_items) : items_per_page; + + try_call( [&] { + head_page.store(make_copy(allocator, srcp, index, end_in_first_page, g_index, construct_item), std::memory_order_relaxed); + }).on_exception( [&] { + head_counter.store(0, std::memory_order_relaxed); + tail_counter.store(0, std::memory_order_relaxed); + }); + padded_page* cur_page = head_page.load(std::memory_order_relaxed); + + try_call( [&] { + if (srcp != src.tail_page.load(std::memory_order_relaxed)) { + for (srcp = srcp->next; srcp != src.tail_page.load(std::memory_order_relaxed); srcp=srcp->next ) { + cur_page->next = make_copy( allocator, srcp, 0, items_per_page, g_index, construct_item ); + cur_page = cur_page->next; + } + + __TBB_ASSERT(srcp == src.tail_page.load(std::memory_order_relaxed), nullptr ); + size_type last_index = modulo_power_of_two(tail_counter.load(std::memory_order_relaxed) / queue_rep_type::n_queue, items_per_page); + if( last_index==0 ) last_index = items_per_page; + + cur_page->next = make_copy( allocator, srcp, 0, last_index, g_index, construct_item ); + cur_page = cur_page->next; + } + tail_page.store(cur_page, std::memory_order_relaxed); + }).on_exception( [&] { + padded_page* invalid_page = reinterpret_cast(std::uintptr_t(1)); + tail_page.store(invalid_page, std::memory_order_relaxed); + }); + } else { + head_page.store(nullptr, std::memory_order_relaxed); + tail_page.store(nullptr, std::memory_order_relaxed); + } + return *this; + } + + padded_page* make_copy( queue_allocator_type& allocator, const padded_page* src_page, size_type begin_in_page, + size_type end_in_page, ticket_type& g_index, item_constructor_type construct_item ) + { + page_allocator_type page_allocator(allocator); + padded_page* new_page = page_allocator_traits::allocate(page_allocator, 1); + new_page->next = nullptr; + new_page->mask.store(src_page->mask.load(std::memory_order_relaxed), std::memory_order_relaxed); + for (; begin_in_page!=end_in_page; ++begin_in_page, ++g_index) { + if (new_page->mask.load(std::memory_order_relaxed) & uintptr_t(1) << begin_in_page) { + copy_item(*new_page, begin_in_page, *src_page, begin_in_page, construct_item); + } + } + return new_page; + } + + void invalidate_page( ticket_type k ) { + // Append an invalid page at address 1 so that no more pushes are allowed. + padded_page* invalid_page = reinterpret_cast(std::uintptr_t(1)); + { + spin_mutex::scoped_lock lock( page_mutex ); + tail_counter.store(k + queue_rep_type::n_queue + 1, std::memory_order_relaxed); + padded_page* q = tail_page.load(std::memory_order_relaxed); + if (is_valid_page(q)) { + q->next = invalid_page; + } else { + head_page.store(invalid_page, std::memory_order_relaxed); + } + tail_page.store(invalid_page, std::memory_order_relaxed); + } + } + + padded_page* get_head_page() { + return head_page.load(std::memory_order_relaxed); + } + + void clear(queue_allocator_type& allocator, padded_page* new_head = nullptr, padded_page* new_tail = nullptr) { + padded_page* curr_page = get_head_page(); + size_type index = (head_counter.load(std::memory_order_relaxed) / queue_rep_type::n_queue) % items_per_page; + page_allocator_type page_allocator(allocator); + + while (curr_page && is_valid_page(curr_page)) { + while (index != items_per_page) { + if (curr_page->mask.load(std::memory_order_relaxed) & (std::uintptr_t(1) << index)) { + page_allocator_traits::destroy(page_allocator, &curr_page->operator[](index)); + } + ++index; + } + + index = 0; + padded_page* next_page = curr_page->next; + page_allocator_traits::destroy(page_allocator, curr_page); + page_allocator_traits::deallocate(page_allocator, curr_page, 1); + curr_page = next_page; + } + head_counter.store(0, std::memory_order_relaxed); + tail_counter.store(0, std::memory_order_relaxed); + head_page.store(new_head, std::memory_order_relaxed); + tail_page.store(new_tail, std::memory_order_relaxed); + } + + void clear_and_invalidate(queue_allocator_type& allocator) { + padded_page* invalid_page = reinterpret_cast(std::uintptr_t(1)); + clear(allocator, invalid_page, invalid_page); + } + +private: + // template + friend class micro_queue_pop_finalizer; + + // Class used to ensure exception-safety of method "pop" + class destroyer { + value_type& my_value; + public: + destroyer( reference value ) : my_value(value) {} + destroyer( const destroyer& ) = delete; + destroyer& operator=( const destroyer& ) = delete; + ~destroyer() {my_value.~T();} + }; // class destroyer + + void copy_item( padded_page& dst, size_type dindex, const padded_page& src, size_type sindex, + item_constructor_type construct_item ) + { + auto& src_item = src[sindex]; + construct_item( &dst[dindex], static_cast(&src_item) ); + } + + void assign_and_destroy_item( void* dst, padded_page& src, size_type index ) { + auto& from = src[index]; + destroyer d(from); + *static_cast(dst) = std::move(from); + } + + void spin_wait_until_my_turn( std::atomic& counter, ticket_type k, queue_rep_type& rb ) const { + for (atomic_backoff b{};; b.pause()) { + ticket_type c = counter.load(std::memory_order_acquire); + if (c == k) return; + else if (c & 1) { + ++rb.n_invalid_entries; + throw_exception( exception_id::bad_last_alloc); + } + } + } + + std::atomic head_page{}; + std::atomic head_counter{}; + + std::atomic tail_page{}; + std::atomic tail_counter{}; + + spin_mutex page_mutex{}; +}; // class micro_queue + +#if _MSC_VER && !defined(__INTEL_COMPILER) +// #pragma warning( pop ) +#endif // warning 4146 is back + +template +class micro_queue_pop_finalizer { +public: + using padded_page = typename Container::padded_page; + using allocator_type = Allocator; + using allocator_traits_type = tbb::detail::allocator_traits; + + micro_queue_pop_finalizer( Container& queue, Allocator& alloc, ticket_type k, padded_page* p ) : + my_ticket_type(k), my_queue(queue), my_page(p), allocator(alloc) + {} + + micro_queue_pop_finalizer( const micro_queue_pop_finalizer& ) = delete; + micro_queue_pop_finalizer& operator=( const micro_queue_pop_finalizer& ) = delete; + + ~micro_queue_pop_finalizer() { + padded_page* p = my_page; + if( is_valid_page(p) ) { + spin_mutex::scoped_lock lock( my_queue.page_mutex ); + padded_page* q = p->next; + my_queue.head_page.store(q, std::memory_order_relaxed); + if( !is_valid_page(q) ) { + my_queue.tail_page.store(nullptr, std::memory_order_relaxed); + } + } + my_queue.head_counter.store(my_ticket_type, std::memory_order_release); + if ( is_valid_page(p) ) { + allocator_traits_type::destroy(allocator, static_cast(p)); + allocator_traits_type::deallocate(allocator, static_cast(p), 1); + } + } +private: + ticket_type my_ticket_type; + Container& my_queue; + padded_page* my_page; + Allocator& allocator; +}; // class micro_queue_pop_finalizer + +#if _MSC_VER && !defined(__INTEL_COMPILER) +// structure was padded due to alignment specifier +// #pragma warning( push ) +// #pragma warning( disable: 4324 ) +#endif + +template +struct concurrent_queue_rep { + using self_type = concurrent_queue_rep; + using size_type = std::size_t; + using micro_queue_type = micro_queue; + using allocator_type = Allocator; + using allocator_traits_type = tbb::detail::allocator_traits; + using padded_page = typename micro_queue_type::padded_page; + using page_allocator_type = typename micro_queue_type::page_allocator_type; + using item_constructor_type = typename micro_queue_type::item_constructor_type; +private: + using page_allocator_traits = tbb::detail::allocator_traits; + using queue_allocator_type = typename allocator_traits_type::template rebind_alloc; + +public: + // must be power of 2 + static constexpr size_type n_queue = 8; + // Approximately n_queue/golden ratio + static constexpr size_type phi = 3; + static constexpr size_type item_size = micro_queue_type::item_size; + static constexpr size_type items_per_page = micro_queue_type::items_per_page; + + concurrent_queue_rep() {} + + concurrent_queue_rep( const concurrent_queue_rep& ) = delete; + concurrent_queue_rep& operator=( const concurrent_queue_rep& ) = delete; + + void clear( queue_allocator_type& alloc ) { + for (size_type index = 0; index < n_queue; ++index) { + array[index].clear(alloc); + } + head_counter.store(0, std::memory_order_relaxed); + tail_counter.store(0, std::memory_order_relaxed); + n_invalid_entries.store(0, std::memory_order_relaxed); + } + + void assign( const concurrent_queue_rep& src, queue_allocator_type& alloc, item_constructor_type construct_item ) { + head_counter.store(src.head_counter.load(std::memory_order_relaxed), std::memory_order_relaxed); + tail_counter.store(src.tail_counter.load(std::memory_order_relaxed), std::memory_order_relaxed); + n_invalid_entries.store(src.n_invalid_entries.load(std::memory_order_relaxed), std::memory_order_relaxed); + + // copy or move micro_queues + size_type queue_idx = 0; + try_call( [&] { + for (; queue_idx < n_queue; ++queue_idx) { + array[queue_idx].assign(src.array[queue_idx], alloc, construct_item); + } + }).on_exception( [&] { + for (size_type i = 0; i < queue_idx + 1; ++i) { + array[i].clear_and_invalidate(alloc); + } + head_counter.store(0, std::memory_order_relaxed); + tail_counter.store(0, std::memory_order_relaxed); + n_invalid_entries.store(0, std::memory_order_relaxed); + }); + + __TBB_ASSERT(head_counter.load(std::memory_order_relaxed) == src.head_counter.load(std::memory_order_relaxed) && + tail_counter.load(std::memory_order_relaxed) == src.tail_counter.load(std::memory_order_relaxed), + "the source concurrent queue should not be concurrently modified." ); + } + + bool empty() const { + ticket_type tc = tail_counter.load(std::memory_order_acquire); + ticket_type hc = head_counter.load(std::memory_order_relaxed); + // if tc!=r.tail_counter, the queue was not empty at some point between the two reads. + return tc == tail_counter.load(std::memory_order_relaxed) && + std::ptrdiff_t(tc - hc - n_invalid_entries.load(std::memory_order_relaxed)) <= 0; + } + + std::ptrdiff_t size() const { + __TBB_ASSERT(sizeof(std::ptrdiff_t) <= sizeof(size_type), nullptr); + std::ptrdiff_t hc = head_counter.load(std::memory_order_acquire); + std::ptrdiff_t tc = tail_counter.load(std::memory_order_relaxed); + std::ptrdiff_t nie = n_invalid_entries.load(std::memory_order_relaxed); + + return tc - hc - nie; + } + + friend class micro_queue; + + // Map ticket_type to an array index + static size_type index( ticket_type k ) { + return k * phi % n_queue; + } + + micro_queue_type& choose( ticket_type k ) { + // The formula here approximates LRU in a cache-oblivious way. + return array[index(k)]; + } + + alignas(max_nfs_size) micro_queue_type array[n_queue]; + + alignas(max_nfs_size) std::atomic head_counter{}; + alignas(max_nfs_size) std::atomic tail_counter{}; + alignas(max_nfs_size) std::atomic n_invalid_entries{}; +}; // class concurrent_queue_rep + +#if _MSC_VER && !defined(__INTEL_COMPILER) +// #pragma warning( pop ) +#endif + +template +class concurrent_queue_iterator_base { + using queue_rep_type = concurrent_queue_rep; + using padded_page = typename queue_rep_type::padded_page; +protected: + concurrent_queue_iterator_base() = default; + + concurrent_queue_iterator_base( const concurrent_queue_iterator_base& other ) { + assign(other); + } + + concurrent_queue_iterator_base( queue_rep_type* queue_rep ) + : my_queue_rep(queue_rep), + my_head_counter(my_queue_rep->head_counter.load(std::memory_order_relaxed)) + { + for (std::size_t i = 0; i < queue_rep_type::n_queue; ++i) { + my_array[i] = my_queue_rep->array[i].get_head_page(); + } + + if (!get_item(my_item, my_head_counter)) advance(); + } + + void assign( const concurrent_queue_iterator_base& other ) { + my_item = other.my_item; + my_queue_rep = other.my_queue_rep; + + if (my_queue_rep != nullptr) { + my_head_counter = other.my_head_counter; + + for (std::size_t i = 0; i < queue_rep_type::n_queue; ++i) { + my_array[i] = other.my_array[i]; + } + } + } + + void advance() { + __TBB_ASSERT(my_item, "Attempt to increment iterator past end of the queue"); + std::size_t k = my_head_counter; +#if TBB_USE_ASSERT + Value* tmp; + get_item(tmp, k); + __TBB_ASSERT(my_item == tmp, nullptr); +#endif + std::size_t i = modulo_power_of_two(k / queue_rep_type::n_queue, my_queue_rep->items_per_page); + if (i == my_queue_rep->items_per_page - 1) { + padded_page*& root = my_array[queue_rep_type::index(k)]; + root = root->next; + } + // Advance k + my_head_counter = ++k; + if (!get_item(my_item, k)) advance(); + } + + concurrent_queue_iterator_base& operator=( const concurrent_queue_iterator_base& other ) { + this->assign(other); + return *this; + } + + bool get_item( Value*& item, std::size_t k ) { + if (k == my_queue_rep->tail_counter.load(std::memory_order_relaxed)) { + item = nullptr; + return true; + } else { + padded_page* p = my_array[queue_rep_type::index(k)]; + __TBB_ASSERT(p, nullptr); + std::size_t i = modulo_power_of_two(k / queue_rep_type::n_queue, my_queue_rep->items_per_page); + item = &(*p)[i]; + return (p->mask & uintptr_t(1) << i) != 0; + } + } + + Value* my_item{ nullptr }; + queue_rep_type* my_queue_rep{ nullptr }; + ticket_type my_head_counter{}; + padded_page* my_array[queue_rep_type::n_queue]{}; +}; // class concurrent_queue_iterator_base + +struct concurrent_queue_iterator_provider { + template + static Iterator get( const Container& container ) { + return Iterator(container); + } +}; // struct concurrent_queue_iterator_provider + +template +class concurrent_queue_iterator : public concurrent_queue_iterator_base::type, Allocator> { + using base_type = concurrent_queue_iterator_base::type, Allocator>; +public: + using value_type = Value; + using pointer = value_type*; + using reference = value_type&; + using difference_type = std::ptrdiff_t; + using iterator_category = std::forward_iterator_tag; + + concurrent_queue_iterator() = default; + + /** If Value==Container::value_type, then this routine is the copy constructor. + If Value==const Container::value_type, then this routine is a conversion constructor. */ + concurrent_queue_iterator( const concurrent_queue_iterator& other ) + : base_type(other) {} + +private: + concurrent_queue_iterator( const Container& container ) + : base_type(container.my_queue_representation) {} +public: + concurrent_queue_iterator& operator=( const concurrent_queue_iterator& other ) { + this->assign(other); + return *this; + } + + reference operator*() const { + return *static_cast(this->my_item); + } + + pointer operator->() const { return &operator*(); } + + concurrent_queue_iterator& operator++() { + this->advance(); + return *this; + } + + concurrent_queue_iterator operator++(int) { + concurrent_queue_iterator tmp = *this; + ++*this; + return tmp; + } + + friend bool operator==( const concurrent_queue_iterator& lhs, const concurrent_queue_iterator& rhs ) { + return lhs.my_item == rhs.my_item; + } + + friend bool operator!=( const concurrent_queue_iterator& lhs, const concurrent_queue_iterator& rhs ) { + return lhs.my_item != rhs.my_item; + } +private: + friend struct concurrent_queue_iterator_provider; +}; // class concurrent_queue_iterator + +} // namespace d2 +} // namespace detail +} // tbb + +#endif // __TBB_detail__concurrent_queue_base_H diff --git a/src/tbb/include/oneapi/tbb/detail/_concurrent_skip_list.h b/src/tbb/include/oneapi/tbb/detail/_concurrent_skip_list.h new file mode 100644 index 000000000..096a44df9 --- /dev/null +++ b/src/tbb/include/oneapi/tbb/detail/_concurrent_skip_list.h @@ -0,0 +1,1290 @@ +/* + Copyright (c) 2019-2022 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_detail__concurrent_skip_list_H +#define __TBB_detail__concurrent_skip_list_H + +#if !defined(__TBB_concurrent_map_H) && !defined(__TBB_concurrent_set_H) +#error Do not #include this internal file directly; use public TBB headers instead. +#endif + +#include "_config.h" +#include "_range_common.h" +#include "_allocator_traits.h" +#include "_template_helpers.h" +#include "_node_handle.h" +#include "_containers_helpers.h" +#include "_assert.h" +#include "_exception.h" +#include "../enumerable_thread_specific.h" +#include +#include +#include +#include +#include +#include // Need std::geometric_distribution +#include // Need std::equal and std::lexicographical_compare +#include +#if __TBB_CPP20_COMPARISONS_PRESENT +#include +#endif + +#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) +// #pragma warning(push) +// #pragma warning(disable: 4127) // warning C4127: conditional expression is constant +#endif + +namespace tbb { +namespace detail { +namespace d2 { + +template +class skip_list_node { + using node_ptr = skip_list_node*; +public: + using value_type = Value; + using atomic_node_ptr = std::atomic; + using size_type = std::size_t; + using container_allocator_type = Allocator; + + using reference = value_type&; + using const_reference = const value_type&; +private: + using allocator_traits = tbb::detail::allocator_traits; + + // Allocator is the same as the container allocator=> allocates unitptr_t + // It is required to rebind it to value_type to get the correct pointer and const_pointer + using value_allocator_traits = typename allocator_traits::template rebind_traits; +public: + using pointer = typename value_allocator_traits::pointer; + using const_pointer = typename value_allocator_traits::const_pointer; + + //In perfect world these constructor and destructor would have been private, + //however this seems technically impractical due to use of allocator_traits. + + //Should not be called directly, instead use create method + skip_list_node( size_type levels ) + : my_height(levels), my_index_number(0) + {} + + //Should not be called directly, instead use destroy method + ~skip_list_node() {} + + skip_list_node( const skip_list_node& ) = delete; + skip_list_node( skip_list_node&& ) = delete; + skip_list_node& operator=( const skip_list_node& ) = delete; + skip_list_node& operator=( skip_list_node&& ) = delete; + + static skip_list_node* create( container_allocator_type& alloc, size_type height ) { + size_type sz = calc_node_size(height); + static_assert(std::is_same::value, "skip_list_node assumes that passed in allocator operates on bytes"); + auto* node = reinterpret_cast(allocator_traits::allocate(alloc, sz)); + + //Construct the node itself + allocator_traits::construct(alloc, node, height); + + //Construct the level pointers + for (size_type l = 0; l < height; ++l) { + allocator_traits::construct(alloc, &node->get_atomic_next(l), nullptr); + } + + return node; + } + + static void destroy( container_allocator_type& alloc, skip_list_node* node ) { + //Destroy the level pointers + for (size_type l = 0; l < node->height(); ++l) { + allocator_traits::destroy(alloc, &node->atomic_next(l)); + } + size_type sz = calc_node_size(node->height()); + // Destroy the node itself + allocator_traits::destroy(alloc, node); + + // Deallocate the node + allocator_traits::deallocate(alloc, reinterpret_cast(node), sz); + } + + + pointer storage() { + return &my_value; + } + + reference value() { + return *storage(); + } + + node_ptr next( size_type level ) const { + node_ptr res = get_atomic_next(level).load(std::memory_order_acquire); + __TBB_ASSERT(res == nullptr || res->height() > level, "Broken internal structure"); + return res; + } + + atomic_node_ptr& atomic_next( size_type level ) { + atomic_node_ptr& res = get_atomic_next(level); +#if TBB_USE_DEBUG + node_ptr node = res.load(std::memory_order_acquire); + __TBB_ASSERT(node == nullptr || node->height() > level, "Broken internal structure"); +#endif + return res; + } + + void set_next( size_type level, node_ptr n ) { + __TBB_ASSERT(n == nullptr || n->height() > level, "Broken internal structure"); + get_atomic_next(level).store(n, std::memory_order_relaxed); + } + + size_type height() const { + return my_height; + } + + void set_index_number( size_type index_num ) { + my_index_number = index_num; + } + + size_type index_number() const { + return my_index_number; + } + +private: + static size_type calc_node_size( size_type height ) { + static_assert(alignof(skip_list_node) >= alignof(atomic_node_ptr), "Incorrect alignment"); + return sizeof(skip_list_node) + height * sizeof(atomic_node_ptr); + } + + atomic_node_ptr& get_atomic_next( size_type level ) { + atomic_node_ptr* arr = reinterpret_cast(this + 1); + return arr[level]; + } + + const atomic_node_ptr& get_atomic_next( size_type level ) const { + const atomic_node_ptr* arr = reinterpret_cast(this + 1); + return arr[level]; + } + + union { + value_type my_value; + }; + size_type my_height; + size_type my_index_number; +}; // class skip_list_node + +template +class skip_list_iterator { + using node_type = NodeType; + using node_ptr = node_type*; +public: + using iterator_category = std::forward_iterator_tag; + using value_type = ValueType; + + using difference_type = std::ptrdiff_t; + using pointer = value_type*; + using reference = value_type&; + + skip_list_iterator() : skip_list_iterator(nullptr) {} + + skip_list_iterator( const skip_list_iterator& other ) + : my_node_ptr(other.my_node_ptr) {} + + skip_list_iterator& operator=( const skip_list_iterator& other ) { + my_node_ptr = other.my_node_ptr; + return *this; + } + + reference operator*() const { return my_node_ptr->value(); } + pointer operator->() const { return my_node_ptr->storage(); } + + skip_list_iterator& operator++() { + __TBB_ASSERT(my_node_ptr != nullptr, nullptr); + my_node_ptr = my_node_ptr->next(0); + return *this; + } + + skip_list_iterator operator++(int) { + skip_list_iterator tmp = *this; + ++*this; + return tmp; + } + +private: + skip_list_iterator(node_type* n) : my_node_ptr(n) {} + + node_ptr my_node_ptr; + + template + friend class concurrent_skip_list; + + template + friend class skip_list_iterator; + + friend class const_range; + friend class range; + + friend bool operator==( const skip_list_iterator& lhs, const skip_list_iterator& rhs ) { + return lhs.my_node_ptr == rhs.my_node_ptr; + } + + friend bool operator!=( const skip_list_iterator& lhs, const skip_list_iterator& rhs ) { + return lhs.my_node_ptr != rhs.my_node_ptr; + } +}; // class skip_list_iterator + +template +class concurrent_skip_list { +protected: + using container_traits = Traits; + using self_type = concurrent_skip_list; + using allocator_type = typename container_traits::allocator_type; + using allocator_traits_type = tbb::detail::allocator_traits; + using key_compare = typename container_traits::compare_type; + using value_compare = typename container_traits::value_compare; + using key_type = typename container_traits::key_type; + using value_type = typename container_traits::value_type; + static_assert(std::is_same::value, + "value_type of the container should be the same as its allocator"); + + using size_type = std::size_t; + using difference_type = std::ptrdiff_t; + + static constexpr size_type max_level = container_traits::max_level; + + using node_allocator_type = typename allocator_traits_type::template rebind_alloc; + using node_allocator_traits = tbb::detail::allocator_traits; + + using list_node_type = skip_list_node; + using node_type = d1::node_handle; + + using iterator = skip_list_iterator; + using const_iterator = skip_list_iterator; + + using reference = value_type&; + using const_reference = const value_type&; + using pointer = typename allocator_traits_type::pointer; + using const_pointer = typename allocator_traits_type::const_pointer; + + using random_level_generator_type = typename container_traits::random_level_generator_type; + + using node_ptr = list_node_type*; + + using array_type = std::array; +private: + template + using is_transparent = dependent_bool, T>; +public: + static constexpr bool allow_multimapping = container_traits::allow_multimapping; + + concurrent_skip_list() : my_head_ptr(nullptr), my_size(0), my_max_height(0) {} + + explicit concurrent_skip_list( const key_compare& comp, const allocator_type& alloc = allocator_type() ) + : my_node_allocator(alloc), my_compare(comp), my_head_ptr(nullptr), my_size(0), my_max_height(0) {} + + explicit concurrent_skip_list( const allocator_type& alloc ) + : concurrent_skip_list(key_compare(), alloc) {} + + template + concurrent_skip_list( InputIterator first, InputIterator last, const key_compare& comp = key_compare(), + const allocator_type& alloc = allocator_type() ) + : concurrent_skip_list(comp, alloc) + { + internal_copy(first, last); + } + + template + concurrent_skip_list( InputIterator first, InputIterator last, const allocator_type& alloc ) + : concurrent_skip_list(first, last, key_compare(), alloc) {} + + concurrent_skip_list( std::initializer_list init, const key_compare& comp = key_compare(), + const allocator_type& alloc = allocator_type() ) + : concurrent_skip_list(init.begin(), init.end(), comp, alloc) {} + + concurrent_skip_list( std::initializer_list init, const allocator_type& alloc ) + : concurrent_skip_list(init, key_compare(), alloc) {} + + concurrent_skip_list( const concurrent_skip_list& other ) + : my_node_allocator(node_allocator_traits::select_on_container_copy_construction(other.get_allocator())), + my_compare(other.my_compare), my_rng(other.my_rng), my_head_ptr(nullptr), + my_size(0), my_max_height(0) + { + internal_copy(other); + __TBB_ASSERT(my_size == other.my_size, "Wrong size of copy-constructed container"); + } + + concurrent_skip_list( const concurrent_skip_list& other, const allocator_type& alloc ) + : my_node_allocator(alloc), my_compare(other.my_compare), my_rng(other.my_rng), my_head_ptr(nullptr), + my_size(0), my_max_height(0) + { + internal_copy(other); + __TBB_ASSERT(my_size == other.my_size, "Wrong size of copy-constructed container"); + } + + concurrent_skip_list( concurrent_skip_list&& other ) + : my_node_allocator(std::move(other.my_node_allocator)), my_compare(other.my_compare), + my_rng(std::move(other.my_rng)), my_head_ptr(nullptr) // my_head_ptr would be stored in internal_move + { + internal_move(std::move(other)); + } + + concurrent_skip_list( concurrent_skip_list&& other, const allocator_type& alloc ) + : my_node_allocator(alloc), my_compare(other.my_compare), + my_rng(std::move(other.my_rng)), my_head_ptr(nullptr) + { + using is_always_equal = typename allocator_traits_type::is_always_equal; + internal_move_construct_with_allocator(std::move(other), is_always_equal()); + } + + ~concurrent_skip_list() { + clear(); + delete_head(); + } + + concurrent_skip_list& operator=( const concurrent_skip_list& other ) { + if (this != &other) { + clear(); + copy_assign_allocators(my_node_allocator, other.my_node_allocator); + my_compare = other.my_compare; + my_rng = other.my_rng; + internal_copy(other); + } + return *this; + } + + concurrent_skip_list& operator=( concurrent_skip_list&& other ) { + if (this != &other) { + clear(); + delete_head(); + + my_compare = std::move(other.my_compare); + my_rng = std::move(other.my_rng); + + move_assign_allocators(my_node_allocator, other.my_node_allocator); + using pocma_type = typename node_allocator_traits::propagate_on_container_move_assignment; + using is_always_equal = typename node_allocator_traits::is_always_equal; + internal_move_assign(std::move(other), tbb::detail::disjunction()); + } + return *this; + } + + concurrent_skip_list& operator=( std::initializer_list il ) + { + clear(); + insert(il.begin(),il.end()); + return *this; + } + + std::pair insert( const value_type& value ) { + return internal_insert(value); + } + + std::pair insert( value_type&& value ) { + return internal_insert(std::move(value)); + } + + iterator insert( const_iterator, const_reference value ) { + // Ignore hint + return insert(value).first; + } + + iterator insert( const_iterator, value_type&& value ) { + // Ignore hint + return insert(std::move(value)).first; + } + + template + void insert( InputIterator first, InputIterator last ) { + while (first != last) { + insert(*first); + ++first; + } + } + + void insert( std::initializer_list init ) { + insert(init.begin(), init.end()); + } + + std::pair insert( node_type&& nh ) { + if (!nh.empty()) { + auto insert_node = d1::node_handle_accessor::get_node_ptr(nh); + std::pair insert_result = internal_insert_node(insert_node); + if (insert_result.second) { + d1::node_handle_accessor::deactivate(nh); + } + return insert_result; + } + return std::pair(end(), false); + } + + iterator insert( const_iterator, node_type&& nh ) { + // Ignore hint + return insert(std::move(nh)).first; + } + + template + std::pair emplace( Args&&... args ) { + return internal_insert(std::forward(args)...); + } + + template + iterator emplace_hint( const_iterator, Args&&... args ) { + // Ignore hint + return emplace(std::forward(args)...).first; + } + + iterator unsafe_erase( iterator pos ) { + std::pair extract_result = internal_extract(pos); + if (extract_result.first) { // node was extracted + delete_value_node(extract_result.first); + return extract_result.second; + } + return end(); + } + + iterator unsafe_erase( const_iterator pos ) { + return unsafe_erase(get_iterator(pos)); + } + + iterator unsafe_erase( const_iterator first, const_iterator last ) { + while (first != last) { + // Unsafe erase returns the iterator which follows the erased one + first = unsafe_erase(first); + } + return get_iterator(first); + } + + size_type unsafe_erase( const key_type& key ) { + return internal_erase(key); + } + + template + typename std::enable_if::value + && !std::is_convertible::value + && !std::is_convertible::value, + size_type>::type unsafe_erase( const K& key ) + { + return internal_erase(key); + } + + node_type unsafe_extract( const_iterator pos ) { + std::pair extract_result = internal_extract(pos); + return extract_result.first ? d1::node_handle_accessor::construct(extract_result.first) : node_type(); + } + + node_type unsafe_extract( iterator pos ) { + return unsafe_extract(const_iterator(pos)); + } + + node_type unsafe_extract( const key_type& key ) { + return unsafe_extract(find(key)); + } + + template + typename std::enable_if::value + && !std::is_convertible::value + && !std::is_convertible::value, + node_type>::type unsafe_extract( const K& key ) + { + return unsafe_extract(find(key)); + } + + iterator lower_bound( const key_type& key ) { + return iterator(internal_get_bound(key, my_compare)); + } + + const_iterator lower_bound( const key_type& key ) const { + return const_iterator(internal_get_bound(key, my_compare)); + } + + template + typename std::enable_if::value, iterator>::type lower_bound( const K& key ) { + return iterator(internal_get_bound(key, my_compare)); + } + + template + typename std::enable_if::value, const_iterator>::type lower_bound( const K& key ) const { + return const_iterator(internal_get_bound(key, my_compare)); + } + + iterator upper_bound( const key_type& key ) { + return iterator(internal_get_bound(key, not_greater_compare(my_compare))); + } + + const_iterator upper_bound( const key_type& key ) const { + return const_iterator(internal_get_bound(key, not_greater_compare(my_compare))); + } + + template + typename std::enable_if::value, iterator>::type upper_bound( const K& key ) { + return iterator(internal_get_bound(key, not_greater_compare(my_compare))); + } + + template + typename std::enable_if::value, const_iterator>::type upper_bound( const K& key ) const { + return const_iterator(internal_get_bound(key, not_greater_compare(my_compare))); + } + + iterator find( const key_type& key ) { + return iterator(internal_find(key)); + } + + const_iterator find( const key_type& key ) const { + return const_iterator(internal_find(key)); + } + + template + typename std::enable_if::value, iterator>::type find( const K& key ) { + return iterator(internal_find(key)); + } + + template + typename std::enable_if::value, const_iterator>::type find( const K& key ) const { + return const_iterator(internal_find(key)); + } + + size_type count( const key_type& key ) const { + return internal_count(key); + } + + template + typename std::enable_if::value, size_type>::type count( const K& key ) const { + return internal_count(key); + } + + bool contains( const key_type& key ) const { + return find(key) != end(); + } + + template + typename std::enable_if::value, bool>::type contains( const K& key ) const { + return find(key) != end(); + } + + void clear() noexcept { + // clear is not thread safe - load can be relaxed + node_ptr head = my_head_ptr.load(std::memory_order_relaxed); + + if (head == nullptr) return; // Head is not allocated => container is empty + + node_ptr current = head->next(0); + + // Delete all value nodes in the container + while (current) { + node_ptr next = current->next(0); + delete_value_node(current); + current = next; + } + + for (size_type level = 0; level < head->height(); ++level) { + head->set_next(level, nullptr); + } + + my_size.store(0, std::memory_order_relaxed); + my_max_height.store(0, std::memory_order_relaxed); + } + + iterator begin() { + return iterator(internal_begin()); + } + + const_iterator begin() const { + return const_iterator(internal_begin()); + } + + const_iterator cbegin() const { + return const_iterator(internal_begin()); + } + + iterator end() { + return iterator(nullptr); + } + + const_iterator end() const { + return const_iterator(nullptr); + } + + const_iterator cend() const { + return const_iterator(nullptr); + } + + size_type size() const { + return my_size.load(std::memory_order_relaxed); + } + + size_type max_size() const { + return node_allocator_traits::max_size(my_node_allocator); + } + + __TBB_nodiscard bool empty() const { + return 0 == size(); + } + + allocator_type get_allocator() const { + return my_node_allocator; + } + + void swap(concurrent_skip_list& other) { + if (this != &other) { + using pocs_type = typename node_allocator_traits::propagate_on_container_swap; + using is_always_equal = typename node_allocator_traits::is_always_equal; + internal_swap(other, tbb::detail::disjunction()); + } + } + + std::pair equal_range(const key_type& key) { + return internal_equal_range(key); + } + + std::pair equal_range(const key_type& key) const { + return internal_equal_range(key); + } + + template + typename std::enable_if::value, std::pair>::type equal_range( const K& key ) { + return internal_equal_range(key); + } + + template + typename std::enable_if::value, std::pair>::type equal_range( const K& key ) const { + return internal_equal_range(key); + } + + key_compare key_comp() const { return my_compare; } + + value_compare value_comp() const { return container_traits::value_comp(my_compare); } + + class const_range_type { + public: + using size_type = typename concurrent_skip_list::size_type; + using difference_type = typename concurrent_skip_list::difference_type; + using iterator = typename concurrent_skip_list::const_iterator; + using value_type = typename iterator::value_type; + using reference = typename iterator::reference; + + bool empty() const { + return my_begin.my_node_ptr ? (my_begin.my_node_ptr->next(0) == my_end.my_node_ptr) + : true; + } + + bool is_divisible() const { + return my_begin.my_node_ptr && my_level != 0 + ? my_begin.my_node_ptr->next(my_level - 1) != my_end.my_node_ptr + : false; + } + + size_type size() const { return std::distance(my_begin, my_end); } + + const_range_type( const_range_type& r, split) + : my_end(r.my_end) { + if (r.empty()) { + __TBB_ASSERT(my_end.my_node_ptr == nullptr, nullptr); + my_begin = my_end; + my_level = 0; + } else { + my_begin = iterator(r.my_begin.my_node_ptr->next(r.my_level - 1)); + my_level = my_begin.my_node_ptr->height(); + } + r.my_end = my_begin; + } + + const_range_type( const concurrent_skip_list& l) + : my_end(l.end()), my_begin(l.begin()), + my_level(my_begin.my_node_ptr ? my_begin.my_node_ptr->height() : 0) {} + + iterator begin() const { return my_begin; } + iterator end() const { return my_end; } + size_type grainsize() const { return 1; } + + private: + const_iterator my_end; + const_iterator my_begin; + size_type my_level; + }; // class const_range_type + + class range_type : public const_range_type { + public: + using iterator = typename concurrent_skip_list::iterator; + using value_type = typename iterator::value_type; + using reference = typename iterator::reference; + + range_type(range_type& r, split) : const_range_type(r, split()) {} + range_type(const concurrent_skip_list& l) : const_range_type(l) {} + + iterator begin() const { + node_ptr node = const_range_type::begin().my_node_ptr; + return iterator(node); + } + + iterator end() const { + node_ptr node = const_range_type::end().my_node_ptr; + return iterator(node); + } + }; // class range_type + + range_type range() { return range_type(*this); } + const_range_type range() const { return const_range_type(*this); } + +private: + node_ptr internal_begin() const { + node_ptr head = get_head(); + return head == nullptr ? head : head->next(0); + } + + void internal_move(concurrent_skip_list&& other) { + my_head_ptr.store(other.my_head_ptr.load(std::memory_order_relaxed), std::memory_order_relaxed); + other.my_head_ptr.store(nullptr, std::memory_order_relaxed); + + my_size.store(other.my_size.load(std::memory_order_relaxed), std::memory_order_relaxed); + other.my_size.store(0, std::memory_order_relaxed); + + my_max_height.store(other.my_max_height.load(std::memory_order_relaxed), std::memory_order_relaxed); + other.my_max_height.store(0, std::memory_order_relaxed); + } + + void internal_move_construct_with_allocator(concurrent_skip_list&& other, + /*is_always_equal = */std::true_type) { + internal_move(std::move(other)); + } + + void internal_move_construct_with_allocator(concurrent_skip_list&& other, + /*is_always_equal = */std::false_type) { + if (my_node_allocator == other.get_allocator()) { + internal_move(std::move(other)); + } else { + my_size.store(0, std::memory_order_relaxed); + my_max_height.store(other.my_max_height.load(std::memory_order_relaxed), std::memory_order_relaxed); + internal_copy(std::make_move_iterator(other.begin()), std::make_move_iterator(other.end())); + } + } + + static const key_type& get_key( node_ptr n ) { + __TBB_ASSERT(n, nullptr); + return container_traits::get_key(static_cast(n)->value()); + } + + template + bool found( node_ptr node, const K& key ) const { + return node != nullptr && !my_compare(key, get_key(node)); + } + + template + node_ptr internal_find(const K& key) const { + return allow_multimapping ? internal_find_multi(key) : internal_find_unique(key); + } + + template + node_ptr internal_find_multi( const K& key ) const { + node_ptr prev = get_head(); + if (prev == nullptr) return nullptr; // If the head node is not allocated - exit + + node_ptr curr = nullptr; + node_ptr old_curr = curr; + + for (size_type h = my_max_height.load(std::memory_order_acquire); h > 0; --h) { + curr = internal_find_position(h - 1, prev, key, my_compare); + + if (curr != old_curr && found(curr, key)) { + return curr; + } + old_curr = curr; + } + return nullptr; + } + + template + node_ptr internal_find_unique( const K& key ) const { + const_iterator it = lower_bound(key); + return (it == end() || my_compare(key, container_traits::get_key(*it))) ? nullptr : it.my_node_ptr; + } + + template + size_type internal_count( const K& key ) const { + if (allow_multimapping) { + // TODO: reimplement without double traversal + std::pair r = equal_range(key); + return std::distance(r.first, r.second); + } + return size_type(contains(key) ? 1 : 0); + } + + template + std::pair internal_equal_range(const K& key) const { + iterator lb = get_iterator(lower_bound(key)); + auto result = std::make_pair(lb, lb); + + // If the lower bound points to the node with the requested key + if (found(lb.my_node_ptr, key)) { + + if (!allow_multimapping) { + // For unique containers - move the second iterator forward and exit + ++result.second; + } else { + // For multi containers - find the upper bound starting from the lower bound + node_ptr prev = lb.my_node_ptr; + node_ptr curr = nullptr; + not_greater_compare cmp(my_compare); + + // Start from the lower bound of the range + for (size_type h = prev->height(); h > 0; --h) { + curr = prev->next(h - 1); + while (curr && cmp(get_key(curr), key)) { + prev = curr; + // If the height of the next node is greater than the current one - jump to its height + if (h < curr->height()) { + h = curr->height(); + } + curr = prev->next(h - 1); + } + } + result.second = iterator(curr); + } + } + + return result; + } + + // Finds position on the level using comparator cmp starting from the node prev + template + node_ptr internal_find_position( size_type level, node_ptr& prev, const K& key, + const Comparator& cmp ) const { + __TBB_ASSERT(level < prev->height(), "Wrong level to find position"); + node_ptr curr = prev->next(level); + + while (curr && cmp(get_key(curr), key)) { + prev = curr; + __TBB_ASSERT(level < prev->height(), nullptr); + curr = prev->next(level); + } + + return curr; + } + + // The same as previous overload, but allows index_number comparison + template + node_ptr internal_find_position( size_type level, node_ptr& prev, node_ptr node, + const Comparator& cmp ) const { + __TBB_ASSERT(level < prev->height(), "Wrong level to find position"); + node_ptr curr = prev->next(level); + + while (curr && cmp(get_key(curr), get_key(node))) { + if (allow_multimapping && cmp(get_key(node), get_key(curr)) && curr->index_number() > node->index_number()) { + break; + } + + prev = curr; + __TBB_ASSERT(level < prev->height(), nullptr); + curr = prev->next(level); + } + return curr; + } + + template + void fill_prev_curr_arrays(array_type& prev_nodes, array_type& curr_nodes, node_ptr node, const key_type& key, + const Comparator& cmp, node_ptr head ) { + + size_type curr_max_height = my_max_height.load(std::memory_order_acquire); + size_type node_height = node->height(); + if (curr_max_height < node_height) { + std::fill(prev_nodes.begin() + curr_max_height, prev_nodes.begin() + node_height, head); + std::fill(curr_nodes.begin() + curr_max_height, curr_nodes.begin() + node_height, nullptr); + } + + node_ptr prev = head; + for (size_type level = curr_max_height; level > 0; --level) { + node_ptr curr = internal_find_position(level - 1, prev, key, cmp); + prev_nodes[level - 1] = prev; + curr_nodes[level - 1] = curr; + } + } + + void fill_prev_array_for_existing_node( array_type& prev_nodes, node_ptr node ) { + node_ptr head = create_head_if_necessary(); + prev_nodes.fill(head); + + node_ptr prev = head; + for (size_type level = node->height(); level > 0; --level) { + while (prev->next(level - 1) != node) { + prev = prev->next(level - 1); + } + prev_nodes[level - 1] = prev; + } + } + + struct not_greater_compare { + const key_compare& my_less_compare; + + not_greater_compare( const key_compare& less_compare ) : my_less_compare(less_compare) {} + + template + bool operator()( const K1& first, const K2& second ) const { + return !my_less_compare(second, first); + } + }; + + not_greater_compare select_comparator( /*allow_multimapping = */ std::true_type ) { + return not_greater_compare(my_compare); + } + + key_compare select_comparator( /*allow_multimapping = */ std::false_type ) { + return my_compare; + } + + template + std::pair internal_insert( Args&&... args ) { + node_ptr new_node = create_value_node(std::forward(args)...); + std::pair insert_result = internal_insert_node(new_node); + if (!insert_result.second) { + delete_value_node(new_node); + } + return insert_result; + } + + std::pair internal_insert_node( node_ptr new_node ) { + array_type prev_nodes; + array_type curr_nodes; + size_type new_height = new_node->height(); + auto compare = select_comparator(std::integral_constant{}); + + node_ptr head_node = create_head_if_necessary(); + + for (;;) { + fill_prev_curr_arrays(prev_nodes, curr_nodes, new_node, get_key(new_node), compare, head_node); + + node_ptr prev = prev_nodes[0]; + node_ptr next = curr_nodes[0]; + + if (allow_multimapping) { + new_node->set_index_number(prev->index_number() + 1); + } else { + if (found(next, get_key(new_node))) { + return std::pair(iterator(next), false); + } + } + + new_node->set_next(0, next); + if (!prev->atomic_next(0).compare_exchange_strong(next, new_node)) { + continue; + } + + // If the node was successfully linked on the first level - it will be linked on other levels + // Insertion cannot fail starting from this point + + // If the height of inserted node is greater than maximum - increase maximum + size_type max_height = my_max_height.load(std::memory_order_acquire); + for (;;) { + if (new_height <= max_height || my_max_height.compare_exchange_strong(max_height, new_height)) { + // If the maximum was successfully updated by current thread + // or by an other thread for the value, greater or equal to new_height + break; + } + } + + for (std::size_t level = 1; level < new_height; ++level) { + // Link the node on upper levels + for (;;) { + prev = prev_nodes[level]; + next = static_cast(curr_nodes[level]); + + new_node->set_next(level, next); + __TBB_ASSERT(new_node->height() > level, "Internal structure break"); + if (prev->atomic_next(level).compare_exchange_strong(next, new_node)) { + break; + } + + for (size_type lev = level; lev != new_height; ++lev ) { + curr_nodes[lev] = internal_find_position(lev, prev_nodes[lev], new_node, compare); + } + } + } + ++my_size; + return std::pair(iterator(new_node), true); + } + } + + template + node_ptr internal_get_bound( const K& key, const Comparator& cmp ) const { + node_ptr prev = get_head(); + if (prev == nullptr) return nullptr; // If the head node is not allocated - exit + + node_ptr curr = nullptr; + + for (size_type h = my_max_height.load(std::memory_order_acquire); h > 0; --h) { + curr = internal_find_position(h - 1, prev, key, cmp); + } + + return curr; + } + + template + size_type internal_erase( const K& key ) { + auto eq = equal_range(key); + size_type old_size = size(); + unsafe_erase(eq.first, eq.second); + return old_size - size(); + } + + // Returns node_ptr to the extracted node and node_ptr to the next node after the extracted + std::pair internal_extract( const_iterator it ) { + std::pair result(nullptr, nullptr); + if ( it != end() ) { + array_type prev_nodes; + + node_ptr erase_node = it.my_node_ptr; + node_ptr next_node = erase_node->next(0); + fill_prev_array_for_existing_node(prev_nodes, erase_node); + + for (size_type level = 0; level < erase_node->height(); ++level) { + prev_nodes[level]->set_next(level, erase_node->next(level)); + erase_node->set_next(level, nullptr); + } + my_size.fetch_sub(1, std::memory_order_relaxed); + + result.first = erase_node; + result.second = next_node; + } + return result; + } + +protected: + template + void internal_merge( SourceType&& source ) { + using source_type = typename std::decay::type; + using source_iterator = typename source_type::iterator; + static_assert((std::is_same::value), "Incompatible containers cannot be merged"); + + for (source_iterator it = source.begin(); it != source.end();) { + source_iterator where = it++; + if (allow_multimapping || !contains(container_traits::get_key(*where))) { + node_type handle = source.unsafe_extract(where); + __TBB_ASSERT(!handle.empty(), "Extracted handle in merge is empty"); + + if (!insert(std::move(handle)).second) { + __TBB_ASSERT(!handle.empty(), "Handle should not be empty if insert fails"); + //If the insertion fails - return the node into source + source.insert(std::move(handle)); + } + __TBB_ASSERT(handle.empty(), "Node handle should be empty after the insertion"); + } + } + } + +private: + void internal_copy( const concurrent_skip_list& other ) { + internal_copy(other.begin(), other.end()); + } + + template + void internal_copy( Iterator first, Iterator last ) { + try_call([&] { + for (auto it = first; it != last; ++it) { + insert(*it); + } + }).on_exception([&] { + clear(); + delete_head(); + }); + } + + node_ptr create_node( size_type height ) { + return list_node_type::create(my_node_allocator, height); + } + + template + node_ptr create_value_node( Args&&... args ) { + node_ptr node = create_node(my_rng()); + + // try_call API is not convenient here due to broken + // variadic capture on GCC 4.8.5 + auto value_guard = make_raii_guard([&] { + delete_node(node); + }); + + // Construct the value inside the node + node_allocator_traits::construct(my_node_allocator, node->storage(), std::forward(args)...); + value_guard.dismiss(); + return node; + } + + node_ptr create_head_node() { + return create_node(max_level); + } + + void delete_head() { + node_ptr head = my_head_ptr.load(std::memory_order_relaxed); + if (head != nullptr) { + delete_node(head); + my_head_ptr.store(nullptr, std::memory_order_relaxed); + } + } + + void delete_node( node_ptr node ) { + list_node_type::destroy(my_node_allocator, node); + } + + void delete_value_node( node_ptr node ) { + // Destroy the value inside the node + node_allocator_traits::destroy(my_node_allocator, node->storage()); + delete_node(node); + } + + node_ptr get_head() const { + return my_head_ptr.load(std::memory_order_acquire); + } + + node_ptr create_head_if_necessary() { + node_ptr current_head = get_head(); + if (current_head == nullptr) { + // Head node was not created - create it + node_ptr new_head = create_head_node(); + if (my_head_ptr.compare_exchange_strong(current_head, new_head)) { + current_head = new_head; + } else { + // If an other thread has already created the head node - destroy new_head + // current_head now points to the actual head node + delete_node(new_head); + } + } + __TBB_ASSERT(my_head_ptr.load(std::memory_order_relaxed) != nullptr, nullptr); + __TBB_ASSERT(current_head != nullptr, nullptr); + return current_head; + } + + static iterator get_iterator( const_iterator it ) { + return iterator(it.my_node_ptr); + } + + void internal_move_assign( concurrent_skip_list&& other, /*POCMA || is_always_equal =*/std::true_type ) { + internal_move(std::move(other)); + } + + void internal_move_assign( concurrent_skip_list&& other, /*POCMA || is_always_equal =*/std::false_type ) { + if (my_node_allocator == other.my_node_allocator) { + internal_move(std::move(other)); + } else { + internal_copy(std::make_move_iterator(other.begin()), std::make_move_iterator(other.end())); + } + } + + void internal_swap_fields( concurrent_skip_list& other ) { + using std::swap; + swap_allocators(my_node_allocator, other.my_node_allocator); + swap(my_compare, other.my_compare); + swap(my_rng, other.my_rng); + + swap_atomics_relaxed(my_head_ptr, other.my_head_ptr); + swap_atomics_relaxed(my_size, other.my_size); + swap_atomics_relaxed(my_max_height, other.my_max_height); + } + + void internal_swap( concurrent_skip_list& other, /*POCMA || is_always_equal =*/std::true_type ) { + internal_swap_fields(other); + } + + void internal_swap( concurrent_skip_list& other, /*POCMA || is_always_equal =*/std::false_type ) { + __TBB_ASSERT(my_node_allocator == other.my_node_allocator, "Swapping with unequal allocators is not allowed"); + internal_swap_fields(other); + } + + node_allocator_type my_node_allocator; + key_compare my_compare; + random_level_generator_type my_rng; + std::atomic my_head_ptr; + std::atomic my_size; + std::atomic my_max_height; + + template + friend class concurrent_skip_list; +}; // class concurrent_skip_list + +template +bool operator==( const concurrent_skip_list& lhs, const concurrent_skip_list& rhs ) { + if (lhs.size() != rhs.size()) return false; +#if _MSC_VER + // Passing "unchecked" iterators to std::equal with 3 parameters + // causes compiler warnings. + // The workaround is to use overload with 4 parameters, which is + // available since C++14 - minimally supported version on MSVC + return std::equal(lhs.begin(), lhs.end(), rhs.begin(), rhs.end()); +#else + return std::equal(lhs.begin(), lhs.end(), rhs.begin()); +#endif +} + +#if !__TBB_CPP20_COMPARISONS_PRESENT +template +bool operator!=( const concurrent_skip_list& lhs, const concurrent_skip_list& rhs ) { + return !(lhs == rhs); +} +#endif + +#if __TBB_CPP20_COMPARISONS_PRESENT && __TBB_CPP20_CONCEPTS_PRESENT +template +tbb::detail::synthesized_three_way_result +operator<=>( const concurrent_skip_list& lhs, const concurrent_skip_list& rhs ) { + return std::lexicographical_compare_three_way(lhs.begin(), lhs.end(), + rhs.begin(), rhs.end(), + tbb::detail::synthesized_three_way_comparator{}); +} +#else +template +bool operator<( const concurrent_skip_list& lhs, const concurrent_skip_list& rhs ) { + return std::lexicographical_compare(lhs.begin(), lhs.end(), rhs.begin(), rhs.end()); +} + +template +bool operator>( const concurrent_skip_list& lhs, const concurrent_skip_list& rhs ) { + return rhs < lhs; +} + +template +bool operator<=( const concurrent_skip_list& lhs, const concurrent_skip_list& rhs ) { + return !(rhs < lhs); +} + +template +bool operator>=( const concurrent_skip_list& lhs, const concurrent_skip_list& rhs ) { + return !(lhs < rhs); +} +#endif // __TBB_CPP20_COMPARISONS_PRESENT && __TBB_CPP20_CONCEPTS_PRESENT + +// Generates a number from the interval [0, MaxLevel). +template +class concurrent_geometric_level_generator { +public: + static constexpr std::size_t max_level = MaxLevel; + // TODO: modify the algorithm to accept other values of max_level + static_assert(max_level == 32, "Incompatible max_level for rng"); + + concurrent_geometric_level_generator() : engines(std::minstd_rand::result_type(time(nullptr))) {} + + std::size_t operator()() { + // +1 is required to pass at least 1 into log2 (log2(0) is undefined) + // -1 is required to have an ability to return 0 from the generator (max_level - log2(2^31) - 1) + std::size_t result = max_level - std::size_t(tbb::detail::log2(engines.local()() + 1)) - 1; + __TBB_ASSERT(result <= max_level, nullptr); + return result; + } + +private: + tbb::enumerable_thread_specific engines; +}; + +} // namespace d2 + +} // namespace detail +} // namespace tbb + +#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) +// #pragma warning(pop) // warning 4127 is back +#endif + +#endif // __TBB_detail__concurrent_skip_list_H diff --git a/src/tbb/include/oneapi/tbb/detail/_concurrent_unordered_base.h b/src/tbb/include/oneapi/tbb/detail/_concurrent_unordered_base.h new file mode 100644 index 000000000..5e34d997c --- /dev/null +++ b/src/tbb/include/oneapi/tbb/detail/_concurrent_unordered_base.h @@ -0,0 +1,1515 @@ +/* + Copyright (c) 2005-2024 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_detail__concurrent_unordered_base_H +#define __TBB_detail__concurrent_unordered_base_H + +#if !defined(__TBB_concurrent_unordered_map_H) && !defined(__TBB_concurrent_unordered_set_H) +#error Do not #include this internal file directly; use public TBB headers instead. +#endif + +#include "_range_common.h" +#include "_containers_helpers.h" +#include "_segment_table.h" +#include "_hash_compare.h" +#include "_allocator_traits.h" +#include "_node_handle.h" +#include "_assert.h" +#include "_utils.h" +#include "_exception.h" +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) +// #pragma warning(push) +// #pragma warning(disable: 4127) // warning C4127: conditional expression is constant +#endif + +namespace tbb { +namespace detail { +namespace d2 { + +template +class concurrent_unordered_base; + +template +class solist_iterator { +private: + using node_ptr = typename Container::value_node_ptr; + template + friend class split_ordered_list; + template + friend class solist_iterator; + template + friend class concurrent_unordered_base; + template + friend bool operator==( const solist_iterator& i, const solist_iterator& j ); + template + friend bool operator!=( const solist_iterator& i, const solist_iterator& j ); +public: + using value_type = Value; + using difference_type = typename Container::difference_type; + using pointer = value_type*; + using reference = value_type&; + using iterator_category = std::forward_iterator_tag; + + solist_iterator() : my_node_ptr(nullptr) {} + solist_iterator( const solist_iterator& other ) + : my_node_ptr(other.my_node_ptr) {} + + solist_iterator& operator=( const solist_iterator& other ) { + my_node_ptr = other.my_node_ptr; + return *this; + } + + reference operator*() const { + return my_node_ptr->value(); + } + + pointer operator->() const { + return my_node_ptr->storage(); + } + + solist_iterator& operator++() { + auto next_node = my_node_ptr->next(); + while(next_node && next_node->is_dummy()) { + next_node = next_node->next(); + } + my_node_ptr = static_cast(next_node); + return *this; + } + + solist_iterator operator++(int) { + solist_iterator tmp = *this; + ++*this; + return tmp; + } + +private: + solist_iterator( node_ptr pnode ) : my_node_ptr(pnode) {} + + node_ptr get_node_ptr() const { return my_node_ptr; } + + node_ptr my_node_ptr; +}; + +template +bool operator==( const solist_iterator& i, const solist_iterator& j ) { + return i.my_node_ptr == j.my_node_ptr; +} + +template +bool operator!=( const solist_iterator& i, const solist_iterator& j ) { + return i.my_node_ptr != j.my_node_ptr; +} + +template +class list_node { +public: + using node_ptr = list_node*; + using sokey_type = SokeyType; + + list_node(sokey_type key) : my_next(nullptr), my_order_key(key) {} + + void init( sokey_type key ) { + my_order_key = key; + } + + sokey_type order_key() const { + return my_order_key; + } + + bool is_dummy() { + // The last bit of order key is unset for dummy nodes + return (my_order_key & 0x1) == 0; + } + + node_ptr next() const { + return my_next.load(std::memory_order_acquire); + } + + void set_next( node_ptr next_node ) { + my_next.store(next_node, std::memory_order_release); + } + + bool try_set_next( node_ptr expected_next, node_ptr new_next ) { + return my_next.compare_exchange_strong(expected_next, new_next); + } + +private: + std::atomic my_next; + sokey_type my_order_key; +}; // class list_node + +template +class value_node : public list_node +{ +public: + using base_type = list_node; + using sokey_type = typename base_type::sokey_type; + using value_type = ValueType; + + value_node( sokey_type ord_key ) : base_type(ord_key) {} + ~value_node() {} + value_type* storage() { + return &my_value; + } + + value_type& value() { + return *storage(); + } + +private: + union { + value_type my_value; + }; +}; // class value_node + +template +class concurrent_unordered_base { + using self_type = concurrent_unordered_base; + using traits_type = Traits; + using hash_compare_type = typename traits_type::hash_compare_type; + class unordered_segment_table; +public: + using value_type = typename traits_type::value_type; + using key_type = typename traits_type::key_type; + using allocator_type = typename traits_type::allocator_type; + +private: + using allocator_traits_type = tbb::detail::allocator_traits; + // TODO: check assert conditions for different C++ standards + static_assert(std::is_same::value, + "value_type of the container must be the same as its allocator"); + using sokey_type = std::size_t; + +public: + using size_type = std::size_t; + using difference_type = std::ptrdiff_t; + + using iterator = solist_iterator; + using const_iterator = solist_iterator; + using local_iterator = iterator; + using const_local_iterator = const_iterator; + + using reference = value_type&; + using const_reference = const value_type&; + using pointer = typename allocator_traits_type::pointer; + using const_pointer = typename allocator_traits_type::const_pointer; + + using hasher = typename hash_compare_type::hasher; + using key_equal = typename hash_compare_type::key_equal; + +private: + using list_node_type = list_node; + using value_node_type = value_node; + using node_ptr = list_node_type*; + using value_node_ptr = value_node_type*; + + using value_node_allocator_type = typename allocator_traits_type::template rebind_alloc; + using node_allocator_type = typename allocator_traits_type::template rebind_alloc; + + using node_allocator_traits = tbb::detail::allocator_traits; + using value_node_allocator_traits = tbb::detail::allocator_traits; + + static constexpr size_type round_up_to_power_of_two( size_type bucket_count ) { + return size_type(1) << size_type(tbb::detail::log2(uintptr_t(bucket_count == 0 ? 1 : bucket_count) * 2 - 1)); + } + + template + using is_transparent = dependent_bool, T>; +public: + using node_type = d1::node_handle; + + explicit concurrent_unordered_base( size_type bucket_count, const hasher& hash = hasher(), + const key_equal& equal = key_equal(), const allocator_type& alloc = allocator_type() ) + : my_size(0), + my_bucket_count(round_up_to_power_of_two(bucket_count)), + my_max_load_factor(float(initial_max_load_factor)), + my_hash_compare(hash, equal), + my_head(sokey_type(0)), + my_segments(alloc) {} + + concurrent_unordered_base() : concurrent_unordered_base(initial_bucket_count) {} + + concurrent_unordered_base( size_type bucket_count, const allocator_type& alloc ) + : concurrent_unordered_base(bucket_count, hasher(), key_equal(), alloc) {} + + concurrent_unordered_base( size_type bucket_count, const hasher& hash, const allocator_type& alloc ) + : concurrent_unordered_base(bucket_count, hash, key_equal(), alloc) {} + + explicit concurrent_unordered_base( const allocator_type& alloc ) + : concurrent_unordered_base(initial_bucket_count, hasher(), key_equal(), alloc) {} + + template + concurrent_unordered_base( InputIterator first, InputIterator last, + size_type bucket_count = initial_bucket_count, const hasher& hash = hasher(), + const key_equal& equal = key_equal(), const allocator_type& alloc = allocator_type() ) + : concurrent_unordered_base(bucket_count, hash, equal, alloc) + { + insert(first, last); + } + + template + concurrent_unordered_base( InputIterator first, InputIterator last, + size_type bucket_count, const allocator_type& alloc ) + : concurrent_unordered_base(first, last, bucket_count, hasher(), key_equal(), alloc) {} + + template + concurrent_unordered_base( InputIterator first, InputIterator last, + size_type bucket_count, const hasher& hash, const allocator_type& alloc ) + : concurrent_unordered_base(first, last, bucket_count, hash, key_equal(), alloc) {} + + concurrent_unordered_base( const concurrent_unordered_base& other ) + : my_size(other.my_size.load(std::memory_order_relaxed)), + my_bucket_count(other.my_bucket_count.load(std::memory_order_relaxed)), + my_max_load_factor(other.my_max_load_factor), + my_hash_compare(other.my_hash_compare), + my_head(other.my_head.order_key()), + my_segments(other.my_segments) + { + try_call( [&] { + internal_copy(other); + } ).on_exception( [&] { + clear(); + }); + } + + concurrent_unordered_base( const concurrent_unordered_base& other, const allocator_type& alloc ) + : my_size(other.my_size.load(std::memory_order_relaxed)), + my_bucket_count(other.my_bucket_count.load(std::memory_order_relaxed)), + my_max_load_factor(other.my_max_load_factor), + my_hash_compare(other.my_hash_compare), + my_head(other.my_head.order_key()), + my_segments(other.my_segments, alloc) + { + try_call( [&] { + internal_copy(other); + } ).on_exception( [&] { + clear(); + }); + } + + concurrent_unordered_base( concurrent_unordered_base&& other ) + : my_size(other.my_size.load(std::memory_order_relaxed)), + my_bucket_count(other.my_bucket_count.load(std::memory_order_relaxed)), + my_max_load_factor(std::move(other.my_max_load_factor)), + my_hash_compare(std::move(other.my_hash_compare)), + my_head(other.my_head.order_key()), + my_segments(std::move(other.my_segments)) + { + move_content(std::move(other)); + } + + concurrent_unordered_base( concurrent_unordered_base&& other, const allocator_type& alloc ) + : my_size(other.my_size.load(std::memory_order_relaxed)), + my_bucket_count(other.my_bucket_count.load(std::memory_order_relaxed)), + my_max_load_factor(std::move(other.my_max_load_factor)), + my_hash_compare(std::move(other.my_hash_compare)), + my_head(other.my_head.order_key()), + my_segments(std::move(other.my_segments), alloc) + { + using is_always_equal = typename allocator_traits_type::is_always_equal; + internal_move_construct_with_allocator(std::move(other), alloc, is_always_equal()); + } + + concurrent_unordered_base( std::initializer_list init, + size_type bucket_count = initial_bucket_count, + const hasher& hash = hasher(), const key_equal& equal = key_equal(), + const allocator_type& alloc = allocator_type() ) + : concurrent_unordered_base(init.begin(), init.end(), bucket_count, hash, equal, alloc) {} + + concurrent_unordered_base( std::initializer_list init, + size_type bucket_count, const allocator_type& alloc ) + : concurrent_unordered_base(init, bucket_count, hasher(), key_equal(), alloc) {} + + concurrent_unordered_base( std::initializer_list init, + size_type bucket_count, const hasher& hash, const allocator_type& alloc ) + : concurrent_unordered_base(init, bucket_count, hash, key_equal(), alloc) {} + + ~concurrent_unordered_base() { + internal_clear(); + } + + concurrent_unordered_base& operator=( const concurrent_unordered_base& other ) { + if (this != &other) { + clear(); + my_size.store(other.my_size.load(std::memory_order_relaxed), std::memory_order_relaxed); + my_bucket_count.store(other.my_bucket_count.load(std::memory_order_relaxed), std::memory_order_relaxed); + my_max_load_factor = other.my_max_load_factor; + my_hash_compare = other.my_hash_compare; + my_segments = other.my_segments; + internal_copy(other); // TODO: guards for exceptions? + } + return *this; + } + + concurrent_unordered_base& operator=( concurrent_unordered_base&& other ) noexcept(unordered_segment_table::is_noexcept_assignment) { + if (this != &other) { + clear(); + my_size.store(other.my_size.load(std::memory_order_relaxed), std::memory_order_relaxed); + my_bucket_count.store(other.my_bucket_count.load(std::memory_order_relaxed), std::memory_order_relaxed); + my_max_load_factor = std::move(other.my_max_load_factor); + my_hash_compare = std::move(other.my_hash_compare); + my_segments = std::move(other.my_segments); + + using pocma_type = typename allocator_traits_type::propagate_on_container_move_assignment; + using is_always_equal = typename allocator_traits_type::is_always_equal; + internal_move_assign(std::move(other), tbb::detail::disjunction()); + } + return *this; + } + + concurrent_unordered_base& operator=( std::initializer_list init ) { + clear(); + insert(init); + return *this; + } + + void swap( concurrent_unordered_base& other ) noexcept(unordered_segment_table::is_noexcept_swap) { + if (this != &other) { + using pocs_type = typename allocator_traits_type::propagate_on_container_swap; + using is_always_equal = typename allocator_traits_type::is_always_equal; + internal_swap(other, tbb::detail::disjunction()); + } + } + + allocator_type get_allocator() const noexcept { return my_segments.get_allocator(); } + + iterator begin() noexcept { return iterator(first_value_node(&my_head)); } + const_iterator begin() const noexcept { return const_iterator(first_value_node(const_cast(&my_head))); } + const_iterator cbegin() const noexcept { return const_iterator(first_value_node(const_cast(&my_head))); } + + iterator end() noexcept { return iterator(nullptr); } + const_iterator end() const noexcept { return const_iterator(nullptr); } + const_iterator cend() const noexcept { return const_iterator(nullptr); } + + __TBB_nodiscard bool empty() const noexcept { return size() == 0; } + size_type size() const noexcept { return my_size.load(std::memory_order_relaxed); } + size_type max_size() const noexcept { return allocator_traits_type::max_size(get_allocator()); } + + void clear() noexcept { + internal_clear(); + } + + std::pair insert( const value_type& value ) { + return internal_insert_value(value); + } + + std::pair insert( value_type&& value ) { + return internal_insert_value(std::move(value)); + } + + iterator insert( const_iterator, const value_type& value ) { + // Ignore hint + return insert(value).first; + } + + iterator insert( const_iterator, value_type&& value ) { + // Ignore hint + return insert(std::move(value)).first; + } + + template + void insert( InputIterator first, InputIterator last ) { + for (; first != last; ++first) { + insert(*first); + } + } + + void insert( std::initializer_list init ) { + insert(init.begin(), init.end()); + } + + std::pair insert( node_type&& nh ) { + if (!nh.empty()) { + value_node_ptr insert_node = d1::node_handle_accessor::get_node_ptr(nh); + auto init_node = [&insert_node]( sokey_type order_key )->value_node_ptr { + insert_node->init(order_key); + return insert_node; + }; + auto insert_result = internal_insert(insert_node->value(), init_node); + if (insert_result.inserted) { + // If the insertion succeeded - set node handle to the empty state + __TBB_ASSERT(insert_result.remaining_node == nullptr, + "internal_insert_node should not return the remaining node if the insertion succeeded"); + d1::node_handle_accessor::deactivate(nh); + } + return { iterator(insert_result.node_with_equal_key), insert_result.inserted }; + } + return {end(), false}; + } + + iterator insert( const_iterator, node_type&& nh ) { + // Ignore hint + return insert(std::move(nh)).first; + } + + template + std::pair emplace( Args&&... args ) { + // Create a node with temporary order_key 0, which will be reinitialize + // in internal_insert after the hash calculation + value_node_ptr insert_node = create_node(0, std::forward(args)...); + + auto init_node = [&insert_node]( sokey_type order_key )->value_node_ptr { + insert_node->init(order_key); + return insert_node; + }; + + auto insert_result = internal_insert(insert_node->value(), init_node); + + if (!insert_result.inserted) { + // If the insertion failed - destroy the node which was created + insert_node->init(split_order_key_regular(1)); + destroy_node(insert_node); + } + + return { iterator(insert_result.node_with_equal_key), insert_result.inserted }; + } + + template + iterator emplace_hint( const_iterator, Args&&... args ) { + // Ignore hint + return emplace(std::forward(args)...).first; + } + + iterator unsafe_erase( const_iterator pos ) { + return iterator(first_value_node(internal_erase(pos.get_node_ptr()))); + } + + iterator unsafe_erase( iterator pos ) { + return iterator(first_value_node(internal_erase(pos.get_node_ptr()))); + } + + iterator unsafe_erase( const_iterator first, const_iterator last ) { + while(first != last) { + first = unsafe_erase(first); + } + return iterator(first.get_node_ptr()); + } + + size_type unsafe_erase( const key_type& key ) { + return internal_erase_by_key(key); + } + + template + typename std::enable_if::value + && !std::is_convertible::value + && !std::is_convertible::value, + size_type>::type unsafe_erase( const K& key ) + { + return internal_erase_by_key(key); + } + + node_type unsafe_extract( const_iterator pos ) { + internal_extract(pos.get_node_ptr()); + return d1::node_handle_accessor::construct(pos.get_node_ptr()); + } + + node_type unsafe_extract( iterator pos ) { + internal_extract(pos.get_node_ptr()); + return d1::node_handle_accessor::construct(pos.get_node_ptr()); + } + + node_type unsafe_extract( const key_type& key ) { + iterator item = find(key); + return item == end() ? node_type() : unsafe_extract(item); + } + + template + typename std::enable_if::value + && !std::is_convertible::value + && !std::is_convertible::value, + node_type>::type unsafe_extract( const K& key ) + { + iterator item = find(key); + return item == end() ? node_type() : unsafe_extract(item); + } + + // Lookup functions + iterator find( const key_type& key ) { + value_node_ptr result = internal_find(key); + return result == nullptr ? end() : iterator(result); + } + + const_iterator find( const key_type& key ) const { + value_node_ptr result = const_cast(this)->internal_find(key); + return result == nullptr ? end() : const_iterator(result); + } + + template + typename std::enable_if::value, iterator>::type find( const K& key ) { + value_node_ptr result = internal_find(key); + return result == nullptr ? end() : iterator(result); + } + + template + typename std::enable_if::value, const_iterator>::type find( const K& key ) const { + value_node_ptr result = const_cast(this)->internal_find(key); + return result == nullptr ? end() : const_iterator(result); + } + + std::pair equal_range( const key_type& key ) { + auto result = internal_equal_range(key); + return std::make_pair(iterator(result.first), iterator(result.second)); + } + + std::pair equal_range( const key_type& key ) const { + auto result = const_cast(this)->internal_equal_range(key); + return std::make_pair(const_iterator(result.first), const_iterator(result.second)); + } + + template + typename std::enable_if::value, std::pair>::type equal_range( const K& key ) { + auto result = internal_equal_range(key); + return std::make_pair(iterator(result.first), iterator(result.second)); + } + + template + typename std::enable_if::value, std::pair>::type equal_range( const K& key ) const { + auto result = const_cast(this)->internal_equal_range(key); + return std::make_pair(iterator(result.first), iterator(result.second)); + } + + size_type count( const key_type& key ) const { + return internal_count(key); + } + + template + typename std::enable_if::value, size_type>::type count( const K& key ) const { + return internal_count(key); + } + + bool contains( const key_type& key ) const { + return find(key) != end(); + } + + template + typename std::enable_if::value, bool>::type contains( const K& key ) const { + return find(key) != end(); + } + + // Bucket interface + local_iterator unsafe_begin( size_type n ) { + return local_iterator(first_value_node(get_bucket(n))); + } + + const_local_iterator unsafe_begin( size_type n ) const { + auto bucket_begin = first_value_node(const_cast(this)->get_bucket(n)); + return const_local_iterator(bucket_begin); + } + + const_local_iterator unsafe_cbegin( size_type n ) const { + auto bucket_begin = first_value_node(const_cast(this)->get_bucket(n)); + return const_local_iterator(bucket_begin); + } + + local_iterator unsafe_end( size_type n ) { + size_type bucket_count = my_bucket_count.load(std::memory_order_relaxed); + return n != bucket_count - 1 ? unsafe_begin(get_next_bucket_index(n)) : local_iterator(nullptr); + } + + const_local_iterator unsafe_end( size_type n ) const { + size_type bucket_count = my_bucket_count.load(std::memory_order_relaxed); + return n != bucket_count - 1 ? unsafe_begin(get_next_bucket_index(n)) : const_local_iterator(nullptr); + } + + const_local_iterator unsafe_cend( size_type n ) const { + size_type bucket_count = my_bucket_count.load(std::memory_order_relaxed); + return n != bucket_count - 1 ? unsafe_begin(get_next_bucket_index(n)) : const_local_iterator(nullptr); + } + + size_type unsafe_bucket_count() const { return my_bucket_count.load(std::memory_order_relaxed); } + + size_type unsafe_max_bucket_count() const { + return max_size(); + } + + size_type unsafe_bucket_size( size_type n ) const { + return size_type(std::distance(unsafe_begin(n), unsafe_end(n))); + } + + size_type unsafe_bucket( const key_type& key ) const { + return my_hash_compare(key) % my_bucket_count.load(std::memory_order_relaxed); + } + + // Hash policy + float load_factor() const { + return float(size() / float(my_bucket_count.load(std::memory_order_acquire))); + } + + float max_load_factor() const { return my_max_load_factor; } + + void max_load_factor( float mlf ) { + if (mlf != mlf || mlf < 0) { + tbb::detail::throw_exception(exception_id::invalid_load_factor); + } + my_max_load_factor = mlf; + } // TODO: unsafe? + + void rehash( size_type bucket_count ) { + size_type current_bucket_count = my_bucket_count.load(std::memory_order_acquire); + if (current_bucket_count < bucket_count) { + // TODO: do we need do-while here? + my_bucket_count.compare_exchange_strong(current_bucket_count, round_up_to_power_of_two(bucket_count)); + } + } + + void reserve( size_type elements_count ) { + size_type current_bucket_count = my_bucket_count.load(std::memory_order_acquire); + size_type necessary_bucket_count = current_bucket_count; + + // max_load_factor() is currently unsafe, so we can assume that my_max_load_factor + // would not be changed during the calculation + // TODO: Log2 seems useful here + while (necessary_bucket_count * max_load_factor() < elements_count) { + necessary_bucket_count <<= 1; + } + + while (!my_bucket_count.compare_exchange_strong(current_bucket_count, necessary_bucket_count)) { + if (current_bucket_count >= necessary_bucket_count) + break; + } + } + + // Observers + hasher hash_function() const { return my_hash_compare.hash_function(); } + key_equal key_eq() const { return my_hash_compare.key_eq(); } + + class const_range_type { + private: + const concurrent_unordered_base& my_instance; + node_ptr my_begin_node; // may be node* const + node_ptr my_end_node; + mutable node_ptr my_midpoint_node; + public: + using size_type = typename concurrent_unordered_base::size_type; + using value_type = typename concurrent_unordered_base::value_type; + using reference = typename concurrent_unordered_base::reference; + using difference_type = typename concurrent_unordered_base::difference_type; + using iterator = typename concurrent_unordered_base::const_iterator; + + bool empty() const { return my_begin_node == my_end_node; } + + bool is_divisible() const { + return my_midpoint_node != my_end_node; + } + + size_type grainsize() const { return 1; } + + const_range_type( const_range_type& range, split ) + : my_instance(range.my_instance), + my_begin_node(range.my_midpoint_node), + my_end_node(range.my_end_node) + { + range.my_end_node = my_begin_node; + __TBB_ASSERT(!empty(), "Splitting despite the range is not divisible"); + __TBB_ASSERT(!range.empty(), "Splitting despite the range is not divisible"); + set_midpoint(); + range.set_midpoint(); + } + + iterator begin() const { return iterator(my_instance.first_value_node(my_begin_node)); } + iterator end() const { return iterator(my_instance.first_value_node(my_end_node)); } + + const_range_type( const concurrent_unordered_base& table ) + : my_instance(table), my_begin_node(my_instance.first_value_node(const_cast(&table.my_head))), my_end_node(nullptr) + { + set_midpoint(); + } + private: + void set_midpoint() const { + if (empty()) { + my_midpoint_node = my_end_node; + } else { + sokey_type invalid_key = ~sokey_type(0); + sokey_type begin_key = my_begin_node != nullptr ? my_begin_node->order_key() : invalid_key; + sokey_type end_key = my_end_node != nullptr ? my_end_node->order_key() : invalid_key; + + size_type mid_bucket = reverse_bits(begin_key + (end_key - begin_key) / 2) % + my_instance.my_bucket_count.load(std::memory_order_relaxed); + while( my_instance.my_segments[mid_bucket].load(std::memory_order_relaxed) == nullptr) { + mid_bucket = my_instance.get_parent(mid_bucket); + } + if (reverse_bits(mid_bucket) > begin_key) { + // Found a dummy node between begin and end + my_midpoint_node = my_instance.first_value_node( + my_instance.my_segments[mid_bucket].load(std::memory_order_relaxed)); + } else { + // Didn't find a dummy node between begin and end + my_midpoint_node = my_end_node; + } + } + } + }; // class const_range_type + + class range_type : public const_range_type { + public: + using iterator = typename concurrent_unordered_base::iterator; + using const_range_type::const_range_type; + + iterator begin() const { return iterator(const_range_type::begin().get_node_ptr()); } + iterator end() const { return iterator(const_range_type::end().get_node_ptr()); } + }; // class range_type + + // Parallel iteration + range_type range() { + return range_type(*this); + } + + const_range_type range() const { + return const_range_type(*this); + } +protected: + static constexpr bool allow_multimapping = traits_type::allow_multimapping; + +private: + static constexpr size_type initial_bucket_count = 8; + static constexpr float initial_max_load_factor = 4; // TODO: consider 1? + static constexpr size_type pointers_per_embedded_table = sizeof(size_type) * 8 - 1; + + class unordered_segment_table + : public d1::segment_table, allocator_type, unordered_segment_table, pointers_per_embedded_table> + { + using self_type = unordered_segment_table; + using atomic_node_ptr = std::atomic; + using base_type = d1::segment_table, allocator_type, unordered_segment_table, pointers_per_embedded_table>; + using segment_type = typename base_type::segment_type; + using base_allocator_type = typename base_type::allocator_type; + + using segment_allocator_type = typename allocator_traits_type::template rebind_alloc; + using segment_allocator_traits = tbb::detail::allocator_traits; + public: + // Segment table for unordered containers should not be extended in the wait- free implementation + static constexpr bool allow_table_extending = false; + static constexpr bool is_noexcept_assignment = std::is_nothrow_move_assignable::value && + std::is_nothrow_move_assignable::value && + segment_allocator_traits::is_always_equal::value; + static constexpr bool is_noexcept_swap = tbb::detail::is_nothrow_swappable::value && + tbb::detail::is_nothrow_swappable::value && + segment_allocator_traits::is_always_equal::value; + + // TODO: using base_type::base_type is not compiling on Windows and Intel Compiler - investigate + unordered_segment_table( const base_allocator_type& alloc = base_allocator_type() ) + : base_type(alloc) {} + + unordered_segment_table( const unordered_segment_table& ) = default; + + unordered_segment_table( const unordered_segment_table& other, const base_allocator_type& alloc ) + : base_type(other, alloc) {} + + unordered_segment_table( unordered_segment_table&& ) = default; + + unordered_segment_table( unordered_segment_table&& other, const base_allocator_type& alloc ) + : base_type(std::move(other), alloc) {} + + unordered_segment_table& operator=( const unordered_segment_table& ) = default; + + unordered_segment_table& operator=( unordered_segment_table&& ) = default; + + segment_type create_segment( typename base_type::segment_table_type, typename base_type::segment_index_type segment_index, size_type ) { + segment_allocator_type alloc(this->get_allocator()); + size_type seg_size = this->segment_size(segment_index); + segment_type new_segment = segment_allocator_traits::allocate(alloc, seg_size); + for (size_type i = 0; i != seg_size; ++i) { + segment_allocator_traits::construct(alloc, new_segment + i, nullptr); + } + return new_segment; + } + + segment_type nullify_segment( typename base_type::segment_table_type table, size_type segment_index ) { + segment_type target_segment = table[segment_index].load(std::memory_order_relaxed); + table[segment_index].store(nullptr, std::memory_order_relaxed); + return target_segment; + } + + // deallocate_segment is required by the segment_table base class, but + // in unordered, it is also necessary to call the destructor during deallocation + void deallocate_segment( segment_type address, size_type index ) { + destroy_segment(address, index); + } + + void destroy_segment( segment_type address, size_type index ) { + segment_allocator_type alloc(this->get_allocator()); + for (size_type i = 0; i != this->segment_size(index); ++i) { + segment_allocator_traits::destroy(alloc, address + i); + } + segment_allocator_traits::deallocate(alloc, address, this->segment_size(index)); + } + + + void copy_segment( size_type index, segment_type, segment_type to ) { + if (index == 0) { + // The first element in the first segment is embedded into the table (my_head) + // so the first pointer should not be stored here + // It would be stored during move ctor/assignment operation + to[1].store(nullptr, std::memory_order_relaxed); + } else { + for (size_type i = 0; i != this->segment_size(index); ++i) { + to[i].store(nullptr, std::memory_order_relaxed); + } + } + } + + void move_segment( size_type index, segment_type from, segment_type to ) { + if (index == 0) { + // The first element in the first segment is embedded into the table (my_head) + // so the first pointer should not be stored here + // It would be stored during move ctor/assignment operation + to[1].store(from[1].load(std::memory_order_relaxed), std::memory_order_relaxed); + } else { + for (size_type i = 0; i != this->segment_size(index); ++i) { + to[i].store(from[i].load(std::memory_order_relaxed), std::memory_order_relaxed); + from[i].store(nullptr, std::memory_order_relaxed); + } + } + } + + // allocate_long_table is required by the segment_table base class, but unused for unordered containers + typename base_type::segment_table_type allocate_long_table( const typename base_type::atomic_segment*, size_type ) { + __TBB_ASSERT(false, "This method should never been called"); + // TableType is a pointer + return nullptr; + } + + // destroy_elements is required by the segment_table base class, but unused for unordered containers + // this function call but do nothing + void destroy_elements() {} + }; // struct unordered_segment_table + + void internal_clear() { + // TODO: consider usefulness of two versions of clear() - with dummy nodes deallocation and without it + node_ptr next = my_head.next(); + node_ptr curr = next; + + my_head.set_next(nullptr); + + while (curr != nullptr) { + next = curr->next(); + destroy_node(curr); + curr = next; + } + + my_size.store(0, std::memory_order_relaxed); + my_segments.clear(); + } + + void destroy_node( node_ptr node ) { + if (node->is_dummy()) { + node_allocator_type dummy_node_allocator(my_segments.get_allocator()); + // Destroy the node + node_allocator_traits::destroy(dummy_node_allocator, node); + // Deallocate the memory + node_allocator_traits::deallocate(dummy_node_allocator, node, 1); + } else { + // GCC 11.1 issues a warning here that incorrect destructor might be called for dummy_nodes + #if (__TBB_GCC_VERSION >= 110100 && __TBB_GCC_VERSION < 150000 ) && !__clang__ && !__INTEL_COMPILER + volatile + #endif + value_node_ptr val_node = static_cast(node); + value_node_allocator_type value_node_allocator(my_segments.get_allocator()); + // Destroy the value + value_node_allocator_traits::destroy(value_node_allocator, val_node->storage()); + // Destroy the node + value_node_allocator_traits::destroy(value_node_allocator, val_node); + // Deallocate the memory + value_node_allocator_traits::deallocate(value_node_allocator, val_node, 1); + } + } + + struct internal_insert_return_type { + // If the insertion failed - the remaining_node points to the node, which was failed to insert + // This node can be allocated in process of insertion + value_node_ptr remaining_node; + // If the insertion failed - node_with_equal_key points to the node in the list with the + // key, equivalent to the inserted, otherwise it points to the node, which was inserted. + value_node_ptr node_with_equal_key; + // Insertion status + // NOTE: if it is true - remaining_node should be nullptr + bool inserted; + }; // struct internal_insert_return_type + + // Inserts the value into the split ordered list + template + std::pair internal_insert_value( ValueType&& value ) { + + auto create_value_node = [&value, this]( sokey_type order_key )->value_node_ptr { + return create_node(order_key, std::forward(value)); + }; + + auto insert_result = internal_insert(value, create_value_node); + + if (insert_result.remaining_node != nullptr) { + // If the insertion fails - destroy the node which was failed to insert if it exist + __TBB_ASSERT(!insert_result.inserted, + "remaining_node should be nullptr if the node was successfully inserted"); + destroy_node(insert_result.remaining_node); + } + + return { iterator(insert_result.node_with_equal_key), insert_result.inserted }; + } + + // Inserts the node into the split ordered list + // Creates a node using the specified callback after the place for insertion was found + // Returns internal_insert_return_type object, where: + // - If the insertion succeeded: + // - remaining_node is nullptr + // - node_with_equal_key point to the inserted node + // - inserted is true + // - If the insertion failed: + // - remaining_node points to the node, that was failed to insert if it was created. + // nullptr if the node was not created, because the requested key was already + // presented in the list + // - node_with_equal_key point to the element in the list with the key, equivalent to + // to the requested key + // - inserted is false + template + internal_insert_return_type internal_insert( ValueType&& value, CreateInsertNode create_insert_node ) { + static_assert(std::is_same::type, value_type>::value, + "Incorrect type in internal_insert"); + const key_type& key = traits_type::get_key(value); + sokey_type hash_key = sokey_type(my_hash_compare(key)); + + sokey_type order_key = split_order_key_regular(hash_key); + node_ptr prev = prepare_bucket(hash_key); + __TBB_ASSERT(prev != nullptr, "Invalid head node"); + + auto search_result = search_after(prev, order_key, key); + + if (search_result.second) { + return internal_insert_return_type{ nullptr, search_result.first, false }; + } + + value_node_ptr new_node = create_insert_node(order_key); + node_ptr curr = search_result.first; + + while (!try_insert(prev, new_node, curr)) { + search_result = search_after(prev, order_key, key); + if (search_result.second) { + return internal_insert_return_type{ new_node, search_result.first, false }; + } + curr = search_result.first; + } + + auto sz = my_size.fetch_add(1); + adjust_table_size(sz + 1, my_bucket_count.load(std::memory_order_acquire)); + return internal_insert_return_type{ nullptr, static_cast(new_node), true }; + } + + // Searches the node with the key, equivalent to key with requested order key after the node prev + // Returns the existing node and true if the node is already in the list + // Returns the first node with the order key, greater than requested and false if the node is not presented in the list + std::pair search_after( node_ptr& prev, sokey_type order_key, const key_type& key ) { + // NOTE: static_cast(curr) should be done only after we would ensure + // that the node is not a dummy node + + node_ptr curr = prev->next(); + + while (curr != nullptr && (curr->order_key() < order_key || + (curr->order_key() == order_key && !my_hash_compare(traits_type::get_key(static_cast(curr)->value()), key)))) + { + prev = curr; + curr = curr->next(); + } + + if (curr != nullptr && curr->order_key() == order_key && !allow_multimapping) { + return { static_cast(curr), true }; + } + return { static_cast(curr), false }; + } + + void adjust_table_size( size_type total_elements, size_type current_size ) { + // Grow the table by a factor of 2 if possible and needed + if ( (float(total_elements) / float(current_size)) > my_max_load_factor ) { + // Double the size of the hash only if size hash not changed in between loads + my_bucket_count.compare_exchange_strong(current_size, 2u * current_size); + } + } + + node_ptr insert_dummy_node( node_ptr parent_dummy_node, sokey_type order_key ) { + node_ptr prev_node = parent_dummy_node; + + node_ptr dummy_node = create_dummy_node(order_key); + node_ptr next_node; + + do { + next_node = prev_node->next(); + // Move forward through the list while the order key is less than requested + while (next_node != nullptr && next_node->order_key() < order_key) { + prev_node = next_node; + next_node = next_node->next(); + } + + if (next_node != nullptr && next_node->order_key() == order_key) { + // Another dummy node with the same order key was inserted by another thread + // Destroy the node and exit + destroy_node(dummy_node); + return next_node; + } + } while (!try_insert(prev_node, dummy_node, next_node)); + + return dummy_node; + } + + // Try to insert a node between prev_node and expected next + // If the next is not equal to expected next - return false + static bool try_insert( node_ptr prev_node, node_ptr new_node, node_ptr current_next_node ) { + new_node->set_next(current_next_node); + return prev_node->try_set_next(current_next_node, new_node); + } + + // Returns the bucket, associated with the hash_key + node_ptr prepare_bucket( sokey_type hash_key ) { + size_type bucket = hash_key % my_bucket_count.load(std::memory_order_acquire); + return get_bucket(bucket); + } + + // Initialize the corresponding bucket if it is not initialized + node_ptr get_bucket( size_type bucket_index ) { + if (my_segments[bucket_index].load(std::memory_order_acquire) == nullptr) { + init_bucket(bucket_index); + } + return my_segments[bucket_index].load(std::memory_order_acquire); + } + + void init_bucket( size_type bucket ) { + if (bucket == 0) { + // Atomicaly store the first bucket into my_head + node_ptr disabled = nullptr; + my_segments[0].compare_exchange_strong(disabled, &my_head); + return; + } + + size_type parent_bucket = get_parent(bucket); + + while (my_segments[parent_bucket].load(std::memory_order_acquire) == nullptr) { + // Initialize all of the parent buckets + init_bucket(parent_bucket); + } + + __TBB_ASSERT(my_segments[parent_bucket].load(std::memory_order_acquire) != nullptr, "Parent bucket should be initialized"); + node_ptr parent = my_segments[parent_bucket].load(std::memory_order_acquire); + + // Insert dummy node into the list + node_ptr dummy_node = insert_dummy_node(parent, split_order_key_dummy(bucket)); + // TODO: consider returning pair to avoid store operation if the bucket was stored by an other thread + // or move store to insert_dummy_node + // Add dummy_node into the segment table + my_segments[bucket].store(dummy_node, std::memory_order_release); + } + + node_ptr create_dummy_node( sokey_type order_key ) { + node_allocator_type dummy_node_allocator(my_segments.get_allocator()); + node_ptr dummy_node = node_allocator_traits::allocate(dummy_node_allocator, 1); + node_allocator_traits::construct(dummy_node_allocator, dummy_node, order_key); + return dummy_node; + } + + template + value_node_ptr create_node( sokey_type order_key, Args&&... args ) { + value_node_allocator_type value_node_allocator(my_segments.get_allocator()); + // Allocate memory for the value_node + value_node_ptr new_node = value_node_allocator_traits::allocate(value_node_allocator, 1); + // Construct the node + value_node_allocator_traits::construct(value_node_allocator, new_node, order_key); + + // try_call API is not convenient here due to broken + // variadic capture on GCC 4.8.5 + auto value_guard = make_raii_guard([&] { + value_node_allocator_traits::destroy(value_node_allocator, new_node); + value_node_allocator_traits::deallocate(value_node_allocator, new_node, 1); + }); + + // Construct the value in the node + value_node_allocator_traits::construct(value_node_allocator, new_node->storage(), std::forward(args)...); + value_guard.dismiss(); + return new_node; + } + + value_node_ptr first_value_node( node_ptr first_node ) const { + while (first_node != nullptr && first_node->is_dummy()) { + first_node = first_node->next(); + } + return static_cast(first_node); + } + + // Unsafe method, which removes the node from the list and returns the next node + node_ptr internal_erase( value_node_ptr node_to_erase ) { + __TBB_ASSERT(node_to_erase != nullptr, "Invalid iterator for erase"); + node_ptr next_node = node_to_erase->next(); + internal_extract(node_to_erase); + destroy_node(node_to_erase); + return next_node; + } + + template + size_type internal_erase_by_key( const K& key ) { + // TODO: consider reimplementation without equal_range - it is not effective to perform lookup over a bucket + // for each unsafe_erase call + auto eq_range = equal_range(key); + size_type erased_count = 0; + + for (auto it = eq_range.first; it != eq_range.second;) { + it = unsafe_erase(it); + ++erased_count; + } + return erased_count; + } + + // Unsafe method, which extracts the node from the list + void internal_extract( value_node_ptr node_to_extract ) { + const key_type& key = traits_type::get_key(node_to_extract->value()); + sokey_type hash_key = sokey_type(my_hash_compare(key)); + + node_ptr prev_node = prepare_bucket(hash_key); + + for (node_ptr node = prev_node->next(); node != nullptr; prev_node = node, node = node->next()) { + if (node == node_to_extract) { + unlink_node(prev_node, node, node_to_extract->next()); + my_size.store(my_size.load(std::memory_order_relaxed) - 1, std::memory_order_relaxed); + return; + } + __TBB_ASSERT(node->order_key() <= node_to_extract->order_key(), + "node, which is going to be extracted should be presented in the list"); + } + } + +protected: + template + void internal_merge( SourceType&& source ) { + static_assert(std::is_same::type::node_type>::value, + "Incompatible containers cannot be merged"); + + for (node_ptr source_prev = &source.my_head; source_prev->next() != nullptr;) { + if (!source_prev->next()->is_dummy()) { + value_node_ptr curr = static_cast(source_prev->next()); + // If the multimapping is allowed, or the key is not presented + // in the *this container - extract the node from the list + if (allow_multimapping || !contains(traits_type::get_key(curr->value()))) { + node_ptr next_node = curr->next(); + source.unlink_node(source_prev, curr, next_node); + + // Remember the old order key + sokey_type old_order_key = curr->order_key(); + + // Node handle with curr cannot be used directly in insert call, because + // the destructor of node_type will destroy curr + node_type curr_node = d1::node_handle_accessor::construct(curr); + + // If the insertion fails - return ownership of the node to the source + if (!insert(std::move(curr_node)).second) { + __TBB_ASSERT(!allow_multimapping, "Insertion should succeed for multicontainer"); + __TBB_ASSERT(source_prev->next() == next_node, + "Concurrent operations with the source container in merge are prohibited"); + + // Initialize the node with the old order key, because the order key + // can change during the insertion + curr->init(old_order_key); + __TBB_ASSERT(old_order_key >= source_prev->order_key() && + (next_node == nullptr || old_order_key <= next_node->order_key()), + "Wrong nodes order in the source container"); + // Merge is unsafe for source container, so the insertion back can be done without compare_exchange + curr->set_next(next_node); + source_prev->set_next(curr); + source_prev = curr; + d1::node_handle_accessor::deactivate(curr_node); + } else { + source.my_size.fetch_sub(1, std::memory_order_relaxed); + } + } else { + source_prev = curr; + } + } else { + source_prev = source_prev->next(); + } + } + } + +private: + // Unsafe method, which unlinks the node between prev and next + void unlink_node( node_ptr prev_node, node_ptr node_to_unlink, node_ptr next_node ) { + __TBB_ASSERT(prev_node->next() == node_to_unlink && + node_to_unlink->next() == next_node, + "erasing and extracting nodes from the containers are unsafe in concurrent mode"); + prev_node->set_next(next_node); + node_to_unlink->set_next(nullptr); + } + + template + value_node_ptr internal_find( const K& key ) { + sokey_type hash_key = sokey_type(my_hash_compare(key)); + sokey_type order_key = split_order_key_regular(hash_key); + + node_ptr curr = prepare_bucket(hash_key); + + while (curr != nullptr) { + if (curr->order_key() > order_key) { + // If the order key is greater than the requested order key, + // the element is not in the hash table + return nullptr; + } else if (curr->order_key() == order_key && + my_hash_compare(traits_type::get_key(static_cast(curr)->value()), key)) { + // The fact that order keys match does not mean that the element is found. + // Key function comparison has to be performed to check whether this is the + // right element. If not, keep searching while order key is the same. + return static_cast(curr); + } + curr = curr->next(); + } + + return nullptr; + } + + template + std::pair internal_equal_range( const K& key ) { + sokey_type hash_key = sokey_type(my_hash_compare(key)); + sokey_type order_key = split_order_key_regular(hash_key); + + node_ptr curr = prepare_bucket(hash_key); + + while (curr != nullptr) { + if (curr->order_key() > order_key) { + // If the order key is greater than the requested order key, + // the element is not in the hash table + return std::make_pair(nullptr, nullptr); + } else if (curr->order_key() == order_key && + my_hash_compare(traits_type::get_key(static_cast(curr)->value()), key)) { + value_node_ptr first = static_cast(curr); + node_ptr last = first; + do { + last = last->next(); + } while (allow_multimapping && last != nullptr && !last->is_dummy() && + my_hash_compare(traits_type::get_key(static_cast(last)->value()), key)); + return std::make_pair(first, first_value_node(last)); + } + curr = curr->next(); + } + return {nullptr, nullptr}; + } + + template + size_type internal_count( const K& key ) const { + if (allow_multimapping) { + // TODO: consider reimplementing the internal_equal_range with elements counting to avoid std::distance + auto eq_range = equal_range(key); + return std::distance(eq_range.first, eq_range.second); + } else { + return contains(key) ? 1 : 0; + } + } + + void internal_copy( const concurrent_unordered_base& other ) { + node_ptr last_node = &my_head; + my_segments[0].store(&my_head, std::memory_order_relaxed); + + for (node_ptr node = other.my_head.next(); node != nullptr; node = node->next()) { + node_ptr new_node; + if (!node->is_dummy()) { + // The node in the right table contains a value + new_node = create_node(node->order_key(), static_cast(node)->value()); + } else { + // The node in the right table is a dummy node + new_node = create_dummy_node(node->order_key()); + my_segments[reverse_bits(node->order_key())].store(new_node, std::memory_order_relaxed); + } + + last_node->set_next(new_node); + last_node = new_node; + } + } + + void internal_move( concurrent_unordered_base&& other ) { + node_ptr last_node = &my_head; + my_segments[0].store(&my_head, std::memory_order_relaxed); + + for (node_ptr node = other.my_head.next(); node != nullptr; node = node->next()) { + node_ptr new_node; + if (!node->is_dummy()) { + // The node in the right table contains a value + new_node = create_node(node->order_key(), std::move(static_cast(node)->value())); + } else { + // TODO: do we need to destroy a dummy node in the right container? + // The node in the right table is a dummy_node + new_node = create_dummy_node(node->order_key()); + my_segments[reverse_bits(node->order_key())].store(new_node, std::memory_order_relaxed); + } + + last_node->set_next(new_node); + last_node = new_node; + } + } + + void move_content( concurrent_unordered_base&& other ) { + // NOTE: allocators should be equal + my_head.set_next(other.my_head.next()); + other.my_head.set_next(nullptr); + my_segments[0].store(&my_head, std::memory_order_relaxed); + + other.my_bucket_count.store(initial_bucket_count, std::memory_order_relaxed); + other.my_max_load_factor = initial_max_load_factor; + other.my_size.store(0, std::memory_order_relaxed); + } + + void internal_move_construct_with_allocator( concurrent_unordered_base&& other, const allocator_type&, + /*is_always_equal = */std::true_type ) { + // Allocators are always equal - no need to compare for equality + move_content(std::move(other)); + } + + void internal_move_construct_with_allocator( concurrent_unordered_base&& other, const allocator_type& alloc, + /*is_always_equal = */std::false_type ) { + // Allocators are not always equal + if (alloc == other.my_segments.get_allocator()) { + move_content(std::move(other)); + } else { + try_call( [&] { + internal_move(std::move(other)); + } ).on_exception( [&] { + clear(); + }); + } + } + + // Move assigns the hash table to other is any instances of allocator_type are always equal + // or propagate_on_container_move_assignment is true + void internal_move_assign( concurrent_unordered_base&& other, /*is_always_equal || POCMA = */std::true_type ) { + move_content(std::move(other)); + } + + // Move assigns the hash table to other is any instances of allocator_type are not always equal + // and propagate_on_container_move_assignment is false + void internal_move_assign( concurrent_unordered_base&& other, /*is_always_equal || POCMA = */std::false_type ) { + if (my_segments.get_allocator() == other.my_segments.get_allocator()) { + move_content(std::move(other)); + } else { + // TODO: guards for exceptions + internal_move(std::move(other)); + } + } + + void internal_swap( concurrent_unordered_base& other, /*is_always_equal || POCS = */std::true_type ) { + internal_swap_fields(other); + } + + void internal_swap( concurrent_unordered_base& other, /*is_always_equal || POCS = */std::false_type ) { + __TBB_ASSERT(my_segments.get_allocator() == other.my_segments.get_allocator(), + "Swapping with unequal allocators is not allowed"); + internal_swap_fields(other); + } + + void internal_swap_fields( concurrent_unordered_base& other ) { + node_ptr first_node = my_head.next(); + my_head.set_next(other.my_head.next()); + other.my_head.set_next(first_node); + + size_type current_size = my_size.load(std::memory_order_relaxed); + my_size.store(other.my_size.load(std::memory_order_relaxed), std::memory_order_relaxed); + other.my_size.store(current_size, std::memory_order_relaxed); + + size_type bucket_count = my_bucket_count.load(std::memory_order_relaxed); + my_bucket_count.store(other.my_bucket_count.load(std::memory_order_relaxed), std::memory_order_relaxed); + other.my_bucket_count.store(bucket_count, std::memory_order_relaxed); + + using std::swap; + swap(my_max_load_factor, other.my_max_load_factor); + swap(my_hash_compare, other.my_hash_compare); + my_segments.swap(other.my_segments); + + // swap() method from segment table swaps all of the segments including the first segment + // We should restore it to my_head. Without it the first segment of the container will point + // to other.my_head. + my_segments[0].store(&my_head, std::memory_order_relaxed); + other.my_segments[0].store(&other.my_head, std::memory_order_relaxed); + } + + // A regular order key has its original hash value reversed and the last bit set + static constexpr sokey_type split_order_key_regular( sokey_type hash ) { + return reverse_bits(hash) | 0x1; + } + + // A dummy order key has its original hash value reversed and the last bit unset + static constexpr sokey_type split_order_key_dummy( sokey_type hash ) { + return reverse_bits(hash) & ~sokey_type(0x1); + } + + size_type get_parent( size_type bucket ) const { + // Unset bucket's most significant turned-on bit + __TBB_ASSERT(bucket != 0, "Unable to get_parent of the bucket 0"); + size_type msb = tbb::detail::log2(bucket); + return bucket & ~(size_type(1) << msb); + } + + size_type get_next_bucket_index( size_type bucket ) const { + size_type bits = tbb::detail::log2(my_bucket_count.load(std::memory_order_relaxed)); + size_type reversed_next = reverse_n_bits(bucket, bits) + 1; + return reverse_n_bits(reversed_next, bits); + } + + std::atomic my_size; + std::atomic my_bucket_count; + float my_max_load_factor; + hash_compare_type my_hash_compare; + + list_node_type my_head; // Head node for split ordered list + unordered_segment_table my_segments; // Segment table of pointers to nodes + + template + friend class solist_iterator; + + template + friend class concurrent_unordered_base; +}; // class concurrent_unordered_base + +template +bool operator==( const concurrent_unordered_base& lhs, + const concurrent_unordered_base& rhs ) { + if (&lhs == &rhs) { return true; } + if (lhs.size() != rhs.size()) { return false; } + +#if _MSC_VER + // Passing "unchecked" iterators to std::permutation with 3 parameters + // causes compiler warnings. + // The workaround is to use overload with 4 parameters, which is + // available since C++14 - minimally supported version on MSVC + return std::is_permutation(lhs.begin(), lhs.end(), rhs.begin(), rhs.end()); +#else + return std::is_permutation(lhs.begin(), lhs.end(), rhs.begin()); +#endif +} + +#if !__TBB_CPP20_COMPARISONS_PRESENT +template +bool operator!=( const concurrent_unordered_base& lhs, + const concurrent_unordered_base& rhs ) { + return !(lhs == rhs); +} +#endif + +#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) +// #pragma warning(pop) // warning 4127 is back +#endif + +} // namespace d2 +} // namespace detail +} // namespace tbb + +#endif // __TBB_detail__concurrent_unordered_base_H diff --git a/src/tbb/include/oneapi/tbb/detail/_config.h b/src/tbb/include/oneapi/tbb/detail/_config.h new file mode 100644 index 000000000..e676b1558 --- /dev/null +++ b/src/tbb/include/oneapi/tbb/detail/_config.h @@ -0,0 +1,537 @@ +/* + Copyright (c) 2005-2024 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_detail__config_H +#define __TBB_detail__config_H + +/** This header is supposed to contain macro definitions only. + The macros defined here are intended to control such aspects of TBB build as + - presence of compiler features + - compilation modes + - feature sets + - known compiler/platform issues +**/ + +/* Check which standard library we use. */ +#include + +#ifdef __has_include +#if __has_include() +#include +#endif +#endif + +#include "_export.h" + +#if _MSC_VER + #define __TBB_EXPORTED_FUNC __cdecl + #define __TBB_EXPORTED_METHOD __thiscall +#else + #define __TBB_EXPORTED_FUNC + #define __TBB_EXPORTED_METHOD +#endif + +#if defined(_MSVC_LANG) + #define __TBB_LANG _MSVC_LANG +#else + #define __TBB_LANG __cplusplus +#endif // _MSVC_LANG + +#define __TBB_CPP14_PRESENT (__TBB_LANG >= 201402L) +#define __TBB_CPP17_PRESENT (__TBB_LANG >= 201703L) +#define __TBB_CPP20_PRESENT (__TBB_LANG >= 202002L) + +#if __INTEL_COMPILER || _MSC_VER + #define __TBB_NOINLINE(decl) __declspec(noinline) decl +#elif __GNUC__ + #define __TBB_NOINLINE(decl) decl __attribute__ ((noinline)) +#else + #define __TBB_NOINLINE(decl) decl +#endif + +#define __TBB_STRING_AUX(x) #x +#define __TBB_STRING(x) __TBB_STRING_AUX(x) + +// Note that when ICC or Clang is in use, __TBB_GCC_VERSION might not fully match +// the actual GCC version on the system. +#define __TBB_GCC_VERSION (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) + +/* Check which standard library we use. */ + +// Prior to GCC 7, GNU libstdc++ did not have a convenient version macro. +// Therefore we use different ways to detect its version. +#ifdef TBB_USE_GLIBCXX_VERSION + // The version is explicitly specified in our public TBB_USE_GLIBCXX_VERSION macro. + // Its format should match the __TBB_GCC_VERSION above, e.g. 70301 for libstdc++ coming with GCC 7.3.1. + #define __TBB_GLIBCXX_VERSION TBB_USE_GLIBCXX_VERSION +#elif _GLIBCXX_RELEASE && _GLIBCXX_RELEASE != __GNUC__ + // Reported versions of GCC and libstdc++ do not match; trust the latter + #define __TBB_GLIBCXX_VERSION (_GLIBCXX_RELEASE*10000) +#elif __GLIBCPP__ || __GLIBCXX__ + // The version macro is not defined or matches the GCC version; use __TBB_GCC_VERSION + #define __TBB_GLIBCXX_VERSION __TBB_GCC_VERSION +#endif + +#if __clang__ + // according to clang documentation, version can be vendor specific + #define __TBB_CLANG_VERSION (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) +#endif + +/** Macro helpers **/ + +#define __TBB_CONCAT_AUX(A,B) A##B +// The additional level of indirection is needed to expand macros A and B (not to get the AB macro). +// See [cpp.subst] and [cpp.concat] for more details. +#define __TBB_CONCAT(A,B) __TBB_CONCAT_AUX(A,B) +// The IGNORED argument and comma are needed to always have 2 arguments (even when A is empty). +#define __TBB_IS_MACRO_EMPTY(A,IGNORED) __TBB_CONCAT_AUX(__TBB_MACRO_EMPTY,A) +#define __TBB_MACRO_EMPTY 1 + +#if _M_X64 || _M_ARM64 + #define __TBB_W(name) name##64 +#else + #define __TBB_W(name) name +#endif + +/** User controlled TBB features & modes **/ + +#ifndef TBB_USE_DEBUG + /* + There are four cases that are supported: + 1. "_DEBUG is undefined" means "no debug"; + 2. "_DEBUG defined to something that is evaluated to 0" (including "garbage", as per [cpp.cond]) means "no debug"; + 3. "_DEBUG defined to something that is evaluated to a non-zero value" means "debug"; + 4. "_DEBUG defined to nothing (empty)" means "debug". + */ + #ifdef _DEBUG + // Check if _DEBUG is empty. + #define __TBB_IS__DEBUG_EMPTY (__TBB_IS_MACRO_EMPTY(_DEBUG,IGNORED)==__TBB_MACRO_EMPTY) + #if __TBB_IS__DEBUG_EMPTY + #define TBB_USE_DEBUG 1 + #else + #define TBB_USE_DEBUG _DEBUG + #endif // __TBB_IS__DEBUG_EMPTY + #else + #define TBB_USE_DEBUG 0 + #endif // _DEBUG +#endif // TBB_USE_DEBUG + +#ifndef TBB_USE_ASSERT + #define TBB_USE_ASSERT TBB_USE_DEBUG +#endif // TBB_USE_ASSERT + +#ifndef TBB_USE_PROFILING_TOOLS +#if TBB_USE_DEBUG + #define TBB_USE_PROFILING_TOOLS 2 +#else // TBB_USE_DEBUG + #define TBB_USE_PROFILING_TOOLS 0 +#endif // TBB_USE_DEBUG +#endif // TBB_USE_PROFILING_TOOLS + +// Exceptions support cases +#if !(__EXCEPTIONS || defined(_CPPUNWIND) || __SUNPRO_CC) + #if TBB_USE_EXCEPTIONS + #error Compilation settings do not support exception handling. Please do not set TBB_USE_EXCEPTIONS macro or set it to 0. + #elif !defined(TBB_USE_EXCEPTIONS) + #define TBB_USE_EXCEPTIONS 0 + #endif +#elif !defined(TBB_USE_EXCEPTIONS) + #define TBB_USE_EXCEPTIONS 1 +#endif + +/** Preprocessor symbols to determine HW architecture **/ + +#if _WIN32 || _WIN64 + #if defined(_M_X64) || defined(__x86_64__) // the latter for MinGW support + #define __TBB_x86_64 1 + #elif defined(_M_IA64) + #define __TBB_ipf 1 + #elif defined(_M_IX86) || defined(__i386__) // the latter for MinGW support + #define __TBB_x86_32 1 + #else + #define __TBB_generic_arch 1 + #endif +#else /* Assume generic Unix */ + #if __x86_64__ + #define __TBB_x86_64 1 + #elif __ia64__ + #define __TBB_ipf 1 + #elif __i386__||__i386 // __i386 is for Sun OS + #define __TBB_x86_32 1 + #else + #define __TBB_generic_arch 1 + #endif +#endif + +/** Windows API or POSIX API **/ + +#if _WIN32 || _WIN64 + #define __TBB_USE_WINAPI 1 +#else + #define __TBB_USE_POSIX 1 +#endif + +/** Internal TBB features & modes **/ + +/** __TBB_DYNAMIC_LOAD_ENABLED describes the system possibility to load shared libraries at run time **/ +#ifndef __TBB_DYNAMIC_LOAD_ENABLED + #define __TBB_DYNAMIC_LOAD_ENABLED (!__EMSCRIPTEN__) +#endif + +/** __TBB_WIN8UI_SUPPORT enables support of Windows* Store Apps and limit a possibility to load + shared libraries at run time only from application container **/ +#if defined(WINAPI_FAMILY) && WINAPI_FAMILY == WINAPI_FAMILY_APP + #define __TBB_WIN8UI_SUPPORT 1 +#else + #define __TBB_WIN8UI_SUPPORT 0 +#endif + +/** __TBB_WEAK_SYMBOLS_PRESENT denotes that the system supports the weak symbol mechanism **/ +#ifndef __TBB_WEAK_SYMBOLS_PRESENT + #define __TBB_WEAK_SYMBOLS_PRESENT ( !__EMSCRIPTEN__ && !_WIN32 && !__APPLE__ && !__sun && (__TBB_GCC_VERSION >= 40000 || __INTEL_COMPILER ) ) +#endif + +/** Presence of compiler features **/ + +#if __clang__ && !__INTEL_COMPILER + #define __TBB_USE_OPTIONAL_RTTI __has_feature(cxx_rtti) +#elif defined(_CPPRTTI) + #define __TBB_USE_OPTIONAL_RTTI 1 +#else + #define __TBB_USE_OPTIONAL_RTTI (__GXX_RTTI || __RTTI || __INTEL_RTTI__) +#endif + +/** Address sanitizer detection **/ +#ifdef __SANITIZE_ADDRESS__ + #define __TBB_USE_ADDRESS_SANITIZER 1 +#elif defined(__has_feature) +#if __has_feature(address_sanitizer) + #define __TBB_USE_ADDRESS_SANITIZER 1 +#endif +#endif + +/** Library features presence macros **/ + +#define __TBB_CPP14_INTEGER_SEQUENCE_PRESENT (__TBB_LANG >= 201402L) +#define __TBB_CPP17_INVOKE_PRESENT (__TBB_LANG >= 201703L) + +// TODO: Remove the condition(__INTEL_COMPILER > 2021) from the __TBB_CPP17_DEDUCTION_GUIDES_PRESENT +// macro when this feature start working correctly on this compiler. +#if __INTEL_COMPILER && (!_MSC_VER || __INTEL_CXX11_MOVE__) + #define __TBB_CPP14_VARIABLE_TEMPLATES_PRESENT (__TBB_LANG >= 201402L) + #define __TBB_CPP17_DEDUCTION_GUIDES_PRESENT (__INTEL_COMPILER > 2021 && __TBB_LANG >= 201703L) + #define __TBB_CPP20_CONCEPTS_PRESENT 0 // TODO: add a mechanism for future addition +#elif __clang__ + #define __TBB_CPP14_VARIABLE_TEMPLATES_PRESENT (__has_feature(cxx_variable_templates)) + #define __TBB_CPP20_CONCEPTS_PRESENT 0 // TODO: add a mechanism for future addition + #ifdef __cpp_deduction_guides + #define __TBB_CPP17_DEDUCTION_GUIDES_PRESENT (__cpp_deduction_guides >= 201611L) + #else + #define __TBB_CPP17_DEDUCTION_GUIDES_PRESENT 0 + #endif +#elif __GNUC__ + #define __TBB_CPP14_VARIABLE_TEMPLATES_PRESENT (__TBB_LANG >= 201402L && __TBB_GCC_VERSION >= 50000) + #define __TBB_CPP17_DEDUCTION_GUIDES_PRESENT (__cpp_deduction_guides >= 201606L) + #define __TBB_CPP20_CONCEPTS_PRESENT (__TBB_LANG >= 201709L && __TBB_GCC_VERSION >= 100201) +#elif _MSC_VER + #define __TBB_CPP14_VARIABLE_TEMPLATES_PRESENT (_MSC_FULL_VER >= 190023918 && (!__INTEL_COMPILER || __INTEL_COMPILER >= 1700)) + #define __TBB_CPP17_DEDUCTION_GUIDES_PRESENT (_MSC_VER >= 1914 && __TBB_LANG >= 201703L && (!__INTEL_COMPILER || __INTEL_COMPILER > 2021)) + #define __TBB_CPP20_CONCEPTS_PRESENT (_MSC_VER >= 1923 && __TBB_LANG >= 202002L) // TODO: INTEL_COMPILER? +#else + #define __TBB_CPP14_VARIABLE_TEMPLATES_PRESENT (__TBB_LANG >= 201402L) + #define __TBB_CPP17_DEDUCTION_GUIDES_PRESENT (__TBB_LANG >= 201703L) + #define __TBB_CPP20_CONCEPTS_PRESENT (__TBB_LANG >= 202002L) +#endif + +// GCC4.8 on RHEL7 does not support std::get_new_handler +#define __TBB_CPP11_GET_NEW_HANDLER_PRESENT (_MSC_VER >= 1900 || __TBB_GLIBCXX_VERSION >= 40900 && __GXX_EXPERIMENTAL_CXX0X__ || _LIBCPP_VERSION) +// GCC4.8 on RHEL7 does not support std::is_trivially_copyable +#define __TBB_CPP11_TYPE_PROPERTIES_PRESENT (_LIBCPP_VERSION || _MSC_VER >= 1700 || (__TBB_GLIBCXX_VERSION >= 50000 && __GXX_EXPERIMENTAL_CXX0X__)) + +#define __TBB_CPP17_MEMORY_RESOURCE_PRESENT (_MSC_VER >= 1913 && (__TBB_LANG > 201402L) || \ + __TBB_GLIBCXX_VERSION >= 90000 && __TBB_LANG >= 201703L) +#define __TBB_CPP17_HW_INTERFERENCE_SIZE_PRESENT (_MSC_VER >= 1911) +#define __TBB_CPP17_LOGICAL_OPERATIONS_PRESENT (__TBB_LANG >= 201703L) +#define __TBB_CPP17_ALLOCATOR_IS_ALWAYS_EQUAL_PRESENT (__TBB_LANG >= 201703L) +#define __TBB_CPP17_IS_SWAPPABLE_PRESENT (__TBB_LANG >= 201703L) + +#if defined(__cpp_impl_three_way_comparison) && defined(__cpp_lib_three_way_comparison) + #define __TBB_CPP20_COMPARISONS_PRESENT ((__cpp_impl_three_way_comparison >= 201907L) && (__cpp_lib_three_way_comparison >= 201907L)) +#else + #define __TBB_CPP20_COMPARISONS_PRESENT 0 +#endif + +#define __TBB_RESUMABLE_TASKS (!__TBB_WIN8UI_SUPPORT && !__ANDROID__ && !__QNXNTO__ && (!__linux__ || __GLIBC__)) + +/* This macro marks incomplete code or comments describing ideas which are considered for the future. + * See also for plain comment with TODO and FIXME marks for small improvement opportunities. + */ +#define __TBB_TODO 0 + +/* Check which standard library we use. */ +/* __TBB_SYMBOL is defined only while processing exported symbols list where C++ is not allowed. */ +#if !defined(__TBB_SYMBOL) && !__TBB_CONFIG_PREPROC_ONLY + #include +#endif + +/** Target OS is either iOS* or iOS* simulator **/ +#if __ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__ + #define __TBB_IOS 1 +#endif + +#if __APPLE__ + #if __INTEL_COMPILER && __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ > 1099 \ + && __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 101000 + // ICC does not correctly set the macro if -mmacosx-min-version is not specified + #define __TBB_MACOS_TARGET_VERSION (100000 + 10*(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ - 1000)) + #else + #define __TBB_MACOS_TARGET_VERSION __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ + #endif +#endif + +#if defined(__GNUC__) && !defined(__INTEL_COMPILER) + #define __TBB_GCC_WARNING_IGNORED_ATTRIBUTES_PRESENT (__TBB_GCC_VERSION >= 60100) +#endif + +#if __GNUC__ && !__INTEL_COMPILER && !__clang__ + #define __TBB_GCC_PARAMETER_PACK_IN_LAMBDAS_BROKEN (__TBB_GCC_VERSION <= 40805) +#endif + +#define __TBB_CPP17_FALLTHROUGH_PRESENT (__TBB_LANG >= 201703L) +#define __TBB_CPP17_NODISCARD_PRESENT (__TBB_LANG >= 201703L) +#define __TBB_FALLTHROUGH_PRESENT (__TBB_GCC_VERSION >= 70000 && !__INTEL_COMPILER) + +#if __TBB_CPP17_FALLTHROUGH_PRESENT + #define __TBB_fallthrough [[fallthrough]] +#elif __TBB_FALLTHROUGH_PRESENT + #define __TBB_fallthrough __attribute__ ((fallthrough)) +#else + #define __TBB_fallthrough +#endif + +#if __TBB_CPP17_NODISCARD_PRESENT + #define __TBB_nodiscard [[nodiscard]] +#elif __clang__ || __GNUC__ + #define __TBB_nodiscard __attribute__((warn_unused_result)) +#else + #define __TBB_nodiscard +#endif + +#define __TBB_CPP17_UNCAUGHT_EXCEPTIONS_PRESENT (_MSC_VER >= 1900 || __GLIBCXX__ && __cpp_lib_uncaught_exceptions \ + || _LIBCPP_VERSION >= 3700 && (!__TBB_MACOS_TARGET_VERSION || __TBB_MACOS_TARGET_VERSION >= 101200)) + +#define __TBB_TSX_INTRINSICS_PRESENT (__RTM__ || __INTEL_COMPILER || (_MSC_VER>=1700 && (__TBB_x86_64 || __TBB_x86_32))) + +#define __TBB_WAITPKG_INTRINSICS_PRESENT ((__INTEL_COMPILER >= 1900 || (__TBB_GCC_VERSION >= 110000 && __TBB_GNU_ASM_VERSION >= 2032) || __TBB_CLANG_VERSION >= 120000) \ + && (_WIN32 || _WIN64 || __unix__ || __APPLE__) && (__TBB_x86_32 || __TBB_x86_64) && !__ANDROID__) + +/** Internal TBB features & modes **/ + +/** __TBB_SOURCE_DIRECTLY_INCLUDED is a mode used in whitebox testing when + it's necessary to test internal functions not exported from TBB DLLs +**/ +#if (_WIN32||_WIN64) && (__TBB_SOURCE_DIRECTLY_INCLUDED || TBB_USE_PREVIEW_BINARY) + #define __TBB_NO_IMPLICIT_LINKAGE 1 + #define __TBBMALLOC_NO_IMPLICIT_LINKAGE 1 +#endif + +#if (__TBB_BUILD || __TBBMALLOC_BUILD || __TBBMALLOCPROXY_BUILD || __TBBBIND_BUILD) && !defined(__TBB_NO_IMPLICIT_LINKAGE) + #define __TBB_NO_IMPLICIT_LINKAGE 1 +#endif + +#if _MSC_VER + #if !__TBB_NO_IMPLICIT_LINKAGE + #ifdef _DEBUG + #pragma comment(lib, "tbb12_debug.lib") + #else + #pragma comment(lib, "tbb12.lib") + #endif + #endif +#endif + +#ifndef __TBB_SCHEDULER_OBSERVER + #define __TBB_SCHEDULER_OBSERVER 1 +#endif /* __TBB_SCHEDULER_OBSERVER */ + +#ifndef __TBB_FP_CONTEXT + #define __TBB_FP_CONTEXT 1 +#endif /* __TBB_FP_CONTEXT */ + +#define __TBB_RECYCLE_TO_ENQUEUE __TBB_BUILD // keep non-official + +#ifndef __TBB_ARENA_OBSERVER + #define __TBB_ARENA_OBSERVER __TBB_SCHEDULER_OBSERVER +#endif /* __TBB_ARENA_OBSERVER */ + +#ifndef __TBB_ARENA_BINDING + #define __TBB_ARENA_BINDING 1 +#endif + +// Thread pinning is not available on macOS* +#define __TBB_CPUBIND_PRESENT (__TBB_ARENA_BINDING && !__APPLE__) + +#ifndef __TBB_ENQUEUE_ENFORCED_CONCURRENCY + #define __TBB_ENQUEUE_ENFORCED_CONCURRENCY 1 +#endif + +#if !defined(__TBB_SURVIVE_THREAD_SWITCH) && \ + (_WIN32 || _WIN64 || __APPLE__ || (defined(__unix__) && !__ANDROID__)) + #define __TBB_SURVIVE_THREAD_SWITCH 1 +#endif /* __TBB_SURVIVE_THREAD_SWITCH */ + +#ifndef TBB_PREVIEW_FLOW_GRAPH_FEATURES + #define TBB_PREVIEW_FLOW_GRAPH_FEATURES __TBB_CPF_BUILD +#endif + +#ifndef __TBB_DEFAULT_PARTITIONER + #define __TBB_DEFAULT_PARTITIONER tbb::auto_partitioner +#endif + +#ifndef __TBB_FLOW_TRACE_CODEPTR + #define __TBB_FLOW_TRACE_CODEPTR __TBB_CPF_BUILD +#endif + +// Intel(R) C++ Compiler starts analyzing usages of the deprecated content at the template +// instantiation site, which is too late for suppression of the corresponding messages for internal +// stuff. +#if !defined(__INTEL_COMPILER) && (!defined(TBB_SUPPRESS_DEPRECATED_MESSAGES) || (TBB_SUPPRESS_DEPRECATED_MESSAGES == 0)) + #if (__TBB_LANG >= 201402L && (!defined(_MSC_VER) || _MSC_VER >= 1920)) + #define __TBB_DEPRECATED [[deprecated]] + #define __TBB_DEPRECATED_MSG(msg) [[deprecated(msg)]] + #elif _MSC_VER + #define __TBB_DEPRECATED __declspec(deprecated) + #define __TBB_DEPRECATED_MSG(msg) __declspec(deprecated(msg)) + #elif (__GNUC__ && __TBB_GCC_VERSION >= 40805) || __clang__ + #define __TBB_DEPRECATED __attribute__((deprecated)) + #define __TBB_DEPRECATED_MSG(msg) __attribute__((deprecated(msg))) + #endif +#endif // !defined(TBB_SUPPRESS_DEPRECATED_MESSAGES) || (TBB_SUPPRESS_DEPRECATED_MESSAGES == 0) + +#if !defined(__TBB_DEPRECATED) + #define __TBB_DEPRECATED + #define __TBB_DEPRECATED_MSG(msg) +#elif !defined(__TBB_SUPPRESS_INTERNAL_DEPRECATED_MESSAGES) + // Suppress deprecated messages from self + #define __TBB_SUPPRESS_INTERNAL_DEPRECATED_MESSAGES 1 +#endif + +#if defined(TBB_SUPPRESS_DEPRECATED_MESSAGES) && (TBB_SUPPRESS_DEPRECATED_MESSAGES == 0) + #define __TBB_DEPRECATED_VERBOSE __TBB_DEPRECATED + #define __TBB_DEPRECATED_VERBOSE_MSG(msg) __TBB_DEPRECATED_MSG(msg) +#else + #define __TBB_DEPRECATED_VERBOSE + #define __TBB_DEPRECATED_VERBOSE_MSG(msg) +#endif // (TBB_SUPPRESS_DEPRECATED_MESSAGES == 0) + +#if (!defined(TBB_SUPPRESS_DEPRECATED_MESSAGES) || (TBB_SUPPRESS_DEPRECATED_MESSAGES == 0)) && !(__TBB_LANG >= 201103L || _MSC_VER >= 1900) + #pragma message("TBB Warning: Support for C++98/03 is deprecated. Please use the compiler that supports C++11 features at least.") +#endif + +#ifdef _VARIADIC_MAX + #define __TBB_VARIADIC_MAX _VARIADIC_MAX +#else + #if _MSC_VER == 1700 + #define __TBB_VARIADIC_MAX 5 // VS11 setting, issue resolved in VS12 + #elif _MSC_VER == 1600 + #define __TBB_VARIADIC_MAX 10 // VS10 setting + #else + #define __TBB_VARIADIC_MAX 15 + #endif +#endif + +#if __SANITIZE_THREAD__ + #define __TBB_USE_THREAD_SANITIZER 1 +#elif defined(__has_feature) +#if __has_feature(thread_sanitizer) + #define __TBB_USE_THREAD_SANITIZER 1 +#endif +#endif + +#ifndef __TBB_USE_SANITIZERS +#define __TBB_USE_SANITIZERS (__TBB_USE_THREAD_SANITIZER || __TBB_USE_ADDRESS_SANITIZER) +#endif + +#ifndef __TBB_RESUMABLE_TASKS_USE_THREADS +#define __TBB_RESUMABLE_TASKS_USE_THREADS __TBB_USE_SANITIZERS +#endif + +#ifndef __TBB_USE_CONSTRAINTS +#define __TBB_USE_CONSTRAINTS 1 +#endif + +#ifndef __TBB_STRICT_CONSTRAINTS +#define __TBB_STRICT_CONSTRAINTS 1 +#endif + +#if __TBB_CPP20_CONCEPTS_PRESENT && __TBB_USE_CONSTRAINTS + #define __TBB_requires(...) requires __VA_ARGS__ +#else // __TBB_CPP20_CONCEPTS_PRESENT + #define __TBB_requires(...) +#endif // __TBB_CPP20_CONCEPTS_PRESENT + +/** Macros of the form __TBB_XXX_BROKEN denote known issues that are caused by + the bugs in compilers, standard or OS specific libraries. They should be + removed as soon as the corresponding bugs are fixed or the buggy OS/compiler + versions go out of the support list. +**/ + +// Some STL containers not support allocator traits in old GCC versions +#if __GXX_EXPERIMENTAL_CXX0X__ && __TBB_GLIBCXX_VERSION <= 50301 + #define TBB_ALLOCATOR_TRAITS_BROKEN 1 +#endif + +// GCC 4.8 C++ standard library implements std::this_thread::yield as no-op. +#if __TBB_GLIBCXX_VERSION >= 40800 && __TBB_GLIBCXX_VERSION < 40900 + #define __TBB_GLIBCXX_THIS_THREAD_YIELD_BROKEN 1 +#endif + +/** End of __TBB_XXX_BROKEN macro section **/ + +#if defined(_MSC_VER) && _MSC_VER>=1500 && !defined(__INTEL_COMPILER) + // A macro to suppress erroneous or benign "unreachable code" MSVC warning (4702) + #define __TBB_MSVC_UNREACHABLE_CODE_IGNORED 1 +#endif + +// Many OS versions (Android 4.0.[0-3] for example) need workaround for dlopen to avoid non-recursive loader lock hang +// Setting the workaround for all compile targets ($APP_PLATFORM) below Android 4.4 (android-19) +#if __ANDROID__ + #include +#endif + +#define __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING (TBB_PREVIEW_FLOW_GRAPH_FEATURES) + +#ifndef __TBB_PREVIEW_CRITICAL_TASKS +#define __TBB_PREVIEW_CRITICAL_TASKS 1 +#endif + +#ifndef __TBB_PREVIEW_FLOW_GRAPH_NODE_SET +#define __TBB_PREVIEW_FLOW_GRAPH_NODE_SET (TBB_PREVIEW_FLOW_GRAPH_FEATURES) +#endif + +#ifndef __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT +#define __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT (TBB_PREVIEW_FLOW_GRAPH_FEATURES \ + || TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT) +#endif + +#if TBB_PREVIEW_CONCURRENT_HASH_MAP_EXTENSIONS +#define __TBB_PREVIEW_CONCURRENT_HASH_MAP_EXTENSIONS 1 +#endif + +#if TBB_PREVIEW_TASK_GROUP_EXTENSIONS || __TBB_BUILD +#define __TBB_PREVIEW_TASK_GROUP_EXTENSIONS 1 +#endif + +#endif // __TBB_detail__config_H diff --git a/src/tbb/include/oneapi/tbb/detail/_containers_helpers.h b/src/tbb/include/oneapi/tbb/detail/_containers_helpers.h new file mode 100644 index 000000000..4dca07fa1 --- /dev/null +++ b/src/tbb/include/oneapi/tbb/detail/_containers_helpers.h @@ -0,0 +1,67 @@ +/* + Copyright (c) 2005-2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_detail__containers_helpers_H +#define __TBB_detail__containers_helpers_H + +#include "_template_helpers.h" +#include "_allocator_traits.h" +#include +#include +#include + +namespace tbb { +namespace detail { +inline namespace d0 { + +template +struct comp_is_transparent : std::false_type {}; + +template +struct comp_is_transparent> : std::true_type {}; + +template +struct has_transparent_key_equal : std::false_type { using type = KeyEqual; }; + +template +struct has_transparent_key_equal> : std::true_type { + using type = typename Hasher::transparent_key_equal; + static_assert(comp_is_transparent::value, "Hash::transparent_key_equal::is_transparent is not valid or does not denote a type."); + static_assert((std::is_same>::value || + std::is_same::value), "KeyEqual is a different type than equal_to or Hash::transparent_key_equal."); + }; + +struct is_iterator_impl { +template +using iter_traits_category = typename std::iterator_traits::iterator_category; + +template +using input_iter_category = typename std::enable_if>::value>::type; +}; // struct is_iterator_impl + +template +using is_input_iterator = supports; + +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT +template +inline constexpr bool is_input_iterator_v = is_input_iterator::value; +#endif + +} // inline namespace d0 +} // namespace detail +} // namespace tbb + +#endif // __TBB_detail__containers_helpers_H diff --git a/src/tbb/include/oneapi/tbb/detail/_exception.h b/src/tbb/include/oneapi/tbb/detail/_exception.h new file mode 100644 index 000000000..d1a8b18c4 --- /dev/null +++ b/src/tbb/include/oneapi/tbb/detail/_exception.h @@ -0,0 +1,88 @@ +/* + Copyright (c) 2005-2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB__exception_H +#define __TBB__exception_H + +#include "_config.h" + +#include // std::bad_alloc +#include // std::exception +#include // std::runtime_error + +namespace tbb { +namespace detail { +inline namespace d0 { +enum class exception_id { + bad_alloc = 1, + bad_last_alloc, + user_abort, + nonpositive_step, + out_of_range, + reservation_length_error, + missing_wait, + invalid_load_factor, + invalid_key, + bad_tagged_msg_cast, + unsafe_wait, + last_entry +}; +} // namespace d0 + +#if _MSC_VER + // #pragma warning(disable: 4275) +#endif + +namespace r1 { +//! Exception for concurrent containers +class TBB_EXPORT bad_last_alloc : public std::bad_alloc { +public: + const char* __TBB_EXPORTED_METHOD what() const noexcept(true) override; +}; + +//! Exception for user-initiated abort +class TBB_EXPORT user_abort : public std::exception { +public: + const char* __TBB_EXPORTED_METHOD what() const noexcept(true) override; +}; + +//! Exception for missing wait on structured_task_group +class TBB_EXPORT missing_wait : public std::exception { +public: + const char* __TBB_EXPORTED_METHOD what() const noexcept(true) override; +}; + +//! Exception for impossible finalization of task_sheduler_handle +class TBB_EXPORT unsafe_wait : public std::runtime_error { +public: + unsafe_wait(const char* msg) : std::runtime_error(msg) {} +}; + +//! Gathers all throw operators in one place. +/** Its purpose is to minimize code bloat that can be caused by throw operators + scattered in multiple places, especially in templates. **/ +TBB_EXPORT void __TBB_EXPORTED_FUNC throw_exception ( exception_id ); +} // namespace r1 + +inline namespace d0 { +using r1::throw_exception; +} // namespace d0 + +} // namespace detail +} // namespace tbb + +#endif // __TBB__exception_H + diff --git a/src/tbb/include/oneapi/tbb/detail/_export.h b/src/tbb/include/oneapi/tbb/detail/_export.h new file mode 100644 index 000000000..4c015223b --- /dev/null +++ b/src/tbb/include/oneapi/tbb/detail/_export.h @@ -0,0 +1,46 @@ +/* + Copyright (c) 2005-2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_detail__export_H +#define __TBB_detail__export_H + +#if defined(__MINGW32__) + #define _EXPORT __declspec(dllexport) +#elif defined(_WIN32) || defined(__unix__) || defined(__APPLE__) // Use .def files for these + #define _EXPORT +#else + #error "Unknown platform/compiler" +#endif + +#if __TBB_BUILD + #define TBB_EXPORT _EXPORT +#else + #define TBB_EXPORT +#endif + +#if __TBBMALLOC_BUILD + #define TBBMALLOC_EXPORT _EXPORT +#else + #define TBBMALLOC_EXPORT +#endif + +#if __TBBBIND_BUILD + #define TBBBIND_EXPORT _EXPORT +#else + #define TBBBIND_EXPORT +#endif + +#endif diff --git a/src/tbb/include/oneapi/tbb/detail/_flow_graph_body_impl.h b/src/tbb/include/oneapi/tbb/detail/_flow_graph_body_impl.h new file mode 100644 index 000000000..21da06ce0 --- /dev/null +++ b/src/tbb/include/oneapi/tbb/detail/_flow_graph_body_impl.h @@ -0,0 +1,429 @@ +/* + Copyright (c) 2005-2024 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB__flow_graph_body_impl_H +#define __TBB__flow_graph_body_impl_H + +#ifndef __TBB_flow_graph_H +#error Do not #include this internal file directly; use public TBB headers instead. +#endif + +// included in namespace tbb::detail::d2 (in flow_graph.h) + +typedef std::uint64_t tag_value; + + +// TODO revamp: find out if there is already helper for has_policy. +template struct Policy {}; + +template struct has_policy; + +template +struct has_policy : + std::integral_constant::value || + has_policy::value> {}; + +template +struct has_policy : + std::integral_constant::value> {}; + +template +struct has_policy > : has_policy {}; + +namespace graph_policy_namespace { + + struct rejecting { }; + struct reserving { }; + struct queueing { }; + struct lightweight { }; + + // K == type of field used for key-matching. Each tag-matching port will be provided + // functor that, given an object accepted by the port, will return the + /// field of type K being used for matching. + template::type > > + __TBB_requires(tbb::detail::hash_compare) + struct key_matching { + typedef K key_type; + typedef typename std::decay::type base_key_type; + typedef KHash hash_compare_type; + }; + + // old tag_matching join's new specifier + typedef key_matching tag_matching; + + // Aliases for Policy combinations + typedef Policy queueing_lightweight; + typedef Policy rejecting_lightweight; + +} // namespace graph_policy_namespace + +// -------------- function_body containers ---------------------- + +//! A functor that takes no input and generates a value of type Output +template< typename Output > +class input_body : no_assign { +public: + virtual ~input_body() {} + virtual Output operator()(d1::flow_control& fc) = 0; + virtual input_body* clone() = 0; +}; + +//! The leaf for input_body +template< typename Output, typename Body> +class input_body_leaf : public input_body { +public: + input_body_leaf( const Body &_body ) : body(_body) { } + Output operator()(d1::flow_control& fc) override { return body(fc); } + input_body_leaf* clone() override { + return new input_body_leaf< Output, Body >(body); + } + Body get_body() { return body; } +private: + Body body; +}; + +//! A functor that takes an Input and generates an Output +template< typename Input, typename Output > +class function_body : no_assign { +public: + virtual ~function_body() {} + virtual Output operator()(const Input &input) = 0; + virtual function_body* clone() = 0; +}; + +//! the leaf for function_body +template +class function_body_leaf : public function_body< Input, Output > { +public: + function_body_leaf( const B &_body ) : body(_body) { } + Output operator()(const Input &i) override { return tbb::detail::invoke(body,i); } + B get_body() { return body; } + function_body_leaf* clone() override { + return new function_body_leaf< Input, Output, B >(body); + } +private: + B body; +}; + +//! the leaf for function_body specialized for Input and output of continue_msg +template +class function_body_leaf< continue_msg, continue_msg, B> : public function_body< continue_msg, continue_msg > { +public: + function_body_leaf( const B &_body ) : body(_body) { } + continue_msg operator()( const continue_msg &i ) override { + body(i); + return i; + } + B get_body() { return body; } + function_body_leaf* clone() override { + return new function_body_leaf< continue_msg, continue_msg, B >(body); + } +private: + B body; +}; + +//! the leaf for function_body specialized for Output of continue_msg +template +class function_body_leaf< Input, continue_msg, B> : public function_body< Input, continue_msg > { +public: + function_body_leaf( const B &_body ) : body(_body) { } + continue_msg operator()(const Input &i) override { + body(i); + return continue_msg(); + } + B get_body() { return body; } + function_body_leaf* clone() override { + return new function_body_leaf< Input, continue_msg, B >(body); + } +private: + B body; +}; + +//! the leaf for function_body specialized for Input of continue_msg +template +class function_body_leaf< continue_msg, Output, B > : public function_body< continue_msg, Output > { +public: + function_body_leaf( const B &_body ) : body(_body) { } + Output operator()(const continue_msg &i) override { + return body(i); + } + B get_body() { return body; } + function_body_leaf* clone() override { + return new function_body_leaf< continue_msg, Output, B >(body); + } +private: + B body; +}; + +//! function_body that takes an Input and a set of output ports +template +class multifunction_body : no_assign { +public: + virtual ~multifunction_body () {} + virtual void operator()(const Input &/* input*/, OutputSet &/*oset*/) = 0; + virtual multifunction_body* clone() = 0; + virtual void* get_body_ptr() = 0; +}; + +//! leaf for multifunction. OutputSet can be a std::tuple or a vector. +template +class multifunction_body_leaf : public multifunction_body { +public: + multifunction_body_leaf(const B &_body) : body(_body) { } + void operator()(const Input &input, OutputSet &oset) override { + tbb::detail::invoke(body, input, oset); // body may explicitly put() to one or more of oset. + } + void* get_body_ptr() override { return &body; } + multifunction_body_leaf* clone() override { + return new multifunction_body_leaf(body); + } + +private: + B body; +}; + +// ------ function bodies for hash_buffers and key-matching joins. + +template +class type_to_key_function_body : no_assign { + public: + virtual ~type_to_key_function_body() {} + virtual Output operator()(const Input &input) = 0; // returns an Output + virtual type_to_key_function_body* clone() = 0; +}; + +// specialization for ref output +template +class type_to_key_function_body : no_assign { + public: + virtual ~type_to_key_function_body() {} + virtual const Output & operator()(const Input &input) = 0; // returns a const Output& + virtual type_to_key_function_body* clone() = 0; +}; + +template +class type_to_key_function_body_leaf : public type_to_key_function_body { +public: + type_to_key_function_body_leaf( const B &_body ) : body(_body) { } + Output operator()(const Input &i) override { return tbb::detail::invoke(body, i); } + type_to_key_function_body_leaf* clone() override { + return new type_to_key_function_body_leaf< Input, Output, B>(body); + } +private: + B body; +}; + +template +class type_to_key_function_body_leaf : public type_to_key_function_body< Input, Output&> { +public: + type_to_key_function_body_leaf( const B &_body ) : body(_body) { } + const Output& operator()(const Input &i) override { + return tbb::detail::invoke(body, i); + } + type_to_key_function_body_leaf* clone() override { + return new type_to_key_function_body_leaf< Input, Output&, B>(body); + } +private: + B body; +}; + +// --------------------------- end of function_body containers ------------------------ + +// --------------------------- node task bodies --------------------------------------- + +//! A task that calls a node's forward_task function +template< typename NodeType > +class forward_task_bypass : public graph_task { + NodeType &my_node; +public: + forward_task_bypass( graph& g, d1::small_object_allocator& allocator, NodeType &n + , node_priority_t node_priority = no_priority + ) : graph_task(g, allocator, node_priority), + my_node(n) {} + + d1::task* execute(d1::execution_data& ed) override { + graph_task* next_task = my_node.forward_task(); + if (SUCCESSFULLY_ENQUEUED == next_task) + next_task = nullptr; + else if (next_task) + next_task = prioritize_task(my_node.graph_reference(), *next_task); + finalize(ed); + return next_task; + } + + d1::task* cancel(d1::execution_data& ed) override { + finalize(ed); + return nullptr; + } +}; + +//! A task that calls a node's apply_body_bypass function, passing in an input of type Input +// return the task* unless it is SUCCESSFULLY_ENQUEUED, in which case return nullptr +template< typename NodeType, typename Input, typename BaseTaskType = graph_task> +class apply_body_task_bypass + : public BaseTaskType +{ + NodeType &my_node; + Input my_input; + + using check_metainfo = std::is_same; + using without_metainfo = std::true_type; + using with_metainfo = std::false_type; + + graph_task* call_apply_body_bypass_impl(without_metainfo) { + return my_node.apply_body_bypass(my_input + __TBB_FLOW_GRAPH_METAINFO_ARG(message_metainfo{})); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + graph_task* call_apply_body_bypass_impl(with_metainfo) { + return my_node.apply_body_bypass(my_input, message_metainfo{this->get_msg_wait_context_vertices()}); + } +#endif + + graph_task* call_apply_body_bypass() { + return call_apply_body_bypass_impl(check_metainfo{}); + } + +public: +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + template + apply_body_task_bypass( graph& g, d1::small_object_allocator& allocator, NodeType &n, const Input &i, + node_priority_t node_priority, Metainfo&& metainfo ) + : BaseTaskType(g, allocator, node_priority, std::forward(metainfo).waiters()) + , my_node(n), my_input(i) {} +#endif + + apply_body_task_bypass( graph& g, d1::small_object_allocator& allocator, NodeType& n, const Input& i, + node_priority_t node_priority = no_priority ) + : BaseTaskType(g, allocator, node_priority), my_node(n), my_input(i) {} + + d1::task* execute(d1::execution_data& ed) override { + graph_task* next_task = call_apply_body_bypass(); + if (SUCCESSFULLY_ENQUEUED == next_task) + next_task = nullptr; + else if (next_task) + next_task = prioritize_task(my_node.graph_reference(), *next_task); + BaseTaskType::template finalize(ed); + return next_task; + } + + d1::task* cancel(d1::execution_data& ed) override { + BaseTaskType::template finalize(ed); + return nullptr; + } +}; + +//! A task that calls a node's apply_body_bypass function with no input +template< typename NodeType > +class input_node_task_bypass : public graph_task { + NodeType &my_node; +public: + input_node_task_bypass( graph& g, d1::small_object_allocator& allocator, NodeType &n ) + : graph_task(g, allocator), my_node(n) {} + + d1::task* execute(d1::execution_data& ed) override { + graph_task* next_task = my_node.apply_body_bypass( ); + if (SUCCESSFULLY_ENQUEUED == next_task) + next_task = nullptr; + else if (next_task) + next_task = prioritize_task(my_node.graph_reference(), *next_task); + finalize(ed); + return next_task; + } + + d1::task* cancel(d1::execution_data& ed) override { + finalize(ed); + return nullptr; + } +}; + +// ------------------------ end of node task bodies ----------------------------------- + +template +class threshold_regulator; + +template +class threshold_regulator::value>::type> + : public receiver, no_copy +{ + T* my_node; +protected: + + graph_task* try_put_task( const DecrementType& value ) override { + graph_task* result = my_node->decrement_counter( value ); + if( !result ) + result = SUCCESSFULLY_ENQUEUED; + return result; + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + // Intentionally ignore the metainformation + // If there are more items associated with passed metainfo to be processed + // They should be stored in the buffer before the limiter_node + graph_task* try_put_task(const DecrementType& value, const message_metainfo&) override { + return try_put_task(value); + } +#endif + + graph& graph_reference() const override { + return my_node->my_graph; + } + + template friend class limiter_node; + void reset_receiver( reset_flags ) {} + +public: + threshold_regulator(T* owner) : my_node(owner) { + // Do not work with the passed pointer here as it may not be fully initialized yet + } +}; + +template +class threshold_regulator : public continue_receiver, no_copy { + + T *my_node; + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + // Intentionally ignore the metainformation + // If there are more items associated with passed metainfo to be processed + // They should be stored in the buffer before the limiter_node + graph_task* execute(const message_metainfo&) override { +#else + graph_task* execute() override { +#endif + return my_node->decrement_counter( 1 ); + } + +protected: + + graph& graph_reference() const override { + return my_node->my_graph; + } + +public: + + typedef continue_msg input_type; + typedef continue_msg output_type; + threshold_regulator(T* owner) + : continue_receiver( /*number_of_predecessors=*/0, no_priority ), my_node(owner) + { + // Do not work with the passed pointer here as it may not be fully initialized yet + } +}; + +#endif // __TBB__flow_graph_body_impl_H diff --git a/src/tbb/include/oneapi/tbb/detail/_flow_graph_cache_impl.h b/src/tbb/include/oneapi/tbb/detail/_flow_graph_cache_impl.h new file mode 100644 index 000000000..647f3dc1b --- /dev/null +++ b/src/tbb/include/oneapi/tbb/detail/_flow_graph_cache_impl.h @@ -0,0 +1,501 @@ +/* + Copyright (c) 2005-2024 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB__flow_graph_cache_impl_H +#define __TBB__flow_graph_cache_impl_H + +#ifndef __TBB_flow_graph_H +#error Do not #include this internal file directly; use public TBB headers instead. +#endif + +// included in namespace tbb::detail::d2 (in flow_graph.h) + +//! A node_cache maintains a std::queue of elements of type T. Each operation is protected by a lock. +template< typename T, typename M=spin_mutex > +class node_cache { + public: + + typedef size_t size_type; + + bool empty() { + typename mutex_type::scoped_lock lock( my_mutex ); + return internal_empty(); + } + + void add( T &n ) { + typename mutex_type::scoped_lock lock( my_mutex ); + internal_push(n); + } + + void remove( T &n ) { + typename mutex_type::scoped_lock lock( my_mutex ); + for ( size_t i = internal_size(); i != 0; --i ) { + T &s = internal_pop(); + if ( &s == &n ) + break; // only remove one predecessor per request + internal_push(s); + } + } + + void clear() { + while( !my_q.empty()) (void)my_q.pop(); + } + +protected: + + typedef M mutex_type; + mutex_type my_mutex; + std::queue< T * > my_q; + + // Assumes lock is held + inline bool internal_empty( ) { + return my_q.empty(); + } + + // Assumes lock is held + inline size_type internal_size( ) { + return my_q.size(); + } + + // Assumes lock is held + inline void internal_push( T &n ) { + my_q.push(&n); + } + + // Assumes lock is held + inline T &internal_pop() { + T *v = my_q.front(); + my_q.pop(); + return *v; + } + +}; + +//! A cache of predecessors that only supports try_get +template< typename T, typename M=spin_mutex > +class predecessor_cache : public node_cache< sender, M > { +public: + typedef M mutex_type; + typedef T output_type; + typedef sender predecessor_type; + typedef receiver successor_type; + + predecessor_cache( successor_type* owner ) : my_owner( owner ) { + __TBB_ASSERT( my_owner, "predecessor_cache should have an owner." ); + // Do not work with the passed pointer here as it may not be fully initialized yet + } + +private: + bool get_item_impl( output_type& v + __TBB_FLOW_GRAPH_METAINFO_ARG(message_metainfo* metainfo_ptr = nullptr) ) + { + + bool successful_get = false; + + do { + predecessor_type *src; + { + typename mutex_type::scoped_lock lock(this->my_mutex); + if ( this->internal_empty() ) { + break; + } + src = &this->internal_pop(); + } + + // Try to get from this sender +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + if (metainfo_ptr) { + successful_get = src->try_get( v, *metainfo_ptr ); + } else +#endif + { + successful_get = src->try_get( v ); + } + + if (successful_get == false) { + // Relinquish ownership of the edge + register_successor(*src, *my_owner); + } else { + // Retain ownership of the edge + this->add(*src); + } + } while ( successful_get == false ); + return successful_get; + } +public: + bool get_item( output_type& v ) { + return get_item_impl(v); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + bool get_item( output_type& v, message_metainfo& metainfo ) { + return get_item_impl(v, &metainfo); + } +#endif + + // If we are removing arcs (rf_clear_edges), call clear() rather than reset(). + void reset() { + for(;;) { + predecessor_type *src; + { + if (this->internal_empty()) break; + src = &this->internal_pop(); + } + register_successor(*src, *my_owner); + } + } + +protected: + successor_type* my_owner; +}; + +//! An cache of predecessors that supports requests and reservations +template< typename T, typename M=spin_mutex > +class reservable_predecessor_cache : public predecessor_cache< T, M > { +public: + typedef M mutex_type; + typedef T output_type; + typedef sender predecessor_type; + typedef receiver successor_type; + + reservable_predecessor_cache( successor_type* owner ) + : predecessor_cache(owner), reserved_src(nullptr) + { + // Do not work with the passed pointer here as it may not be fully initialized yet + } + +private: + bool try_reserve_impl( output_type &v __TBB_FLOW_GRAPH_METAINFO_ARG(message_metainfo* metainfo) ) { + bool successful_reserve = false; + + do { + predecessor_type* pred = nullptr; + { + typename mutex_type::scoped_lock lock(this->my_mutex); + if ( reserved_src.load(std::memory_order_relaxed) || this->internal_empty() ) + return false; + + pred = &this->internal_pop(); + reserved_src.store(pred, std::memory_order_relaxed); + } + + // Try to get from this sender +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + if (metainfo) { + successful_reserve = pred->try_reserve( v, *metainfo ); + } else +#endif + { + successful_reserve = pred->try_reserve( v ); + } + + if (successful_reserve == false) { + typename mutex_type::scoped_lock lock(this->my_mutex); + // Relinquish ownership of the edge + register_successor( *pred, *this->my_owner ); + reserved_src.store(nullptr, std::memory_order_relaxed); + } else { + // Retain ownership of the edge + this->add( *pred); + } + } while ( successful_reserve == false ); + + return successful_reserve; + } +public: + bool try_reserve( output_type& v ) { + return try_reserve_impl(v __TBB_FLOW_GRAPH_METAINFO_ARG(nullptr)); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + bool try_reserve( output_type& v, message_metainfo& metainfo ) { + return try_reserve_impl(v, &metainfo); + } +#endif + + bool try_release() { + reserved_src.load(std::memory_order_relaxed)->try_release(); + reserved_src.store(nullptr, std::memory_order_relaxed); + return true; + } + + bool try_consume() { + reserved_src.load(std::memory_order_relaxed)->try_consume(); + reserved_src.store(nullptr, std::memory_order_relaxed); + return true; + } + + void reset() { + reserved_src.store(nullptr, std::memory_order_relaxed); + predecessor_cache::reset(); + } + + void clear() { + reserved_src.store(nullptr, std::memory_order_relaxed); + predecessor_cache::clear(); + } + +private: + std::atomic reserved_src; +}; + + +//! An abstract cache of successors +template +class successor_cache : no_copy { +protected: + + typedef M mutex_type; + mutex_type my_mutex; + + typedef receiver successor_type; + typedef receiver* pointer_type; + typedef sender owner_type; + // TODO revamp: introduce heapified collection of successors for strict priorities + typedef std::list< pointer_type > successors_type; + successors_type my_successors; + + owner_type* my_owner; + +public: + successor_cache( owner_type* owner ) : my_owner(owner) { + // Do not work with the passed pointer here as it may not be fully initialized yet + } + + virtual ~successor_cache() {} + + void register_successor( successor_type& r ) { + typename mutex_type::scoped_lock l(my_mutex, true); + if( r.priority() != no_priority ) + my_successors.push_front( &r ); + else + my_successors.push_back( &r ); + } + + void remove_successor( successor_type& r ) { + typename mutex_type::scoped_lock l(my_mutex, true); + for ( typename successors_type::iterator i = my_successors.begin(); + i != my_successors.end(); ++i ) { + if ( *i == & r ) { + my_successors.erase(i); + break; + } + } + } + + bool empty() { + typename mutex_type::scoped_lock l(my_mutex, false); + return my_successors.empty(); + } + + void clear() { + my_successors.clear(); + } + + virtual graph_task* try_put_task( const T& t ) = 0; +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + virtual graph_task* try_put_task( const T& t, const message_metainfo& metainfo ) = 0; +#endif +}; // successor_cache + +//! An abstract cache of successors, specialized to continue_msg +template +class successor_cache< continue_msg, M > : no_copy { +protected: + + typedef M mutex_type; + mutex_type my_mutex; + + typedef receiver successor_type; + typedef receiver* pointer_type; + typedef sender owner_type; + typedef std::list< pointer_type > successors_type; + successors_type my_successors; + owner_type* my_owner; + +public: + successor_cache( sender* owner ) : my_owner(owner) { + // Do not work with the passed pointer here as it may not be fully initialized yet + } + + virtual ~successor_cache() {} + + void register_successor( successor_type& r ) { + typename mutex_type::scoped_lock l(my_mutex, true); + if( r.priority() != no_priority ) + my_successors.push_front( &r ); + else + my_successors.push_back( &r ); + __TBB_ASSERT( my_owner, "Cache of successors must have an owner." ); + if ( r.is_continue_receiver() ) { + r.register_predecessor( *my_owner ); + } + } + + void remove_successor( successor_type& r ) { + typename mutex_type::scoped_lock l(my_mutex, true); + for ( successors_type::iterator i = my_successors.begin(); i != my_successors.end(); ++i ) { + if ( *i == &r ) { + __TBB_ASSERT(my_owner, "Cache of successors must have an owner."); + // TODO: check if we need to test for continue_receiver before removing from r. + r.remove_predecessor( *my_owner ); + my_successors.erase(i); + break; + } + } + } + + bool empty() { + typename mutex_type::scoped_lock l(my_mutex, false); + return my_successors.empty(); + } + + void clear() { + my_successors.clear(); + } + + virtual graph_task* try_put_task( const continue_msg& t ) = 0; +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + virtual graph_task* try_put_task( const continue_msg& t, const message_metainfo& metainfo ) = 0; +#endif +}; // successor_cache< continue_msg > + +//! A cache of successors that are broadcast to +template +class broadcast_cache : public successor_cache { + typedef successor_cache base_type; + typedef M mutex_type; + typedef typename successor_cache::successors_type successors_type; + + graph_task* try_put_task_impl( const T& t __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& metainfo) ) { + graph_task * last_task = nullptr; + typename mutex_type::scoped_lock l(this->my_mutex, /*write=*/true); + typename successors_type::iterator i = this->my_successors.begin(); + while ( i != this->my_successors.end() ) { + graph_task *new_task = (*i)->try_put_task(t __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo)); + // workaround for icc bug + graph& graph_ref = (*i)->graph_reference(); + last_task = combine_tasks(graph_ref, last_task, new_task); // enqueue if necessary + if(new_task) { + ++i; + } + else { // failed + if ( (*i)->register_predecessor(*this->my_owner) ) { + i = this->my_successors.erase(i); + } else { + ++i; + } + } + } + return last_task; + } +public: + + broadcast_cache( typename base_type::owner_type* owner ): base_type(owner) { + // Do not work with the passed pointer here as it may not be fully initialized yet + } + + graph_task* try_put_task( const T &t ) override { + return try_put_task_impl(t __TBB_FLOW_GRAPH_METAINFO_ARG(message_metainfo{})); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + graph_task* try_put_task( const T &t, const message_metainfo& metainfo ) override { + return try_put_task_impl(t, metainfo); + } +#endif + + // call try_put_task and return list of received tasks + bool gather_successful_try_puts( const T &t, graph_task_list& tasks ) { + bool is_at_least_one_put_successful = false; + typename mutex_type::scoped_lock l(this->my_mutex, /*write=*/true); + typename successors_type::iterator i = this->my_successors.begin(); + while ( i != this->my_successors.end() ) { + graph_task * new_task = (*i)->try_put_task(t); + if(new_task) { + ++i; + if(new_task != SUCCESSFULLY_ENQUEUED) { + tasks.push_back(*new_task); + } + is_at_least_one_put_successful = true; + } + else { // failed + if ( (*i)->register_predecessor(*this->my_owner) ) { + i = this->my_successors.erase(i); + } else { + ++i; + } + } + } + return is_at_least_one_put_successful; + } +}; + +//! A cache of successors that are put in a round-robin fashion +template +class round_robin_cache : public successor_cache { + typedef successor_cache base_type; + typedef size_t size_type; + typedef M mutex_type; + typedef typename successor_cache::successors_type successors_type; + +public: + + round_robin_cache( typename base_type::owner_type* owner ): base_type(owner) { + // Do not work with the passed pointer here as it may not be fully initialized yet + } + + size_type size() { + typename mutex_type::scoped_lock l(this->my_mutex, false); + return this->my_successors.size(); + } + +private: + + graph_task* try_put_task_impl( const T &t + __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& metainfo) ) + { + typename mutex_type::scoped_lock l(this->my_mutex, /*write=*/true); + typename successors_type::iterator i = this->my_successors.begin(); + while ( i != this->my_successors.end() ) { + graph_task* new_task = (*i)->try_put_task(t __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo)); + if ( new_task ) { + return new_task; + } else { + if ( (*i)->register_predecessor(*this->my_owner) ) { + i = this->my_successors.erase(i); + } + else { + ++i; + } + } + } + return nullptr; + } + +public: + graph_task* try_put_task(const T& t) override { + return try_put_task_impl(t __TBB_FLOW_GRAPH_METAINFO_ARG(message_metainfo{})); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + graph_task* try_put_task( const T& t, const message_metainfo& metainfo ) override { + return try_put_task_impl(t, metainfo); + } +#endif +}; + +#endif // __TBB__flow_graph_cache_impl_H diff --git a/src/tbb/include/oneapi/tbb/detail/_flow_graph_impl.h b/src/tbb/include/oneapi/tbb/detail/_flow_graph_impl.h new file mode 100644 index 000000000..19e00a8ef --- /dev/null +++ b/src/tbb/include/oneapi/tbb/detail/_flow_graph_impl.h @@ -0,0 +1,570 @@ +/* + Copyright (c) 2005-2024 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_flow_graph_impl_H +#define __TBB_flow_graph_impl_H + +// #include "../config.h" +#include "_task.h" +#include "../task_group.h" +#include "../task_arena.h" +#include "../flow_graph_abstractions.h" + +#include "../concurrent_priority_queue.h" + +#include + +namespace tbb { +namespace detail { + +namespace d2 { + +class graph_task; +static graph_task* const SUCCESSFULLY_ENQUEUED = (graph_task*)-1; +typedef unsigned int node_priority_t; +static const node_priority_t no_priority = node_priority_t(0); + +class graph; +class graph_node; + +template +class graph_iterator { + friend class graph; + friend class graph_node; +public: + typedef size_t size_type; + typedef GraphNodeType value_type; + typedef GraphNodeType* pointer; + typedef GraphNodeType& reference; + typedef const GraphNodeType& const_reference; + typedef std::forward_iterator_tag iterator_category; + + //! Copy constructor + graph_iterator(const graph_iterator& other) : + my_graph(other.my_graph), current_node(other.current_node) + {} + + //! Assignment + graph_iterator& operator=(const graph_iterator& other) { + if (this != &other) { + my_graph = other.my_graph; + current_node = other.current_node; + } + return *this; + } + + //! Dereference + reference operator*() const; + + //! Dereference + pointer operator->() const; + + //! Equality + bool operator==(const graph_iterator& other) const { + return ((my_graph == other.my_graph) && (current_node == other.current_node)); + } + +#if !__TBB_CPP20_COMPARISONS_PRESENT + //! Inequality + bool operator!=(const graph_iterator& other) const { return !(operator==(other)); } +#endif + + //! Pre-increment + graph_iterator& operator++() { + internal_forward(); + return *this; + } + + //! Post-increment + graph_iterator operator++(int) { + graph_iterator result = *this; + operator++(); + return result; + } + +private: + // the graph over which we are iterating + GraphContainerType *my_graph; + // pointer into my_graph's my_nodes list + pointer current_node; + + //! Private initializing constructor for begin() and end() iterators + graph_iterator(GraphContainerType *g, bool begin); + void internal_forward(); +}; // class graph_iterator + +// flags to modify the behavior of the graph reset(). Can be combined. +enum reset_flags { + rf_reset_protocol = 0, + rf_reset_bodies = 1 << 0, // delete the current node body, reset to a copy of the initial node body. + rf_clear_edges = 1 << 1 // delete edges +}; + +void activate_graph(graph& g); +void deactivate_graph(graph& g); +bool is_graph_active(graph& g); +graph_task* prioritize_task(graph& g, graph_task& arena_task); +void spawn_in_graph_arena(graph& g, graph_task& arena_task); +void enqueue_in_graph_arena(graph &g, graph_task& arena_task); + +class graph; + +//! Base class for tasks generated by graph nodes. +class graph_task : public d1::task { +public: + graph_task(graph& g, d1::small_object_allocator& allocator, + node_priority_t node_priority = no_priority); + + graph& my_graph; // graph instance the task belongs to + // TODO revamp: rename to my_priority + node_priority_t priority; + template + void destruct_and_deallocate(const d1::execution_data& ed); +protected: + template + void finalize(const d1::execution_data& ed); +private: + // To organize task_list + graph_task* my_next{ nullptr }; + d1::small_object_allocator my_allocator; + d1::wait_tree_vertex_interface* my_reference_vertex; + // TODO revamp: elaborate internal interfaces to avoid friends declarations + friend class graph_task_list; + friend graph_task* prioritize_task(graph& g, graph_task& gt); +}; + +inline bool is_this_thread_in_graph_arena(graph& g); + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT +class trackable_messages_graph_task : public graph_task { +public: + trackable_messages_graph_task(graph& g, d1::small_object_allocator& allocator, + node_priority_t node_priority, + const std::forward_list& msg_waiters) + : graph_task(g, allocator, node_priority) + , my_msg_wait_context_vertices(msg_waiters) + { + auto last_iterator = my_msg_reference_vertices.cbefore_begin(); + + for (auto& msg_waiter : my_msg_wait_context_vertices) { + // If the task is created by the thread outside the graph arena, the lifetime of the thread reference vertex + // may be shorter that the lifetime of the task, so thread reference vertex approach cannot be used + // and the task should be associated with the msg wait context itself + d1::wait_tree_vertex_interface* ref_vertex = is_this_thread_in_graph_arena(g) ? + r1::get_thread_reference_vertex(msg_waiter) : + msg_waiter; + last_iterator = my_msg_reference_vertices.emplace_after(last_iterator, + ref_vertex); + ref_vertex->reserve(1); + } + } + + trackable_messages_graph_task(graph& g, d1::small_object_allocator& allocator, + node_priority_t node_priority, + std::forward_list&& msg_waiters) + : graph_task(g, allocator, node_priority) + , my_msg_wait_context_vertices(std::move(msg_waiters)) + { + } + + const std::forward_list get_msg_wait_context_vertices() const { + return my_msg_wait_context_vertices; + } + +protected: + template + void finalize(const d1::execution_data& ed) { + auto wait_context_vertices = std::move(my_msg_wait_context_vertices); + auto msg_reference_vertices = std::move(my_msg_reference_vertices); + graph_task::finalize(ed); + + // If there is no thread reference vertices associated with the task + // then this task was created by transferring the ownership from other metainfo + // instance (e.g. while taking from the buffer) + if (msg_reference_vertices.empty()) { + for (auto& msg_waiter : wait_context_vertices) { + msg_waiter->release(1); + } + } else { + for (auto& msg_waiter : msg_reference_vertices) { + msg_waiter->release(1); + } + } + } +private: + // Each task that holds information about single message wait_contexts should hold two lists + // The first one is wait_contexts associated with the message itself. They are needed + // to be able to broadcast the list of wait_contexts to the node successors while executing the task. + // The second list is a list of reference vertices for each wait_context_vertex in the first list + // to support the distributed reference counting schema + std::forward_list my_msg_wait_context_vertices; + std::forward_list my_msg_reference_vertices; +}; // class trackable_messages_graph_task +#endif // __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + +struct graph_task_comparator { + bool operator()(const graph_task* left, const graph_task* right) { + return left->priority < right->priority; + } +}; + +typedef tbb::concurrent_priority_queue graph_task_priority_queue_t; + +class priority_task_selector : public d1::task { +public: + priority_task_selector(graph_task_priority_queue_t& priority_queue, d1::small_object_allocator& allocator) + : my_priority_queue(priority_queue), my_allocator(allocator), my_task() {} + task* execute(d1::execution_data& ed) override { + next_task(); + __TBB_ASSERT(my_task, nullptr); + task* t_next = my_task->execute(ed); + my_allocator.delete_object(this, ed); + return t_next; + } + task* cancel(d1::execution_data& ed) override { + if (!my_task) { + next_task(); + } + __TBB_ASSERT(my_task, nullptr); + task* t_next = my_task->cancel(ed); + my_allocator.delete_object(this, ed); + return t_next; + } +private: + void next_task() { + // TODO revamp: hold functors in priority queue instead of real tasks + bool result = my_priority_queue.try_pop(my_task); + __TBB_ASSERT_EX(result, "Number of critical tasks for scheduler and tasks" + " in graph's priority queue mismatched"); + __TBB_ASSERT(my_task && my_task != SUCCESSFULLY_ENQUEUED, + "Incorrect task submitted to graph priority queue"); + __TBB_ASSERT(my_task->priority != no_priority, + "Tasks from graph's priority queue must have priority"); + } + + graph_task_priority_queue_t& my_priority_queue; + d1::small_object_allocator my_allocator; + graph_task* my_task; +}; + +template class run_and_put_task; +template class run_task; + +//******************************************************************************** +// graph tasks helpers +//******************************************************************************** + +//! The list of graph tasks +class graph_task_list : no_copy { +private: + graph_task* my_first; + graph_task** my_next_ptr; +public: + //! Construct empty list + graph_task_list() : my_first(nullptr), my_next_ptr(&my_first) {} + + //! True if list is empty; false otherwise. + bool empty() const { return !my_first; } + + //! Push task onto back of list. + void push_back(graph_task& task) { + task.my_next = nullptr; + *my_next_ptr = &task; + my_next_ptr = &task.my_next; + } + + //! Pop the front task from the list. + graph_task& pop_front() { + __TBB_ASSERT(!empty(), "attempt to pop item from empty task_list"); + graph_task* result = my_first; + my_first = result->my_next; + if (!my_first) { + my_next_ptr = &my_first; + } + return *result; + } +}; + +//! The graph class +/** This class serves as a handle to the graph */ +class graph : no_copy, public graph_proxy { + friend class graph_node; + + void prepare_task_arena(bool reinit = false) { + if (reinit) { + __TBB_ASSERT(my_task_arena, "task arena is nullptr"); + my_task_arena->terminate(); + my_task_arena->initialize(task_arena::attach()); + } + else { + __TBB_ASSERT(my_task_arena == nullptr, "task arena is not nullptr"); + my_task_arena = new task_arena(task_arena::attach()); + } + if (!my_task_arena->is_active()) // failed to attach + my_task_arena->initialize(); // create a new, default-initialized arena + __TBB_ASSERT(my_task_arena->is_active(), "task arena is not active"); + } + +public: + //! Constructs a graph with isolated task_group_context + graph(); + + //! Constructs a graph with use_this_context as context + explicit graph(task_group_context& use_this_context); + + //! Destroys the graph. + /** Calls wait_for_all, then destroys the root task and context. */ + ~graph(); + + //! Used to register that an external entity may still interact with the graph. + /** The graph will not return from wait_for_all until a matching number of release_wait calls is + made. */ + void reserve_wait() override; + + //! Deregisters an external entity that may have interacted with the graph. + /** The graph will not return from wait_for_all until all the number of reserve_wait calls + matches the number of release_wait calls. */ + void release_wait() override; + + //! Wait until graph is idle and the number of release_wait calls equals to the number of + //! reserve_wait calls. + /** The waiting thread will go off and steal work while it is blocked in the wait_for_all. */ + void wait_for_all() { + cancelled = false; + caught_exception = false; + try_call([this] { + my_task_arena->execute([this] { + wait(my_wait_context_vertex.get_context(), *my_context); + }); + cancelled = my_context->is_group_execution_cancelled(); + }).on_exception([this] { + my_context->reset(); + caught_exception = true; + cancelled = true; + }); + // TODO: the "if" condition below is just a work-around to support the concurrent wait + // mode. The cancellation and exception mechanisms are still broken in this mode. + // Consider using task group not to re-implement the same functionality. + if (!(my_context->traits() & task_group_context::concurrent_wait)) { + my_context->reset(); // consistent with behavior in catch() + } + } + + // TODO revamp: consider adding getter for task_group_context. + + // ITERATORS + template + friend class graph_iterator; + + // Graph iterator typedefs + typedef graph_iterator iterator; + typedef graph_iterator const_iterator; + + // Graph iterator constructors + //! start iterator + iterator begin(); + //! end iterator + iterator end(); + //! start const iterator + const_iterator begin() const; + //! end const iterator + const_iterator end() const; + //! start const iterator + const_iterator cbegin() const; + //! end const iterator + const_iterator cend() const; + + // thread-unsafe state reset. + void reset(reset_flags f = rf_reset_protocol); + + //! cancels execution of the associated task_group_context + void cancel(); + + //! return status of graph execution + bool is_cancelled() { return cancelled; } + bool exception_thrown() { return caught_exception; } + +private: + d1::wait_context_vertex my_wait_context_vertex; + task_group_context *my_context; + bool own_context; + bool cancelled; + bool caught_exception; + bool my_is_active; + + graph_node *my_nodes, *my_nodes_last; + + tbb::spin_mutex nodelist_mutex; + void register_node(graph_node *n); + void remove_node(graph_node *n); + + task_arena* my_task_arena; + + graph_task_priority_queue_t my_priority_queue; + + d1::wait_context_vertex& get_wait_context_vertex() { return my_wait_context_vertex; } + + friend void activate_graph(graph& g); + friend void deactivate_graph(graph& g); + friend bool is_graph_active(graph& g); + friend bool is_this_thread_in_graph_arena(graph& g); + friend graph_task* prioritize_task(graph& g, graph_task& arena_task); + friend void spawn_in_graph_arena(graph& g, graph_task& arena_task); + friend void enqueue_in_graph_arena(graph &g, graph_task& arena_task); + + friend class d1::task_arena_base; + friend class graph_task; + + template + friend class receiver; +}; // class graph + +template +inline void graph_task::destruct_and_deallocate(const d1::execution_data& ed) { + auto allocator = my_allocator; + // TODO: investigate if direct call of derived destructor gives any benefits. + this->~graph_task(); + allocator.deallocate(static_cast(this), ed); +} + +template +inline void graph_task::finalize(const d1::execution_data& ed) { + d1::wait_tree_vertex_interface* reference_vertex = my_reference_vertex; + destruct_and_deallocate(ed); + reference_vertex->release(); +} + +inline graph_task::graph_task(graph& g, d1::small_object_allocator& allocator, + node_priority_t node_priority) + : my_graph(g) + , priority(node_priority) + , my_allocator(allocator) +{ + // If the task is created by the thread outside the graph arena, the lifetime of the thread reference vertex + // may be shorter that the lifetime of the task, so thread reference vertex approach cannot be used + // and the task should be associated with the graph wait context itself + // TODO: consider how reference counting can be improved for such a use case. Most common example is the async_node + d1::wait_context_vertex* graph_wait_context_vertex = &my_graph.get_wait_context_vertex(); + my_reference_vertex = is_this_thread_in_graph_arena(g) ? r1::get_thread_reference_vertex(graph_wait_context_vertex) + : graph_wait_context_vertex; + __TBB_ASSERT(my_reference_vertex, nullptr); + my_reference_vertex->reserve(); +} + +//******************************************************************************** +// end of graph tasks helpers +//******************************************************************************** + + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET +class get_graph_helper; +#endif + +//! The base of all graph nodes. +class graph_node : no_copy { + friend class graph; + template + friend class graph_iterator; + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + friend class get_graph_helper; +#endif + +protected: + graph& my_graph; + graph& graph_reference() const { + // TODO revamp: propagate graph_reference() method to all the reference places. + return my_graph; + } + graph_node* next = nullptr; + graph_node* prev = nullptr; +public: + explicit graph_node(graph& g); + + virtual ~graph_node(); + +protected: + // performs the reset on an individual node. + virtual void reset_node(reset_flags f = rf_reset_protocol) = 0; +}; // class graph_node + +inline void activate_graph(graph& g) { + g.my_is_active = true; +} + +inline void deactivate_graph(graph& g) { + g.my_is_active = false; +} + +inline bool is_graph_active(graph& g) { + return g.my_is_active; +} + +inline bool is_this_thread_in_graph_arena(graph& g) { + __TBB_ASSERT(g.my_task_arena && g.my_task_arena->is_active(), nullptr); + return r1::execution_slot(*g.my_task_arena) != d1::slot_id(-1); +} + +inline graph_task* prioritize_task(graph& g, graph_task& gt) { + if( no_priority == gt.priority ) + return > + + //! Non-preemptive priority pattern. The original task is submitted as a work item to the + //! priority queue, and a new critical task is created to take and execute a work item with + //! the highest known priority. The reference counting responsibility is transferred to + //! the new task. + d1::task* critical_task = gt.my_allocator.new_object(g.my_priority_queue, gt.my_allocator); + __TBB_ASSERT( critical_task, "bad_alloc?" ); + g.my_priority_queue.push(>); + using tbb::detail::d1::submit; + submit( *critical_task, *g.my_task_arena, *g.my_context, /*as_critical=*/true ); + return nullptr; +} + +//! Spawns a task inside graph arena +inline void spawn_in_graph_arena(graph& g, graph_task& arena_task) { + if (is_graph_active(g)) { + d1::task* gt = prioritize_task(g, arena_task); + if( !gt ) + return; + + __TBB_ASSERT(g.my_task_arena && g.my_task_arena->is_active(), nullptr); + submit( *gt, *g.my_task_arena, *g.my_context +#if __TBB_PREVIEW_CRITICAL_TASKS + , /*as_critical=*/false +#endif + ); + } +} + +// TODO revamp: unify *_in_graph_arena functions + +//! Enqueues a task inside graph arena +inline void enqueue_in_graph_arena(graph &g, graph_task& arena_task) { + if (is_graph_active(g)) { + __TBB_ASSERT( g.my_task_arena && g.my_task_arena->is_active(), "Is graph's arena initialized and active?" ); + + // TODO revamp: decide on the approach that does not postpone critical task + if( d1::task* gt = prioritize_task(g, arena_task) ) + submit( *gt, *g.my_task_arena, *g.my_context, /*as_critical=*/false); + } +} + +} // namespace d2 +} // namespace detail +} // namespace tbb + +#endif // __TBB_flow_graph_impl_H diff --git a/src/tbb/include/oneapi/tbb/detail/_flow_graph_indexer_impl.h b/src/tbb/include/oneapi/tbb/detail/_flow_graph_indexer_impl.h new file mode 100644 index 000000000..a74331007 --- /dev/null +++ b/src/tbb/include/oneapi/tbb/detail/_flow_graph_indexer_impl.h @@ -0,0 +1,370 @@ +/* + Copyright (c) 2005-2024 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB__flow_graph_indexer_impl_H +#define __TBB__flow_graph_indexer_impl_H + +#ifndef __TBB_flow_graph_H +#error Do not #include this internal file directly; use public TBB headers instead. +#endif + +// included in namespace tbb::detail::d2 + +#include "_flow_graph_types_impl.h" + + // Output of the indexer_node is a tbb::flow::tagged_msg, and will be of + // the form tagged_msg + // where the value of tag will indicate which result was put to the + // successor. + + template + graph_task* do_try_put(const T &v, void *p __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& metainfo)) { + typename IndexerNodeBaseType::output_type o(K, v); + return reinterpret_cast(p)->try_put_task(&o __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo)); + } + + template + struct indexer_helper { + template + static inline void set_indexer_node_pointer(PortTuple &my_input, IndexerNodeBaseType *p, graph& g) { + typedef typename std::tuple_element::type T; + auto indexer_node_put_task = do_try_put; + std::get(my_input).set_up(p, indexer_node_put_task, g); + indexer_helper::template set_indexer_node_pointer(my_input, p, g); + } + }; + + template + struct indexer_helper { + template + static inline void set_indexer_node_pointer(PortTuple &my_input, IndexerNodeBaseType *p, graph& g) { + typedef typename std::tuple_element<0, TupleTypes>::type T; + auto indexer_node_put_task = do_try_put; + std::get<0>(my_input).set_up(p, indexer_node_put_task, g); + } + }; + + template + class indexer_input_port : public receiver { + private: + void* my_indexer_ptr; + typedef graph_task* (* forward_function_ptr)(T const &, void* + __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo&)); + forward_function_ptr my_try_put_task; + graph* my_graph; + public: + void set_up(void* p, forward_function_ptr f, graph& g) { + my_indexer_ptr = p; + my_try_put_task = f; + my_graph = &g; + } + + protected: + template< typename R, typename B > friend class run_and_put_task; + template friend class broadcast_cache; + template friend class round_robin_cache; + graph_task* try_put_task(const T &v) override { + return my_try_put_task(v, my_indexer_ptr __TBB_FLOW_GRAPH_METAINFO_ARG(message_metainfo{})); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + graph_task* try_put_task(const T& v, const message_metainfo& metainfo) override { + return my_try_put_task(v, my_indexer_ptr, metainfo); + } +#endif + + graph& graph_reference() const override { + return *my_graph; + } + }; + + template + class indexer_node_FE { + public: + static const int N = std::tuple_size::value; + typedef OutputType output_type; + typedef InputTuple input_type; + + // Some versions of Intel(R) C++ Compiler fail to generate an implicit constructor for the class which has std::tuple as a member. + indexer_node_FE() : my_inputs() {} + + input_type &input_ports() { return my_inputs; } + protected: + input_type my_inputs; + }; + + //! indexer_node_base + template + class indexer_node_base : public graph_node, public indexer_node_FE, + public sender { + protected: + using graph_node::my_graph; + public: + static const size_t N = std::tuple_size::value; + typedef OutputType output_type; + typedef StructTypes tuple_types; + typedef typename sender::successor_type successor_type; + typedef indexer_node_FE input_ports_type; + + private: + // ----------- Aggregator ------------ + enum op_type { reg_succ, rem_succ, try__put_task + }; + typedef indexer_node_base class_type; + + class indexer_node_base_operation : public d1::aggregated_operation { + public: + char type; + union { + output_type const *my_arg; + successor_type *my_succ; + graph_task* bypass_t; + }; +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + message_metainfo const* metainfo; +#endif + indexer_node_base_operation(const output_type* e, op_type t) : + type(char(t)), my_arg(e) __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo(nullptr)) + {} +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + indexer_node_base_operation(const output_type* e, op_type t, const message_metainfo& info) + : type(char(t)), my_arg(e), metainfo(&info) {} +#endif + indexer_node_base_operation(const successor_type &s, op_type t) : type(char(t)), + my_succ(const_cast(&s)) {} + }; + + typedef d1::aggregating_functor handler_type; + friend class d1::aggregating_functor; + d1::aggregator my_aggregator; + + void handle_operations(indexer_node_base_operation* op_list) { + indexer_node_base_operation *current; + while(op_list) { + current = op_list; + op_list = op_list->next; + switch(current->type) { + + case reg_succ: + my_successors.register_successor(*(current->my_succ)); + current->status.store( SUCCEEDED, std::memory_order_release); + break; + + case rem_succ: + my_successors.remove_successor(*(current->my_succ)); + current->status.store( SUCCEEDED, std::memory_order_release); + break; + case try__put_task: { + current->bypass_t = my_successors.try_put_task(*(current->my_arg) + __TBB_FLOW_GRAPH_METAINFO_ARG(*(current->metainfo))); + current->status.store( SUCCEEDED, std::memory_order_release); // return of try_put_task actual return value + } + break; + } + } + } + // ---------- end aggregator ----------- + public: + indexer_node_base(graph& g) : graph_node(g), input_ports_type(), my_successors(this) { + indexer_helper::set_indexer_node_pointer(this->my_inputs, this, g); + my_aggregator.initialize_handler(handler_type(this)); + } + + indexer_node_base(const indexer_node_base& other) + : graph_node(other.my_graph), input_ports_type(), sender(), my_successors(this) + { + indexer_helper::set_indexer_node_pointer(this->my_inputs, this, other.my_graph); + my_aggregator.initialize_handler(handler_type(this)); + } + + bool register_successor(successor_type &r) override { + indexer_node_base_operation op_data(r, reg_succ); + my_aggregator.execute(&op_data); + return op_data.status == SUCCEEDED; + } + + bool remove_successor( successor_type &r) override { + indexer_node_base_operation op_data(r, rem_succ); + my_aggregator.execute(&op_data); + return op_data.status == SUCCEEDED; + } + + // not a virtual method in this class + graph_task* try_put_task(output_type const *v + __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& metainfo)) + { + indexer_node_base_operation op_data(v, try__put_task __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo)); + my_aggregator.execute(&op_data); + return op_data.bypass_t; + } + + protected: + void reset_node(reset_flags f) override { + if(f & rf_clear_edges) { + my_successors.clear(); + } + } + + private: + broadcast_cache my_successors; + }; //indexer_node_base + + + template struct input_types; + + template + struct input_types<1, InputTuple> { + typedef typename std::tuple_element<0, InputTuple>::type first_type; + typedef tagged_msg type; + }; + + template + struct input_types<2, InputTuple> { + typedef typename std::tuple_element<0, InputTuple>::type first_type; + typedef typename std::tuple_element<1, InputTuple>::type second_type; + typedef tagged_msg type; + }; + + template + struct input_types<3, InputTuple> { + typedef typename std::tuple_element<0, InputTuple>::type first_type; + typedef typename std::tuple_element<1, InputTuple>::type second_type; + typedef typename std::tuple_element<2, InputTuple>::type third_type; + typedef tagged_msg type; + }; + + template + struct input_types<4, InputTuple> { + typedef typename std::tuple_element<0, InputTuple>::type first_type; + typedef typename std::tuple_element<1, InputTuple>::type second_type; + typedef typename std::tuple_element<2, InputTuple>::type third_type; + typedef typename std::tuple_element<3, InputTuple>::type fourth_type; + typedef tagged_msg type; + }; + + template + struct input_types<5, InputTuple> { + typedef typename std::tuple_element<0, InputTuple>::type first_type; + typedef typename std::tuple_element<1, InputTuple>::type second_type; + typedef typename std::tuple_element<2, InputTuple>::type third_type; + typedef typename std::tuple_element<3, InputTuple>::type fourth_type; + typedef typename std::tuple_element<4, InputTuple>::type fifth_type; + typedef tagged_msg type; + }; + + template + struct input_types<6, InputTuple> { + typedef typename std::tuple_element<0, InputTuple>::type first_type; + typedef typename std::tuple_element<1, InputTuple>::type second_type; + typedef typename std::tuple_element<2, InputTuple>::type third_type; + typedef typename std::tuple_element<3, InputTuple>::type fourth_type; + typedef typename std::tuple_element<4, InputTuple>::type fifth_type; + typedef typename std::tuple_element<5, InputTuple>::type sixth_type; + typedef tagged_msg type; + }; + + template + struct input_types<7, InputTuple> { + typedef typename std::tuple_element<0, InputTuple>::type first_type; + typedef typename std::tuple_element<1, InputTuple>::type second_type; + typedef typename std::tuple_element<2, InputTuple>::type third_type; + typedef typename std::tuple_element<3, InputTuple>::type fourth_type; + typedef typename std::tuple_element<4, InputTuple>::type fifth_type; + typedef typename std::tuple_element<5, InputTuple>::type sixth_type; + typedef typename std::tuple_element<6, InputTuple>::type seventh_type; + typedef tagged_msg type; + }; + + + template + struct input_types<8, InputTuple> { + typedef typename std::tuple_element<0, InputTuple>::type first_type; + typedef typename std::tuple_element<1, InputTuple>::type second_type; + typedef typename std::tuple_element<2, InputTuple>::type third_type; + typedef typename std::tuple_element<3, InputTuple>::type fourth_type; + typedef typename std::tuple_element<4, InputTuple>::type fifth_type; + typedef typename std::tuple_element<5, InputTuple>::type sixth_type; + typedef typename std::tuple_element<6, InputTuple>::type seventh_type; + typedef typename std::tuple_element<7, InputTuple>::type eighth_type; + typedef tagged_msg type; + }; + + + template + struct input_types<9, InputTuple> { + typedef typename std::tuple_element<0, InputTuple>::type first_type; + typedef typename std::tuple_element<1, InputTuple>::type second_type; + typedef typename std::tuple_element<2, InputTuple>::type third_type; + typedef typename std::tuple_element<3, InputTuple>::type fourth_type; + typedef typename std::tuple_element<4, InputTuple>::type fifth_type; + typedef typename std::tuple_element<5, InputTuple>::type sixth_type; + typedef typename std::tuple_element<6, InputTuple>::type seventh_type; + typedef typename std::tuple_element<7, InputTuple>::type eighth_type; + typedef typename std::tuple_element<8, InputTuple>::type nineth_type; + typedef tagged_msg type; + }; + + template + struct input_types<10, InputTuple> { + typedef typename std::tuple_element<0, InputTuple>::type first_type; + typedef typename std::tuple_element<1, InputTuple>::type second_type; + typedef typename std::tuple_element<2, InputTuple>::type third_type; + typedef typename std::tuple_element<3, InputTuple>::type fourth_type; + typedef typename std::tuple_element<4, InputTuple>::type fifth_type; + typedef typename std::tuple_element<5, InputTuple>::type sixth_type; + typedef typename std::tuple_element<6, InputTuple>::type seventh_type; + typedef typename std::tuple_element<7, InputTuple>::type eighth_type; + typedef typename std::tuple_element<8, InputTuple>::type nineth_type; + typedef typename std::tuple_element<9, InputTuple>::type tenth_type; + typedef tagged_msg type; + }; + + // type generators + template + struct indexer_types : public input_types::value, OutputTuple> { + static const int N = std::tuple_size::value; + typedef typename input_types::type output_type; + typedef typename wrap_tuple_elements::type input_ports_type; + typedef indexer_node_FE indexer_FE_type; + typedef indexer_node_base indexer_base_type; + }; + + template + class unfolded_indexer_node : public indexer_types::indexer_base_type { + public: + typedef typename indexer_types::input_ports_type input_ports_type; + typedef OutputTuple tuple_types; + typedef typename indexer_types::output_type output_type; + private: + typedef typename indexer_types::indexer_base_type base_type; + public: + unfolded_indexer_node(graph& g) : base_type(g) {} + unfolded_indexer_node(const unfolded_indexer_node &other) : base_type(other) {} + }; + +#endif /* __TBB__flow_graph_indexer_impl_H */ diff --git a/src/tbb/include/oneapi/tbb/detail/_flow_graph_item_buffer_impl.h b/src/tbb/include/oneapi/tbb/detail/_flow_graph_item_buffer_impl.h new file mode 100644 index 000000000..cf7c54b85 --- /dev/null +++ b/src/tbb/include/oneapi/tbb/detail/_flow_graph_item_buffer_impl.h @@ -0,0 +1,422 @@ +/* + Copyright (c) 2005-2024 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB__flow_graph_item_buffer_impl_H +#define __TBB__flow_graph_item_buffer_impl_H + +#ifndef __TBB_flow_graph_H +#error Do not #include this internal file directly; use public TBB headers instead. +#endif + +#include "_aligned_space.h" + +// in namespace tbb::flow::interfaceX (included in _flow_graph_node_impl.h) + +//! Expandable buffer of items. The possible operations are push, pop, +//* tests for empty and so forth. No mutual exclusion is built in. +//* objects are constructed into and explicitly-destroyed. get_my_item gives +// a read-only reference to the item in the buffer. set_my_item may be called +// with either an empty or occupied slot. + +template > +class item_buffer { +public: + typedef T item_type; + enum buffer_item_state { no_item=0, has_item=1, reserved_item=2 }; +protected: + struct aligned_space_item { + item_type item; + buffer_item_state state; +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + message_metainfo metainfo; +#endif + }; + typedef size_t size_type; + typedef aligned_space buffer_item_type; + typedef typename allocator_traits::template rebind_alloc allocator_type; + buffer_item_type *my_array; + size_type my_array_size; + static const size_type initial_buffer_size = 4; + size_type my_head; + size_type my_tail; + + bool buffer_empty() const { return my_head == my_tail; } + + aligned_space_item &element(size_type i) { + __TBB_ASSERT(!(size_type(&(my_array[i&(my_array_size-1)].begin()->state))%alignment_of::value), nullptr); + __TBB_ASSERT(!(size_type(&(my_array[i&(my_array_size-1)].begin()->item))%alignment_of::value), nullptr); +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + __TBB_ASSERT(!(size_type(&(my_array[i&(my_array_size-1)].begin()->metainfo))%alignment_of::value), nullptr); +#endif + return *my_array[i & (my_array_size - 1) ].begin(); + } + + const aligned_space_item &element(size_type i) const { + __TBB_ASSERT(!(size_type(&(my_array[i&(my_array_size-1)].begin()->state))%alignment_of::value), nullptr); + __TBB_ASSERT(!(size_type(&(my_array[i&(my_array_size-1)].begin()->item))%alignment_of::value), nullptr); +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + __TBB_ASSERT(!(size_type(&(my_array[i&(my_array_size-1)].begin()->metainfo))%alignment_of::value), nullptr); +#endif + return *my_array[i & (my_array_size-1)].begin(); + } + + bool my_item_valid(size_type i) const { return (i < my_tail) && (i >= my_head) && (element(i).state != no_item); } +#if TBB_USE_ASSERT + bool my_item_reserved(size_type i) const { return element(i).state == reserved_item; } +#endif + + // object management in buffer + const item_type &get_my_item(size_t i) const { + __TBB_ASSERT(my_item_valid(i),"attempt to get invalid item"); + return element(i).item; + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + message_metainfo& get_my_metainfo(size_t i) { + __TBB_ASSERT(my_item_valid(i), "attempt to get invalid item"); + return element(i).metainfo; + } +#endif + + // may be called with an empty slot or a slot that has already been constructed into. + void set_my_item(size_t i, const item_type &o + __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& metainfo)) + { + if(element(i).state != no_item) { + destroy_item(i); + } + new(&(element(i).item)) item_type(o); + element(i).state = has_item; +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + new(&element(i).metainfo) message_metainfo(metainfo); + + for (auto& waiter : metainfo.waiters()) { + waiter->reserve(1); + } +#endif + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + void set_my_item(size_t i, const item_type& o, message_metainfo&& metainfo) { + if(element(i).state != no_item) { + destroy_item(i); + } + + new(&(element(i).item)) item_type(o); + new(&element(i).metainfo) message_metainfo(std::move(metainfo)); + // Skipping the reservation on metainfo.waiters since the ownership + // is moving from metainfo to the cache + element(i).state = has_item; + } +#endif + + // destructively-fetch an object from the buffer +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + void fetch_item(size_t i, item_type& o, message_metainfo& metainfo) { + __TBB_ASSERT(my_item_valid(i), "Trying to fetch an empty slot"); + o = get_my_item(i); // could have std::move assign semantics + metainfo = std::move(get_my_metainfo(i)); + destroy_item(i); + } +#else + void fetch_item(size_t i, item_type &o) { + __TBB_ASSERT(my_item_valid(i), "Trying to fetch an empty slot"); + o = get_my_item(i); // could have std::move assign semantics + destroy_item(i); + } +#endif + + // move an existing item from one slot to another. The moved-to slot must be unoccupied, + // the moved-from slot must exist and not be reserved. The after, from will be empty, + // to will be occupied but not reserved + void move_item(size_t to, size_t from) { + __TBB_ASSERT(!my_item_valid(to), "Trying to move to a non-empty slot"); + __TBB_ASSERT(my_item_valid(from), "Trying to move from an empty slot"); + // could have std::move semantics + set_my_item(to, get_my_item(from) __TBB_FLOW_GRAPH_METAINFO_ARG(get_my_metainfo(from))); + destroy_item(from); + } + + // put an item in an empty slot. Return true if successful, else false +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + template + bool place_item(size_t here, const item_type &me, Metainfo&& metainfo) { +#if !TBB_DEPRECATED_SEQUENCER_DUPLICATES + if(my_item_valid(here)) return false; +#endif + set_my_item(here, me, std::forward(metainfo)); + return true; + } +#else + bool place_item(size_t here, const item_type &me) { +#if !TBB_DEPRECATED_SEQUENCER_DUPLICATES + if(my_item_valid(here)) return false; +#endif + set_my_item(here, me); + return true; + } +#endif + + // could be implemented with std::move semantics + void swap_items(size_t i, size_t j) { + __TBB_ASSERT(my_item_valid(i) && my_item_valid(j), "attempt to swap invalid item(s)"); + item_type temp = get_my_item(i); +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + message_metainfo temp_metainfo = get_my_metainfo(i); + set_my_item(i, get_my_item(j), get_my_metainfo(j)); + set_my_item(j, temp, temp_metainfo); +#else + set_my_item(i, get_my_item(j)); + set_my_item(j, temp); +#endif + } + + void destroy_item(size_type i) { + __TBB_ASSERT(my_item_valid(i), "destruction of invalid item"); + + auto& e = element(i); + e.item.~item_type(); + e.state = no_item; + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + for (auto& msg_waiter : e.metainfo.waiters()) { + msg_waiter->release(1); + } + + e.metainfo.~message_metainfo(); +#endif + } + + // returns the front element + const item_type& front() const + { + __TBB_ASSERT(my_item_valid(my_head), "attempt to fetch head non-item"); + return get_my_item(my_head); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + const message_metainfo& front_metainfo() const + { + __TBB_ASSERT(my_item_valid(my_head), "attempt to fetch head non-item"); + return element(my_head).metainfo; + } +#endif + + // returns the back element + const item_type& back() const + { + __TBB_ASSERT(my_item_valid(my_tail - 1), "attempt to fetch head non-item"); + return get_my_item(my_tail - 1); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + const message_metainfo& back_metainfo() const { + __TBB_ASSERT(my_item_valid(my_tail - 1), "attempt to fetch head non-item"); + return element(my_tail - 1).metainfo; + } +#endif + + // following methods are for reservation of the front of a buffer. + void reserve_item(size_type i) { + __TBB_ASSERT(my_item_valid(i) && !my_item_reserved(i), "item cannot be reserved"); + element(i).state = reserved_item; + } + + void release_item(size_type i) { + __TBB_ASSERT(my_item_reserved(i), "item is not reserved"); + element(i).state = has_item; + } + + void destroy_front() { destroy_item(my_head); ++my_head; } + void destroy_back() { destroy_item(my_tail-1); --my_tail; } + + // we have to be able to test against a new tail value without changing my_tail + // grow_array doesn't work if we change my_tail when the old array is too small + size_type size(size_t new_tail = 0) { return (new_tail ? new_tail : my_tail) - my_head; } + size_type capacity() { return my_array_size; } + // sequencer_node does not use this method, so we don't + // need a version that passes in the new_tail value. + bool buffer_full() { return size() >= capacity(); } + + //! Grows the internal array. + void grow_my_array( size_t minimum_size ) { + // test that we haven't made the structure inconsistent. + __TBB_ASSERT(capacity() >= my_tail - my_head, "total items exceed capacity"); + size_type new_size = my_array_size ? 2*my_array_size : initial_buffer_size; + while( new_sizestate = no_item; } + + for( size_type i=my_head; iitem); + (void)new(new_space) item_type(get_my_item(i)); + new_array[i&(new_size-1)].begin()->state = element(i).state; +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + char* meta_space = (char *)&(new_array[i&(new_size-1)].begin()->metainfo); + ::new(meta_space) message_metainfo(std::move(element(i).metainfo)); +#endif + } + } + + clean_up_buffer(/*reset_pointers*/false); + + my_array = new_array; + my_array_size = new_size; + } + + bool push_back(item_type& v + __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& metainfo)) + { + if (buffer_full()) { + grow_my_array(size() + 1); + } + set_my_item(my_tail, v __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo)); + ++my_tail; + return true; + } + + bool pop_back(item_type& v + __TBB_FLOW_GRAPH_METAINFO_ARG(message_metainfo& metainfo)) + { + if (!my_item_valid(my_tail - 1)) { + return false; + } + auto& e = element(my_tail - 1); + v = e.item; +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + metainfo = std::move(e.metainfo); +#endif + + destroy_back(); + return true; + } + + bool pop_front(item_type& v + __TBB_FLOW_GRAPH_METAINFO_ARG(message_metainfo& metainfo)) + { + if (!my_item_valid(my_head)) { + return false; + } + auto& e = element(my_head); + v = e.item; +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + metainfo = std::move(e.metainfo); +#endif + + destroy_front(); + return true; + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + bool pop_back(item_type& v) { + message_metainfo metainfo; + return pop_back(v, metainfo); + } + + bool pop_front(item_type& v) { + message_metainfo metainfo; + return pop_front(v, metainfo); + } +#endif + + // This is used both for reset and for grow_my_array. In the case of grow_my_array + // we want to retain the values of the head and tail. + void clean_up_buffer(bool reset_pointers) { + if (my_array) { + for( size_type i=my_head; i > +class reservable_item_buffer : public item_buffer { +protected: + using item_buffer::my_item_valid; + using item_buffer::my_head; + +public: + reservable_item_buffer() : item_buffer(), my_reserved(false) {} + void reset() {my_reserved = false; item_buffer::reset(); } +protected: + + bool reserve_front(T &v) { + if(my_reserved || !my_item_valid(this->my_head)) return false; + my_reserved = true; + // reserving the head + v = this->front(); + this->reserve_item(this->my_head); + return true; + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + bool reserve_front(T& v, message_metainfo& metainfo) { + if (my_reserved || !my_item_valid(this->my_head)) return false; + my_reserved = true; + // reserving the head + v = this->front(); + metainfo = this->front_metainfo(); + this->reserve_item(this->my_head); + return true; + } +#endif + + void consume_front() { + __TBB_ASSERT(my_reserved, "Attempt to consume a non-reserved item"); + this->destroy_front(); + my_reserved = false; + } + + void release_front() { + __TBB_ASSERT(my_reserved, "Attempt to release a non-reserved item"); + this->release_item(this->my_head); + my_reserved = false; + } + + bool my_reserved; +}; + +#endif // __TBB__flow_graph_item_buffer_impl_H diff --git a/src/tbb/include/oneapi/tbb/detail/_flow_graph_join_impl.h b/src/tbb/include/oneapi/tbb/detail/_flow_graph_join_impl.h new file mode 100644 index 000000000..8bca9a2c4 --- /dev/null +++ b/src/tbb/include/oneapi/tbb/detail/_flow_graph_join_impl.h @@ -0,0 +1,1970 @@ +/* + Copyright (c) 2005-2024 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB__flow_graph_join_impl_H +#define __TBB__flow_graph_join_impl_H + +#ifndef __TBB_flow_graph_H +#error Do not #include this internal file directly; use public TBB headers instead. +#endif + +// included into namespace tbb::detail::d2 + + struct forwarding_base : no_assign { + forwarding_base(graph &g) : graph_ref(g) {} + virtual ~forwarding_base() {} + graph& graph_ref; + }; + + struct queueing_forwarding_base : forwarding_base { + using forwarding_base::forwarding_base; + // decrement_port_count may create a forwarding task. If we cannot handle the task + // ourselves, ask decrement_port_count to deal with it. + virtual graph_task* decrement_port_count(bool handle_task) = 0; + }; + + struct reserving_forwarding_base : forwarding_base { + using forwarding_base::forwarding_base; + // decrement_port_count may create a forwarding task. If we cannot handle the task + // ourselves, ask decrement_port_count to deal with it. + virtual graph_task* decrement_port_count() = 0; + virtual void increment_port_count() = 0; + }; + + // specialization that lets us keep a copy of the current_key for building results. + // KeyType can be a reference type. + template + struct matching_forwarding_base : public forwarding_base { + typedef typename std::decay::type current_key_type; + matching_forwarding_base(graph &g) : forwarding_base(g) { } + virtual graph_task* increment_key_count(current_key_type const & /*t*/) = 0; + current_key_type current_key; // so ports can refer to FE's desired items + }; + + template< int N > + struct join_helper { + + template< typename TupleType, typename PortType > + static inline void set_join_node_pointer(TupleType &my_input, PortType *port) { + std::get( my_input ).set_join_node_pointer(port); + join_helper::set_join_node_pointer( my_input, port ); + } + template< typename TupleType > + static inline void consume_reservations( TupleType &my_input ) { + std::get( my_input ).consume(); + join_helper::consume_reservations( my_input ); + } + + template< typename TupleType > + static inline void release_my_reservation( TupleType &my_input ) { + std::get( my_input ).release(); + } + + template + static inline void release_reservations( TupleType &my_input) { + join_helper::release_reservations(my_input); + release_my_reservation(my_input); + } + + template< typename InputTuple, typename OutputTuple > + static inline bool reserve( InputTuple &my_input, OutputTuple &out) { + if ( !std::get( my_input ).reserve( std::get( out ) ) ) return false; + if ( !join_helper::reserve( my_input, out ) ) { + release_my_reservation( my_input ); + return false; + } + return true; + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + template + static inline bool reserve(InputTuple& my_input, OutputTuple& out, message_metainfo& metainfo) { + message_metainfo element_metainfo; + if (!std::get(my_input).reserve(std::get(out), element_metainfo)) return false; + if (!join_helper::reserve(my_input, out, metainfo)) { + release_my_reservation(my_input); + return false; + } + metainfo.merge(element_metainfo); + return true; + + } +#endif + + template + static inline bool get_my_item( InputTuple &my_input, OutputTuple &out) { + bool res = std::get(my_input).get_item(std::get(out) ); // may fail + return join_helper::get_my_item(my_input, out) && res; // do get on other inputs before returning + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + template + static inline bool get_my_item(InputTuple& my_input, OutputTuple& out, message_metainfo& metainfo) { + message_metainfo element_metainfo; + bool res = std::get(my_input).get_item(std::get(out), element_metainfo); + metainfo.merge(element_metainfo); + return join_helper::get_my_item(my_input, out, metainfo) && res; + } +#endif + + template + static inline bool get_items(InputTuple &my_input, OutputTuple &out) { + return get_my_item(my_input, out); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + template + static inline bool get_items(InputTuple& my_input, OutputTuple& out, message_metainfo& metainfo) { + return get_my_item(my_input, out, metainfo); + } +#endif + + template + static inline void reset_my_port(InputTuple &my_input) { + join_helper::reset_my_port(my_input); + std::get(my_input).reset_port(); + } + + template + static inline void reset_ports(InputTuple& my_input) { + reset_my_port(my_input); + } + + template + static inline void set_key_functors(InputTuple &my_input, KeyFuncTuple &my_key_funcs) { + std::get(my_input).set_my_key_func(std::get(my_key_funcs)); + std::get(my_key_funcs) = nullptr; + join_helper::set_key_functors(my_input, my_key_funcs); + } + + template< typename KeyFuncTuple> + static inline void copy_key_functors(KeyFuncTuple &my_inputs, KeyFuncTuple &other_inputs) { + __TBB_ASSERT( + std::get(other_inputs).get_my_key_func(), + "key matching join node should not be instantiated without functors." + ); + std::get(my_inputs).set_my_key_func(std::get(other_inputs).get_my_key_func()->clone()); + join_helper::copy_key_functors(my_inputs, other_inputs); + } + + template + static inline void reset_inputs(InputTuple &my_input, reset_flags f) { + join_helper::reset_inputs(my_input, f); + std::get(my_input).reset_receiver(f); + } + }; // join_helper + + template< > + struct join_helper<1> { + + template< typename TupleType, typename PortType > + static inline void set_join_node_pointer(TupleType &my_input, PortType *port) { + std::get<0>( my_input ).set_join_node_pointer(port); + } + + template< typename TupleType > + static inline void consume_reservations( TupleType &my_input ) { + std::get<0>( my_input ).consume(); + } + + template< typename TupleType > + static inline void release_my_reservation( TupleType &my_input ) { + std::get<0>( my_input ).release(); + } + + template + static inline void release_reservations( TupleType &my_input) { + release_my_reservation(my_input); + } + + template< typename InputTuple, typename OutputTuple > + static inline bool reserve( InputTuple &my_input, OutputTuple &out) { + return std::get<0>( my_input ).reserve( std::get<0>( out ) ); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + template + static inline bool reserve(InputTuple& my_input, OutputTuple& out, message_metainfo& metainfo) { + message_metainfo element_metainfo; + bool result = std::get<0>(my_input).reserve(std::get<0>(out), element_metainfo); + metainfo.merge(element_metainfo); + return result; + } +#endif + + template + static inline bool get_my_item( InputTuple &my_input, OutputTuple &out) { + return std::get<0>(my_input).get_item(std::get<0>(out)); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + template + static inline bool get_my_item(InputTuple& my_input, OutputTuple& out, message_metainfo& metainfo) { + message_metainfo element_metainfo; + bool res = std::get<0>(my_input).get_item(std::get<0>(out), element_metainfo); + metainfo.merge(element_metainfo); + return res; + } +#endif + + template + static inline bool get_items(InputTuple &my_input, OutputTuple &out) { + return get_my_item(my_input, out); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + template + static inline bool get_items(InputTuple& my_input, OutputTuple& out, message_metainfo& metainfo) { + return get_my_item(my_input, out, metainfo); + } +#endif + + template + static inline void reset_my_port(InputTuple &my_input) { + std::get<0>(my_input).reset_port(); + } + + template + static inline void reset_ports(InputTuple& my_input) { + reset_my_port(my_input); + } + + template + static inline void set_key_functors(InputTuple &my_input, KeyFuncTuple &my_key_funcs) { + std::get<0>(my_input).set_my_key_func(std::get<0>(my_key_funcs)); + std::get<0>(my_key_funcs) = nullptr; + } + + template< typename KeyFuncTuple> + static inline void copy_key_functors(KeyFuncTuple &my_inputs, KeyFuncTuple &other_inputs) { + __TBB_ASSERT( + std::get<0>(other_inputs).get_my_key_func(), + "key matching join node should not be instantiated without functors." + ); + std::get<0>(my_inputs).set_my_key_func(std::get<0>(other_inputs).get_my_key_func()->clone()); + } + template + static inline void reset_inputs(InputTuple &my_input, reset_flags f) { + std::get<0>(my_input).reset_receiver(f); + } + }; // join_helper<1> + + //! The two-phase join port + template< typename T > + class reserving_port : public receiver { + public: + typedef T input_type; + typedef typename receiver::predecessor_type predecessor_type; + + private: + // ----------- Aggregator ------------ + enum op_type { reg_pred, rem_pred, res_item, rel_res, con_res + }; + typedef reserving_port class_type; + + class reserving_port_operation : public d1::aggregated_operation { + public: + char type; + union { + T *my_arg; + predecessor_type *my_pred; + }; +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + message_metainfo* metainfo; +#endif + reserving_port_operation(const T& e, op_type t __TBB_FLOW_GRAPH_METAINFO_ARG(message_metainfo& info)) : + type(char(t)), my_arg(const_cast(&e)) + __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo(&info)) {} +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + reserving_port_operation(const T& e, op_type t) + : type(char(t)), my_arg(const_cast(&e)), metainfo(nullptr) {} +#endif + reserving_port_operation(const predecessor_type &s, op_type t) : type(char(t)), + my_pred(const_cast(&s)) {} + reserving_port_operation(op_type t) : type(char(t)) {} + }; + + typedef d1::aggregating_functor handler_type; + friend class d1::aggregating_functor; + d1::aggregator my_aggregator; + + void handle_operations(reserving_port_operation* op_list) { + reserving_port_operation *current; + bool was_missing_predecessors = false; + while(op_list) { + current = op_list; + op_list = op_list->next; + switch(current->type) { + case reg_pred: + was_missing_predecessors = my_predecessors.empty(); + my_predecessors.add(*(current->my_pred)); + if ( was_missing_predecessors ) { + (void) my_join->decrement_port_count(); // may try to forward + } + current->status.store( SUCCEEDED, std::memory_order_release); + break; + case rem_pred: + if ( !my_predecessors.empty() ) { + my_predecessors.remove(*(current->my_pred)); + if ( my_predecessors.empty() ) // was the last predecessor + my_join->increment_port_count(); + } + // TODO: consider returning failure if there were no predecessors to remove + current->status.store( SUCCEEDED, std::memory_order_release ); + break; + case res_item: + if ( reserved ) { + current->status.store( FAILED, std::memory_order_release); + } + else { + bool reserve_result = false; +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + if (current->metainfo) { + reserve_result = my_predecessors.try_reserve(*(current->my_arg), + *(current->metainfo)); + } else +#endif + { + reserve_result = my_predecessors.try_reserve(*(current->my_arg)); + } + if (reserve_result) { + reserved = true; + current->status.store( SUCCEEDED, std::memory_order_release); + } else { + if ( my_predecessors.empty() ) { + my_join->increment_port_count(); + } + current->status.store( FAILED, std::memory_order_release); + } + } + break; + case rel_res: + reserved = false; + my_predecessors.try_release( ); + current->status.store( SUCCEEDED, std::memory_order_release); + break; + case con_res: + reserved = false; + my_predecessors.try_consume( ); + current->status.store( SUCCEEDED, std::memory_order_release); + break; + } + } + } + + protected: + template< typename R, typename B > friend class run_and_put_task; + template friend class broadcast_cache; + template friend class round_robin_cache; + graph_task* try_put_task( const T & ) override { + return nullptr; + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + graph_task* try_put_task(const T&, const message_metainfo&) override { return nullptr; } +#endif + + graph& graph_reference() const override { + return my_join->graph_ref; + } + + public: + + //! Constructor + reserving_port() : my_join(nullptr), my_predecessors(this), reserved(false) { + my_aggregator.initialize_handler(handler_type(this)); + } + + // copy constructor + reserving_port(const reserving_port& /* other */) = delete; + + void set_join_node_pointer(reserving_forwarding_base *join) { + my_join = join; + } + + //! Add a predecessor + bool register_predecessor( predecessor_type &src ) override { + reserving_port_operation op_data(src, reg_pred); + my_aggregator.execute(&op_data); + return op_data.status == SUCCEEDED; + } + + //! Remove a predecessor + bool remove_predecessor( predecessor_type &src ) override { + reserving_port_operation op_data(src, rem_pred); + my_aggregator.execute(&op_data); + return op_data.status == SUCCEEDED; + } + + //! Reserve an item from the port + bool reserve( T &v ) { + reserving_port_operation op_data(v, res_item); + my_aggregator.execute(&op_data); + return op_data.status == SUCCEEDED; + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + bool reserve( T& v, message_metainfo& metainfo ) { + reserving_port_operation op_data(v, res_item, metainfo); + my_aggregator.execute(&op_data); + return op_data.status == SUCCEEDED; + } +#endif + + //! Release the port + void release( ) { + reserving_port_operation op_data(rel_res); + my_aggregator.execute(&op_data); + } + + //! Complete use of the port + void consume( ) { + reserving_port_operation op_data(con_res); + my_aggregator.execute(&op_data); + } + + void reset_receiver( reset_flags f) { + if(f & rf_clear_edges) my_predecessors.clear(); + else + my_predecessors.reset(); + reserved = false; + __TBB_ASSERT(!(f&rf_clear_edges) || my_predecessors.empty(), "port edges not removed"); + } + + private: +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + friend class get_graph_helper; +#endif + + reserving_forwarding_base *my_join; + reservable_predecessor_cache< T, null_mutex > my_predecessors; + bool reserved; + }; // reserving_port + + //! queueing join_port + template + class queueing_port : public receiver, public item_buffer { + public: + typedef T input_type; + typedef typename receiver::predecessor_type predecessor_type; + typedef queueing_port class_type; + + // ----------- Aggregator ------------ + private: + enum op_type { get__item, res_port, try__put_task + }; + + class queueing_port_operation : public d1::aggregated_operation { + public: + char type; + T my_val; + T* my_arg; + graph_task* bypass_t; +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + message_metainfo* metainfo; +#endif + // constructor for value parameter + queueing_port_operation(const T& e, op_type t __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& info)) + : type(char(t)), my_val(e), my_arg(nullptr) + , bypass_t(nullptr) + __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo(const_cast(&info))) + {} + // constructor for pointer parameter + queueing_port_operation(const T* p, op_type t __TBB_FLOW_GRAPH_METAINFO_ARG(message_metainfo& info)) : + type(char(t)), my_arg(const_cast(p)) + , bypass_t(nullptr) + __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo(&info)) + {} +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + queueing_port_operation(const T* p, op_type t) + : type(char(t)), my_arg(const_cast(p)), bypass_t(nullptr), metainfo(nullptr) + {} +#endif + // constructor with no parameter + queueing_port_operation(op_type t) : type(char(t)), my_arg(nullptr) + , bypass_t(nullptr) + __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo(nullptr)) + {} + }; + + typedef d1::aggregating_functor handler_type; + friend class d1::aggregating_functor; + d1::aggregator my_aggregator; + + void handle_operations(queueing_port_operation* op_list) { + queueing_port_operation *current; + bool was_empty; + while(op_list) { + current = op_list; + op_list = op_list->next; + switch(current->type) { + case try__put_task: { + graph_task* rtask = nullptr; + was_empty = this->buffer_empty(); +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + __TBB_ASSERT(current->metainfo, nullptr); + this->push_back(current->my_val, *(current->metainfo)); +#else + this->push_back(current->my_val); +#endif + if (was_empty) rtask = my_join->decrement_port_count(false); + else + rtask = SUCCESSFULLY_ENQUEUED; + current->bypass_t = rtask; + current->status.store( SUCCEEDED, std::memory_order_release); + } + break; + case get__item: + if(!this->buffer_empty()) { + __TBB_ASSERT(current->my_arg, nullptr); + *(current->my_arg) = this->front(); +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + if (current->metainfo) { + *(current->metainfo) = this->front_metainfo(); + } +#endif + current->status.store( SUCCEEDED, std::memory_order_release); + } + else { + current->status.store( FAILED, std::memory_order_release); + } + break; + case res_port: + __TBB_ASSERT(this->my_item_valid(this->my_head), "No item to reset"); + this->destroy_front(); + if(this->my_item_valid(this->my_head)) { + (void)my_join->decrement_port_count(true); + } + current->status.store( SUCCEEDED, std::memory_order_release); + break; + } + } + } + // ------------ End Aggregator --------------- + + protected: + template< typename R, typename B > friend class run_and_put_task; + template friend class broadcast_cache; + template friend class round_robin_cache; + + private: + graph_task* try_put_task_impl(const T& v __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& metainfo)) { + queueing_port_operation op_data(v, try__put_task __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo)); + my_aggregator.execute(&op_data); + __TBB_ASSERT(op_data.status == SUCCEEDED || !op_data.bypass_t, "inconsistent return from aggregator"); + if(!op_data.bypass_t) return SUCCESSFULLY_ENQUEUED; + return op_data.bypass_t; + } + + protected: + graph_task* try_put_task(const T &v) override { + return try_put_task_impl(v __TBB_FLOW_GRAPH_METAINFO_ARG(message_metainfo{})); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + graph_task* try_put_task(const T& v, const message_metainfo& metainfo) override { + return try_put_task_impl(v, metainfo); + } +#endif + + graph& graph_reference() const override { + return my_join->graph_ref; + } + + public: + + //! Constructor + queueing_port() : item_buffer() { + my_join = nullptr; + my_aggregator.initialize_handler(handler_type(this)); + } + + //! copy constructor + queueing_port(const queueing_port& /* other */) = delete; + + //! record parent for tallying available items + void set_join_node_pointer(queueing_forwarding_base *join) { + my_join = join; + } + + bool get_item( T &v ) { + queueing_port_operation op_data(&v, get__item); + my_aggregator.execute(&op_data); + return op_data.status == SUCCEEDED; + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + bool get_item( T& v, message_metainfo& metainfo ) { + queueing_port_operation op_data(&v, get__item, metainfo); + my_aggregator.execute(&op_data); + return op_data.status == SUCCEEDED; + } +#endif + + // reset_port is called when item is accepted by successor, but + // is initiated by join_node. + void reset_port() { + queueing_port_operation op_data(res_port); + my_aggregator.execute(&op_data); + return; + } + + void reset_receiver(reset_flags) { + item_buffer::reset(); + } + + private: +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + friend class get_graph_helper; +#endif + + queueing_forwarding_base *my_join; + }; // queueing_port + +#include "_flow_graph_tagged_buffer_impl.h" + + template + struct count_element { + K my_key; + size_t my_value; + }; + + // method to access the key in the counting table + // the ref has already been removed from K + template< typename K > + struct key_to_count_functor { + typedef count_element table_item_type; + const K& operator()(const table_item_type& v) { return v.my_key; } + }; + + template + struct key_matching_port_base { +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + using type = metainfo_hash_buffer; +#else + using type = hash_buffer; +#endif + }; + + // the ports can have only one template parameter. We wrap the types needed in + // a traits type + template< class TraitsType > + class key_matching_port : + public receiver, + public key_matching_port_base< typename TraitsType::K, typename TraitsType::T, typename TraitsType::TtoK, + typename TraitsType::KHash >::type + { + public: + typedef TraitsType traits; + typedef key_matching_port class_type; + typedef typename TraitsType::T input_type; + typedef typename TraitsType::K key_type; + typedef typename std::decay::type noref_key_type; + typedef typename receiver::predecessor_type predecessor_type; + typedef typename TraitsType::TtoK type_to_key_func_type; + typedef typename TraitsType::KHash hash_compare_type; + typedef typename key_matching_port_base::type buffer_type; + + private: +// ----------- Aggregator ------------ + private: + enum op_type { try__put, get__item, res_port + }; + + class key_matching_port_operation : public d1::aggregated_operation { + public: + char type; + input_type my_val; + input_type *my_arg; +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + message_metainfo* metainfo = nullptr; +#endif + // constructor for value parameter + key_matching_port_operation(const input_type& e, op_type t + __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& info)) + : type(char(t)), my_val(e), my_arg(nullptr) + __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo(const_cast(&info))) {} + + // constructor for pointer parameter + key_matching_port_operation(const input_type* p, op_type t + __TBB_FLOW_GRAPH_METAINFO_ARG(message_metainfo& info)) + : type(char(t)), my_arg(const_cast(p)) + __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo(&info)) {} + + // constructor with no parameter + key_matching_port_operation(op_type t) : type(char(t)), my_arg(nullptr) {} + }; + + typedef d1::aggregating_functor handler_type; + friend class d1::aggregating_functor; + d1::aggregator my_aggregator; + + void handle_operations(key_matching_port_operation* op_list) { + key_matching_port_operation *current; + while(op_list) { + current = op_list; + op_list = op_list->next; + switch(current->type) { + case try__put: { +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + __TBB_ASSERT(current->metainfo, nullptr); + bool was_inserted = this->insert_with_key(current->my_val, *(current->metainfo)); +#else + bool was_inserted = this->insert_with_key(current->my_val); +#endif + // return failure if a duplicate insertion occurs + current->status.store( was_inserted ? SUCCEEDED : FAILED, std::memory_order_release); + } + break; + case get__item: { + // use current_key from FE for item + __TBB_ASSERT(current->my_arg, nullptr); +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + __TBB_ASSERT(current->metainfo, nullptr); + bool find_result = this->find_with_key(my_join->current_key, *(current->my_arg), + *(current->metainfo)); +#else + bool find_result = this->find_with_key(my_join->current_key, *(current->my_arg)); +#endif +#if TBB_USE_DEBUG + if (!find_result) { + __TBB_ASSERT(false, "Failed to find item corresponding to current_key."); + } +#else + tbb::detail::suppress_unused_warning(find_result); +#endif + current->status.store( SUCCEEDED, std::memory_order_release); + } + break; + case res_port: + // use current_key from FE for item + this->delete_with_key(my_join->current_key); + current->status.store( SUCCEEDED, std::memory_order_release); + break; + } + } + } +// ------------ End Aggregator --------------- + protected: + template< typename R, typename B > friend class run_and_put_task; + template friend class broadcast_cache; + template friend class round_robin_cache; + private: + graph_task* try_put_task_impl(const input_type& v __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& metainfo)) { + key_matching_port_operation op_data(v, try__put __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo)); + graph_task* rtask = nullptr; + my_aggregator.execute(&op_data); + if(op_data.status == SUCCEEDED) { + rtask = my_join->increment_key_count((*(this->get_key_func()))(v)); // may spawn + // rtask has to reflect the return status of the try_put + if(!rtask) rtask = SUCCESSFULLY_ENQUEUED; + } + return rtask; + } + protected: + graph_task* try_put_task(const input_type& v) override { + return try_put_task_impl(v __TBB_FLOW_GRAPH_METAINFO_ARG(message_metainfo{})); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + graph_task* try_put_task(const input_type& v, const message_metainfo& metainfo) override { + return try_put_task_impl(v, metainfo); + } +#endif + + graph& graph_reference() const override { + return my_join->graph_ref; + } + + public: + + key_matching_port() : receiver(), buffer_type() { + my_join = nullptr; + my_aggregator.initialize_handler(handler_type(this)); + } + + // copy constructor + key_matching_port(const key_matching_port& /*other*/) = delete; +#if __INTEL_COMPILER <= 2021 + // Suppress superfluous diagnostic about virtual keyword absence in a destructor of an inherited + // class while the parent class has the virtual keyword for the destrocutor. + virtual +#endif + ~key_matching_port() { } + + void set_join_node_pointer(forwarding_base *join) { + my_join = dynamic_cast*>(join); + } + + void set_my_key_func(type_to_key_func_type *f) { this->set_key_func(f); } + + type_to_key_func_type* get_my_key_func() { return this->get_key_func(); } + + bool get_item( input_type &v ) { + // aggregator uses current_key from FE for Key + key_matching_port_operation op_data(&v, get__item); + my_aggregator.execute(&op_data); + return op_data.status == SUCCEEDED; + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + bool get_item( input_type& v, message_metainfo& metainfo ) { + // aggregator uses current_key from FE for Key + key_matching_port_operation op_data(&v, get__item, metainfo); + my_aggregator.execute(&op_data); + return op_data.status == SUCCEEDED; + } +#endif + + // reset_port is called when item is accepted by successor, but + // is initiated by join_node. + void reset_port() { + key_matching_port_operation op_data(res_port); + my_aggregator.execute(&op_data); + return; + } + + void reset_receiver(reset_flags ) { + buffer_type::reset(); + } + + private: + // my_join forwarding base used to count number of inputs that + // received key. + matching_forwarding_base *my_join; + }; // key_matching_port + + using namespace graph_policy_namespace; + + template + class join_node_base; + + //! join_node_FE : implements input port policy + template + class join_node_FE; + + template + class join_node_FE : public reserving_forwarding_base { + private: + static const int N = std::tuple_size::value; + typedef OutputTuple output_type; + typedef InputTuple input_type; + typedef join_node_base base_node_type; // for forwarding + public: + join_node_FE(graph &g) : reserving_forwarding_base(g), my_node(nullptr) { + ports_with_no_inputs = N; + join_helper::set_join_node_pointer(my_inputs, this); + } + + join_node_FE(const join_node_FE& other) : reserving_forwarding_base((other.reserving_forwarding_base::graph_ref)), my_node(nullptr) { + ports_with_no_inputs = N; + join_helper::set_join_node_pointer(my_inputs, this); + } + + void set_my_node(base_node_type *new_my_node) { my_node = new_my_node; } + + void increment_port_count() override { + ++ports_with_no_inputs; + } + + // if all input_ports have predecessors, spawn forward to try and consume tuples + graph_task* decrement_port_count() override { + if(ports_with_no_inputs.fetch_sub(1) == 1) { + if(is_graph_active(this->graph_ref)) { + d1::small_object_allocator allocator{}; + typedef forward_task_bypass task_type; + graph_task* t = allocator.new_object(graph_ref, allocator, *my_node); + spawn_in_graph_arena(this->graph_ref, *t); + } + } + return nullptr; + } + + input_type &input_ports() { return my_inputs; } + + protected: + + void reset( reset_flags f) { + // called outside of parallel contexts + ports_with_no_inputs = N; + join_helper::reset_inputs(my_inputs, f); + } + + // all methods on input ports should be called under mutual exclusion from join_node_base. + + bool tuple_build_may_succeed() { + return !ports_with_no_inputs; + } + + bool try_to_make_tuple(output_type &out) { + if(ports_with_no_inputs) return false; + return join_helper::reserve(my_inputs, out); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + bool try_to_make_tuple(output_type &out, message_metainfo& metainfo) { + if (ports_with_no_inputs) return false; + return join_helper::reserve(my_inputs, out, metainfo); + } +#endif + + void tuple_accepted() { + join_helper::consume_reservations(my_inputs); + } + void tuple_rejected() { + join_helper::release_reservations(my_inputs); + } + + input_type my_inputs; + base_node_type *my_node; + std::atomic ports_with_no_inputs; + }; // join_node_FE + + template + class join_node_FE : public queueing_forwarding_base { + public: + static const int N = std::tuple_size::value; + typedef OutputTuple output_type; + typedef InputTuple input_type; + typedef join_node_base base_node_type; // for forwarding + + join_node_FE(graph &g) : queueing_forwarding_base(g), my_node(nullptr) { + ports_with_no_items = N; + join_helper::set_join_node_pointer(my_inputs, this); + } + + join_node_FE(const join_node_FE& other) : queueing_forwarding_base((other.queueing_forwarding_base::graph_ref)), my_node(nullptr) { + ports_with_no_items = N; + join_helper::set_join_node_pointer(my_inputs, this); + } + + // needed for forwarding + void set_my_node(base_node_type *new_my_node) { my_node = new_my_node; } + + void reset_port_count() { + ports_with_no_items = N; + } + + // if all input_ports have items, spawn forward to try and consume tuples + graph_task* decrement_port_count(bool handle_task) override + { + if(ports_with_no_items.fetch_sub(1) == 1) { + if(is_graph_active(this->graph_ref)) { + d1::small_object_allocator allocator{}; + typedef forward_task_bypass task_type; + graph_task* t = allocator.new_object(graph_ref, allocator, *my_node); + if( !handle_task ) + return t; + spawn_in_graph_arena(this->graph_ref, *t); + } + } + return nullptr; + } + + input_type &input_ports() { return my_inputs; } + + protected: + + void reset( reset_flags f) { + reset_port_count(); + join_helper::reset_inputs(my_inputs, f ); + } + + // all methods on input ports should be called under mutual exclusion from join_node_base. + + bool tuple_build_may_succeed() { + return !ports_with_no_items; + } + + bool try_to_make_tuple(output_type &out) { + if(ports_with_no_items) return false; + return join_helper::get_items(my_inputs, out); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + bool try_to_make_tuple(output_type &out, message_metainfo& metainfo) { + if(ports_with_no_items) return false; + return join_helper::get_items(my_inputs, out, metainfo); + } +#endif + + void tuple_accepted() { + reset_port_count(); + join_helper::reset_ports(my_inputs); + } + void tuple_rejected() { + // nothing to do. + } + + input_type my_inputs; + base_node_type *my_node; + std::atomic ports_with_no_items; + }; // join_node_FE + + // key_matching join front-end. + template + class join_node_FE, InputTuple, OutputTuple> : public matching_forwarding_base, + // buffer of key value counts + public hash_buffer< // typedefed below to key_to_count_buffer_type + typename std::decay::type&, // force ref type on K + count_element::type>, + type_to_key_function_body< + count_element::type>, + typename std::decay::type& >, + KHash >, + // buffer of output items + public item_buffer { + public: + static const int N = std::tuple_size::value; + typedef OutputTuple output_type; + typedef InputTuple input_type; + typedef K key_type; + typedef typename std::decay::type unref_key_type; + typedef KHash key_hash_compare; + // must use K without ref. + typedef count_element count_element_type; + // method that lets us refer to the key of this type. + typedef key_to_count_functor key_to_count_func; + typedef type_to_key_function_body< count_element_type, unref_key_type&> TtoK_function_body_type; + typedef type_to_key_function_body_leaf TtoK_function_body_leaf_type; + // this is the type of the special table that keeps track of the number of discrete + // elements corresponding to each key that we've seen. + typedef hash_buffer< unref_key_type&, count_element_type, TtoK_function_body_type, key_hash_compare > + key_to_count_buffer_type; + typedef item_buffer output_buffer_type; + typedef join_node_base, InputTuple, OutputTuple> base_node_type; // for forwarding + typedef matching_forwarding_base forwarding_base_type; + +// ----------- Aggregator ------------ + // the aggregator is only needed to serialize the access to the hash table. + // and the output_buffer_type base class + private: + enum op_type { res_count, inc_count, may_succeed, try_make }; + typedef join_node_FE, InputTuple, OutputTuple> class_type; + + class key_matching_FE_operation : public d1::aggregated_operation { + public: + char type; + unref_key_type my_val; + output_type* my_output; + graph_task* bypass_t; +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + message_metainfo* metainfo = nullptr; +#endif + // constructor for value parameter + key_matching_FE_operation(const unref_key_type& e , op_type t) : type(char(t)), my_val(e), + my_output(nullptr), bypass_t(nullptr) {} + key_matching_FE_operation(output_type *p, op_type t) : type(char(t)), my_output(p), bypass_t(nullptr) {} +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + key_matching_FE_operation(output_type *p, op_type t, message_metainfo& info) + : type(char(t)), my_output(p), bypass_t(nullptr), metainfo(&info) {} +#endif + // constructor with no parameter + key_matching_FE_operation(op_type t) : type(char(t)), my_output(nullptr), bypass_t(nullptr) {} + }; + + typedef d1::aggregating_functor handler_type; + friend class d1::aggregating_functor; + d1::aggregator my_aggregator; + + // called from aggregator, so serialized + // returns a task pointer if the a task would have been enqueued but we asked that + // it be returned. Otherwise returns nullptr. + graph_task* fill_output_buffer(unref_key_type &t) { + output_type l_out; + graph_task* rtask = nullptr; + bool do_fwd = this->buffer_empty() && is_graph_active(this->graph_ref); + this->current_key = t; + this->delete_with_key(this->current_key); // remove the key +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + message_metainfo metainfo; +#endif + if(join_helper::get_items(my_inputs, l_out __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo))) { // <== call back + this->push_back(l_out __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo)); + if(do_fwd) { // we enqueue if receiving an item from predecessor, not if successor asks for item + d1::small_object_allocator allocator{}; + typedef forward_task_bypass task_type; + rtask = allocator.new_object(this->graph_ref, allocator, *my_node); + do_fwd = false; + } + // retire the input values + join_helper::reset_ports(my_inputs); // <== call back + } + else { + __TBB_ASSERT(false, "should have had something to push"); + } + return rtask; + } + + void handle_operations(key_matching_FE_operation* op_list) { + key_matching_FE_operation *current; + while(op_list) { + current = op_list; + op_list = op_list->next; + switch(current->type) { + case res_count: // called from BE + { + this->destroy_front(); + current->status.store( SUCCEEDED, std::memory_order_release); + } + break; + case inc_count: { // called from input ports + count_element_type *p = nullptr; + unref_key_type &t = current->my_val; + if(!(this->find_ref_with_key(t,p))) { + count_element_type ev; + ev.my_key = t; + ev.my_value = 0; + this->insert_with_key(ev); + bool found = this->find_ref_with_key(t, p); + __TBB_ASSERT_EX(found, "should find key after inserting it"); + } + if(++(p->my_value) == size_t(N)) { + current->bypass_t = fill_output_buffer(t); + } + } + current->status.store( SUCCEEDED, std::memory_order_release); + break; + case may_succeed: // called from BE + current->status.store( this->buffer_empty() ? FAILED : SUCCEEDED, std::memory_order_release); + break; + case try_make: // called from BE + if(this->buffer_empty()) { + current->status.store( FAILED, std::memory_order_release); + } + else { + *(current->my_output) = this->front(); +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + if (current->metainfo) { + *(current->metainfo) = this->front_metainfo(); + } +#endif + current->status.store( SUCCEEDED, std::memory_order_release); + } + break; + } + } + } +// ------------ End Aggregator --------------- + + public: + template + join_node_FE(graph &g, FunctionTuple &TtoK_funcs) : forwarding_base_type(g), my_node(nullptr) { + join_helper::set_join_node_pointer(my_inputs, this); + join_helper::set_key_functors(my_inputs, TtoK_funcs); + my_aggregator.initialize_handler(handler_type(this)); + TtoK_function_body_type *cfb = new TtoK_function_body_leaf_type(key_to_count_func()); + this->set_key_func(cfb); + } + + join_node_FE(const join_node_FE& other) : forwarding_base_type((other.forwarding_base_type::graph_ref)), key_to_count_buffer_type(), + output_buffer_type() { + my_node = nullptr; + join_helper::set_join_node_pointer(my_inputs, this); + join_helper::copy_key_functors(my_inputs, const_cast(other.my_inputs)); + my_aggregator.initialize_handler(handler_type(this)); + TtoK_function_body_type *cfb = new TtoK_function_body_leaf_type(key_to_count_func()); + this->set_key_func(cfb); + } + + // needed for forwarding + void set_my_node(base_node_type *new_my_node) { my_node = new_my_node; } + + void reset_port_count() { // called from BE + key_matching_FE_operation op_data(res_count); + my_aggregator.execute(&op_data); + return; + } + + // if all input_ports have items, spawn forward to try and consume tuples + // return a task if we are asked and did create one. + graph_task *increment_key_count(unref_key_type const & t) override { // called from input_ports + key_matching_FE_operation op_data(t, inc_count); + my_aggregator.execute(&op_data); + return op_data.bypass_t; + } + + input_type &input_ports() { return my_inputs; } + + protected: + + void reset( reset_flags f ) { + // called outside of parallel contexts + join_helper::reset_inputs(my_inputs, f); + + key_to_count_buffer_type::reset(); + output_buffer_type::reset(); + } + + // all methods on input ports should be called under mutual exclusion from join_node_base. + + bool tuple_build_may_succeed() { // called from back-end + key_matching_FE_operation op_data(may_succeed); + my_aggregator.execute(&op_data); + return op_data.status == SUCCEEDED; + } + + // cannot lock while calling back to input_ports. current_key will only be set + // and reset under the aggregator, so it will remain consistent. + bool try_to_make_tuple(output_type &out) { + key_matching_FE_operation op_data(&out,try_make); + my_aggregator.execute(&op_data); + return op_data.status == SUCCEEDED; + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + bool try_to_make_tuple(output_type &out, message_metainfo& metainfo) { + key_matching_FE_operation op_data(&out, try_make, metainfo); + my_aggregator.execute(&op_data); + return op_data.status == SUCCEEDED; + } +#endif + + void tuple_accepted() { + reset_port_count(); // reset current_key after ports reset. + } + + void tuple_rejected() { + // nothing to do. + } + + input_type my_inputs; // input ports + base_node_type *my_node; + }; // join_node_FE, InputTuple, OutputTuple> + + //! join_node_base + template + class join_node_base : public graph_node, public join_node_FE, + public sender { + protected: + using graph_node::my_graph; + public: + typedef OutputTuple output_type; + + typedef typename sender::successor_type successor_type; + typedef join_node_FE input_ports_type; + using input_ports_type::tuple_build_may_succeed; + using input_ports_type::try_to_make_tuple; + using input_ports_type::tuple_accepted; + using input_ports_type::tuple_rejected; + + private: + // ----------- Aggregator ------------ + enum op_type { reg_succ, rem_succ, try__get, do_fwrd, do_fwrd_bypass + }; + typedef join_node_base class_type; + + class join_node_base_operation : public d1::aggregated_operation { + public: + char type; + union { + output_type *my_arg; + successor_type *my_succ; + }; + graph_task* bypass_t; +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + message_metainfo* metainfo; +#endif + join_node_base_operation(const output_type& e, op_type t __TBB_FLOW_GRAPH_METAINFO_ARG(message_metainfo& info)) + : type(char(t)), my_arg(const_cast(&e)), bypass_t(nullptr) + __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo(&info)) {} +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + join_node_base_operation(const output_type& e, op_type t) + : type(char(t)), my_arg(const_cast(&e)), bypass_t(nullptr), metainfo(nullptr) {} +#endif + join_node_base_operation(const successor_type &s, op_type t) : type(char(t)), + my_succ(const_cast(&s)), bypass_t(nullptr) {} + join_node_base_operation(op_type t) : type(char(t)), bypass_t(nullptr) {} + }; + + typedef d1::aggregating_functor handler_type; + friend class d1::aggregating_functor; + bool forwarder_busy; + d1::aggregator my_aggregator; + + void handle_operations(join_node_base_operation* op_list) { + join_node_base_operation *current; + while(op_list) { + current = op_list; + op_list = op_list->next; + switch(current->type) { + case reg_succ: { + my_successors.register_successor(*(current->my_succ)); + if(tuple_build_may_succeed() && !forwarder_busy && is_graph_active(my_graph)) { + d1::small_object_allocator allocator{}; + typedef forward_task_bypass< join_node_base > task_type; + graph_task* t = allocator.new_object(my_graph, allocator, *this); + spawn_in_graph_arena(my_graph, *t); + forwarder_busy = true; + } + current->status.store( SUCCEEDED, std::memory_order_release); + } + break; + case rem_succ: + my_successors.remove_successor(*(current->my_succ)); + current->status.store( SUCCEEDED, std::memory_order_release); + break; + case try__get: + if(tuple_build_may_succeed()) { + bool make_tuple_result = false; +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + if (current->metainfo) { + make_tuple_result = try_to_make_tuple(*(current->my_arg), *(current->metainfo)); + } else +#endif + { + make_tuple_result = try_to_make_tuple(*(current->my_arg)); + } + if(make_tuple_result) { +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + if (current->metainfo) { + // Since elements would be removed from queues while calling to tuple_accepted + // together with corresponding message_metainfo objects + // we need to prolong the wait until the successor would create a task for removed elements + for (auto waiter : current->metainfo->waiters()) { + waiter->reserve(1); + } + } +#endif + tuple_accepted(); + current->status.store( SUCCEEDED, std::memory_order_release); + } + else current->status.store( FAILED, std::memory_order_release); + } + else current->status.store( FAILED, std::memory_order_release); + break; + case do_fwrd_bypass: { + bool build_succeeded; + graph_task *last_task = nullptr; + output_type out; + // forwarding must be exclusive, because try_to_make_tuple and tuple_accepted + // are separate locked methods in the FE. We could conceivably fetch the front + // of the FE queue, then be swapped out, have someone else consume the FE's + // object, then come back, forward, and then try to remove it from the queue + // again. Without reservation of the FE, the methods accessing it must be locked. + // We could remember the keys of the objects we forwarded, and then remove + // them from the input ports after forwarding is complete? + if(tuple_build_may_succeed()) { // checks output queue of FE + do { +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + message_metainfo metainfo; +#endif + // fetch front_end of queue + build_succeeded = try_to_make_tuple(out __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo)); + if(build_succeeded) { + graph_task *new_task = + my_successors.try_put_task(out __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo)); + last_task = combine_tasks(my_graph, last_task, new_task); + if(new_task) { + tuple_accepted(); + } + else { + tuple_rejected(); + build_succeeded = false; + } + } + } while(build_succeeded); + } + current->bypass_t = last_task; + current->status.store( SUCCEEDED, std::memory_order_release); + forwarder_busy = false; + } + break; + } + } + } + // ---------- end aggregator ----------- + public: + join_node_base(graph &g) + : graph_node(g), input_ports_type(g), forwarder_busy(false), my_successors(this) + { + input_ports_type::set_my_node(this); + my_aggregator.initialize_handler(handler_type(this)); + } + + join_node_base(const join_node_base& other) : + graph_node(other.graph_node::my_graph), input_ports_type(other), + sender(), forwarder_busy(false), my_successors(this) + { + input_ports_type::set_my_node(this); + my_aggregator.initialize_handler(handler_type(this)); + } + + template + join_node_base(graph &g, FunctionTuple f) + : graph_node(g), input_ports_type(g, f), forwarder_busy(false), my_successors(this) + { + input_ports_type::set_my_node(this); + my_aggregator.initialize_handler(handler_type(this)); + } + + bool register_successor(successor_type &r) override { + join_node_base_operation op_data(r, reg_succ); + my_aggregator.execute(&op_data); + return op_data.status == SUCCEEDED; + } + + bool remove_successor( successor_type &r) override { + join_node_base_operation op_data(r, rem_succ); + my_aggregator.execute(&op_data); + return op_data.status == SUCCEEDED; + } + + bool try_get( output_type &v) override { + join_node_base_operation op_data(v, try__get); + my_aggregator.execute(&op_data); + return op_data.status == SUCCEEDED; + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + bool try_get( output_type &v, message_metainfo& metainfo) override { + join_node_base_operation op_data(v, try__get, metainfo); + my_aggregator.execute(&op_data); + return op_data.status == SUCCEEDED; + } +#endif + + protected: + void reset_node(reset_flags f) override { + input_ports_type::reset(f); + if(f & rf_clear_edges) my_successors.clear(); + } + + private: + broadcast_cache my_successors; + + friend class forward_task_bypass< join_node_base >; + graph_task *forward_task() { + join_node_base_operation op_data(do_fwrd_bypass); + my_aggregator.execute(&op_data); + return op_data.bypass_t; + } + + }; // join_node_base + + // join base class type generator + template class PT, typename OutputTuple, typename JP> + struct join_base { + typedef join_node_base::type, OutputTuple> type; + }; + + template + struct join_base > { + typedef key_matching key_traits_type; + typedef K key_type; + typedef KHash key_hash_compare; + typedef join_node_base< key_traits_type, + // ports type + typename wrap_key_tuple_elements::type, + OutputTuple > type; + }; + + //! unfolded_join_node : passes input_ports_type to join_node_base. We build the input port type + // using tuple_element. The class PT is the port type (reserving_port, queueing_port, key_matching_port) + // and should match the typename. + + template class PT, typename OutputTuple, typename JP> + class unfolded_join_node : public join_base::type { + public: + typedef typename wrap_tuple_elements::type input_ports_type; + typedef OutputTuple output_type; + private: + typedef join_node_base base_type; + public: + unfolded_join_node(graph &g) : base_type(g) {} + unfolded_join_node(const unfolded_join_node &other) : base_type(other) {} + }; + +#if __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING + template + struct key_from_message_body { + K operator()(const T& t) const { + return key_from_message(t); + } + }; + // Adds const to reference type + template + struct key_from_message_body { + const K& operator()(const T& t) const { + return key_from_message(t); + } + }; +#endif /* __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING */ + // key_matching unfolded_join_node. This must be a separate specialization because the constructors + // differ. + + template + class unfolded_join_node<2,key_matching_port,OutputTuple,key_matching > : public + join_base<2,key_matching_port,OutputTuple,key_matching >::type { + typedef typename std::tuple_element<0, OutputTuple>::type T0; + typedef typename std::tuple_element<1, OutputTuple>::type T1; + public: + typedef typename wrap_key_tuple_elements<2,key_matching_port,key_matching,OutputTuple>::type input_ports_type; + typedef OutputTuple output_type; + private: + typedef join_node_base, input_ports_type, output_type > base_type; + typedef type_to_key_function_body *f0_p; + typedef type_to_key_function_body *f1_p; + typedef std::tuple< f0_p, f1_p > func_initializer_type; + public: +#if __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING + unfolded_join_node(graph &g) : base_type(g, + func_initializer_type( + new type_to_key_function_body_leaf >(key_from_message_body()), + new type_to_key_function_body_leaf >(key_from_message_body()) + ) ) { + } +#endif /* __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING */ + template + unfolded_join_node(graph &g, Body0 body0, Body1 body1) : base_type(g, + func_initializer_type( + new type_to_key_function_body_leaf(body0), + new type_to_key_function_body_leaf(body1) + ) ) { + static_assert(std::tuple_size::value == 2, "wrong number of body initializers"); + } + unfolded_join_node(const unfolded_join_node &other) : base_type(other) {} + }; + + template + class unfolded_join_node<3,key_matching_port,OutputTuple,key_matching > : public + join_base<3,key_matching_port,OutputTuple,key_matching >::type { + typedef typename std::tuple_element<0, OutputTuple>::type T0; + typedef typename std::tuple_element<1, OutputTuple>::type T1; + typedef typename std::tuple_element<2, OutputTuple>::type T2; + public: + typedef typename wrap_key_tuple_elements<3,key_matching_port,key_matching,OutputTuple>::type input_ports_type; + typedef OutputTuple output_type; + private: + typedef join_node_base, input_ports_type, output_type > base_type; + typedef type_to_key_function_body *f0_p; + typedef type_to_key_function_body *f1_p; + typedef type_to_key_function_body *f2_p; + typedef std::tuple< f0_p, f1_p, f2_p > func_initializer_type; + public: +#if __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING + unfolded_join_node(graph &g) : base_type(g, + func_initializer_type( + new type_to_key_function_body_leaf >(key_from_message_body()), + new type_to_key_function_body_leaf >(key_from_message_body()), + new type_to_key_function_body_leaf >(key_from_message_body()) + ) ) { + } +#endif /* __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING */ + template + unfolded_join_node(graph &g, Body0 body0, Body1 body1, Body2 body2) : base_type(g, + func_initializer_type( + new type_to_key_function_body_leaf(body0), + new type_to_key_function_body_leaf(body1), + new type_to_key_function_body_leaf(body2) + ) ) { + static_assert(std::tuple_size::value == 3, "wrong number of body initializers"); + } + unfolded_join_node(const unfolded_join_node &other) : base_type(other) {} + }; + + template + class unfolded_join_node<4,key_matching_port,OutputTuple,key_matching > : public + join_base<4,key_matching_port,OutputTuple,key_matching >::type { + typedef typename std::tuple_element<0, OutputTuple>::type T0; + typedef typename std::tuple_element<1, OutputTuple>::type T1; + typedef typename std::tuple_element<2, OutputTuple>::type T2; + typedef typename std::tuple_element<3, OutputTuple>::type T3; + public: + typedef typename wrap_key_tuple_elements<4,key_matching_port,key_matching,OutputTuple>::type input_ports_type; + typedef OutputTuple output_type; + private: + typedef join_node_base, input_ports_type, output_type > base_type; + typedef type_to_key_function_body *f0_p; + typedef type_to_key_function_body *f1_p; + typedef type_to_key_function_body *f2_p; + typedef type_to_key_function_body *f3_p; + typedef std::tuple< f0_p, f1_p, f2_p, f3_p > func_initializer_type; + public: +#if __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING + unfolded_join_node(graph &g) : base_type(g, + func_initializer_type( + new type_to_key_function_body_leaf >(key_from_message_body()), + new type_to_key_function_body_leaf >(key_from_message_body()), + new type_to_key_function_body_leaf >(key_from_message_body()), + new type_to_key_function_body_leaf >(key_from_message_body()) + ) ) { + } +#endif /* __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING */ + template + unfolded_join_node(graph &g, Body0 body0, Body1 body1, Body2 body2, Body3 body3) : base_type(g, + func_initializer_type( + new type_to_key_function_body_leaf(body0), + new type_to_key_function_body_leaf(body1), + new type_to_key_function_body_leaf(body2), + new type_to_key_function_body_leaf(body3) + ) ) { + static_assert(std::tuple_size::value == 4, "wrong number of body initializers"); + } + unfolded_join_node(const unfolded_join_node &other) : base_type(other) {} + }; + + template + class unfolded_join_node<5,key_matching_port,OutputTuple,key_matching > : public + join_base<5,key_matching_port,OutputTuple,key_matching >::type { + typedef typename std::tuple_element<0, OutputTuple>::type T0; + typedef typename std::tuple_element<1, OutputTuple>::type T1; + typedef typename std::tuple_element<2, OutputTuple>::type T2; + typedef typename std::tuple_element<3, OutputTuple>::type T3; + typedef typename std::tuple_element<4, OutputTuple>::type T4; + public: + typedef typename wrap_key_tuple_elements<5,key_matching_port,key_matching,OutputTuple>::type input_ports_type; + typedef OutputTuple output_type; + private: + typedef join_node_base , input_ports_type, output_type > base_type; + typedef type_to_key_function_body *f0_p; + typedef type_to_key_function_body *f1_p; + typedef type_to_key_function_body *f2_p; + typedef type_to_key_function_body *f3_p; + typedef type_to_key_function_body *f4_p; + typedef std::tuple< f0_p, f1_p, f2_p, f3_p, f4_p > func_initializer_type; + public: +#if __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING + unfolded_join_node(graph &g) : base_type(g, + func_initializer_type( + new type_to_key_function_body_leaf >(key_from_message_body()), + new type_to_key_function_body_leaf >(key_from_message_body()), + new type_to_key_function_body_leaf >(key_from_message_body()), + new type_to_key_function_body_leaf >(key_from_message_body()), + new type_to_key_function_body_leaf >(key_from_message_body()) + ) ) { + } +#endif /* __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING */ + template + unfolded_join_node(graph &g, Body0 body0, Body1 body1, Body2 body2, Body3 body3, Body4 body4) : base_type(g, + func_initializer_type( + new type_to_key_function_body_leaf(body0), + new type_to_key_function_body_leaf(body1), + new type_to_key_function_body_leaf(body2), + new type_to_key_function_body_leaf(body3), + new type_to_key_function_body_leaf(body4) + ) ) { + static_assert(std::tuple_size::value == 5, "wrong number of body initializers"); + } + unfolded_join_node(const unfolded_join_node &other) : base_type(other) {} + }; + +#if __TBB_VARIADIC_MAX >= 6 + template + class unfolded_join_node<6,key_matching_port,OutputTuple,key_matching > : public + join_base<6,key_matching_port,OutputTuple,key_matching >::type { + typedef typename std::tuple_element<0, OutputTuple>::type T0; + typedef typename std::tuple_element<1, OutputTuple>::type T1; + typedef typename std::tuple_element<2, OutputTuple>::type T2; + typedef typename std::tuple_element<3, OutputTuple>::type T3; + typedef typename std::tuple_element<4, OutputTuple>::type T4; + typedef typename std::tuple_element<5, OutputTuple>::type T5; + public: + typedef typename wrap_key_tuple_elements<6,key_matching_port,key_matching,OutputTuple>::type input_ports_type; + typedef OutputTuple output_type; + private: + typedef join_node_base , input_ports_type, output_type > base_type; + typedef type_to_key_function_body *f0_p; + typedef type_to_key_function_body *f1_p; + typedef type_to_key_function_body *f2_p; + typedef type_to_key_function_body *f3_p; + typedef type_to_key_function_body *f4_p; + typedef type_to_key_function_body *f5_p; + typedef std::tuple< f0_p, f1_p, f2_p, f3_p, f4_p, f5_p > func_initializer_type; + public: +#if __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING + unfolded_join_node(graph &g) : base_type(g, + func_initializer_type( + new type_to_key_function_body_leaf >(key_from_message_body()), + new type_to_key_function_body_leaf >(key_from_message_body()), + new type_to_key_function_body_leaf >(key_from_message_body()), + new type_to_key_function_body_leaf >(key_from_message_body()), + new type_to_key_function_body_leaf >(key_from_message_body()), + new type_to_key_function_body_leaf >(key_from_message_body()) + ) ) { + } +#endif /* __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING */ + template + unfolded_join_node(graph &g, Body0 body0, Body1 body1, Body2 body2, Body3 body3, Body4 body4, Body5 body5) + : base_type(g, func_initializer_type( + new type_to_key_function_body_leaf(body0), + new type_to_key_function_body_leaf(body1), + new type_to_key_function_body_leaf(body2), + new type_to_key_function_body_leaf(body3), + new type_to_key_function_body_leaf(body4), + new type_to_key_function_body_leaf(body5) + ) ) { + static_assert(std::tuple_size::value == 6, "wrong number of body initializers"); + } + unfolded_join_node(const unfolded_join_node &other) : base_type(other) {} + }; +#endif + +#if __TBB_VARIADIC_MAX >= 7 + template + class unfolded_join_node<7,key_matching_port,OutputTuple,key_matching > : public + join_base<7,key_matching_port,OutputTuple,key_matching >::type { + typedef typename std::tuple_element<0, OutputTuple>::type T0; + typedef typename std::tuple_element<1, OutputTuple>::type T1; + typedef typename std::tuple_element<2, OutputTuple>::type T2; + typedef typename std::tuple_element<3, OutputTuple>::type T3; + typedef typename std::tuple_element<4, OutputTuple>::type T4; + typedef typename std::tuple_element<5, OutputTuple>::type T5; + typedef typename std::tuple_element<6, OutputTuple>::type T6; + public: + typedef typename wrap_key_tuple_elements<7,key_matching_port,key_matching,OutputTuple>::type input_ports_type; + typedef OutputTuple output_type; + private: + typedef join_node_base , input_ports_type, output_type > base_type; + typedef type_to_key_function_body *f0_p; + typedef type_to_key_function_body *f1_p; + typedef type_to_key_function_body *f2_p; + typedef type_to_key_function_body *f3_p; + typedef type_to_key_function_body *f4_p; + typedef type_to_key_function_body *f5_p; + typedef type_to_key_function_body *f6_p; + typedef std::tuple< f0_p, f1_p, f2_p, f3_p, f4_p, f5_p, f6_p > func_initializer_type; + public: +#if __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING + unfolded_join_node(graph &g) : base_type(g, + func_initializer_type( + new type_to_key_function_body_leaf >(key_from_message_body()), + new type_to_key_function_body_leaf >(key_from_message_body()), + new type_to_key_function_body_leaf >(key_from_message_body()), + new type_to_key_function_body_leaf >(key_from_message_body()), + new type_to_key_function_body_leaf >(key_from_message_body()), + new type_to_key_function_body_leaf >(key_from_message_body()), + new type_to_key_function_body_leaf >(key_from_message_body()) + ) ) { + } +#endif /* __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING */ + template + unfolded_join_node(graph &g, Body0 body0, Body1 body1, Body2 body2, Body3 body3, Body4 body4, + Body5 body5, Body6 body6) : base_type(g, func_initializer_type( + new type_to_key_function_body_leaf(body0), + new type_to_key_function_body_leaf(body1), + new type_to_key_function_body_leaf(body2), + new type_to_key_function_body_leaf(body3), + new type_to_key_function_body_leaf(body4), + new type_to_key_function_body_leaf(body5), + new type_to_key_function_body_leaf(body6) + ) ) { + static_assert(std::tuple_size::value == 7, "wrong number of body initializers"); + } + unfolded_join_node(const unfolded_join_node &other) : base_type(other) {} + }; +#endif + +#if __TBB_VARIADIC_MAX >= 8 + template + class unfolded_join_node<8,key_matching_port,OutputTuple,key_matching > : public + join_base<8,key_matching_port,OutputTuple,key_matching >::type { + typedef typename std::tuple_element<0, OutputTuple>::type T0; + typedef typename std::tuple_element<1, OutputTuple>::type T1; + typedef typename std::tuple_element<2, OutputTuple>::type T2; + typedef typename std::tuple_element<3, OutputTuple>::type T3; + typedef typename std::tuple_element<4, OutputTuple>::type T4; + typedef typename std::tuple_element<5, OutputTuple>::type T5; + typedef typename std::tuple_element<6, OutputTuple>::type T6; + typedef typename std::tuple_element<7, OutputTuple>::type T7; + public: + typedef typename wrap_key_tuple_elements<8,key_matching_port,key_matching,OutputTuple>::type input_ports_type; + typedef OutputTuple output_type; + private: + typedef join_node_base , input_ports_type, output_type > base_type; + typedef type_to_key_function_body *f0_p; + typedef type_to_key_function_body *f1_p; + typedef type_to_key_function_body *f2_p; + typedef type_to_key_function_body *f3_p; + typedef type_to_key_function_body *f4_p; + typedef type_to_key_function_body *f5_p; + typedef type_to_key_function_body *f6_p; + typedef type_to_key_function_body *f7_p; + typedef std::tuple< f0_p, f1_p, f2_p, f3_p, f4_p, f5_p, f6_p, f7_p > func_initializer_type; + public: +#if __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING + unfolded_join_node(graph &g) : base_type(g, + func_initializer_type( + new type_to_key_function_body_leaf >(key_from_message_body()), + new type_to_key_function_body_leaf >(key_from_message_body()), + new type_to_key_function_body_leaf >(key_from_message_body()), + new type_to_key_function_body_leaf >(key_from_message_body()), + new type_to_key_function_body_leaf >(key_from_message_body()), + new type_to_key_function_body_leaf >(key_from_message_body()), + new type_to_key_function_body_leaf >(key_from_message_body()), + new type_to_key_function_body_leaf >(key_from_message_body()) + ) ) { + } +#endif /* __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING */ + template + unfolded_join_node(graph &g, Body0 body0, Body1 body1, Body2 body2, Body3 body3, Body4 body4, + Body5 body5, Body6 body6, Body7 body7) : base_type(g, func_initializer_type( + new type_to_key_function_body_leaf(body0), + new type_to_key_function_body_leaf(body1), + new type_to_key_function_body_leaf(body2), + new type_to_key_function_body_leaf(body3), + new type_to_key_function_body_leaf(body4), + new type_to_key_function_body_leaf(body5), + new type_to_key_function_body_leaf(body6), + new type_to_key_function_body_leaf(body7) + ) ) { + static_assert(std::tuple_size::value == 8, "wrong number of body initializers"); + } + unfolded_join_node(const unfolded_join_node &other) : base_type(other) {} + }; +#endif + +#if __TBB_VARIADIC_MAX >= 9 + template + class unfolded_join_node<9,key_matching_port,OutputTuple,key_matching > : public + join_base<9,key_matching_port,OutputTuple,key_matching >::type { + typedef typename std::tuple_element<0, OutputTuple>::type T0; + typedef typename std::tuple_element<1, OutputTuple>::type T1; + typedef typename std::tuple_element<2, OutputTuple>::type T2; + typedef typename std::tuple_element<3, OutputTuple>::type T3; + typedef typename std::tuple_element<4, OutputTuple>::type T4; + typedef typename std::tuple_element<5, OutputTuple>::type T5; + typedef typename std::tuple_element<6, OutputTuple>::type T6; + typedef typename std::tuple_element<7, OutputTuple>::type T7; + typedef typename std::tuple_element<8, OutputTuple>::type T8; + public: + typedef typename wrap_key_tuple_elements<9,key_matching_port,key_matching,OutputTuple>::type input_ports_type; + typedef OutputTuple output_type; + private: + typedef join_node_base , input_ports_type, output_type > base_type; + typedef type_to_key_function_body *f0_p; + typedef type_to_key_function_body *f1_p; + typedef type_to_key_function_body *f2_p; + typedef type_to_key_function_body *f3_p; + typedef type_to_key_function_body *f4_p; + typedef type_to_key_function_body *f5_p; + typedef type_to_key_function_body *f6_p; + typedef type_to_key_function_body *f7_p; + typedef type_to_key_function_body *f8_p; + typedef std::tuple< f0_p, f1_p, f2_p, f3_p, f4_p, f5_p, f6_p, f7_p, f8_p > func_initializer_type; + public: +#if __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING + unfolded_join_node(graph &g) : base_type(g, + func_initializer_type( + new type_to_key_function_body_leaf >(key_from_message_body()), + new type_to_key_function_body_leaf >(key_from_message_body()), + new type_to_key_function_body_leaf >(key_from_message_body()), + new type_to_key_function_body_leaf >(key_from_message_body()), + new type_to_key_function_body_leaf >(key_from_message_body()), + new type_to_key_function_body_leaf >(key_from_message_body()), + new type_to_key_function_body_leaf >(key_from_message_body()), + new type_to_key_function_body_leaf >(key_from_message_body()), + new type_to_key_function_body_leaf >(key_from_message_body()) + ) ) { + } +#endif /* __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING */ + template + unfolded_join_node(graph &g, Body0 body0, Body1 body1, Body2 body2, Body3 body3, Body4 body4, + Body5 body5, Body6 body6, Body7 body7, Body8 body8) : base_type(g, func_initializer_type( + new type_to_key_function_body_leaf(body0), + new type_to_key_function_body_leaf(body1), + new type_to_key_function_body_leaf(body2), + new type_to_key_function_body_leaf(body3), + new type_to_key_function_body_leaf(body4), + new type_to_key_function_body_leaf(body5), + new type_to_key_function_body_leaf(body6), + new type_to_key_function_body_leaf(body7), + new type_to_key_function_body_leaf(body8) + ) ) { + static_assert(std::tuple_size::value == 9, "wrong number of body initializers"); + } + unfolded_join_node(const unfolded_join_node &other) : base_type(other) {} + }; +#endif + +#if __TBB_VARIADIC_MAX >= 10 + template + class unfolded_join_node<10,key_matching_port,OutputTuple,key_matching > : public + join_base<10,key_matching_port,OutputTuple,key_matching >::type { + typedef typename std::tuple_element<0, OutputTuple>::type T0; + typedef typename std::tuple_element<1, OutputTuple>::type T1; + typedef typename std::tuple_element<2, OutputTuple>::type T2; + typedef typename std::tuple_element<3, OutputTuple>::type T3; + typedef typename std::tuple_element<4, OutputTuple>::type T4; + typedef typename std::tuple_element<5, OutputTuple>::type T5; + typedef typename std::tuple_element<6, OutputTuple>::type T6; + typedef typename std::tuple_element<7, OutputTuple>::type T7; + typedef typename std::tuple_element<8, OutputTuple>::type T8; + typedef typename std::tuple_element<9, OutputTuple>::type T9; + public: + typedef typename wrap_key_tuple_elements<10,key_matching_port,key_matching,OutputTuple>::type input_ports_type; + typedef OutputTuple output_type; + private: + typedef join_node_base , input_ports_type, output_type > base_type; + typedef type_to_key_function_body *f0_p; + typedef type_to_key_function_body *f1_p; + typedef type_to_key_function_body *f2_p; + typedef type_to_key_function_body *f3_p; + typedef type_to_key_function_body *f4_p; + typedef type_to_key_function_body *f5_p; + typedef type_to_key_function_body *f6_p; + typedef type_to_key_function_body *f7_p; + typedef type_to_key_function_body *f8_p; + typedef type_to_key_function_body *f9_p; + typedef std::tuple< f0_p, f1_p, f2_p, f3_p, f4_p, f5_p, f6_p, f7_p, f8_p, f9_p > func_initializer_type; + public: +#if __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING + unfolded_join_node(graph &g) : base_type(g, + func_initializer_type( + new type_to_key_function_body_leaf >(key_from_message_body()), + new type_to_key_function_body_leaf >(key_from_message_body()), + new type_to_key_function_body_leaf >(key_from_message_body()), + new type_to_key_function_body_leaf >(key_from_message_body()), + new type_to_key_function_body_leaf >(key_from_message_body()), + new type_to_key_function_body_leaf >(key_from_message_body()), + new type_to_key_function_body_leaf >(key_from_message_body()), + new type_to_key_function_body_leaf >(key_from_message_body()), + new type_to_key_function_body_leaf >(key_from_message_body()), + new type_to_key_function_body_leaf >(key_from_message_body()) + ) ) { + } +#endif /* __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING */ + template + unfolded_join_node(graph &g, Body0 body0, Body1 body1, Body2 body2, Body3 body3, Body4 body4, + Body5 body5, Body6 body6, Body7 body7, Body8 body8, Body9 body9) : base_type(g, func_initializer_type( + new type_to_key_function_body_leaf(body0), + new type_to_key_function_body_leaf(body1), + new type_to_key_function_body_leaf(body2), + new type_to_key_function_body_leaf(body3), + new type_to_key_function_body_leaf(body4), + new type_to_key_function_body_leaf(body5), + new type_to_key_function_body_leaf(body6), + new type_to_key_function_body_leaf(body7), + new type_to_key_function_body_leaf(body8), + new type_to_key_function_body_leaf(body9) + ) ) { + static_assert(std::tuple_size::value == 10, "wrong number of body initializers"); + } + unfolded_join_node(const unfolded_join_node &other) : base_type(other) {} + }; +#endif + + //! templated function to refer to input ports of the join node + template + typename std::tuple_element::type &input_port(JNT &jn) { + return std::get(jn.input_ports()); + } + +#endif // __TBB__flow_graph_join_impl_H diff --git a/src/tbb/include/oneapi/tbb/detail/_flow_graph_node_impl.h b/src/tbb/include/oneapi/tbb/detail/_flow_graph_node_impl.h new file mode 100644 index 000000000..b136ba89a --- /dev/null +++ b/src/tbb/include/oneapi/tbb/detail/_flow_graph_node_impl.h @@ -0,0 +1,894 @@ +/* + Copyright (c) 2005-2024 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB__flow_graph_node_impl_H +#define __TBB__flow_graph_node_impl_H + +#ifndef __TBB_flow_graph_H +#error Do not #include this internal file directly; use public TBB headers instead. +#endif + +#include "_flow_graph_item_buffer_impl.h" + +template< typename T, typename A > +class function_input_queue : public item_buffer { +public: + bool empty() const { + return this->buffer_empty(); + } + + const T& front() const { + return this->item_buffer::front(); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + const message_metainfo& front_metainfo() const { + return this->item_buffer::front_metainfo(); + } +#endif + + void pop() { + this->destroy_front(); + } + + bool push( T& t ) { + return this->push_back( t ); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + bool push( T& t, const message_metainfo& metainfo ) { + return this->push_back(t, metainfo); + } +#endif +}; + +//! Input and scheduling for a function node that takes a type Input as input +// The only up-ref is apply_body_impl, which should implement the function +// call and any handling of the result. +template< typename Input, typename Policy, typename A, typename ImplType > +class function_input_base : public receiver, no_assign { + enum op_type {reg_pred, rem_pred, try_fwd, tryput_bypass, app_body_bypass, occupy_concurrency + }; + typedef function_input_base class_type; + +public: + + //! The input type of this receiver + typedef Input input_type; + typedef typename receiver::predecessor_type predecessor_type; + typedef predecessor_cache predecessor_cache_type; + typedef function_input_queue input_queue_type; + typedef typename allocator_traits::template rebind_alloc allocator_type; + static_assert(!has_policy::value || !has_policy::value, ""); + + //! Constructor for function_input_base + function_input_base( graph &g, size_t max_concurrency, node_priority_t a_priority, bool is_no_throw ) + : my_graph_ref(g), my_max_concurrency(max_concurrency) + , my_concurrency(0), my_priority(a_priority), my_is_no_throw(is_no_throw) + , my_queue(!has_policy::value ? new input_queue_type() : nullptr) + , my_predecessors(this) + , forwarder_busy(false) + { + my_aggregator.initialize_handler(handler_type(this)); + } + + //! Copy constructor + function_input_base( const function_input_base& src ) + : function_input_base(src.my_graph_ref, src.my_max_concurrency, src.my_priority, src.my_is_no_throw) {} + + //! Destructor + // The queue is allocated by the constructor for {multi}function_node. + // TODO: pass the graph_buffer_policy to the base so it can allocate the queue instead. + // This would be an interface-breaking change. + virtual ~function_input_base() { + delete my_queue; + my_queue = nullptr; + } + + graph_task* try_put_task( const input_type& t) override { + return try_put_task_base(t __TBB_FLOW_GRAPH_METAINFO_ARG(message_metainfo{})); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + graph_task* try_put_task( const input_type& t, const message_metainfo& metainfo ) override { + return try_put_task_base(t, metainfo); + } +#endif // __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + + //! Adds src to the list of cached predecessors. + bool register_predecessor( predecessor_type &src ) override { + operation_type op_data(reg_pred); + op_data.r = &src; + my_aggregator.execute(&op_data); + return true; + } + + //! Removes src from the list of cached predecessors. + bool remove_predecessor( predecessor_type &src ) override { + operation_type op_data(rem_pred); + op_data.r = &src; + my_aggregator.execute(&op_data); + return true; + } + +protected: + + void reset_function_input_base( reset_flags f) { + my_concurrency = 0; + if(my_queue) { + my_queue->reset(); + } + reset_receiver(f); + forwarder_busy = false; + } + + graph& my_graph_ref; + const size_t my_max_concurrency; + size_t my_concurrency; + node_priority_t my_priority; + const bool my_is_no_throw; + input_queue_type *my_queue; + predecessor_cache my_predecessors; + + void reset_receiver( reset_flags f) { + if( f & rf_clear_edges) my_predecessors.clear(); + else + my_predecessors.reset(); + __TBB_ASSERT(!(f & rf_clear_edges) || my_predecessors.empty(), "function_input_base reset failed"); + } + + graph& graph_reference() const override { + return my_graph_ref; + } + + graph_task* try_get_postponed_task(const input_type& i) { + operation_type op_data(i, app_body_bypass); // tries to pop an item or get_item + my_aggregator.execute(&op_data); + return op_data.bypass_t; + } + +private: + + friend class apply_body_task_bypass< class_type, input_type >; +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + friend class apply_body_task_bypass< class_type, input_type, trackable_messages_graph_task >; +#endif + friend class forward_task_bypass< class_type >; + + class operation_type : public d1::aggregated_operation< operation_type > { + public: + char type; + union { + input_type *elem; + predecessor_type *r; + }; + graph_task* bypass_t; +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + message_metainfo* metainfo; +#endif + operation_type(const input_type& e, op_type t) : + type(char(t)), elem(const_cast(&e)), bypass_t(nullptr) +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + , metainfo(nullptr) +#endif + {} +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + operation_type(const input_type& e, op_type t, const message_metainfo& info) : + type(char(t)), elem(const_cast(&e)), bypass_t(nullptr), + metainfo(const_cast(&info)) {} +#endif + operation_type(op_type t) : type(char(t)), r(nullptr), bypass_t(nullptr) {} + }; + + bool forwarder_busy; + typedef d1::aggregating_functor handler_type; + friend class d1::aggregating_functor; + d1::aggregator< handler_type, operation_type > my_aggregator; + + graph_task* perform_queued_requests() { + graph_task* new_task = nullptr; + if(my_queue) { + if(!my_queue->empty()) { + ++my_concurrency; + // TODO: consider removing metainfo from the queue using move semantics to avoid + // ref counter increase + new_task = create_body_task(my_queue->front() + __TBB_FLOW_GRAPH_METAINFO_ARG(my_queue->front_metainfo())); + + my_queue->pop(); + } + } + else { + input_type i; +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + message_metainfo metainfo; +#endif + if(my_predecessors.get_item(i __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo))) { + ++my_concurrency; + new_task = create_body_task(i __TBB_FLOW_GRAPH_METAINFO_ARG(std::move(metainfo))); + } + } + return new_task; + } + void handle_operations(operation_type *op_list) { + operation_type* tmp; + while (op_list) { + tmp = op_list; + op_list = op_list->next; + switch (tmp->type) { + case reg_pred: + my_predecessors.add(*(tmp->r)); + tmp->status.store(SUCCEEDED, std::memory_order_release); + if (!forwarder_busy) { + forwarder_busy = true; + spawn_forward_task(); + } + break; + case rem_pred: + my_predecessors.remove(*(tmp->r)); + tmp->status.store(SUCCEEDED, std::memory_order_release); + break; + case app_body_bypass: { + tmp->bypass_t = nullptr; + __TBB_ASSERT(my_max_concurrency != 0, nullptr); + --my_concurrency; + if(my_concurrencybypass_t = perform_queued_requests(); + tmp->status.store(SUCCEEDED, std::memory_order_release); + } + break; + case tryput_bypass: internal_try_put_task(tmp); break; + case try_fwd: internal_forward(tmp); break; + case occupy_concurrency: + if (my_concurrency < my_max_concurrency) { + ++my_concurrency; + tmp->status.store(SUCCEEDED, std::memory_order_release); + } else { + tmp->status.store(FAILED, std::memory_order_release); + } + break; + } + } + } + + //! Put to the node, but return the task instead of enqueueing it + void internal_try_put_task(operation_type *op) { + __TBB_ASSERT(my_max_concurrency != 0, nullptr); + if (my_concurrency < my_max_concurrency) { + ++my_concurrency; + graph_task* new_task = create_body_task(*(op->elem) + __TBB_FLOW_GRAPH_METAINFO_ARG(*(op->metainfo))); + op->bypass_t = new_task; + op->status.store(SUCCEEDED, std::memory_order_release); + } else if ( my_queue && my_queue->push(*(op->elem) + __TBB_FLOW_GRAPH_METAINFO_ARG(*(op->metainfo))) ) + { + op->bypass_t = SUCCESSFULLY_ENQUEUED; + op->status.store(SUCCEEDED, std::memory_order_release); + } else { + op->bypass_t = nullptr; + op->status.store(FAILED, std::memory_order_release); + } + } + + //! Creates tasks for postponed messages if available and if concurrency allows + void internal_forward(operation_type *op) { + op->bypass_t = nullptr; + if (my_concurrency < my_max_concurrency) + op->bypass_t = perform_queued_requests(); + if(op->bypass_t) + op->status.store(SUCCEEDED, std::memory_order_release); + else { + forwarder_busy = false; + op->status.store(FAILED, std::memory_order_release); + } + } + + graph_task* internal_try_put_bypass( const input_type& t + __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& metainfo)) + { + operation_type op_data(t, tryput_bypass __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo)); + my_aggregator.execute(&op_data); + if( op_data.status == SUCCEEDED ) { + return op_data.bypass_t; + } + return nullptr; + } + + graph_task* try_put_task_base(const input_type& t + __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& metainfo)) + { + if ( my_is_no_throw ) + return try_put_task_impl(t, has_policy() + __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo)); + else + return try_put_task_impl(t, std::false_type() + __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo)); + } + + graph_task* try_put_task_impl( const input_type& t, /*lightweight=*/std::true_type + __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& metainfo)) + { + if( my_max_concurrency == 0 ) { + return apply_body_bypass(t __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo)); + } else { + operation_type check_op(t, occupy_concurrency); + my_aggregator.execute(&check_op); + if( check_op.status == SUCCEEDED ) { + return apply_body_bypass(t __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo)); + } + return internal_try_put_bypass(t __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo)); + } + } + + graph_task* try_put_task_impl( const input_type& t, /*lightweight=*/std::false_type + __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& metainfo)) + { + if( my_max_concurrency == 0 ) { + return create_body_task(t __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo)); + } else { + return internal_try_put_bypass(t __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo)); + } + } + + //! Applies the body to the provided input + // then decides if more work is available + graph_task* apply_body_bypass( const input_type &i + __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& metainfo)) + + { + return static_cast(this)->apply_body_impl_bypass(i __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo)); + } + + //! allocates a task to apply a body +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + template + graph_task* create_body_task( const input_type &input, Metainfo&& metainfo ) +#else + graph_task* create_body_task( const input_type &input ) +#endif + { + if (!is_graph_active(my_graph_ref)) { + return nullptr; + } + // TODO revamp: extract helper for common graph task allocation part + d1::small_object_allocator allocator{}; + graph_task* t = nullptr; +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + if (!metainfo.empty()) { + using task_type = apply_body_task_bypass; + t = allocator.new_object(my_graph_ref, allocator, *this, input, my_priority, std::forward(metainfo)); + } else +#endif + { + using task_type = apply_body_task_bypass; + t = allocator.new_object(my_graph_ref, allocator, *this, input, my_priority); + } + return t; + } + + //! This is executed by an enqueued task, the "forwarder" + graph_task* forward_task() { + operation_type op_data(try_fwd); + graph_task* rval = nullptr; + do { + op_data.status = WAIT; + my_aggregator.execute(&op_data); + if(op_data.status == SUCCEEDED) { + graph_task* ttask = op_data.bypass_t; + __TBB_ASSERT( ttask && ttask != SUCCESSFULLY_ENQUEUED, nullptr); + rval = combine_tasks(my_graph_ref, rval, ttask); + } + } while (op_data.status == SUCCEEDED); + return rval; + } + + inline graph_task* create_forward_task() { + if (!is_graph_active(my_graph_ref)) { + return nullptr; + } + d1::small_object_allocator allocator{}; + typedef forward_task_bypass task_type; + graph_task* t = allocator.new_object( graph_reference(), allocator, *this, my_priority ); + return t; + } + + //! Spawns a task that calls forward() + inline void spawn_forward_task() { + graph_task* tp = create_forward_task(); + if(tp) { + spawn_in_graph_arena(graph_reference(), *tp); + } + } + + node_priority_t priority() const override { return my_priority; } +}; // function_input_base + +//! Implements methods for a function node that takes a type Input as input and sends +// a type Output to its successors. +template< typename Input, typename Output, typename Policy, typename A> +class function_input : public function_input_base > { +public: + typedef Input input_type; + typedef Output output_type; + typedef function_body function_body_type; + typedef function_input my_class; + typedef function_input_base base_type; + typedef function_input_queue input_queue_type; + + // constructor + template + function_input( + graph &g, size_t max_concurrency, Body& body, node_priority_t a_priority ) + : base_type(g, max_concurrency, a_priority, noexcept(tbb::detail::invoke(body, input_type()))) + , my_body( new function_body_leaf< input_type, output_type, Body>(body) ) + , my_init_body( new function_body_leaf< input_type, output_type, Body>(body) ) { + } + + //! Copy constructor + function_input( const function_input& src ) : + base_type(src), + my_body( src.my_init_body->clone() ), + my_init_body(src.my_init_body->clone() ) { + } +#if __INTEL_COMPILER <= 2021 + // Suppress superfluous diagnostic about virtual keyword absence in a destructor of an inherited + // class while the parent class has the virtual keyword for the destrocutor. + virtual +#endif + ~function_input() { + delete my_body; + delete my_init_body; + } + + template< typename Body > + Body copy_function_object() { + function_body_type &body_ref = *this->my_body; + return dynamic_cast< function_body_leaf & >(body_ref).get_body(); + } + + output_type apply_body_impl( const input_type& i) { + // There is an extra copied needed to capture the + // body execution without the try_put + fgt_begin_body( my_body ); + output_type v = tbb::detail::invoke(*my_body, i); + fgt_end_body( my_body ); + return v; + } + + //TODO: consider moving into the base class + graph_task* apply_body_impl_bypass( const input_type &i + __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& metainfo)) + { + output_type v = apply_body_impl(i); + graph_task* postponed_task = nullptr; + if( base_type::my_max_concurrency != 0 ) { + postponed_task = base_type::try_get_postponed_task(i); + __TBB_ASSERT( !postponed_task || postponed_task != SUCCESSFULLY_ENQUEUED, nullptr); + } + if( postponed_task ) { + // make the task available for other workers since we do not know successors' + // execution policy + spawn_in_graph_arena(base_type::graph_reference(), *postponed_task); + } + graph_task* successor_task = successors().try_put_task(v __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo)); +#if _MSC_VER && !__INTEL_COMPILER +// #pragma warning (push) +// #pragma warning (disable: 4127) /* suppress conditional expression is constant */ +#endif + if(has_policy::value) { +#if _MSC_VER && !__INTEL_COMPILER +// #pragma warning (pop) +#endif + if(!successor_task) { + // Return confirmative status since current + // node's body has been executed anyway + successor_task = SUCCESSFULLY_ENQUEUED; + } + } + return successor_task; + } + +protected: + + void reset_function_input(reset_flags f) { + base_type::reset_function_input_base(f); + if(f & rf_reset_bodies) { + function_body_type *tmp = my_init_body->clone(); + delete my_body; + my_body = tmp; + } + } + + function_body_type *my_body; + function_body_type *my_init_body; + virtual broadcast_cache &successors() = 0; + +}; // function_input + + +// helper templates to clear the successor edges of the output ports of an multifunction_node +template struct clear_element { + template static void clear_this(P &p) { + (void)std::get(p).successors().clear(); + clear_element::clear_this(p); + } +#if TBB_USE_ASSERT + template static bool this_empty(P &p) { + if(std::get(p).successors().empty()) + return clear_element::this_empty(p); + return false; + } +#endif +}; + +template<> struct clear_element<1> { + template static void clear_this(P &p) { + (void)std::get<0>(p).successors().clear(); + } +#if TBB_USE_ASSERT + template static bool this_empty(P &p) { + return std::get<0>(p).successors().empty(); + } +#endif +}; + +template +struct init_output_ports { + template + static OutputTuple call(graph& g, const std::tuple&) { + return OutputTuple(Args(g)...); + } +}; // struct init_output_ports + +//! Implements methods for a function node that takes a type Input as input +// and has a tuple of output ports specified. +template< typename Input, typename OutputPortSet, typename Policy, typename A> +class multifunction_input : public function_input_base > { +public: + static const int N = std::tuple_size::value; + typedef Input input_type; + typedef OutputPortSet output_ports_type; + typedef multifunction_body multifunction_body_type; + typedef multifunction_input my_class; + typedef function_input_base base_type; + typedef function_input_queue input_queue_type; + + // constructor + template + multifunction_input(graph &g, size_t max_concurrency,Body& body, node_priority_t a_priority ) + : base_type(g, max_concurrency, a_priority, noexcept(tbb::detail::invoke(body, input_type(), my_output_ports))) + , my_body( new multifunction_body_leaf(body) ) + , my_init_body( new multifunction_body_leaf(body) ) + , my_output_ports(init_output_ports::call(g, my_output_ports)){ + } + + //! Copy constructor + multifunction_input( const multifunction_input& src ) : + base_type(src), + my_body( src.my_init_body->clone() ), + my_init_body(src.my_init_body->clone() ), + my_output_ports( init_output_ports::call(src.my_graph_ref, my_output_ports) ) { + } + + ~multifunction_input() { + delete my_body; + delete my_init_body; + } + + template< typename Body > + Body copy_function_object() { + multifunction_body_type &body_ref = *this->my_body; + return *static_cast(dynamic_cast< multifunction_body_leaf & >(body_ref).get_body_ptr()); + } + + // for multifunction nodes we do not have a single successor as such. So we just tell + // the task we were successful. + //TODO: consider moving common parts with implementation in function_input into separate function + graph_task* apply_body_impl_bypass( const input_type &i + __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo&) ) + { + fgt_begin_body( my_body ); + (*my_body)(i, my_output_ports); + fgt_end_body( my_body ); + graph_task* ttask = nullptr; + if(base_type::my_max_concurrency != 0) { + ttask = base_type::try_get_postponed_task(i); + } + return ttask ? ttask : SUCCESSFULLY_ENQUEUED; + } + + output_ports_type &output_ports(){ return my_output_ports; } + +protected: + + void reset(reset_flags f) { + base_type::reset_function_input_base(f); + if(f & rf_clear_edges)clear_element::clear_this(my_output_ports); + if(f & rf_reset_bodies) { + multifunction_body_type* tmp = my_init_body->clone(); + delete my_body; + my_body = tmp; + } + __TBB_ASSERT(!(f & rf_clear_edges) || clear_element::this_empty(my_output_ports), "multifunction_node reset failed"); + } + + multifunction_body_type *my_body; + multifunction_body_type *my_init_body; + output_ports_type my_output_ports; + +}; // multifunction_input + +// template to refer to an output port of a multifunction_node +template +typename std::tuple_element::type &output_port(MOP &op) { + return std::get(op.output_ports()); +} + +inline void check_task_and_spawn(graph& g, graph_task* t) { + if (t && t != SUCCESSFULLY_ENQUEUED) { + spawn_in_graph_arena(g, *t); + } +} + +// helper structs for split_node +template +struct emit_element { + template + static graph_task* emit_this(graph& g, const T &t, P &p) { + // TODO: consider to collect all the tasks in task_list and spawn them all at once + graph_task* last_task = std::get(p).try_put_task(std::get(t)); + check_task_and_spawn(g, last_task); + return emit_element::emit_this(g,t,p); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + template + static graph_task* emit_this(graph& g, const TupleType& t, PortsType& p, + const message_metainfo& metainfo) + { + // TODO: consider to collect all the tasks in task_list and spawn them all at once + graph_task* last_task = std::get(p).try_put_task(std::get(t), metainfo); + check_task_and_spawn(g, last_task); + return emit_element::emit_this(g, t, p, metainfo); + } +#endif +}; + +template<> +struct emit_element<1> { + template + static graph_task* emit_this(graph& g, const T &t, P &p) { + graph_task* last_task = std::get<0>(p).try_put_task(std::get<0>(t)); + check_task_and_spawn(g, last_task); + return SUCCESSFULLY_ENQUEUED; + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + template + static graph_task* emit_this(graph& g, const TupleType& t, PortsType& ports, + const message_metainfo& metainfo) + { + graph_task* last_task = std::get<0>(ports).try_put_task(std::get<0>(t), metainfo); + check_task_and_spawn(g, last_task); + return SUCCESSFULLY_ENQUEUED; + } +#endif +}; + +//! Implements methods for an executable node that takes continue_msg as input +template< typename Output, typename Policy> +class continue_input : public continue_receiver { +public: + + //! The input type of this receiver + typedef continue_msg input_type; + + //! The output type of this receiver + typedef Output output_type; + typedef function_body function_body_type; + typedef continue_input class_type; + + template< typename Body > + continue_input( graph &g, Body& body, node_priority_t a_priority ) + : continue_receiver(/*number_of_predecessors=*/0, a_priority) + , my_graph_ref(g) + , my_body( new function_body_leaf< input_type, output_type, Body>(body) ) + , my_init_body( new function_body_leaf< input_type, output_type, Body>(body) ) + { } + + template< typename Body > + continue_input( graph &g, int number_of_predecessors, + Body& body, node_priority_t a_priority ) + : continue_receiver( number_of_predecessors, a_priority ) + , my_graph_ref(g) + , my_body( new function_body_leaf< input_type, output_type, Body>(body) ) + , my_init_body( new function_body_leaf< input_type, output_type, Body>(body) ) + { } + + continue_input( const continue_input& src ) : continue_receiver(src), + my_graph_ref(src.my_graph_ref), + my_body( src.my_init_body->clone() ), + my_init_body( src.my_init_body->clone() ) {} + + ~continue_input() { + delete my_body; + delete my_init_body; + } + + template< typename Body > + Body copy_function_object() { + function_body_type &body_ref = *my_body; + return dynamic_cast< function_body_leaf & >(body_ref).get_body(); + } + + void reset_receiver( reset_flags f) override { + continue_receiver::reset_receiver(f); + if(f & rf_reset_bodies) { + function_body_type *tmp = my_init_body->clone(); + delete my_body; + my_body = tmp; + } + } + +protected: + + graph& my_graph_ref; + function_body_type *my_body; + function_body_type *my_init_body; + + virtual broadcast_cache &successors() = 0; + + friend class apply_body_task_bypass< class_type, continue_msg >; +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + friend class apply_body_task_bypass< class_type, continue_msg, trackable_messages_graph_task >; +#endif + + //! Applies the body to the provided input + graph_task* apply_body_bypass( input_type __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& metainfo) ) { + // There is an extra copied needed to capture the + // body execution without the try_put + fgt_begin_body( my_body ); + output_type v = (*my_body)( continue_msg() ); + fgt_end_body( my_body ); + return successors().try_put_task( v __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo) ); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + graph_task* execute(const message_metainfo& metainfo) override { +#else + graph_task* execute() override { +#endif + if(!is_graph_active(my_graph_ref)) { + return nullptr; + } +#if _MSC_VER && !__INTEL_COMPILER +// #pragma warning (push) +// #pragma warning (disable: 4127) /* suppress conditional expression is constant */ +#endif + if(has_policy::value) { +#if _MSC_VER && !__INTEL_COMPILER +// #pragma warning (pop) +#endif + return apply_body_bypass( continue_msg() __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo) ); + } + else { + d1::small_object_allocator allocator{}; + graph_task* t = nullptr; +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + if (!metainfo.empty()) { + using task_type = apply_body_task_bypass; + t = allocator.new_object( graph_reference(), allocator, *this, continue_msg(), my_priority, metainfo ); + } else +#endif + { + using task_type = apply_body_task_bypass; + t = allocator.new_object( graph_reference(), allocator, *this, continue_msg(), my_priority ); + } + return t; + } + } + + graph& graph_reference() const override { + return my_graph_ref; + } +}; // continue_input + +//! Implements methods for both executable and function nodes that puts Output to its successors +template< typename Output > +class function_output : public sender { +public: + + template friend struct clear_element; + typedef Output output_type; + typedef typename sender::successor_type successor_type; + typedef broadcast_cache broadcast_cache_type; + + function_output(graph& g) : my_successors(this), my_graph_ref(g) {} + function_output(const function_output& other) = delete; + + //! Adds a new successor to this node + bool register_successor( successor_type &r ) override { + successors().register_successor( r ); + return true; + } + + //! Removes a successor from this node + bool remove_successor( successor_type &r ) override { + successors().remove_successor( r ); + return true; + } + + broadcast_cache_type &successors() { return my_successors; } + + graph& graph_reference() const { return my_graph_ref; } +protected: + broadcast_cache_type my_successors; + graph& my_graph_ref; +}; // function_output + +template< typename Output > +class multifunction_output : public function_output { +public: + typedef Output output_type; + typedef function_output base_type; + using base_type::my_successors; + + multifunction_output(graph& g) : base_type(g) {} + multifunction_output(const multifunction_output& other) : base_type(other.my_graph_ref) {} + + bool try_put(const output_type &i) { + graph_task *res = try_put_task(i); + if( !res ) return false; + if( res != SUCCESSFULLY_ENQUEUED ) { + // wrapping in task_arena::execute() is not needed since the method is called from + // inside task::execute() + spawn_in_graph_arena(graph_reference(), *res); + } + return true; + } + + using base_type::graph_reference; + +protected: + + graph_task* try_put_task(const output_type &i) { + return my_successors.try_put_task(i); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + graph_task* try_put_task(const output_type& i, const message_metainfo& metainfo) { + return my_successors.try_put_task(i, metainfo); + } +#endif + + template friend struct emit_element; + +}; // multifunction_output + +//composite_node +template +void add_nodes_impl(CompositeType*, bool) {} + +template< typename CompositeType, typename NodeType1, typename... NodeTypes > +void add_nodes_impl(CompositeType *c_node, bool visible, const NodeType1& n1, const NodeTypes&... n) { + void *addr = const_cast(&n1); + + fgt_alias_port(c_node, addr, visible); + add_nodes_impl(c_node, visible, n...); +} + +#endif // __TBB__flow_graph_node_impl_H diff --git a/src/tbb/include/oneapi/tbb/detail/_flow_graph_node_set_impl.h b/src/tbb/include/oneapi/tbb/detail/_flow_graph_node_set_impl.h new file mode 100644 index 000000000..8440bd700 --- /dev/null +++ b/src/tbb/include/oneapi/tbb/detail/_flow_graph_node_set_impl.h @@ -0,0 +1,265 @@ +/* + Copyright (c) 2020-2024 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_flow_graph_node_set_impl_H +#define __TBB_flow_graph_node_set_impl_H + +#ifndef __TBB_flow_graph_H +#error Do not #include this internal file directly; use public TBB headers instead. +#endif + +// Included in namespace tbb::detail::d2 (in flow_graph.h) + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET +// Visual Studio 2019 reports an error while calling predecessor_selector::get and successor_selector::get +// Seems like the well-formed expression in trailing decltype is treated as ill-formed +// TODO: investigate problems with decltype in trailing return types or find the cross-platform solution +#define __TBB_MSVC_DISABLE_TRAILING_DECLTYPE (_MSC_VER >= 1900) + +namespace order { +struct undefined {}; +struct following {}; +struct preceding {}; +} + +class get_graph_helper { +public: + // TODO: consider making graph_reference() public and consistent interface to get a reference to the graph + // and remove get_graph_helper + template + static graph& get(const T& object) { + return get_impl(object, std::is_base_of()); + } + +private: + // Get graph from the object of type derived from graph_node + template + static graph& get_impl(const T& object, std::true_type) { + return static_cast(&object)->my_graph; + } + + template + static graph& get_impl(const T& object, std::false_type) { + return object.graph_reference(); + } +}; + +template +struct node_set { + typedef Order order_type; + + std::tuple nodes; + node_set(Nodes&... ns) : nodes(ns...) {} + + template + node_set(const node_set& set) : nodes(set.nodes) {} + + graph& graph_reference() const { + return get_graph_helper::get(std::get<0>(nodes)); + } +}; + +namespace alias_helpers { +template using output_type = typename T::output_type; +template using output_ports_type = typename T::output_ports_type; +template using input_type = typename T::input_type; +template using input_ports_type = typename T::input_ports_type; +} // namespace alias_helpers + +template +using has_output_type = supports; + +template +using has_input_type = supports; + +template +using has_input_ports_type = supports; + +template +using has_output_ports_type = supports; + +template +struct is_sender : std::is_base_of, T> {}; + +template +struct is_receiver : std::is_base_of, T> {}; + +template +struct is_async_node : std::false_type {}; + +template +struct is_async_node> : std::true_type {}; + +template +node_set +follows(FirstPredecessor& first_predecessor, Predecessors&... predecessors) { + static_assert((conjunction, + has_output_type...>::value), + "Not all node's predecessors has output_type typedef"); + static_assert((conjunction, is_sender...>::value), + "Not all node's predecessors are senders"); + return node_set(first_predecessor, predecessors...); +} + +template +node_set +follows(node_set& predecessors_set) { + static_assert((conjunction...>::value), + "Not all nodes in the set has output_type typedef"); + static_assert((conjunction...>::value), + "Not all nodes in the set are senders"); + return node_set(predecessors_set); +} + +template +node_set +precedes(FirstSuccessor& first_successor, Successors&... successors) { + static_assert((conjunction, + has_input_type...>::value), + "Not all node's successors has input_type typedef"); + static_assert((conjunction, is_receiver...>::value), + "Not all node's successors are receivers"); + return node_set(first_successor, successors...); +} + +template +node_set +precedes(node_set& successors_set) { + static_assert((conjunction...>::value), + "Not all nodes in the set has input_type typedef"); + static_assert((conjunction...>::value), + "Not all nodes in the set are receivers"); + return node_set(successors_set); +} + +template +node_set +make_node_set(Node& first_node, Nodes&... nodes) { + return node_set(first_node, nodes...); +} + +template +class successor_selector { + template + static auto get_impl(NodeType& node, std::true_type) -> decltype(input_port(node)) { + return input_port(node); + } + + template + static NodeType& get_impl(NodeType& node, std::false_type) { return node; } + +public: + template +#if __TBB_MSVC_DISABLE_TRAILING_DECLTYPE + static auto& get(NodeType& node) +#else + static auto get(NodeType& node) -> decltype(get_impl(node, has_input_ports_type())) +#endif + { + return get_impl(node, has_input_ports_type()); + } +}; + +template +class predecessor_selector { + template + static auto internal_get(NodeType& node, std::true_type) -> decltype(output_port(node)) { + return output_port(node); + } + + template + static NodeType& internal_get(NodeType& node, std::false_type) { return node;} + + template +#if __TBB_MSVC_DISABLE_TRAILING_DECLTYPE + static auto& get_impl(NodeType& node, std::false_type) +#else + static auto get_impl(NodeType& node, std::false_type) -> decltype(internal_get(node, has_output_ports_type())) +#endif + { + return internal_get(node, has_output_ports_type()); + } + + template + static AsyncNode& get_impl(AsyncNode& node, std::true_type) { return node; } + +public: + template +#if __TBB_MSVC_DISABLE_TRAILING_DECLTYPE + static auto& get(NodeType& node) +#else + static auto get(NodeType& node) -> decltype(get_impl(node, is_async_node())) +#endif + { + return get_impl(node, is_async_node()); + } +}; + +template +class make_edges_helper { +public: + template + static void connect_predecessors(PredecessorsTuple& predecessors, NodeType& node) { + make_edge(std::get(predecessors), successor_selector::get(node)); + make_edges_helper::connect_predecessors(predecessors, node); + } + + template + static void connect_successors(NodeType& node, SuccessorsTuple& successors) { + make_edge(predecessor_selector::get(node), std::get(successors)); + make_edges_helper::connect_successors(node, successors); + } +}; + +template<> +struct make_edges_helper<0> { + template + static void connect_predecessors(PredecessorsTuple& predecessors, NodeType& node) { + make_edge(std::get<0>(predecessors), successor_selector<0>::get(node)); + } + + template + static void connect_successors(NodeType& node, SuccessorsTuple& successors) { + make_edge(predecessor_selector<0>::get(node), std::get<0>(successors)); + } +}; + +// TODO: consider adding an overload for making edges between node sets +template +void make_edges(const node_set& s, NodeType& node) { + const std::size_t SetSize = std::tuple_size::value; + make_edges_helper::connect_predecessors(s.nodes, node); +} + +template +void make_edges(NodeType& node, const node_set& s) { + const std::size_t SetSize = std::tuple_size::value; + make_edges_helper::connect_successors(node, s.nodes); +} + +template +void make_edges_in_order(const node_set& ns, NodeType& node) { + make_edges(ns, node); +} + +template +void make_edges_in_order(const node_set& ns, NodeType& node) { + make_edges(node, ns); +} + +#endif // __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + +#endif // __TBB_flow_graph_node_set_impl_H diff --git a/src/tbb/include/oneapi/tbb/detail/_flow_graph_nodes_deduction.h b/src/tbb/include/oneapi/tbb/detail/_flow_graph_nodes_deduction.h new file mode 100644 index 000000000..47ecfb2a8 --- /dev/null +++ b/src/tbb/include/oneapi/tbb/detail/_flow_graph_nodes_deduction.h @@ -0,0 +1,277 @@ +/* + Copyright (c) 2005-2024 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_flow_graph_nodes_deduction_H +#define __TBB_flow_graph_nodes_deduction_H + +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +namespace tbb { +namespace detail { +namespace d2 { + +template +struct declare_body_types { + using input_type = Input; + using output_type = Output; +}; + +struct NoInputBody {}; + +template +struct declare_body_types { + using output_type = Output; +}; + +template struct body_types; + +template +struct body_types : declare_body_types {}; + +template +struct body_types : declare_body_types {}; + +template +struct body_types : declare_body_types {}; + +template +struct body_types : declare_body_types {}; + +template +struct body_types : declare_body_types {}; + +template +struct body_types : declare_body_types {}; + +template +struct body_types : declare_body_types {}; + +template +struct body_types : declare_body_types {}; + +template +struct body_types : declare_body_types {}; + +template +using input_t = typename body_types::input_type; + +template +using output_t = typename body_types::output_type; + +template +auto decide_on_operator_overload(Output (T::*name)(const Input&) const)->decltype(name); + +template +auto decide_on_operator_overload(Output (T::*name)(const Input&))->decltype(name); + +template +auto decide_on_operator_overload(Output (T::*name)(Input&) const)->decltype(name); + +template +auto decide_on_operator_overload(Output (T::*name)(Input&))->decltype(name); + +template +auto decide_on_operator_overload(Output (*name)(const Input&))->decltype(name); + +template +auto decide_on_operator_overload(Output (*name)(Input&))->decltype(name); + +template +decltype(decide_on_operator_overload(&Body::operator())) decide_on_callable_type(int); + +template +decltype(decide_on_operator_overload(std::declval())) decide_on_callable_type(...); + +// Deduction guides for Flow Graph nodes + +template +input_node(GraphOrSet&&, Body) +->input_node(0))>>; + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + +template +struct decide_on_set; + +template +struct decide_on_set> { + using type = typename Node::output_type; +}; + +template +struct decide_on_set> { + using type = typename Node::input_type; +}; + +template +using decide_on_set_t = typename decide_on_set>::type; + +template +broadcast_node(const NodeSet&) +->broadcast_node>; + +template +buffer_node(const NodeSet&) +->buffer_node>; + +template +queue_node(const NodeSet&) +->queue_node>; +#endif // __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + +template +sequencer_node(GraphOrProxy&&, Sequencer) +->sequencer_node(0))>>; + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET +template +priority_queue_node(const NodeSet&, const Compare&) +->priority_queue_node, Compare>; + +template +priority_queue_node(const NodeSet&) +->priority_queue_node, std::less>>; +#endif // __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + +template +struct join_key { + using type = Key; +}; + +template +struct join_key { + using type = T&; +}; + +template +using join_key_t = typename join_key::type; + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET +template +join_node(const node_set&, Policy) +->join_node, + Policy>; + +template +join_node(const node_set&, Policy) +->join_node; + +template +join_node(const node_set) +->join_node, + queueing>; + +template +join_node(const node_set) +->join_node; +#endif + +template +join_node(GraphOrProxy&&, Body, Bodies...) +->join_node(0))>, + input_t(0))>...>, + key_matching(0))>>>>; + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET +template +indexer_node(const node_set&) +->indexer_node; +#endif + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET +template +limiter_node(const NodeSet&, size_t) +->limiter_node>; + +template +split_node(const node_set&) +->split_node; + +template +split_node(const node_set&) +->split_node>; + +#endif + +template +function_node(GraphOrSet&&, + size_t, Body, + Policy, node_priority_t = no_priority) +->function_node(0))>, + output_t(0))>, + Policy>; + +template +function_node(GraphOrSet&&, size_t, + Body, node_priority_t = no_priority) +->function_node(0))>, + output_t(0))>, + queueing>; + +template +struct continue_output { + using type = Output; +}; + +template <> +struct continue_output { + using type = continue_msg; +}; + +template +using continue_output_t = typename continue_output::type; + +template +continue_node(GraphOrSet&&, Body, + Policy, node_priority_t = no_priority) +->continue_node>, + Policy>; + +template +continue_node(GraphOrSet&&, + int, Body, + Policy, node_priority_t = no_priority) +->continue_node>, + Policy>; + +template +continue_node(GraphOrSet&&, + Body, node_priority_t = no_priority) +->continue_node>, Policy>; + +template +continue_node(GraphOrSet&&, int, + Body, node_priority_t = no_priority) +->continue_node>, + Policy>; + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + +template +overwrite_node(const NodeSet&) +->overwrite_node>; + +template +write_once_node(const NodeSet&) +->write_once_node>; +#endif // __TBB_PREVIEW_FLOW_GRAPH_NODE_SET +} // namespace d2 +} // namespace detail +} // namespace tbb + +#endif // __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +#endif // __TBB_flow_graph_nodes_deduction_H diff --git a/src/tbb/include/oneapi/tbb/detail/_flow_graph_tagged_buffer_impl.h b/src/tbb/include/oneapi/tbb/detail/_flow_graph_tagged_buffer_impl.h new file mode 100644 index 000000000..0f7c0d174 --- /dev/null +++ b/src/tbb/include/oneapi/tbb/detail/_flow_graph_tagged_buffer_impl.h @@ -0,0 +1,380 @@ +/* + Copyright (c) 2005-2024 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// a hash table buffer that can expand, and can support as many deletions as +// additions, list-based, with elements of list held in array (for destruction +// management), multiplicative hashing (like ets). No synchronization built-in. +// + +#ifndef __TBB__flow_graph_hash_buffer_impl_H +#define __TBB__flow_graph_hash_buffer_impl_H + +#ifndef __TBB_flow_graph_H +#error Do not #include this internal file directly; use public TBB headers instead. +#endif + +// included in namespace tbb::flow::interfaceX::internal + +// elements in the table are a simple list; we need pointer to next element to +// traverse the chain + +template +struct hash_buffer_element : public aligned_pair { + using key_type = Key; + using value_type = ValueType; + + value_type* get_value_ptr() { return reinterpret_cast(this->first); } + hash_buffer_element* get_next() { return reinterpret_cast(this->second); } + void set_next(hash_buffer_element* new_next) { this->second = reinterpret_cast(new_next); } + + void create_element(const value_type& v) { + ::new(this->first) value_type(v); + } + + void create_element(hash_buffer_element&& other) { + ::new(this->first) value_type(std::move(*other.get_value_ptr())); + } + + void destroy_element() { + get_value_ptr()->~value_type(); + } +}; + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT +template +struct metainfo_hash_buffer_element : public aligned_triple { + using key_type = Key; + using value_type = ValueType; + + value_type* get_value_ptr() { return reinterpret_cast(this->first); } + metainfo_hash_buffer_element* get_next() { + return reinterpret_cast(this->second); + } + void set_next(metainfo_hash_buffer_element* new_next) { this->second = reinterpret_cast(new_next); } + message_metainfo& get_metainfo() { return this->third; } + + void create_element(const value_type& v, const message_metainfo& metainfo) { + __TBB_ASSERT(this->third.empty(), nullptr); + ::new(this->first) value_type(v); + this->third = metainfo; + + for (auto waiter : metainfo.waiters()) { + waiter->reserve(1); + } + } + + void create_element(metainfo_hash_buffer_element&& other) { + __TBB_ASSERT(this->third.empty(), nullptr); + ::new(this->first) value_type(std::move(*other.get_value_ptr())); + this->third = std::move(other.get_metainfo()); + } + + void destroy_element() { + get_value_ptr()->~value_type(); + + for (auto waiter : get_metainfo().waiters()) { + waiter->release(1); + } + get_metainfo() = message_metainfo{}; + } +}; +#endif + +template + < + typename ElementType, + typename ValueToKey, // abstract method that returns "const Key" or "const Key&" given ValueType + typename HashCompare, // has hash and equal + typename Allocator=tbb::cache_aligned_allocator + > +class hash_buffer_impl : public HashCompare { +public: + static const size_t INITIAL_SIZE = 8; // initial size of the hash pointer table + typedef typename ElementType::key_type key_type; + typedef typename ElementType::value_type value_type; + typedef ElementType element_type; + typedef value_type *pointer_type; + typedef element_type *list_array_type; // array we manage manually + typedef list_array_type *pointer_array_type; + typedef typename std::allocator_traits::template rebind_alloc pointer_array_allocator_type; + typedef typename std::allocator_traits::template rebind_alloc elements_array_allocator; + typedef typename std::decay::type Knoref; + +private: + ValueToKey *my_key; + size_t my_size; + size_t nelements; + pointer_array_type pointer_array; // pointer_array[my_size] + list_array_type elements_array; // elements_array[my_size / 2] + element_type* free_list; + + size_t mask() { return my_size - 1; } + + void set_up_free_list( element_type **p_free_list, list_array_type la, size_t sz) { + for(size_t i=0; i < sz - 1; ++i ) { // construct free list + la[i].set_next(&(la[i + 1])); + } + la[sz - 1].set_next(nullptr); + *p_free_list = (element_type *)&(la[0]); + } + + // cleanup for exceptions + struct DoCleanup { + pointer_array_type *my_pa; + list_array_type *my_elements; + size_t my_size; + + DoCleanup(pointer_array_type &pa, list_array_type &my_els, size_t sz) : + my_pa(&pa), my_elements(&my_els), my_size(sz) { } + ~DoCleanup() { + if(my_pa) { + size_t dont_care = 0; + internal_free_buffer(*my_pa, *my_elements, my_size, dont_care); + } + } + }; + + // exception-safety requires we do all the potentially-throwing operations first + void grow_array() { + size_t new_size = my_size*2; + size_t new_nelements = nelements; // internal_free_buffer zeroes this + list_array_type new_elements_array = nullptr; + pointer_array_type new_pointer_array = nullptr; + list_array_type new_free_list = nullptr; + { + DoCleanup my_cleanup(new_pointer_array, new_elements_array, new_size); + new_elements_array = elements_array_allocator().allocate(my_size); +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + for (std::size_t i = 0; i < my_size; ++i) { + ::new(new_elements_array + i) element_type(); + } +#endif + new_pointer_array = pointer_array_allocator_type().allocate(new_size); + for(size_t i=0; i < new_size; ++i) new_pointer_array[i] = nullptr; + set_up_free_list(&new_free_list, new_elements_array, my_size ); + + for(size_t i=0; i < my_size; ++i) { + for( element_type* op = pointer_array[i]; op; op = (element_type *)(op->get_next())) { + internal_insert_with_key(new_pointer_array, new_size, new_free_list, std::move(*op)); + } + } + my_cleanup.my_pa = nullptr; + my_cleanup.my_elements = nullptr; + } + + internal_free_buffer(pointer_array, elements_array, my_size, nelements); + free_list = new_free_list; + pointer_array = new_pointer_array; + elements_array = new_elements_array; + my_size = new_size; + nelements = new_nelements; + } + + // v should have perfect forwarding if std::move implemented. + // we use this method to move elements in grow_array, so can't use class fields + template + const value_type& get_value_from_pack(const Value& value, const Args&...) { + return value; + } + + template + const value_type& get_value_from_pack(Element&& element) { + return *(element.get_value_ptr()); + } + + template + void internal_insert_with_key( element_type **p_pointer_array, size_t p_sz, list_array_type &p_free_list, + Args&&... args) { + size_t l_mask = p_sz-1; + __TBB_ASSERT(my_key, "Error: value-to-key functor not provided"); + size_t h = this->hash(tbb::detail::invoke(*my_key, get_value_from_pack(args...))) & l_mask; + __TBB_ASSERT(p_free_list, "Error: free list not set up."); + element_type* my_elem = p_free_list; p_free_list = (element_type *)(p_free_list->get_next()); + my_elem->create_element(std::forward(args)...); + my_elem->set_next(p_pointer_array[h]); + p_pointer_array[h] = my_elem; + } + + void internal_initialize_buffer() { + pointer_array = pointer_array_allocator_type().allocate(my_size); + for(size_t i = 0; i < my_size; ++i) pointer_array[i] = nullptr; + elements_array = elements_array_allocator().allocate(my_size / 2); +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + for (std::size_t i = 0; i < my_size / 2; ++i) { + ::new(elements_array + i) element_type(); + } +#endif + set_up_free_list(&free_list, elements_array, my_size / 2); + } + + // made static so an enclosed class can use to properly dispose of the internals + static void internal_free_buffer( pointer_array_type &pa, list_array_type &el, size_t &sz, size_t &ne ) { + if(pa) { + for(size_t i = 0; i < sz; ++i ) { + element_type *p_next; + for( element_type *p = pa[i]; p; p = p_next) { + p_next = p->get_next(); + p->destroy_element(); + } + } + pointer_array_allocator_type().deallocate(pa, sz); + pa = nullptr; + } + // Separate test (if allocation of pa throws, el may be allocated. + // but no elements will be constructed.) + if(el) { +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + for (std::size_t i = 0; i < sz / 2; ++i) { + (el + i)->~element_type(); + } +#endif + elements_array_allocator().deallocate(el, sz / 2); + el = nullptr; + } + sz = INITIAL_SIZE; + ne = 0; + } + +public: + hash_buffer_impl() : my_key(nullptr), my_size(INITIAL_SIZE), nelements(0) { + internal_initialize_buffer(); + } + + ~hash_buffer_impl() { + internal_free_buffer(pointer_array, elements_array, my_size, nelements); + delete my_key; + my_key = nullptr; + } + hash_buffer_impl(const hash_buffer_impl&) = delete; + hash_buffer_impl& operator=(const hash_buffer_impl&) = delete; + + void reset() { + internal_free_buffer(pointer_array, elements_array, my_size, nelements); + internal_initialize_buffer(); + } + + // Take ownership of func object allocated with new. + // This method is only used internally, so can't be misused by user. + void set_key_func(ValueToKey *vtk) { my_key = vtk; } + // pointer is used to clone() + ValueToKey* get_key_func() { return my_key; } + + template + bool insert_with_key(const value_type &v, Args&&... args) { + element_type* p = nullptr; + __TBB_ASSERT(my_key, "Error: value-to-key functor not provided"); + if(find_element_ref_with_key(tbb::detail::invoke(*my_key, v), p)) { + p->destroy_element(); + p->create_element(v, std::forward(args)...); + return false; + } + ++nelements; + if(nelements*2 > my_size) grow_array(); + internal_insert_with_key(pointer_array, my_size, free_list, v, std::forward(args)...); + return true; + } + + bool find_element_ref_with_key(const Knoref& k, element_type*& v) { + size_t i = this->hash(k) & mask(); + for(element_type* p = pointer_array[i]; p; p = (element_type *)(p->get_next())) { + __TBB_ASSERT(my_key, "Error: value-to-key functor not provided"); + if(this->equal(tbb::detail::invoke(*my_key, *p->get_value_ptr()), k)) { + v = p; + return true; + } + } + return false; + } + + // returns true and sets v to array element if found, else returns false. + bool find_ref_with_key(const Knoref& k, pointer_type &v) { + element_type* element_ptr = nullptr; + bool res = find_element_ref_with_key(k, element_ptr); + v = element_ptr->get_value_ptr(); + return res; + } + + bool find_with_key( const Knoref& k, value_type &v) { + value_type *p; + if(find_ref_with_key(k, p)) { + v = *p; + return true; + } + else + return false; + } + + void delete_with_key(const Knoref& k) { + size_t h = this->hash(k) & mask(); + element_type* prev = nullptr; + for(element_type* p = pointer_array[h]; p; prev = p, p = (element_type *)(p->get_next())) { + value_type *vp = p->get_value_ptr(); + __TBB_ASSERT(my_key, "Error: value-to-key functor not provided"); + if(this->equal(tbb::detail::invoke(*my_key, *vp), k)) { + p->destroy_element(); + if(prev) prev->set_next(p->get_next()); + else pointer_array[h] = (element_type *)(p->get_next()); + p->set_next(free_list); + free_list = p; + --nelements; + return; + } + } + __TBB_ASSERT(false, "key not found for delete"); + } +}; + +template + < + typename Key, // type of key within ValueType + typename ValueType, + typename ValueToKey, // abstract method that returns "const Key" or "const Key&" given ValueType + typename HashCompare, // has hash and equal + typename Allocator=tbb::cache_aligned_allocator> + > +using hash_buffer = hash_buffer_impl, + ValueToKey, HashCompare, Allocator>; + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT +template + < + typename Key, // type of key within ValueType + typename ValueType, + typename ValueToKey, // abstract method that returns "const Key" or "const Key&" given ValueType + typename HashCompare, // has hash and equal + typename Allocator=tbb::cache_aligned_allocator> + > +struct metainfo_hash_buffer : public hash_buffer_impl, + ValueToKey, HashCompare, Allocator> +{ +private: + using base_type = hash_buffer_impl, + ValueToKey, HashCompare, Allocator>; +public: + bool find_with_key(const typename base_type::Knoref& k, + typename base_type::value_type& v, message_metainfo& metainfo) + { + typename base_type::element_type* p = nullptr; + bool result = this->find_element_ref_with_key(k, p); + if (result) { + v = *(p->get_value_ptr()); + metainfo = p->get_metainfo(); + } + return result; + } +}; +#endif // __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT +#endif // __TBB__flow_graph_hash_buffer_impl_H diff --git a/src/tbb/include/oneapi/tbb/detail/_flow_graph_trace_impl.h b/src/tbb/include/oneapi/tbb/detail/_flow_graph_trace_impl.h new file mode 100644 index 000000000..74ebf0845 --- /dev/null +++ b/src/tbb/include/oneapi/tbb/detail/_flow_graph_trace_impl.h @@ -0,0 +1,364 @@ +/* + Copyright (c) 2005-2024 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef _FGT_GRAPH_TRACE_IMPL_H +#define _FGT_GRAPH_TRACE_IMPL_H + +#include "../profiling.h" +#if (_MSC_VER >= 1900) + #include +#endif + +namespace tbb { +namespace detail { +namespace d2 { + +template< typename T > class sender; +template< typename T > class receiver; + +#if TBB_USE_PROFILING_TOOLS + #if __TBB_FLOW_TRACE_CODEPTR + #if (_MSC_VER >= 1900) + #define CODEPTR() (_ReturnAddress()) + #elif __TBB_GCC_VERSION >= 40800 + #define CODEPTR() ( __builtin_return_address(0)) + #else + #define CODEPTR() nullptr + #endif + #else + #define CODEPTR() nullptr + #endif /* __TBB_FLOW_TRACE_CODEPTR */ + +static inline void fgt_alias_port(void *node, void *p, bool visible) { + if(visible) + itt_relation_add( d1::ITT_DOMAIN_FLOW, node, FLOW_NODE, __itt_relation_is_parent_of, p, FLOW_NODE ); + else + itt_relation_add( d1::ITT_DOMAIN_FLOW, p, FLOW_NODE, __itt_relation_is_child_of, node, FLOW_NODE ); +} + +static inline void fgt_composite ( void* codeptr, void *node, void *graph ) { + itt_make_task_group( d1::ITT_DOMAIN_FLOW, node, FLOW_NODE, graph, FLOW_GRAPH, FLOW_COMPOSITE_NODE ); + suppress_unused_warning( codeptr ); +#if __TBB_FLOW_TRACE_CODEPTR + if (codeptr != nullptr) { + register_node_addr(d1::ITT_DOMAIN_FLOW, node, FLOW_NODE, CODE_ADDRESS, &codeptr); + } +#endif +} + +static inline void fgt_internal_alias_input_port( void *node, void *p, string_resource_index name_index ) { + itt_make_task_group( d1::ITT_DOMAIN_FLOW, p, FLOW_INPUT_PORT, node, FLOW_NODE, name_index ); + itt_relation_add( d1::ITT_DOMAIN_FLOW, node, FLOW_NODE, __itt_relation_is_parent_of, p, FLOW_INPUT_PORT ); +} + +static inline void fgt_internal_alias_output_port( void *node, void *p, string_resource_index name_index ) { + itt_make_task_group( d1::ITT_DOMAIN_FLOW, p, FLOW_OUTPUT_PORT, node, FLOW_NODE, name_index ); + itt_relation_add( d1::ITT_DOMAIN_FLOW, node, FLOW_NODE, __itt_relation_is_parent_of, p, FLOW_OUTPUT_PORT ); +} + +template +void alias_input_port(void *node, receiver* port, string_resource_index name_index) { + // TODO: Make fgt_internal_alias_input_port a function template? + fgt_internal_alias_input_port( node, port, name_index); +} + +template < typename PortsTuple, int N > +struct fgt_internal_input_alias_helper { + static void alias_port( void *node, PortsTuple &ports ) { + alias_input_port( node, &(std::get(ports)), static_cast(FLOW_INPUT_PORT_0 + N - 1) ); + fgt_internal_input_alias_helper::alias_port( node, ports ); + } +}; + +template < typename PortsTuple > +struct fgt_internal_input_alias_helper { + static void alias_port( void * /* node */, PortsTuple & /* ports */ ) { } +}; + +template +void alias_output_port(void *node, sender* port, string_resource_index name_index) { + // TODO: Make fgt_internal_alias_output_port a function template? + fgt_internal_alias_output_port( node, static_cast(port), name_index); +} + +template < typename PortsTuple, int N > +struct fgt_internal_output_alias_helper { + static void alias_port( void *node, PortsTuple &ports ) { + alias_output_port( node, &(std::get(ports)), static_cast(FLOW_OUTPUT_PORT_0 + N - 1) ); + fgt_internal_output_alias_helper::alias_port( node, ports ); + } +}; + +template < typename PortsTuple > +struct fgt_internal_output_alias_helper { + static void alias_port( void * /*node*/, PortsTuple &/*ports*/ ) { + } +}; + +static inline void fgt_internal_create_input_port( void *node, void *p, string_resource_index name_index ) { + itt_make_task_group( d1::ITT_DOMAIN_FLOW, p, FLOW_INPUT_PORT, node, FLOW_NODE, name_index ); +} + +static inline void fgt_internal_create_output_port( void* codeptr, void *node, void *p, string_resource_index name_index ) { + itt_make_task_group(d1::ITT_DOMAIN_FLOW, p, FLOW_OUTPUT_PORT, node, FLOW_NODE, name_index); + suppress_unused_warning( codeptr ); +#if __TBB_FLOW_TRACE_CODEPTR + if (codeptr != nullptr) { + register_node_addr(d1::ITT_DOMAIN_FLOW, node, FLOW_NODE, CODE_ADDRESS, &codeptr); + } +#endif +} + +template +void register_input_port(void *node, receiver* port, string_resource_index name_index) { + // TODO: Make fgt_internal_create_input_port a function template? + fgt_internal_create_input_port(node, static_cast(port), name_index); +} + +template < typename PortsTuple, int N > +struct fgt_internal_input_helper { + static void register_port( void *node, PortsTuple &ports ) { + register_input_port( node, &(std::get(ports)), static_cast(FLOW_INPUT_PORT_0 + N - 1) ); + fgt_internal_input_helper::register_port( node, ports ); + } +}; + +template < typename PortsTuple > +struct fgt_internal_input_helper { + static void register_port( void *node, PortsTuple &ports ) { + register_input_port( node, &(std::get<0>(ports)), FLOW_INPUT_PORT_0 ); + } +}; + +template +void register_output_port(void* codeptr, void *node, sender* port, string_resource_index name_index) { + // TODO: Make fgt_internal_create_output_port a function template? + fgt_internal_create_output_port( codeptr, node, static_cast(port), name_index); +} + +template < typename PortsTuple, int N > +struct fgt_internal_output_helper { + static void register_port( void* codeptr, void *node, PortsTuple &ports ) { + register_output_port( codeptr, node, &(std::get(ports)), static_cast(FLOW_OUTPUT_PORT_0 + N - 1) ); + fgt_internal_output_helper::register_port( codeptr, node, ports ); + } +}; + +template < typename PortsTuple > +struct fgt_internal_output_helper { + static void register_port( void* codeptr, void *node, PortsTuple &ports ) { + register_output_port( codeptr, node, &(std::get<0>(ports)), FLOW_OUTPUT_PORT_0 ); + } +}; + +template< typename NodeType > +void fgt_multioutput_node_desc( const NodeType *node, const char *desc ) { + void *addr = (void *)( static_cast< receiver< typename NodeType::input_type > * >(const_cast< NodeType *>(node)) ); + itt_metadata_str_add( d1::ITT_DOMAIN_FLOW, addr, FLOW_NODE, FLOW_OBJECT_NAME, desc ); +} + +template< typename NodeType > +void fgt_multiinput_multioutput_node_desc( const NodeType *node, const char *desc ) { + void *addr = const_cast(node); + itt_metadata_str_add( d1::ITT_DOMAIN_FLOW, addr, FLOW_NODE, FLOW_OBJECT_NAME, desc ); +} + +template< typename NodeType > +static inline void fgt_node_desc( const NodeType *node, const char *desc ) { + void *addr = (void *)( static_cast< sender< typename NodeType::output_type > * >(const_cast< NodeType *>(node)) ); + itt_metadata_str_add( d1::ITT_DOMAIN_FLOW, addr, FLOW_NODE, FLOW_OBJECT_NAME, desc ); +} + +static inline void fgt_graph_desc( const void *g, const char *desc ) { + void *addr = const_cast< void *>(g); + itt_metadata_str_add( d1::ITT_DOMAIN_FLOW, addr, FLOW_GRAPH, FLOW_OBJECT_NAME, desc ); +} + +static inline void fgt_body( void *node, void *body ) { + itt_relation_add( d1::ITT_DOMAIN_FLOW, body, FLOW_BODY, __itt_relation_is_child_of, node, FLOW_NODE ); +} + +template< int N, typename PortsTuple > +static inline void fgt_multioutput_node(void* codeptr, string_resource_index t, void *g, void *input_port, PortsTuple &ports ) { + itt_make_task_group( d1::ITT_DOMAIN_FLOW, input_port, FLOW_NODE, g, FLOW_GRAPH, t ); + fgt_internal_create_input_port( input_port, input_port, FLOW_INPUT_PORT_0 ); + fgt_internal_output_helper::register_port(codeptr, input_port, ports ); +} + +template< int N, typename PortsTuple > +static inline void fgt_multioutput_node_with_body( void* codeptr, string_resource_index t, void *g, void *input_port, PortsTuple &ports, void *body ) { + itt_make_task_group( d1::ITT_DOMAIN_FLOW, input_port, FLOW_NODE, g, FLOW_GRAPH, t ); + fgt_internal_create_input_port( input_port, input_port, FLOW_INPUT_PORT_0 ); + fgt_internal_output_helper::register_port( codeptr, input_port, ports ); + fgt_body( input_port, body ); +} + +template< int N, typename PortsTuple > +static inline void fgt_multiinput_node( void* codeptr, string_resource_index t, void *g, PortsTuple &ports, void *output_port) { + itt_make_task_group( d1::ITT_DOMAIN_FLOW, output_port, FLOW_NODE, g, FLOW_GRAPH, t ); + fgt_internal_create_output_port( codeptr, output_port, output_port, FLOW_OUTPUT_PORT_0 ); + fgt_internal_input_helper::register_port( output_port, ports ); +} + +static inline void fgt_multiinput_multioutput_node( void* codeptr, string_resource_index t, void *n, void *g ) { + itt_make_task_group( d1::ITT_DOMAIN_FLOW, n, FLOW_NODE, g, FLOW_GRAPH, t ); + suppress_unused_warning( codeptr ); +#if __TBB_FLOW_TRACE_CODEPTR + if (codeptr != nullptr) { + register_node_addr(d1::ITT_DOMAIN_FLOW, n, FLOW_NODE, CODE_ADDRESS, &codeptr); + } +#endif +} + +static inline void fgt_node( void* codeptr, string_resource_index t, void *g, void *output_port ) { + itt_make_task_group( d1::ITT_DOMAIN_FLOW, output_port, FLOW_NODE, g, FLOW_GRAPH, t ); + fgt_internal_create_output_port( codeptr, output_port, output_port, FLOW_OUTPUT_PORT_0 ); +} + +static void fgt_node_with_body( void* codeptr, string_resource_index t, void *g, void *output_port, void *body ) { + itt_make_task_group( d1::ITT_DOMAIN_FLOW, output_port, FLOW_NODE, g, FLOW_GRAPH, t ); + fgt_internal_create_output_port(codeptr, output_port, output_port, FLOW_OUTPUT_PORT_0 ); + fgt_body( output_port, body ); +} + +static inline void fgt_node( void* codeptr, string_resource_index t, void *g, void *input_port, void *output_port ) { + fgt_node( codeptr, t, g, output_port ); + fgt_internal_create_input_port( output_port, input_port, FLOW_INPUT_PORT_0 ); +} + +static inline void fgt_node_with_body( void* codeptr, string_resource_index t, void *g, void *input_port, void *output_port, void *body ) { + fgt_node_with_body( codeptr, t, g, output_port, body ); + fgt_internal_create_input_port( output_port, input_port, FLOW_INPUT_PORT_0 ); +} + + +static inline void fgt_node( void* codeptr, string_resource_index t, void *g, void *input_port, void *decrement_port, void *output_port ) { + fgt_node( codeptr, t, g, input_port, output_port ); + fgt_internal_create_input_port( output_port, decrement_port, FLOW_INPUT_PORT_1 ); +} + +static inline void fgt_make_edge( void *output_port, void *input_port ) { + itt_relation_add( d1::ITT_DOMAIN_FLOW, output_port, FLOW_OUTPUT_PORT, __itt_relation_is_predecessor_to, input_port, FLOW_INPUT_PORT); +} + +static inline void fgt_remove_edge( void *output_port, void *input_port ) { + itt_relation_add( d1::ITT_DOMAIN_FLOW, output_port, FLOW_OUTPUT_PORT, __itt_relation_is_sibling_of, input_port, FLOW_INPUT_PORT); +} + +static inline void fgt_graph( void *g ) { + itt_make_task_group( d1::ITT_DOMAIN_FLOW, g, FLOW_GRAPH, nullptr, FLOW_NULL, FLOW_GRAPH ); +} + +static inline void fgt_begin_body( void *body ) { + itt_task_begin( d1::ITT_DOMAIN_FLOW, body, FLOW_BODY, nullptr, FLOW_NULL, FLOW_BODY ); +} + +static inline void fgt_end_body( void * ) { + itt_task_end( d1::ITT_DOMAIN_FLOW ); +} + +static inline void fgt_async_try_put_begin( void *node, void *port ) { + itt_task_begin( d1::ITT_DOMAIN_FLOW, port, FLOW_OUTPUT_PORT, node, FLOW_NODE, FLOW_OUTPUT_PORT ); +} + +static inline void fgt_async_try_put_end( void *, void * ) { + itt_task_end( d1::ITT_DOMAIN_FLOW ); +} + +static inline void fgt_async_reserve( void *node, void *graph ) { + itt_region_begin( d1::ITT_DOMAIN_FLOW, node, FLOW_NODE, graph, FLOW_GRAPH, FLOW_NULL ); +} + +static inline void fgt_async_commit( void *node, void * /*graph*/) { + itt_region_end( d1::ITT_DOMAIN_FLOW, node, FLOW_NODE ); +} + +static inline void fgt_reserve_wait( void *graph ) { + itt_region_begin( d1::ITT_DOMAIN_FLOW, graph, FLOW_GRAPH, nullptr, FLOW_NULL, FLOW_NULL ); +} + +static inline void fgt_release_wait( void *graph ) { + itt_region_end( d1::ITT_DOMAIN_FLOW, graph, FLOW_GRAPH ); +} + +#else // TBB_USE_PROFILING_TOOLS + +#define CODEPTR() nullptr + +static inline void fgt_alias_port(void * /*node*/, void * /*p*/, bool /*visible*/ ) { } + +static inline void fgt_composite ( void* /*codeptr*/, void * /*node*/, void * /*graph*/ ) { } + +static inline void fgt_graph( void * /*g*/ ) { } + +template< typename NodeType > +static inline void fgt_multioutput_node_desc( const NodeType * /*node*/, const char * /*desc*/ ) { } + +template< typename NodeType > +static inline void fgt_node_desc( const NodeType * /*node*/, const char * /*desc*/ ) { } + +static inline void fgt_graph_desc( const void * /*g*/, const char * /*desc*/ ) { } + +template< int N, typename PortsTuple > +static inline void fgt_multioutput_node( void* /*codeptr*/, string_resource_index /*t*/, void * /*g*/, void * /*input_port*/, PortsTuple & /*ports*/ ) { } + +template< int N, typename PortsTuple > +static inline void fgt_multioutput_node_with_body( void* /*codeptr*/, string_resource_index /*t*/, void * /*g*/, void * /*input_port*/, PortsTuple & /*ports*/, void * /*body*/ ) { } + +template< int N, typename PortsTuple > +static inline void fgt_multiinput_node( void* /*codeptr*/, string_resource_index /*t*/, void * /*g*/, PortsTuple & /*ports*/, void * /*output_port*/ ) { } + +static inline void fgt_multiinput_multioutput_node( void* /*codeptr*/, string_resource_index /*t*/, void * /*node*/, void * /*graph*/ ) { } + +static inline void fgt_node( void* /*codeptr*/, string_resource_index /*t*/, void * /*g*/, void * /*input_port*/, void * /*output_port*/ ) { } +static inline void fgt_node( void* /*codeptr*/, string_resource_index /*t*/, void * /*g*/, void * /*input_port*/, void * /*decrement_port*/, void * /*output_port*/ ) { } + +static inline void fgt_node_with_body( void* /*codeptr*/, string_resource_index /*t*/, void * /*g*/, void * /*output_port*/, void * /*body*/ ) { } +static inline void fgt_node_with_body( void* /*codeptr*/, string_resource_index /*t*/, void * /*g*/, void * /*input_port*/, void * /*output_port*/, void * /*body*/ ) { } + +static inline void fgt_make_edge( void * /*output_port*/, void * /*input_port*/ ) { } +static inline void fgt_remove_edge( void * /*output_port*/, void * /*input_port*/ ) { } + +static inline void fgt_begin_body( void * /*body*/ ) { } +static inline void fgt_end_body( void * /*body*/) { } + +static inline void fgt_async_try_put_begin( void * /*node*/, void * /*port*/ ) { } +static inline void fgt_async_try_put_end( void * /*node*/ , void * /*port*/ ) { } +static inline void fgt_async_reserve( void * /*node*/, void * /*graph*/ ) { } +static inline void fgt_async_commit( void * /*node*/, void * /*graph*/ ) { } +static inline void fgt_reserve_wait( void * /*graph*/ ) { } +static inline void fgt_release_wait( void * /*graph*/ ) { } + +template< typename NodeType > +void fgt_multiinput_multioutput_node_desc( const NodeType * /*node*/, const char * /*desc*/ ) { } + +template < typename PortsTuple, int N > +struct fgt_internal_input_alias_helper { + static void alias_port( void * /*node*/, PortsTuple & /*ports*/ ) { } +}; + +template < typename PortsTuple, int N > +struct fgt_internal_output_alias_helper { + static void alias_port( void * /*node*/, PortsTuple & /*ports*/ ) { } +}; + +#endif // TBB_USE_PROFILING_TOOLS + +} // d2 +} // namespace detail +} // namespace tbb + +#endif // _FGT_GRAPH_TRACE_IMPL_H diff --git a/src/tbb/include/oneapi/tbb/detail/_flow_graph_types_impl.h b/src/tbb/include/oneapi/tbb/detail/_flow_graph_types_impl.h new file mode 100644 index 000000000..c5634413b --- /dev/null +++ b/src/tbb/include/oneapi/tbb/detail/_flow_graph_types_impl.h @@ -0,0 +1,422 @@ +/* + Copyright (c) 2005-2024 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB__flow_graph_types_impl_H +#define __TBB__flow_graph_types_impl_H + +#ifndef __TBB_flow_graph_H +#error Do not #include this internal file directly; use public TBB headers instead. +#endif + +// included in namespace tbb::detail::d2 + +// the change to key_matching (adding a K and KHash template parameter, making it a class) +// means we have to pass this data to the key_matching_port. All the ports have only one +// template parameter, so we have to wrap the following types in a trait: +// +// . K == key_type +// . KHash == hash and compare for Key +// . TtoK == function_body that given an object of T, returns its K +// . T == type accepted by port, and stored in the hash table +// +// The port will have an additional parameter on node construction, which is a function_body +// that accepts a const T& and returns a K which is the field in T which is its K. +template +struct KeyTrait { + typedef Kp K; + typedef Tp T; + typedef type_to_key_function_body TtoK; + typedef KHashp KHash; +}; + +// wrap each element of a tuple in a template, and make a tuple of the result. +template class PT, typename TypeTuple> +struct wrap_tuple_elements; + +// A wrapper that generates the traits needed for each port of a key-matching join, +// and the type of the tuple of input ports. +template class PT, typename KeyTraits, typename TypeTuple> +struct wrap_key_tuple_elements; + +template class PT, typename... Args> +struct wrap_tuple_elements >{ + typedef typename std::tuple... > type; +}; + +template class PT, typename KeyTraits, typename... Args> +struct wrap_key_tuple_elements > { + typedef typename KeyTraits::key_type K; + typedef typename KeyTraits::hash_compare_type KHash; + typedef typename std::tuple >... > type; +}; + +template< int... S > class sequence {}; + +template< int N, int... S > +struct make_sequence : make_sequence < N - 1, N - 1, S... > {}; + +template< int... S > +struct make_sequence < 0, S... > { + typedef sequence type; +}; + +template struct alignment_of { + typedef struct { char t; U padded; } test_alignment; + static const size_t value = sizeof(test_alignment) - sizeof(U); +}; + +template +struct max_alignment_helper; + +template +struct max_alignment_helper { + using type = typename max_alignment_helper::type>::type; +}; + +template +struct max_alignment_helper { + using type = typename std::conditional::type; +}; + +template +using max_alignment_helper_t = typename max_alignment_helper::type; + +#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) +// #pragma warning(push) +// #pragma warning(disable: 4324) // warning C4324: structure was padded due to alignment specifier +#endif + +// T1, T2 are actual types stored. The space defined for T1 in the type returned +// is a char array of the correct size. Type T2 should be trivially-constructible, +// T1 must be explicitly managed. + +template +struct alignas(alignof(max_alignment_helper_t)) aligned_pair { + char first[sizeof(T1)]; + T2 second; +}; + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT +template +struct alignas(alignof(max_alignment_helper_t)) aligned_triple { + char first[sizeof(T1)]; + T2 second; + T3 third; +}; +#endif + + +#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) +// #pragma warning(pop) // warning 4324 is back +#endif + +// support for variant type +// type we use when we're not storing a value +struct default_constructed { }; + +// type which contains another type, tests for what type is contained, and references to it. +// Wrapper +// void CopyTo( void *newSpace) : builds a Wrapper copy of itself in newSpace + +// struct to allow us to copy and test the type of objects +struct WrapperBase { + virtual ~WrapperBase() {} + virtual void CopyTo(void* /*newSpace*/) const = 0; +}; + +// Wrapper contains a T, with the ability to test what T is. The Wrapper can be +// constructed from a T, can be copy-constructed from another Wrapper, and can be +// examined via value(), but not modified. +template +struct Wrapper: public WrapperBase { + typedef T value_type; + typedef T* pointer_type; +private: + T value_space; +public: + const value_type &value() const { return value_space; } + +private: + Wrapper(); + + // on exception will ensure the Wrapper will contain only a trivially-constructed object + struct _unwind_space { + pointer_type space; + _unwind_space(pointer_type p) : space(p) {} + ~_unwind_space() { + if(space) (void) new (space) Wrapper(default_constructed()); + } + }; +public: + explicit Wrapper( const T& other ) : value_space(other) { } + explicit Wrapper(const Wrapper& other) = delete; + + void CopyTo(void* newSpace) const override { + _unwind_space guard((pointer_type)newSpace); + (void) new(newSpace) Wrapper(value_space); + guard.space = nullptr; + } + ~Wrapper() { } +}; + +// specialization for array objects +template +struct Wrapper : public WrapperBase { + typedef T value_type; + typedef T* pointer_type; + // space must be untyped. + typedef T ArrayType[N]; +private: + // The space is not of type T[N] because when copy-constructing, it would be + // default-initialized and then copied to in some fashion, resulting in two + // constructions and one destruction per element. If the type is char[ ], we + // placement new into each element, resulting in one construction per element. + static const size_t space_size = sizeof(ArrayType); + char value_space[space_size]; + + + // on exception will ensure the already-built objects will be destructed + // (the value_space is a char array, so it is already trivially-destructible.) + struct _unwind_class { + pointer_type space; + int already_built; + _unwind_class(pointer_type p) : space(p), already_built(0) {} + ~_unwind_class() { + if(space) { + for(size_t i = already_built; i > 0 ; --i ) space[i-1].~value_type(); + (void) new(space) Wrapper(default_constructed()); + } + } + }; +public: + const ArrayType &value() const { + char *vp = const_cast(value_space); + return reinterpret_cast(*vp); + } + +private: + Wrapper(); +public: + // have to explicitly construct because other decays to a const value_type* + explicit Wrapper(const ArrayType& other) { + _unwind_class guard((pointer_type)value_space); + pointer_type vp = reinterpret_cast(&value_space); + for(size_t i = 0; i < N; ++i ) { + (void) new(vp++) value_type(other[i]); + ++(guard.already_built); + } + guard.space = nullptr; + } + explicit Wrapper(const Wrapper& other) : WrapperBase() { + // we have to do the heavy lifting to copy contents + _unwind_class guard((pointer_type)value_space); + pointer_type dp = reinterpret_cast(value_space); + pointer_type sp = reinterpret_cast(const_cast(other.value_space)); + for(size_t i = 0; i < N; ++i, ++dp, ++sp) { + (void) new(dp) value_type(*sp); + ++(guard.already_built); + } + guard.space = nullptr; + } + + void CopyTo(void* newSpace) const override { + (void) new(newSpace) Wrapper(*this); // exceptions handled in copy constructor + } + + ~Wrapper() { + // have to destroy explicitly in reverse order + pointer_type vp = reinterpret_cast(&value_space); + for(size_t i = N; i > 0 ; --i ) vp[i-1].~value_type(); + } +}; + +// given a tuple, return the type of the element that has the maximum alignment requirement. +// Given a tuple and that type, return the number of elements of the object with the max +// alignment requirement that is at least as big as the largest object in the tuple. + +template struct pick_one; +template struct pick_one { typedef T1 type; }; +template struct pick_one { typedef T2 type; }; + +template< template class Selector, typename T1, typename T2 > +struct pick_max { + typedef typename pick_one< (Selector::value > Selector::value), T1, T2 >::type type; +}; + +template struct size_of { static const int value = sizeof(T); }; + +template< size_t N, class Tuple, template class Selector > struct pick_tuple_max { + typedef typename pick_tuple_max::type LeftMaxType; + typedef typename std::tuple_element::type ThisType; + typedef typename pick_max::type type; +}; + +template< class Tuple, template class Selector > struct pick_tuple_max<0, Tuple, Selector> { + typedef typename std::tuple_element<0, Tuple>::type type; +}; + +// is the specified type included in a tuple? +template +struct is_element_of { + typedef typename std::tuple_element::type T_i; + static const bool value = std::is_same::value || is_element_of::value; +}; + +template +struct is_element_of { + typedef typename std::tuple_element<0, Tuple>::type T_i; + static const bool value = std::is_same::value; +}; + +// allow the construction of types that are listed tuple. If a disallowed type +// construction is written, a method involving this type is created. The +// type has no definition, so a syntax error is generated. +template struct ERROR_Type_Not_allowed_In_Tagged_Msg_Not_Member_Of_Tuple; + +template struct do_if; +template +struct do_if { + static void construct(void *mySpace, const T& x) { + (void) new(mySpace) Wrapper(x); + } +}; +template +struct do_if { + static void construct(void * /*mySpace*/, const T& x) { + // This method is instantiated when the type T does not match any of the + // element types in the Tuple in variant. + ERROR_Type_Not_allowed_In_Tagged_Msg_Not_Member_Of_Tuple::bad_type(x); + } +}; + +// Tuple tells us the allowed types that variant can hold. It determines the alignment of the space in +// Wrapper, and how big Wrapper is. +// +// the object can only be tested for type, and a read-only reference can be fetched by cast_to(). + +using tbb::detail::punned_cast; +struct tagged_null_type {}; +template +class tagged_msg { + typedef std::tuple= 6 + , T5 + #endif + #if __TBB_VARIADIC_MAX >= 7 + , T6 + #endif + #if __TBB_VARIADIC_MAX >= 8 + , T7 + #endif + #if __TBB_VARIADIC_MAX >= 9 + , T8 + #endif + #if __TBB_VARIADIC_MAX >= 10 + , T9 + #endif + > Tuple; + +private: + class variant { + static const size_t N = std::tuple_size::value; + typedef typename pick_tuple_max::type AlignType; + typedef typename pick_tuple_max::type MaxSizeType; + static const size_t MaxNBytes = (sizeof(Wrapper)+sizeof(AlignType)-1); + static const size_t MaxNElements = MaxNBytes/sizeof(AlignType); + typedef aligned_space SpaceType; + SpaceType my_space; + static const size_t MaxSize = sizeof(SpaceType); + + public: + variant() { (void) new(&my_space) Wrapper(default_constructed()); } + + template + variant( const T& x ) { + do_if::value>::construct(&my_space,x); + } + + variant(const variant& other) { + const WrapperBase * h = punned_cast(&(other.my_space)); + h->CopyTo(&my_space); + } + + // assignment must destroy and re-create the Wrapper type, as there is no way + // to create a Wrapper-to-Wrapper assign even if we find they agree in type. + void operator=( const variant& rhs ) { + if(&rhs != this) { + WrapperBase *h = punned_cast(&my_space); + h->~WrapperBase(); + const WrapperBase *ch = punned_cast(&(rhs.my_space)); + ch->CopyTo(&my_space); + } + } + + template + const U& variant_cast_to() const { + const Wrapper *h = dynamic_cast*>(punned_cast(&my_space)); + if(!h) { + throw_exception(exception_id::bad_tagged_msg_cast); + } + return h->value(); + } + template + bool variant_is_a() const { return dynamic_cast*>(punned_cast(&my_space)) != nullptr; } + + bool variant_is_default_constructed() const {return variant_is_a();} + + ~variant() { + WrapperBase *h = punned_cast(&my_space); + h->~WrapperBase(); + } + }; //class variant + + TagType my_tag; + variant my_msg; + +public: + tagged_msg(): my_tag(TagType(~0)), my_msg(){} + + template + tagged_msg(T const &index, R const &value) : my_tag(index), my_msg(value) {} + + template + tagged_msg(T const &index, R (&value)[N]) : my_tag(index), my_msg(value) {} + + void set_tag(TagType const &index) {my_tag = index;} + TagType tag() const {return my_tag;} + + template + const V& cast_to() const {return my_msg.template variant_cast_to();} + + template + bool is_a() const {return my_msg.template variant_is_a();} + + bool is_default_constructed() const {return my_msg.variant_is_default_constructed();} +}; //class tagged_msg + +// template to simplify cast and test for tagged_msg in template contexts +template +const V& cast_to(T const &t) { return t.template cast_to(); } + +template +bool is_a(T const &t) { return t.template is_a(); } + +enum op_stat { WAIT = 0, SUCCEEDED, FAILED }; + +#endif /* __TBB__flow_graph_types_impl_H */ diff --git a/src/tbb/include/oneapi/tbb/detail/_hash_compare.h b/src/tbb/include/oneapi/tbb/detail/_hash_compare.h new file mode 100644 index 000000000..2060a2c1b --- /dev/null +++ b/src/tbb/include/oneapi/tbb/detail/_hash_compare.h @@ -0,0 +1,147 @@ +/* + Copyright (c) 2005-2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_detail__hash_compare_H +#define __TBB_detail__hash_compare_H + +#include + +#include "_containers_helpers.h" + +namespace tbb { +namespace detail { +namespace d1 { + +template +class hash_compare { + using is_transparent_hash = has_transparent_key_equal; +public: + using hasher = Hash; + using key_equal = typename is_transparent_hash::type; + + hash_compare() = default; + hash_compare( hasher hash, key_equal equal ) : my_hasher(hash), my_equal(equal) {} + + std::size_t operator()( const Key& key ) const { + return std::size_t(my_hasher(key)); + } + + bool operator()( const Key& key1, const Key& key2 ) const { + return my_equal(key1, key2); + } + + template ::type> + std::size_t operator()( const K& key ) const { + return std::size_t(my_hasher(key)); + } + + template ::type> + bool operator()( const K1& key1, const K2& key2 ) const { + return my_equal(key1, key2); + } + + hasher hash_function() const { + return my_hasher; + } + + key_equal key_eq() const { + return my_equal; + } + + +private: + hasher my_hasher; + key_equal my_equal; +}; // class hash_compare + +//! hash_compare that is default argument for concurrent_hash_map +template +class tbb_hash_compare { +public: + std::size_t hash( const Key& a ) const { return my_hash_func(a); } +#if defined(_MSC_VER) && _MSC_VER <= 1900 +// #pragma warning (push) +// MSVC 2015 throws a strange warning: 'std::size_t': forcing value to bool 'true' or 'false' +// #pragma warning (disable: 4800) +#endif + bool equal( const Key& a, const Key& b ) const { return my_key_equal(a, b); } +#if defined(_MSC_VER) && _MSC_VER <= 1900 +// #pragma warning (pop) +#endif +private: + std::hash my_hash_func; + std::equal_to my_key_equal; +}; + +} // namespace d1 +#if __TBB_CPP20_CONCEPTS_PRESENT +inline namespace d0 { + +template +concept hash_compare = std::copy_constructible && + requires( const std::remove_reference_t& hc, const Key& key1, const Key& key2 ) { + { hc.hash(key1) } -> std::same_as; + { hc.equal(key1, key2) } -> std::convertible_to; + }; + +} // namespace d0 +#endif // __TBB_CPP20_CONCEPTS_PRESENT +} // namespace detail +} // namespace tbb + +#if TBB_DEFINE_STD_HASH_SPECIALIZATIONS + +namespace std { + +template +struct hash> { +public: + std::size_t operator()( const std::pair& p ) const { + return first_hash(p.first) ^ second_hash(p.second); + } + +private: + std::hash first_hash; + std::hash second_hash; +}; // struct hash + +// Apple clang and MSVC defines their own specializations for std::hash> +#if !(_LIBCPP_VERSION) && !(_CPPLIB_VER) + +template +struct hash> { +public: + std::size_t operator()( const std::basic_string& s ) const { + std::size_t h = 0; + for ( const CharT* c = s.c_str(); *c; ++c ) { + h = h * hash_multiplier ^ char_hash(*c); + } + return h; + } + +private: + static constexpr std::size_t hash_multiplier = tbb::detail::select_size_t_constant<2654435769U, 11400714819323198485ULL>::value; + + std::hash char_hash; +}; // struct hash + +#endif // !(_LIBCPP_VERSION || _CPPLIB_VER) + +} // namespace std + +#endif // TBB_DEFINE_STD_HASH_SPECIALIZATIONS + +#endif // __TBB_detail__hash_compare_H diff --git a/src/tbb/include/oneapi/tbb/detail/_intrusive_list_node.h b/src/tbb/include/oneapi/tbb/detail/_intrusive_list_node.h new file mode 100644 index 000000000..69286c8fa --- /dev/null +++ b/src/tbb/include/oneapi/tbb/detail/_intrusive_list_node.h @@ -0,0 +1,41 @@ +/* + Copyright (c) 2005-2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef _TBB_detail__intrusive_list_node_H +#define _TBB_detail__intrusive_list_node_H + +namespace tbb { +namespace detail { +namespace d1 { + +//! Data structure to be inherited by the types that can form intrusive lists. +/** Intrusive list is formed by means of the member_intrusive_list template class. + Note that type T must derive from intrusive_list_node either publicly or + declare instantiation member_intrusive_list as a friend. + This class implements a limited subset of std::list interface. **/ +struct intrusive_list_node { + intrusive_list_node* my_prev_node{}; + intrusive_list_node* my_next_node{}; +#if TBB_USE_ASSERT + intrusive_list_node() { my_prev_node = my_next_node = this; } +#endif /* TBB_USE_ASSERT */ +}; + +} // namespace d1 +} // namespace detail +} // namespace tbb + +#endif // _TBB_detail__intrusive_list_node_H diff --git a/src/tbb/include/oneapi/tbb/detail/_machine.h b/src/tbb/include/oneapi/tbb/detail/_machine.h new file mode 100644 index 000000000..ca481380c --- /dev/null +++ b/src/tbb/include/oneapi/tbb/detail/_machine.h @@ -0,0 +1,373 @@ +/* + Copyright (c) 2005-2024 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_detail__machine_H +#define __TBB_detail__machine_H + +#include "_config.h" +#include "_assert.h" + +#include +#include +#include +#include + +#ifdef _WIN32 +#include +#ifdef __TBBMALLOC_BUILD +#define WIN32_LEAN_AND_MEAN +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include // SwitchToThread() +#endif +#ifdef _MSC_VER +#if __TBB_x86_64 || __TBB_x86_32 +#pragma intrinsic(__rdtsc) +#endif +#endif +#endif +#if __TBB_x86_64 || __TBB_x86_32 +#include // _mm_pause +#endif +#if (_WIN32) +#include // _control87 +#endif + +#if __TBB_GLIBCXX_THIS_THREAD_YIELD_BROKEN +#include // sched_yield +#else +#include // std::this_thread::yield() +#endif + +namespace tbb { +namespace detail { +inline namespace d0 { + +//-------------------------------------------------------------------------------------------------- +// Yield implementation +//-------------------------------------------------------------------------------------------------- + +#if __TBB_GLIBCXX_THIS_THREAD_YIELD_BROKEN +static inline void yield() { + int err = sched_yield(); + __TBB_ASSERT_EX(err == 0, "sched_yield has failed"); +} +#elif __TBBMALLOC_BUILD && _WIN32 +// Use Windows API for yield in tbbmalloc to avoid dependency on C++ runtime with some implementations. +static inline void yield() { + SwitchToThread(); +} +#else +using std::this_thread::yield; +#endif + +//-------------------------------------------------------------------------------------------------- +// atomic_fence_seq_cst implementation +//-------------------------------------------------------------------------------------------------- + +static inline void atomic_fence_seq_cst() { +#if (__TBB_x86_64 || __TBB_x86_32) && defined(__GNUC__) && __GNUC__ < 11 + unsigned char dummy = 0u; + __asm__ __volatile__ ("lock; notb %0" : "+m" (dummy) :: "memory"); +#else + std::atomic_thread_fence(std::memory_order_seq_cst); +#endif +} + +//-------------------------------------------------------------------------------------------------- +// Pause implementation +//-------------------------------------------------------------------------------------------------- + +static inline void machine_pause(int32_t delay) { +#if __TBB_x86_64 || __TBB_x86_32 + while (delay-- > 0) { _mm_pause(); } +#elif __ARM_ARCH_7A__ || __aarch64__ + while (delay-- > 0) { __asm__ __volatile__("isb sy" ::: "memory"); } +#else /* Generic */ + (void)delay; // suppress without including _template_helpers.h + yield(); +#endif +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// +// tbb::detail::log2() implementation +//////////////////////////////////////////////////////////////////////////////////////////////////// +// TODO: Use log2p1() function that will be available in C++20 standard + +#if defined(__GNUC__) || defined(__clang__) +namespace gnu_builtins { + inline uintptr_t clz(unsigned int x) { return static_cast(__builtin_clz(x)); } + inline uintptr_t clz(unsigned long int x) { return static_cast(__builtin_clzl(x)); } + inline uintptr_t clz(unsigned long long int x) { return static_cast(__builtin_clzll(x)); } +} +#elif defined(_MSC_VER) +#pragma intrinsic(__TBB_W(_BitScanReverse)) +namespace msvc_intrinsics { + static inline uintptr_t bit_scan_reverse(uintptr_t i) { + unsigned long j; + __TBB_W(_BitScanReverse)( &j, i ); + return j; + } +} +#endif + +template +constexpr std::uintptr_t number_of_bits() { + return sizeof(T) * CHAR_BIT; +} + +// logarithm is the index of the most significant non-zero bit +static inline uintptr_t machine_log2(uintptr_t x) { +#if defined(__GNUC__) || defined(__clang__) + // If P is a power of 2 and x() - 1) ^ gnu_builtins::clz(x); +#elif defined(_MSC_VER) + return msvc_intrinsics::bit_scan_reverse(x); +#elif __i386__ || __i386 /*for Sun OS*/ || __MINGW32__ + uintptr_t j, i = x; + __asm__("bsr %1,%0" : "=r"(j) : "r"(i)); + return j; +#elif __powerpc__ || __POWERPC__ + #if __TBB_WORDSIZE==8 + __asm__ __volatile__ ("cntlzd %0,%0" : "+r"(x)); + return 63 - static_cast(x); + #else + __asm__ __volatile__ ("cntlzw %0,%0" : "+r"(x)); + return 31 - static_cast(x); + #endif /*__TBB_WORDSIZE*/ +#elif __sparc + uint64_t count; + // one hot encode + x |= (x >> 1); + x |= (x >> 2); + x |= (x >> 4); + x |= (x >> 8); + x |= (x >> 16); + x |= (x >> 32); + // count 1's + __asm__ ("popc %1, %0" : "=r"(count) : "r"(x) ); + return count - 1; +#else + intptr_t result = 0; + + if( sizeof(x) > 4 && (uintptr_t tmp = x >> 32) ) { x = tmp; result += 32; } + if( uintptr_t tmp = x >> 16 ) { x = tmp; result += 16; } + if( uintptr_t tmp = x >> 8 ) { x = tmp; result += 8; } + if( uintptr_t tmp = x >> 4 ) { x = tmp; result += 4; } + if( uintptr_t tmp = x >> 2 ) { x = tmp; result += 2; } + + return (x & 2) ? result + 1 : result; +#endif +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// +// tbb::detail::reverse_bits() implementation +//////////////////////////////////////////////////////////////////////////////////////////////////// +#if TBB_USE_CLANG_BITREVERSE_BUILTINS +namespace llvm_builtins { + inline uint8_t builtin_bitreverse(uint8_t x) { return __builtin_bitreverse8 (x); } + inline uint16_t builtin_bitreverse(uint16_t x) { return __builtin_bitreverse16(x); } + inline uint32_t builtin_bitreverse(uint32_t x) { return __builtin_bitreverse32(x); } + inline uint64_t builtin_bitreverse(uint64_t x) { return __builtin_bitreverse64(x); } +} +#else // generic +template +struct reverse { + static const T byte_table[256]; +}; + +template +const T reverse::byte_table[256] = { + 0x00, 0x80, 0x40, 0xC0, 0x20, 0xA0, 0x60, 0xE0, 0x10, 0x90, 0x50, 0xD0, 0x30, 0xB0, 0x70, 0xF0, + 0x08, 0x88, 0x48, 0xC8, 0x28, 0xA8, 0x68, 0xE8, 0x18, 0x98, 0x58, 0xD8, 0x38, 0xB8, 0x78, 0xF8, + 0x04, 0x84, 0x44, 0xC4, 0x24, 0xA4, 0x64, 0xE4, 0x14, 0x94, 0x54, 0xD4, 0x34, 0xB4, 0x74, 0xF4, + 0x0C, 0x8C, 0x4C, 0xCC, 0x2C, 0xAC, 0x6C, 0xEC, 0x1C, 0x9C, 0x5C, 0xDC, 0x3C, 0xBC, 0x7C, 0xFC, + 0x02, 0x82, 0x42, 0xC2, 0x22, 0xA2, 0x62, 0xE2, 0x12, 0x92, 0x52, 0xD2, 0x32, 0xB2, 0x72, 0xF2, + 0x0A, 0x8A, 0x4A, 0xCA, 0x2A, 0xAA, 0x6A, 0xEA, 0x1A, 0x9A, 0x5A, 0xDA, 0x3A, 0xBA, 0x7A, 0xFA, + 0x06, 0x86, 0x46, 0xC6, 0x26, 0xA6, 0x66, 0xE6, 0x16, 0x96, 0x56, 0xD6, 0x36, 0xB6, 0x76, 0xF6, + 0x0E, 0x8E, 0x4E, 0xCE, 0x2E, 0xAE, 0x6E, 0xEE, 0x1E, 0x9E, 0x5E, 0xDE, 0x3E, 0xBE, 0x7E, 0xFE, + 0x01, 0x81, 0x41, 0xC1, 0x21, 0xA1, 0x61, 0xE1, 0x11, 0x91, 0x51, 0xD1, 0x31, 0xB1, 0x71, 0xF1, + 0x09, 0x89, 0x49, 0xC9, 0x29, 0xA9, 0x69, 0xE9, 0x19, 0x99, 0x59, 0xD9, 0x39, 0xB9, 0x79, 0xF9, + 0x05, 0x85, 0x45, 0xC5, 0x25, 0xA5, 0x65, 0xE5, 0x15, 0x95, 0x55, 0xD5, 0x35, 0xB5, 0x75, 0xF5, + 0x0D, 0x8D, 0x4D, 0xCD, 0x2D, 0xAD, 0x6D, 0xED, 0x1D, 0x9D, 0x5D, 0xDD, 0x3D, 0xBD, 0x7D, 0xFD, + 0x03, 0x83, 0x43, 0xC3, 0x23, 0xA3, 0x63, 0xE3, 0x13, 0x93, 0x53, 0xD3, 0x33, 0xB3, 0x73, 0xF3, + 0x0B, 0x8B, 0x4B, 0xCB, 0x2B, 0xAB, 0x6B, 0xEB, 0x1B, 0x9B, 0x5B, 0xDB, 0x3B, 0xBB, 0x7B, 0xFB, + 0x07, 0x87, 0x47, 0xC7, 0x27, 0xA7, 0x67, 0xE7, 0x17, 0x97, 0x57, 0xD7, 0x37, 0xB7, 0x77, 0xF7, + 0x0F, 0x8F, 0x4F, 0xCF, 0x2F, 0xAF, 0x6F, 0xEF, 0x1F, 0x9F, 0x5F, 0xDF, 0x3F, 0xBF, 0x7F, 0xFF +}; + +inline unsigned char reverse_byte(unsigned char src) { + return reverse::byte_table[src]; +} +#endif // TBB_USE_CLANG_BITREVERSE_BUILTINS + +template +T machine_reverse_bits(T src) { +#if TBB_USE_CLANG_BITREVERSE_BUILTINS + return builtin_bitreverse(fixed_width_cast(src)); +#else /* Generic */ + T dst; + unsigned char *original = reinterpret_cast(&src); + unsigned char *reversed = reinterpret_cast(&dst); + + for ( int i = sizeof(T) - 1; i >= 0; i-- ) { + reversed[i] = reverse_byte( original[sizeof(T) - i - 1] ); + } + + return dst; +#endif // TBB_USE_CLANG_BITREVERSE_BUILTINS +} + +} // inline namespace d0 + +namespace d1 { + +#if (_WIN32) +// API to retrieve/update FPU control setting +#define __TBB_CPU_CTL_ENV_PRESENT 1 +struct cpu_ctl_env { + unsigned int x87cw{}; +#if (__TBB_x86_64) + // Changing the infinity mode or the floating-point precision is not supported on x64. + // The attempt causes an assertion. See + // https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/control87-controlfp-control87-2 + static constexpr unsigned int X87CW_CONTROL_MASK = _MCW_DN | _MCW_EM | _MCW_RC; +#else + static constexpr unsigned int X87CW_CONTROL_MASK = ~0U; +#endif +#if (__TBB_x86_32 || __TBB_x86_64) + unsigned int mxcsr{}; + static constexpr unsigned int MXCSR_CONTROL_MASK = ~0x3fu; /* all except last six status bits */ +#endif + + bool operator!=( const cpu_ctl_env& ctl ) const { + return +#if (__TBB_x86_32 || __TBB_x86_64) + mxcsr != ctl.mxcsr || +#endif + x87cw != ctl.x87cw; + } + void get_env() { + x87cw = _control87(0, 0); +#if (__TBB_x86_32 || __TBB_x86_64) + mxcsr = _mm_getcsr(); +#endif + } + void set_env() const { + _control87(x87cw, X87CW_CONTROL_MASK); +#if (__TBB_x86_32 || __TBB_x86_64) + _mm_setcsr(mxcsr & MXCSR_CONTROL_MASK); +#endif + } +}; +#elif (__TBB_x86_32 || __TBB_x86_64) +// API to retrieve/update FPU control setting +#define __TBB_CPU_CTL_ENV_PRESENT 1 +struct cpu_ctl_env { + int mxcsr{}; + short x87cw{}; + static const int MXCSR_CONTROL_MASK = ~0x3f; /* all except last six status bits */ + + bool operator!=(const cpu_ctl_env& ctl) const { + return mxcsr != ctl.mxcsr || x87cw != ctl.x87cw; + } + void get_env() { + __asm__ __volatile__( + "stmxcsr %0\n\t" + "fstcw %1" + : "=m"(mxcsr), "=m"(x87cw) + ); + mxcsr &= MXCSR_CONTROL_MASK; + } + void set_env() const { + __asm__ __volatile__( + "ldmxcsr %0\n\t" + "fldcw %1" + : : "m"(mxcsr), "m"(x87cw) + ); + } +}; +#endif + +} // namespace d1 + +} // namespace detail +} // namespace tbb + +#if !__TBB_CPU_CTL_ENV_PRESENT +#include + +#include + +namespace tbb { +namespace detail { + +namespace r1 { +void* __TBB_EXPORTED_FUNC cache_aligned_allocate(std::size_t size); +void __TBB_EXPORTED_FUNC cache_aligned_deallocate(void* p); +} // namespace r1 + +namespace d1 { + +class cpu_ctl_env { + fenv_t *my_fenv_ptr; +public: + cpu_ctl_env() : my_fenv_ptr(nullptr) {} + ~cpu_ctl_env() { + if ( my_fenv_ptr ) + r1::cache_aligned_deallocate( (void*)my_fenv_ptr ); + } + // It is possible not to copy memory but just to copy pointers but the following issues should be addressed: + // 1. The arena lifetime and the context lifetime are independent; + // 2. The user is allowed to recapture different FPU settings to context so 'current FPU settings' inside + // dispatch loop may become invalid. + // But do we really want to improve the fenv implementation? It seems to be better to replace the fenv implementation + // with a platform specific implementation. + cpu_ctl_env( const cpu_ctl_env &src ) : my_fenv_ptr(nullptr) { + *this = src; + } + cpu_ctl_env& operator=( const cpu_ctl_env &src ) { + __TBB_ASSERT( src.my_fenv_ptr, nullptr); + if ( !my_fenv_ptr ) + my_fenv_ptr = (fenv_t*)r1::cache_aligned_allocate(sizeof(fenv_t)); + *my_fenv_ptr = *src.my_fenv_ptr; + return *this; + } + bool operator!=( const cpu_ctl_env &ctl ) const { + __TBB_ASSERT( my_fenv_ptr, "cpu_ctl_env is not initialized." ); + __TBB_ASSERT( ctl.my_fenv_ptr, "cpu_ctl_env is not initialized." ); + return std::memcmp( (void*)my_fenv_ptr, (void*)ctl.my_fenv_ptr, sizeof(fenv_t) ); + } + void get_env () { + if ( !my_fenv_ptr ) + my_fenv_ptr = (fenv_t*)r1::cache_aligned_allocate(sizeof(fenv_t)); + fegetenv( my_fenv_ptr ); + } + const cpu_ctl_env& set_env () const { + __TBB_ASSERT( my_fenv_ptr, "cpu_ctl_env is not initialized." ); + fesetenv( my_fenv_ptr ); + return *this; + } +}; + +} // namespace d1 +} // namespace detail +} // namespace tbb + +#endif /* !__TBB_CPU_CTL_ENV_PRESENT */ + +#endif // __TBB_detail__machine_H diff --git a/src/tbb/include/oneapi/tbb/detail/_mutex_common.h b/src/tbb/include/oneapi/tbb/detail/_mutex_common.h new file mode 100644 index 000000000..4650c1926 --- /dev/null +++ b/src/tbb/include/oneapi/tbb/detail/_mutex_common.h @@ -0,0 +1,61 @@ +/* + Copyright (c) 2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_detail__mutex_common_H +#define __TBB_detail__mutex_common_H + +#include "_config.h" +#include "_utils.h" + +#if __TBB_CPP20_CONCEPTS_PRESENT +#include + +namespace tbb { +namespace detail { +inline namespace d0 { + +template +concept mutex_scoped_lock = std::default_initializable && + std::constructible_from && + requires( Lock& lock, Mutex& mutex ) { + lock.acquire(mutex); + { lock.try_acquire(mutex) } -> adaptive_same_as; + lock.release(); + }; + +template +concept rw_mutex_scoped_lock = mutex_scoped_lock && + std::constructible_from && + requires( Lock& lock, Mutex& mutex ) { + lock.acquire(mutex, false); + { lock.try_acquire(mutex, false) } -> adaptive_same_as; + { lock.upgrade_to_writer() } -> adaptive_same_as; + { lock.downgrade_to_reader() } -> adaptive_same_as; + }; + +template +concept scoped_lockable = mutex_scoped_lock; + +template +concept rw_scoped_lockable = scoped_lockable && + rw_mutex_scoped_lock; + +} // namespace d0 +} // namespace detail +} // namespace tbb + +#endif // __TBB_CPP20_CONCEPTS_PRESENT +#endif // __TBB_detail__mutex_common_H diff --git a/src/tbb/src/rml/server/lin-rml-export.def b/src/tbb/include/oneapi/tbb/detail/_namespace_injection.h similarity index 69% rename from src/tbb/src/rml/server/lin-rml-export.def rename to src/tbb/include/oneapi/tbb/detail/_namespace_injection.h index 5fb069b46..2e1df3093 100644 --- a/src/tbb/src/rml/server/lin-rml-export.def +++ b/src/tbb/include/oneapi/tbb/detail/_namespace_injection.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2020-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,19 +12,13 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. +*/ +// All public entities of the OneAPI Spec are available under oneapi namespace +// Define tbb namespace first as it might not be known yet +namespace tbb {} - -*/ - -{ -global: -__RML_open_factory; -__RML_close_factory; -__TBB_make_rml_server; -__KMP_make_rml_server; -__TBB_call_with_my_server_info; -__KMP_call_with_my_server_info; -local:*; -}; +namespace oneapi { +namespace tbb = ::tbb; +} diff --git a/src/tbb/include/oneapi/tbb/detail/_node_handle.h b/src/tbb/include/oneapi/tbb/detail/_node_handle.h new file mode 100644 index 000000000..265be0755 --- /dev/null +++ b/src/tbb/include/oneapi/tbb/detail/_node_handle.h @@ -0,0 +1,162 @@ +/* + Copyright (c) 2019-2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_detail__node_handle_H +#define __TBB_detail__node_handle_H + +#include "_allocator_traits.h" +#include "_assert.h" + +namespace tbb { +namespace detail { +namespace d1 { + +// A structure to access private node handle methods in internal TBB classes +// Regular friend declaration is not convenient because classes which use node handle +// can be placed in the different versioning namespaces. +struct node_handle_accessor { + template + static typename NodeHandleType::node* get_node_ptr( NodeHandleType& nh ) { + return nh.get_node_ptr(); + } + + template + static NodeHandleType construct( typename NodeHandleType::node* node_ptr ) { + return NodeHandleType{node_ptr}; + } + + template + static void deactivate( NodeHandleType& nh ) { + nh.deactivate(); + } +}; // struct node_handle_accessor + +template +class node_handle_base { +public: + using allocator_type = Allocator; +protected: + using node = Node; + using allocator_traits_type = tbb::detail::allocator_traits; +public: + + node_handle_base() : my_node(nullptr), my_allocator() {} + node_handle_base(node_handle_base&& nh) : my_node(nh.my_node), + my_allocator(std::move(nh.my_allocator)) { + nh.my_node = nullptr; + } + + __TBB_nodiscard bool empty() const { return my_node == nullptr; } + explicit operator bool() const { return my_node != nullptr; } + + ~node_handle_base() { internal_destroy(); } + + node_handle_base& operator=( node_handle_base&& nh ) { + internal_destroy(); + my_node = nh.my_node; + move_assign_allocators(my_allocator, nh.my_allocator); + nh.deactivate(); + return *this; + } + + void swap( node_handle_base& nh ) { + using std::swap; + swap(my_node, nh.my_node); + swap_allocators(my_allocator, nh.my_allocator); + } + + allocator_type get_allocator() const { + return my_allocator; + } + +protected: + node_handle_base( node* n ) : my_node(n) {} + + void internal_destroy() { + if(my_node != nullptr) { + allocator_traits_type::destroy(my_allocator, my_node->storage()); + typename allocator_traits_type::template rebind_alloc node_allocator(my_allocator); + node_allocator.deallocate(my_node, 1); + } + } + + node* get_node_ptr() { return my_node; } + + void deactivate() { my_node = nullptr; } + + node* my_node; + allocator_type my_allocator; +}; + +// node handle for maps +template +class node_handle : public node_handle_base { + using base_type = node_handle_base; +public: + using key_type = Key; + using mapped_type = typename Value::second_type; + using allocator_type = typename base_type::allocator_type; + + node_handle() = default; + + key_type& key() const { + __TBB_ASSERT(!this->empty(), "Cannot get key from the empty node_type object"); + return *const_cast(&(this->my_node->value().first)); + } + + mapped_type& mapped() const { + __TBB_ASSERT(!this->empty(), "Cannot get mapped value from the empty node_type object"); + return this->my_node->value().second; + } + +private: + friend struct node_handle_accessor; + + node_handle( typename base_type::node* n ) : base_type(n) {} +}; // class node_handle + +// node handle for sets +template +class node_handle : public node_handle_base { + using base_type = node_handle_base; +public: + using value_type = Key; + using allocator_type = typename base_type::allocator_type; + + node_handle() = default; + + value_type& value() const { + __TBB_ASSERT(!this->empty(), "Cannot get value from the empty node_type object"); + return *const_cast(&(this->my_node->value())); + } + +private: + friend struct node_handle_accessor; + + node_handle( typename base_type::node* n ) : base_type(n) {} +}; // class node_handle + +template +void swap( node_handle& lhs, + node_handle& rhs ) { + return lhs.swap(rhs); +} + +} // namespace d1 +} // namespace detail +} // namespace tbb + +#endif // __TBB_detail__node_handle_H diff --git a/src/tbb/include/oneapi/tbb/detail/_pipeline_filters.h b/src/tbb/include/oneapi/tbb/detail/_pipeline_filters.h new file mode 100644 index 000000000..812194672 --- /dev/null +++ b/src/tbb/include/oneapi/tbb/detail/_pipeline_filters.h @@ -0,0 +1,461 @@ +/* + Copyright (c) 2005-2024 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_parallel_filters_H +#define __TBB_parallel_filters_H + +#include "_config.h" +#include "_task.h" +#include "_pipeline_filters_deduction.h" +#include "../tbb_allocator.h" + +#include +#include + +namespace tbb { +namespace detail { + +namespace d1 { +class base_filter; +} + +namespace d2 { +template +__TBB_requires(std::copyable) +class input_node; +} + +namespace r1 { +TBB_EXPORT void __TBB_EXPORTED_FUNC set_end_of_input(d1::base_filter&); +class pipeline; +class stage_task; +class input_buffer; +} + +namespace d1 { +class filter_node; + +//! A stage in a pipeline. +/** @ingroup algorithms */ +class base_filter{ +private: + //! Value used to mark "not in pipeline" + static base_filter* not_in_pipeline() { return reinterpret_cast(std::intptr_t(-1)); } +public: + //! The lowest bit 0 is for parallel vs serial + static constexpr unsigned int filter_is_serial = 0x1; + + //! 2nd bit distinguishes ordered vs unordered filters. + static constexpr unsigned int filter_is_out_of_order = 0x1<<1; + + //! 3rd bit marks input filters emitting small objects + static constexpr unsigned int filter_may_emit_null = 0x1<<2; + + base_filter(const base_filter&) = delete; + base_filter& operator=(const base_filter&) = delete; + +protected: + explicit base_filter( unsigned int m ) : + next_filter_in_pipeline(not_in_pipeline()), + my_input_buffer(nullptr), + my_filter_mode(m), + my_pipeline(nullptr) + {} + + // signal end-of-input for concrete_filters + void set_end_of_input() { + r1::set_end_of_input(*this); + } + +public: + //! True if filter is serial. + bool is_serial() const { + return bool( my_filter_mode & filter_is_serial ); + } + + //! True if filter must receive stream in order. + bool is_ordered() const { + return (my_filter_mode & filter_is_serial) && !(my_filter_mode & filter_is_out_of_order); + } + + //! true if an input filter can emit null + bool object_may_be_null() { + return ( my_filter_mode & filter_may_emit_null ) == filter_may_emit_null; + } + + //! Operate on an item from the input stream, and return item for output stream. + /** Returns nullptr if filter is a sink. */ + virtual void* operator()( void* item ) = 0; + + //! Destroy filter. + virtual ~base_filter() {}; + + //! Destroys item if pipeline was cancelled. + /** Required to prevent memory leaks. + Note it can be called concurrently even for serial filters.*/ + virtual void finalize( void* /*item*/ ) {} + +private: + //! Pointer to next filter in the pipeline. + base_filter* next_filter_in_pipeline; + + //! Buffer for incoming tokens, or nullptr if not required. + /** The buffer is required if the filter is serial. */ + r1::input_buffer* my_input_buffer; + + friend class r1::stage_task; + friend class r1::pipeline; + friend void r1::set_end_of_input(d1::base_filter&); + + //! Storage for filter mode and dynamically checked implementation version. + const unsigned int my_filter_mode; + + //! Pointer to the pipeline. + r1::pipeline* my_pipeline; +}; + +template +class concrete_filter; + +//! input_filter control to signal end-of-input for parallel_pipeline +class flow_control { + bool is_pipeline_stopped = false; + flow_control() = default; + template friend class concrete_filter; + template + __TBB_requires(std::copyable) + friend class d2::input_node; +public: + void stop() { is_pipeline_stopped = true; } +}; + +// Emulate std::is_trivially_copyable (false positives not allowed, false negatives suboptimal but safe). +#if __TBB_CPP11_TYPE_PROPERTIES_PRESENT +template using tbb_trivially_copyable = std::is_trivially_copyable; +#else +template struct tbb_trivially_copyable { enum { value = false }; }; +template struct tbb_trivially_copyable < T* > { enum { value = true }; }; +template<> struct tbb_trivially_copyable < bool > { enum { value = true }; }; +template<> struct tbb_trivially_copyable < char > { enum { value = true }; }; +template<> struct tbb_trivially_copyable < signed char > { enum { value = true }; }; +template<> struct tbb_trivially_copyable { enum { value = true }; }; +template<> struct tbb_trivially_copyable < short > { enum { value = true }; }; +template<> struct tbb_trivially_copyable { enum { value = true }; }; +template<> struct tbb_trivially_copyable < int > { enum { value = true }; }; +template<> struct tbb_trivially_copyable { enum { value = true }; }; +template<> struct tbb_trivially_copyable < long > { enum { value = true }; }; +template<> struct tbb_trivially_copyable { enum { value = true }; }; +template<> struct tbb_trivially_copyable < long long> { enum { value = true }; }; +template<> struct tbb_trivially_copyable { enum { value = true }; }; +template<> struct tbb_trivially_copyable < float > { enum { value = true }; }; +template<> struct tbb_trivially_copyable < double > { enum { value = true }; }; +template<> struct tbb_trivially_copyable < long double > { enum { value = true }; }; +#endif // __TBB_CPP11_TYPE_PROPERTIES_PRESENT + +template +struct use_allocator { + static constexpr bool value = sizeof(T) > sizeof(void *) || !tbb_trivially_copyable::value; +}; + +// A helper class to customize how a type is passed between filters. +// Usage: token_helper::value> +template struct token_helper; + +// using tbb_allocator +template +struct token_helper { + using pointer = T*; + using value_type = T; + static pointer create_token(value_type && source) { + return new (r1::allocate_memory(sizeof(T))) T(std::move(source)); + } + static value_type & token(pointer & t) { return *t; } + static void * cast_to_void_ptr(pointer ref) { return reinterpret_cast(ref); } + static pointer cast_from_void_ptr(void * ref) { return reinterpret_cast(ref); } + static void destroy_token(pointer token) { + token->~value_type(); + r1::deallocate_memory(token); + } +}; + +// pointer specialization +template +struct token_helper { + using pointer = T*; + using value_type = T*; + static pointer create_token(const value_type & source) { return source; } + static value_type & token(pointer & t) { return t; } + static void * cast_to_void_ptr(pointer ref) { return reinterpret_cast(ref); } + static pointer cast_from_void_ptr(void * ref) { return reinterpret_cast(ref); } + static void destroy_token( pointer /*token*/) {} +}; + +// converting type to and from void*, passing objects directly +template +struct token_helper { + typedef union { + T actual_value; + void * void_overlay; + } type_to_void_ptr_map; + using pointer = T; // not really a pointer in this case. + using value_type = T; + static pointer create_token(const value_type & source) { return source; } + static value_type & token(pointer & t) { return t; } + static void * cast_to_void_ptr(pointer ref) { + type_to_void_ptr_map mymap; + mymap.void_overlay = nullptr; + mymap.actual_value = ref; + return mymap.void_overlay; + } + static pointer cast_from_void_ptr(void * ref) { + type_to_void_ptr_map mymap; + mymap.void_overlay = ref; + return mymap.actual_value; + } + static void destroy_token( pointer /*token*/) {} +}; + +// intermediate +template +class concrete_filter: public base_filter { + const Body& my_body; + using input_helper = token_helper::value>; + using input_pointer = typename input_helper::pointer; + using output_helper = token_helper::value>; + using output_pointer = typename output_helper::pointer; + + void* operator()(void* input) override { + input_pointer temp_input = input_helper::cast_from_void_ptr(input); + output_pointer temp_output = output_helper::create_token(tbb::detail::invoke(my_body, std::move(input_helper::token(temp_input)))); + input_helper::destroy_token(temp_input); + return output_helper::cast_to_void_ptr(temp_output); + } + + void finalize(void * input) override { + input_pointer temp_input = input_helper::cast_from_void_ptr(input); + input_helper::destroy_token(temp_input); + } + +public: + concrete_filter(unsigned int m, const Body& body) : base_filter(m), my_body(body) {} +}; + +// input +template +class concrete_filter: public base_filter { + const Body& my_body; + using output_helper = token_helper::value>; + using output_pointer = typename output_helper::pointer; + + void* operator()(void*) override { + flow_control control; + output_pointer temp_output = output_helper::create_token(my_body(control)); + if(control.is_pipeline_stopped) { + output_helper::destroy_token(temp_output); + set_end_of_input(); + return nullptr; + } + return output_helper::cast_to_void_ptr(temp_output); + } + +public: + concrete_filter(unsigned int m, const Body& body) : + base_filter(m | filter_may_emit_null), + my_body(body) + {} +}; + +// output +template +class concrete_filter: public base_filter { + const Body& my_body; + using input_helper = token_helper::value>; + using input_pointer = typename input_helper::pointer; + + void* operator()(void* input) override { + input_pointer temp_input = input_helper::cast_from_void_ptr(input); + tbb::detail::invoke(my_body, std::move(input_helper::token(temp_input))); + input_helper::destroy_token(temp_input); + return nullptr; + } + void finalize(void* input) override { + input_pointer temp_input = input_helper::cast_from_void_ptr(input); + input_helper::destroy_token(temp_input); + } + +public: + concrete_filter(unsigned int m, const Body& body) : base_filter(m), my_body(body) {} +}; + +template +class concrete_filter: public base_filter { + const Body& my_body; + + void* operator()(void*) override { + flow_control control; + my_body(control); + void* output = control.is_pipeline_stopped ? nullptr : (void*)(std::intptr_t)-1; + return output; + } +public: + concrete_filter(unsigned int m, const Body& body) : base_filter(m), my_body(body) {} +}; + +class filter_node_ptr { + filter_node * my_node; + +public: + filter_node_ptr() : my_node(nullptr) {} + filter_node_ptr(filter_node *); + ~filter_node_ptr(); + filter_node_ptr(const filter_node_ptr &); + filter_node_ptr(filter_node_ptr &&); + void operator=(filter_node *); + void operator=(const filter_node_ptr &); + void operator=(filter_node_ptr &&); + filter_node& operator*() const; + operator bool() const; +}; + +//! Abstract base class that represents a node in a parse tree underlying a filter class. +/** These nodes are always heap-allocated and can be shared by filter objects. */ +class filter_node { + /** Count must be atomic because it is hidden state for user, but might be shared by threads. */ + std::atomic ref_count; +public: + filter_node_ptr left; + filter_node_ptr right; +protected: + filter_node() : ref_count(0), left(nullptr), right(nullptr) { +#ifdef __TBB_TEST_FILTER_NODE_COUNT + ++(__TBB_TEST_FILTER_NODE_COUNT); +#endif + } +public: + filter_node(const filter_node_ptr& x, const filter_node_ptr& y) : filter_node(){ + left = x; + right = y; + } + filter_node(const filter_node&) = delete; + filter_node& operator=(const filter_node&) = delete; + + //! Add concrete_filter to pipeline + virtual base_filter* create_filter() const { + __TBB_ASSERT(false, "method of non-leaf was called"); + return nullptr; + } + + //! Increment reference count + void add_ref() { ref_count.fetch_add(1, std::memory_order_relaxed); } + + //! Decrement reference count and delete if it becomes zero. + void remove_ref() { + __TBB_ASSERT(ref_count>0,"ref_count underflow"); + if( ref_count.fetch_sub(1, std::memory_order_relaxed) == 1 ) { + this->~filter_node(); + r1::deallocate_memory(this); + } + } + + virtual ~filter_node() { +#ifdef __TBB_TEST_FILTER_NODE_COUNT + --(__TBB_TEST_FILTER_NODE_COUNT); +#endif + } +}; + +inline filter_node_ptr::filter_node_ptr(filter_node * nd) : my_node(nd) { + if (my_node) { + my_node->add_ref(); + } +} + +inline filter_node_ptr::~filter_node_ptr() { + if (my_node) { + my_node->remove_ref(); + } +} + +inline filter_node_ptr::filter_node_ptr(const filter_node_ptr & rhs) : my_node(rhs.my_node) { + if (my_node) { + my_node->add_ref(); + } +} + +inline filter_node_ptr::filter_node_ptr(filter_node_ptr && rhs) : my_node(rhs.my_node) { + rhs.my_node = nullptr; +} + +inline void filter_node_ptr::operator=(filter_node * rhs) { + // Order of operations below carefully chosen so that reference counts remain correct + // in unlikely event that remove_ref throws exception. + filter_node* old = my_node; + my_node = rhs; + if (my_node) { + my_node->add_ref(); + } + if (old) { + old->remove_ref(); + } +} + +inline void filter_node_ptr::operator=(const filter_node_ptr & rhs) { + *this = rhs.my_node; +} + +inline void filter_node_ptr::operator=(filter_node_ptr && rhs) { + filter_node* old = my_node; + my_node = rhs.my_node; + rhs.my_node = nullptr; + if (old) { + old->remove_ref(); + } +} + +inline filter_node& filter_node_ptr::operator*() const{ + __TBB_ASSERT(my_node,"nullptr node is used"); + return *my_node; +} + +inline filter_node_ptr::operator bool() const { + return my_node != nullptr; +} + +//! Node in parse tree representing result of make_filter. +template +class filter_node_leaf: public filter_node { + const unsigned int my_mode; + const Body my_body; + base_filter* create_filter() const override { + return new(r1::allocate_memory(sizeof(concrete_filter))) concrete_filter(my_mode,my_body); + } +public: + filter_node_leaf( unsigned int m, const Body& b ) : my_mode(m), my_body(b) {} +}; + + +template ::input_type> +using filter_input = typename std::conditional::value, void, Input>::type; + +template +using filter_output = typename filter_body_types::output_type; + +} // namespace d1 +} // namespace detail +} // namespace tbb + + +#endif /* __TBB_parallel_filters_H */ diff --git a/src/tbb/include/oneapi/tbb/detail/_pipeline_filters_deduction.h b/src/tbb/include/oneapi/tbb/detail/_pipeline_filters_deduction.h new file mode 100644 index 000000000..c1a6e8aed --- /dev/null +++ b/src/tbb/include/oneapi/tbb/detail/_pipeline_filters_deduction.h @@ -0,0 +1,46 @@ +/* + Copyright (c) 2005-2023 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB__pipeline_filters_deduction_H +#define __TBB__pipeline_filters_deduction_H + +#include "_config.h" +#include +#include + +namespace tbb { +namespace detail { +namespace d1 { + +template +struct declare_filter_types { + using input_type = typename std::remove_const::type>::type; + using output_type = typename std::remove_const::type>::type; +}; + +template struct filter_body_types; + +template +struct filter_body_types : declare_filter_types {}; + +template +struct filter_body_types : declare_filter_types {}; + +} // namespace d1 +} // namespace detail +} // namespace tbb + +#endif // __TBB__pipeline_filters_deduction_H diff --git a/src/tbb/include/oneapi/tbb/detail/_range_common.h b/src/tbb/include/oneapi/tbb/detail/_range_common.h new file mode 100644 index 000000000..1011f029d --- /dev/null +++ b/src/tbb/include/oneapi/tbb/detail/_range_common.h @@ -0,0 +1,130 @@ +/* + Copyright (c) 2005-2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_detail__range_common_H +#define __TBB_detail__range_common_H + +#include "_config.h" +#include "_utils.h" +#if __TBB_CPP20_CONCEPTS_PRESENT +#include +#endif +#include + +namespace tbb { +namespace detail { +inline namespace d0 { + +//! Dummy type that distinguishes splitting constructor from copy constructor. +/** + * See description of parallel_for and parallel_reduce for example usages. + * @ingroup algorithms + */ +class split {}; + +//! Type enables transmission of splitting proportion from partitioners to range objects +/** + * In order to make use of such facility Range objects must implement + * splitting constructor with this type passed. + */ +class proportional_split : no_assign { +public: + proportional_split(size_t _left = 1, size_t _right = 1) : my_left(_left), my_right(_right) { } + + size_t left() const { return my_left; } + size_t right() const { return my_right; } + + // used when range does not support proportional split + explicit operator split() const { return split(); } + +private: + size_t my_left, my_right; +}; + +template +struct range_split_object_provider { + template + static split get( PartitionerSplitType& ) { return split(); } +}; + +template +struct range_split_object_provider::value>::type> { + template + static PartitionerSplitType& get( PartitionerSplitType& split_obj ) { return split_obj; } +}; + +template +auto get_range_split_object( PartitionerSplitType& split_obj ) +-> decltype(range_split_object_provider::get(split_obj)) { + return range_split_object_provider::get(split_obj); +} + +template +using range_iterator_type = decltype(std::begin(std::declval())); + +#if __TBB_CPP20_CONCEPTS_PRESENT +template +using iterator_reference_type = typename std::iterator_traits::reference; + +template +using range_reference_type = iterator_reference_type>; + +template +concept blocked_range_value = std::copyable && + requires( const std::remove_reference_t& lhs, const std::remove_reference_t& rhs ) { + { lhs < rhs } -> relaxed_convertible_to; + { lhs - rhs } -> std::convertible_to; + { lhs + (rhs - lhs) } -> std::convertible_to; + }; + +template +concept splittable = std::constructible_from; + +template +concept tbb_range = std::copy_constructible && + splittable && + requires( const std::remove_reference_t& range ) { + { range.empty() } -> relaxed_convertible_to; + { range.is_divisible() } -> relaxed_convertible_to; + }; + +template +constexpr bool iterator_concept_helper( std::input_iterator_tag ) { + return std::input_iterator; +} + +template +constexpr bool iterator_concept_helper( std::random_access_iterator_tag ) { + return std::random_access_iterator; +} + +template +concept iterator_satisfies = requires (IteratorTag tag) { + requires iterator_concept_helper(tag); +}; + +template +concept container_based_sequence = requires( Sequence& seq ) { + { std::begin(seq) } -> iterator_satisfies; + { std::end(seq) } -> iterator_satisfies; +}; +#endif // __TBB_CPP20_CONCEPTS_PRESENT +} // namespace d0 +} // namespace detail +} // namespace tbb + +#endif // __TBB_detail__range_common_H diff --git a/src/tbb/include/oneapi/tbb/detail/_rtm_mutex.h b/src/tbb/include/oneapi/tbb/detail/_rtm_mutex.h new file mode 100644 index 000000000..3901a7403 --- /dev/null +++ b/src/tbb/include/oneapi/tbb/detail/_rtm_mutex.h @@ -0,0 +1,162 @@ +/* + Copyright (c) 2005-2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB__rtm_mutex_impl_H +#define __TBB__rtm_mutex_impl_H + +#include "_assert.h" +#include "_utils.h" +#include "../spin_mutex.h" + +#include "../profiling.h" + +namespace tbb { +namespace detail { +namespace r1 { +struct rtm_mutex_impl; +} +namespace d1 { + +#if _MSC_VER && !defined(__INTEL_COMPILER) + // Suppress warning: structure was padded due to alignment specifier + // #pragma warning (push) + // #pragma warning (disable: 4324) +#endif + +/** A rtm_mutex is an speculation-enabled spin mutex. + It should be used for locking short critical sections where the lock is + contended but the data it protects are not. If zero-initialized, the + mutex is considered unheld. + @ingroup synchronization */ +class alignas(max_nfs_size) rtm_mutex : private spin_mutex { +private: + enum class rtm_state { + rtm_none, + rtm_transacting, + rtm_real + }; +public: + //! Constructors + rtm_mutex() noexcept { + create_itt_sync(this, "tbb::speculative_spin_mutex", ""); + } + + //! Destructor + ~rtm_mutex() = default; + + //! Represents acquisition of a mutex. + class scoped_lock { + public: + friend class rtm_mutex; + //! Construct lock that has not acquired a mutex. + /** Equivalent to zero-initialization of *this. */ + constexpr scoped_lock() : m_mutex(nullptr), m_transaction_state(rtm_state::rtm_none) {} + + //! Acquire lock on given mutex. + scoped_lock(rtm_mutex& m) : m_mutex(nullptr), m_transaction_state(rtm_state::rtm_none) { + acquire(m); + } + + //! Release lock (if lock is held). + ~scoped_lock() { + if(m_transaction_state != rtm_state::rtm_none) { + release(); + } + } + + //! No Copy + scoped_lock(const scoped_lock&) = delete; + scoped_lock& operator=(const scoped_lock&) = delete; + + //! Acquire lock on given mutex. + void acquire(rtm_mutex& m); + + //! Try acquire lock on given mutex. + bool try_acquire(rtm_mutex& m); + + //! Release lock + void release(); + + private: + rtm_mutex* m_mutex; + rtm_state m_transaction_state; + friend r1::rtm_mutex_impl; + }; + + //! Mutex traits + static constexpr bool is_rw_mutex = false; + static constexpr bool is_recursive_mutex = false; + static constexpr bool is_fair_mutex = false; +private: + friend r1::rtm_mutex_impl; +}; // end of rtm_mutex +} // namespace d1 + +namespace r1 { + //! Internal acquire lock. + // only_speculate == true if we're doing a try_lock, else false. + TBB_EXPORT void __TBB_EXPORTED_FUNC acquire(d1::rtm_mutex&, d1::rtm_mutex::scoped_lock&, bool only_speculate = false); + //! Internal try_acquire lock. + TBB_EXPORT bool __TBB_EXPORTED_FUNC try_acquire(d1::rtm_mutex&, d1::rtm_mutex::scoped_lock&); + //! Internal release lock. + TBB_EXPORT void __TBB_EXPORTED_FUNC release(d1::rtm_mutex::scoped_lock&); +} // namespace r1 + +namespace d1 { +//! Acquire lock on given mutex. +inline void rtm_mutex::scoped_lock::acquire(rtm_mutex& m) { + __TBB_ASSERT(!m_mutex, "lock is already acquired"); + r1::acquire(m, *this); +} + +//! Try acquire lock on given mutex. +inline bool rtm_mutex::scoped_lock::try_acquire(rtm_mutex& m) { + __TBB_ASSERT(!m_mutex, "lock is already acquired"); + return r1::try_acquire(m, *this); +} + +//! Release lock +inline void rtm_mutex::scoped_lock::release() { + __TBB_ASSERT(m_mutex, "lock is not acquired"); + __TBB_ASSERT(m_transaction_state != rtm_state::rtm_none, "lock is not acquired"); + return r1::release(*this); +} + +#if _MSC_VER && !defined(__INTEL_COMPILER) + // #pragma warning (pop) // 4324 warning +#endif + +#if TBB_USE_PROFILING_TOOLS +inline void set_name(rtm_mutex& obj, const char* name) { + itt_set_sync_name(&obj, name); +} +#if (_WIN32||_WIN64) +inline void set_name(rtm_mutex& obj, const wchar_t* name) { + itt_set_sync_name(&obj, name); +} +#endif // WIN +#else +inline void set_name(rtm_mutex&, const char*) {} +#if (_WIN32||_WIN64) +inline void set_name(rtm_mutex&, const wchar_t*) {} +#endif // WIN +#endif + +} // namespace d1 +} // namespace detail +} // namespace tbb + +#endif /* __TBB__rtm_mutex_impl_H */ diff --git a/src/tbb/include/oneapi/tbb/detail/_rtm_rw_mutex.h b/src/tbb/include/oneapi/tbb/detail/_rtm_rw_mutex.h new file mode 100644 index 000000000..e7536caab --- /dev/null +++ b/src/tbb/include/oneapi/tbb/detail/_rtm_rw_mutex.h @@ -0,0 +1,215 @@ +/* + Copyright (c) 2005-2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_detail__rtm_rw_mutex_H +#define __TBB_detail__rtm_rw_mutex_H + +#include "_assert.h" +#include "_utils.h" +#include "../spin_rw_mutex.h" + +#include + +namespace tbb { +namespace detail { + +namespace r1 { +struct rtm_rw_mutex_impl; +} + +namespace d1 { + +constexpr std::size_t speculation_granularity = 64; +#if _MSC_VER && !defined(__INTEL_COMPILER) + // Suppress warning: structure was padded due to alignment specifier + // #pragma warning (push) + // #pragma warning (disable: 4324) +#endif + +//! Fast, unfair, spinning speculation-enabled reader-writer lock with backoff and writer-preference +/** @ingroup synchronization */ +class alignas(max_nfs_size) rtm_rw_mutex : private spin_rw_mutex { + friend struct r1::rtm_rw_mutex_impl; +private: + enum class rtm_type { + rtm_not_in_mutex, + rtm_transacting_reader, + rtm_transacting_writer, + rtm_real_reader, + rtm_real_writer + }; +public: + //! Constructors + rtm_rw_mutex() noexcept : write_flag(false) { + create_itt_sync(this, "tbb::speculative_spin_rw_mutex", ""); + } + + //! Destructor + ~rtm_rw_mutex() = default; + + //! Represents acquisition of a mutex. + class scoped_lock { + friend struct r1::rtm_rw_mutex_impl; + public: + //! Construct lock that has not acquired a mutex. + /** Equivalent to zero-initialization of *this. */ + constexpr scoped_lock() : m_mutex(nullptr), m_transaction_state(rtm_type::rtm_not_in_mutex) {} + + //! Acquire lock on given mutex. + scoped_lock(rtm_rw_mutex& m, bool write = true) : m_mutex(nullptr), m_transaction_state(rtm_type::rtm_not_in_mutex) { + acquire(m, write); + } + + //! Release lock (if lock is held). + ~scoped_lock() { + if(m_transaction_state != rtm_type::rtm_not_in_mutex) { + release(); + } + } + + //! No Copy + scoped_lock(const scoped_lock&) = delete; + scoped_lock& operator=(const scoped_lock&) = delete; + + //! Acquire lock on given mutex. + inline void acquire(rtm_rw_mutex& m, bool write = true); + + //! Try acquire lock on given mutex. + inline bool try_acquire(rtm_rw_mutex& m, bool write = true); + + //! Release lock + inline void release(); + + //! Upgrade reader to become a writer. + /** Returns whether the upgrade happened without releasing and re-acquiring the lock */ + inline bool upgrade_to_writer(); + + //! Downgrade writer to become a reader. + inline bool downgrade_to_reader(); + + inline bool is_writer() const; + private: + rtm_rw_mutex* m_mutex; + rtm_type m_transaction_state; + }; + + //! Mutex traits + static constexpr bool is_rw_mutex = true; + static constexpr bool is_recursive_mutex = false; + static constexpr bool is_fair_mutex = false; + +private: + alignas(speculation_granularity) std::atomic write_flag; +}; + +#if _MSC_VER && !defined(__INTEL_COMPILER) + // #pragma warning (pop) // 4324 warning +#endif + +} // namespace d1 + +namespace r1 { + //! Internal acquire write lock. + // only_speculate == true if we're doing a try_lock, else false. + TBB_EXPORT void __TBB_EXPORTED_FUNC acquire_writer(d1::rtm_rw_mutex&, d1::rtm_rw_mutex::scoped_lock&, bool only_speculate = false); + //! Internal acquire read lock. + // only_speculate == true if we're doing a try_lock, else false. + TBB_EXPORT void __TBB_EXPORTED_FUNC acquire_reader(d1::rtm_rw_mutex&, d1::rtm_rw_mutex::scoped_lock&, bool only_speculate = false); + //! Internal upgrade reader to become a writer. + TBB_EXPORT bool __TBB_EXPORTED_FUNC upgrade(d1::rtm_rw_mutex::scoped_lock&); + //! Internal downgrade writer to become a reader. + TBB_EXPORT bool __TBB_EXPORTED_FUNC downgrade(d1::rtm_rw_mutex::scoped_lock&); + //! Internal try_acquire write lock. + TBB_EXPORT bool __TBB_EXPORTED_FUNC try_acquire_writer(d1::rtm_rw_mutex&, d1::rtm_rw_mutex::scoped_lock&); + //! Internal try_acquire read lock. + TBB_EXPORT bool __TBB_EXPORTED_FUNC try_acquire_reader(d1::rtm_rw_mutex&, d1::rtm_rw_mutex::scoped_lock&); + //! Internal release lock. + TBB_EXPORT void __TBB_EXPORTED_FUNC release(d1::rtm_rw_mutex::scoped_lock&); +} + +namespace d1 { +//! Acquire lock on given mutex. +void rtm_rw_mutex::scoped_lock::acquire(rtm_rw_mutex& m, bool write) { + __TBB_ASSERT(!m_mutex, "lock is already acquired"); + if (write) { + r1::acquire_writer(m, *this); + } else { + r1::acquire_reader(m, *this); + } +} + +//! Try acquire lock on given mutex. +bool rtm_rw_mutex::scoped_lock::try_acquire(rtm_rw_mutex& m, bool write) { + __TBB_ASSERT(!m_mutex, "lock is already acquired"); + if (write) { + return r1::try_acquire_writer(m, *this); + } else { + return r1::try_acquire_reader(m, *this); + } +} + +//! Release lock +void rtm_rw_mutex::scoped_lock::release() { + __TBB_ASSERT(m_mutex, "lock is not acquired"); + __TBB_ASSERT(m_transaction_state != rtm_type::rtm_not_in_mutex, "lock is not acquired"); + return r1::release(*this); +} + +//! Upgrade reader to become a writer. +/** Returns whether the upgrade happened without releasing and re-acquiring the lock */ +bool rtm_rw_mutex::scoped_lock::upgrade_to_writer() { + __TBB_ASSERT(m_mutex, "lock is not acquired"); + if (m_transaction_state == rtm_type::rtm_transacting_writer || m_transaction_state == rtm_type::rtm_real_writer) { + return true; // Already a writer + } + return r1::upgrade(*this); +} + +//! Downgrade writer to become a reader. +bool rtm_rw_mutex::scoped_lock::downgrade_to_reader() { + __TBB_ASSERT(m_mutex, "lock is not acquired"); + if (m_transaction_state == rtm_type::rtm_transacting_reader || m_transaction_state == rtm_type::rtm_real_reader) { + return true; // Already a reader + } + return r1::downgrade(*this); +} + +bool rtm_rw_mutex::scoped_lock::is_writer() const { + __TBB_ASSERT(m_mutex, "lock is not acquired"); + return m_transaction_state == rtm_type::rtm_transacting_writer || m_transaction_state == rtm_type::rtm_real_writer; +} + +#if TBB_USE_PROFILING_TOOLS +inline void set_name(rtm_rw_mutex& obj, const char* name) { + itt_set_sync_name(&obj, name); +} +#if (_WIN32||_WIN64) +inline void set_name(rtm_rw_mutex& obj, const wchar_t* name) { + itt_set_sync_name(&obj, name); +} +#endif // WIN +#else +inline void set_name(rtm_rw_mutex&, const char*) {} +#if (_WIN32||_WIN64) +inline void set_name(rtm_rw_mutex&, const wchar_t*) {} +#endif // WIN +#endif + +} // namespace d1 +} // namespace detail +} // namespace tbb + +#endif // __TBB_detail__rtm_rw_mutex_H diff --git a/src/tbb/include/oneapi/tbb/detail/_scoped_lock.h b/src/tbb/include/oneapi/tbb/detail/_scoped_lock.h new file mode 100644 index 000000000..a49dcdff5 --- /dev/null +++ b/src/tbb/include/oneapi/tbb/detail/_scoped_lock.h @@ -0,0 +1,174 @@ +/* + Copyright (c) 2005-2022 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_detail_scoped_lock_H +#define __TBB_detail_scoped_lock_H + +namespace tbb { +namespace detail { +namespace d1 { + +// unique_scoped_lock supposes that Mutex operations never throw +template +class unique_scoped_lock { + //! Points to currently held Mutex, or nullptr if no lock is held. + Mutex* m_mutex{}; + +public: + //! Construct without acquiring a Mutex. + constexpr unique_scoped_lock() noexcept : m_mutex(nullptr) {} + + //! Construct and acquire lock on a Mutex. + unique_scoped_lock(Mutex& m) { + acquire(m); + } + + //! No Copy + unique_scoped_lock(const unique_scoped_lock&) = delete; + unique_scoped_lock& operator=(const unique_scoped_lock&) = delete; + + //! Acquire lock. + void acquire(Mutex& m) { + __TBB_ASSERT(m_mutex == nullptr, "The mutex is already acquired"); + m_mutex = &m; + m.lock(); + } + + //! Try acquiring lock (non-blocking) + /** Return true if lock acquired; false otherwise. */ + bool try_acquire(Mutex& m) { + __TBB_ASSERT(m_mutex == nullptr, "The mutex is already acquired"); + bool succeed = m.try_lock(); + if (succeed) { + m_mutex = &m; + } + return succeed; + } + + //! Release lock + void release() { + __TBB_ASSERT(m_mutex, "release on Mutex::unique_scoped_lock that is not holding a lock"); + m_mutex->unlock(); + m_mutex = nullptr; + } + + //! Destroy lock. If holding a lock, releases the lock first. + ~unique_scoped_lock() { + if (m_mutex) { + release(); + } + } +}; + +// rw_scoped_lock supposes that Mutex operations never throw +template +class rw_scoped_lock { +public: + //! Construct lock that has not acquired a mutex. + /** Equivalent to zero-initialization of *this. */ + constexpr rw_scoped_lock() noexcept {} + + //! Acquire lock on given mutex. + rw_scoped_lock(Mutex& m, bool write = true) { + acquire(m, write); + } + + //! Release lock (if lock is held). + ~rw_scoped_lock() { + if (m_mutex) { + release(); + } + } + + //! No Copy + rw_scoped_lock(const rw_scoped_lock&) = delete; + rw_scoped_lock& operator=(const rw_scoped_lock&) = delete; + + //! Acquire lock on given mutex. + void acquire(Mutex& m, bool write = true) { + __TBB_ASSERT(m_mutex == nullptr, "The mutex is already acquired"); + m_is_writer = write; + m_mutex = &m; + if (write) { + m_mutex->lock(); + } else { + m_mutex->lock_shared(); + } + } + + //! Try acquire lock on given mutex. + bool try_acquire(Mutex& m, bool write = true) { + bool succeed = write ? m.try_lock() : m.try_lock_shared(); + if (succeed) { + m_mutex = &m; + m_is_writer = write; + } + return succeed; + } + + //! Release lock. + void release() { + __TBB_ASSERT(m_mutex != nullptr, "The mutex is not acquired"); + Mutex* m = m_mutex; + m_mutex = nullptr; + + if (m_is_writer) { + m->unlock(); + } else { + m->unlock_shared(); + } + } + + //! Upgrade reader to become a writer. + /** Returns whether the upgrade happened without releasing and re-acquiring the lock */ + bool upgrade_to_writer() { + __TBB_ASSERT(m_mutex != nullptr, "The mutex is not acquired"); + if (m_is_writer) { + return true; // Already a writer + } + m_is_writer = true; + return m_mutex->upgrade(); + } + + //! Downgrade writer to become a reader. + bool downgrade_to_reader() { + __TBB_ASSERT(m_mutex != nullptr, "The mutex is not acquired"); + if (m_is_writer) { + m_mutex->downgrade(); + m_is_writer = false; + } + return true; + } + + bool is_writer() const { + __TBB_ASSERT(m_mutex != nullptr, "The mutex is not acquired"); + return m_is_writer; + } + +protected: + //! The pointer to the current mutex that is held, or nullptr if no mutex is held. + Mutex* m_mutex {nullptr}; + + //! If mutex != nullptr, then is_writer is true if holding a writer lock, false if holding a reader lock. + /** Not defined if not holding a lock. */ + bool m_is_writer {false}; +}; + +} // namespace d1 +} // namespace detail +} // namespace tbb + +#endif // __TBB_detail_scoped_lock_H diff --git a/src/tbb/include/oneapi/tbb/detail/_segment_table.h b/src/tbb/include/oneapi/tbb/detail/_segment_table.h new file mode 100644 index 000000000..cd4d35188 --- /dev/null +++ b/src/tbb/include/oneapi/tbb/detail/_segment_table.h @@ -0,0 +1,566 @@ +/* + Copyright (c) 2005-2022 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_detail__segment_table_H +#define __TBB_detail__segment_table_H + +#include "_config.h" +#include "_allocator_traits.h" +#include "_template_helpers.h" +#include "_utils.h" +#include "_assert.h" +#include "_exception.h" +#include +#include +#include +#include + +#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) +// #pragma warning(push) +// #pragma warning(disable: 4127) // warning C4127: conditional expression is constant +#endif + +namespace tbb { +namespace detail { +namespace d1 { + +template +class segment_table { +public: + using value_type = T; + using segment_type = T*; + using atomic_segment = std::atomic; + using segment_table_type = atomic_segment*; + + using size_type = std::size_t; + using segment_index_type = std::size_t; + + using allocator_type = Allocator; + + using allocator_traits_type = tbb::detail::allocator_traits; + using segment_table_allocator_type = typename allocator_traits_type::template rebind_alloc; +protected: + using segment_table_allocator_traits = tbb::detail::allocator_traits; + using derived_type = DerivedType; + + static constexpr size_type pointers_per_embedded_table = PointersPerEmbeddedTable; + static constexpr size_type pointers_per_long_table = sizeof(size_type) * 8; +public: + segment_table( const allocator_type& alloc = allocator_type() ) + : my_segment_table_allocator(alloc), my_segment_table(nullptr) + , my_first_block{}, my_size{}, my_segment_table_allocation_failed{} + { + my_segment_table.store(my_embedded_table, std::memory_order_relaxed); + zero_table(my_embedded_table, pointers_per_embedded_table); + } + + segment_table( const segment_table& other ) + : my_segment_table_allocator(segment_table_allocator_traits:: + select_on_container_copy_construction(other.my_segment_table_allocator)) + , my_segment_table(nullptr), my_first_block{}, my_size{}, my_segment_table_allocation_failed{} + { + my_segment_table.store(my_embedded_table, std::memory_order_relaxed); + zero_table(my_embedded_table, pointers_per_embedded_table); + try_call( [&] { + internal_transfer(other, copy_segment_body_type{*this}); + } ).on_exception( [&] { + clear(); + }); + } + + segment_table( const segment_table& other, const allocator_type& alloc ) + : my_segment_table_allocator(alloc), my_segment_table(nullptr) + , my_first_block{}, my_size{}, my_segment_table_allocation_failed{} + { + my_segment_table.store(my_embedded_table, std::memory_order_relaxed); + zero_table(my_embedded_table, pointers_per_embedded_table); + try_call( [&] { + internal_transfer(other, copy_segment_body_type{*this}); + } ).on_exception( [&] { + clear(); + }); + } + + segment_table( segment_table&& other ) + : my_segment_table_allocator(std::move(other.my_segment_table_allocator)), my_segment_table(nullptr) + , my_first_block{}, my_size{}, my_segment_table_allocation_failed{} + { + my_segment_table.store(my_embedded_table, std::memory_order_relaxed); + zero_table(my_embedded_table, pointers_per_embedded_table); + internal_move(std::move(other)); + } + + segment_table( segment_table&& other, const allocator_type& alloc ) + : my_segment_table_allocator(alloc), my_segment_table(nullptr), my_first_block{} + , my_size{}, my_segment_table_allocation_failed{} + { + my_segment_table.store(my_embedded_table, std::memory_order_relaxed); + zero_table(my_embedded_table, pointers_per_embedded_table); + using is_equal_type = typename segment_table_allocator_traits::is_always_equal; + internal_move_construct_with_allocator(std::move(other), alloc, is_equal_type()); + } + + ~segment_table() { + clear(); + } + + segment_table& operator=( const segment_table& other ) { + if (this != &other) { + copy_assign_allocators(my_segment_table_allocator, other.my_segment_table_allocator); + internal_transfer(other, copy_segment_body_type{*this}); + } + return *this; + } + + segment_table& operator=( segment_table&& other ) + noexcept(derived_type::is_noexcept_assignment) + { + using pocma_type = typename segment_table_allocator_traits::propagate_on_container_move_assignment; + using is_equal_type = typename segment_table_allocator_traits::is_always_equal; + + if (this != &other) { + move_assign_allocators(my_segment_table_allocator, other.my_segment_table_allocator); + internal_move_assign(std::move(other), tbb::detail::disjunction()); + } + return *this; + } + + void swap( segment_table& other ) + noexcept(derived_type::is_noexcept_swap) + { + using is_equal_type = typename segment_table_allocator_traits::is_always_equal; + using pocs_type = typename segment_table_allocator_traits::propagate_on_container_swap; + + if (this != &other) { + swap_allocators(my_segment_table_allocator, other.my_segment_table_allocator); + internal_swap(other, tbb::detail::disjunction()); + } + } + + segment_type get_segment( segment_index_type index ) const { + return get_table()[index] + segment_base(index); + } + + value_type& operator[]( size_type index ) { + return internal_subscript(index); + } + + const value_type& operator[]( size_type index ) const { + return const_cast(this)->internal_subscript(index); + } + + const segment_table_allocator_type& get_allocator() const { + return my_segment_table_allocator; + } + + segment_table_allocator_type& get_allocator() { + return my_segment_table_allocator; + } + + void enable_segment( segment_type& segment, segment_table_type table, segment_index_type seg_index, size_type index ) { + // Allocate new segment + segment_type new_segment = self()->create_segment(table, seg_index, index); + if (new_segment != nullptr) { + // Store (new_segment - segment_base) into the segment table to allow access to the table by index via + // my_segment_table[segment_index_of(index)][index] + segment_type disabled_segment = nullptr; + if (!table[seg_index].compare_exchange_strong(disabled_segment, new_segment - segment_base(seg_index))) { + // compare_exchange failed => some other thread has already enabled this segment + // Deallocate the memory + self()->deallocate_segment(new_segment, seg_index); + } + } + + segment = table[seg_index].load(std::memory_order_acquire); + __TBB_ASSERT(segment != nullptr, "If create_segment returned nullptr, the element should be stored in the table"); + } + + void delete_segment( segment_index_type seg_index ) { + segment_type segment_to_delete = self()->nullify_segment(get_table(), seg_index); + if (segment_to_delete == segment_allocation_failure_tag) { + return; + } + + segment_to_delete += segment_base(seg_index); + + // Deallocate the segment + self()->destroy_segment(segment_to_delete, seg_index); + } + + size_type number_of_segments( segment_table_type table ) const { + // Check for an active table, if it is embedded table - return the number of embedded segments + // Otherwise - return the maximum number of segments + return table == my_embedded_table ? pointers_per_embedded_table : pointers_per_long_table; + } + + size_type capacity() const noexcept { + segment_table_type table = get_table(); + size_type num_segments = number_of_segments(table); + for (size_type seg_index = 0; seg_index < num_segments; ++seg_index) { + // Check if the pointer is valid (allocated) + if (table[seg_index].load(std::memory_order_relaxed) <= segment_allocation_failure_tag) { + return segment_base(seg_index); + } + } + return segment_base(num_segments); + } + + size_type find_last_allocated_segment( segment_table_type table ) const noexcept { + size_type end = 0; + size_type num_segments = number_of_segments(table); + for (size_type seg_index = 0; seg_index < num_segments; ++seg_index) { + // Check if the pointer is valid (allocated) + if (table[seg_index].load(std::memory_order_relaxed) > segment_allocation_failure_tag) { + end = seg_index + 1; + } + } + return end; + } + + void reserve( size_type n ) { + if (n > allocator_traits_type::max_size(my_segment_table_allocator)) { + throw_exception(exception_id::reservation_length_error); + } + + size_type size = my_size.load(std::memory_order_relaxed); + segment_index_type start_seg_idx = size == 0 ? 0 : segment_index_of(size - 1) + 1; + for (segment_index_type seg_idx = start_seg_idx; segment_base(seg_idx) < n; ++seg_idx) { + size_type first_index = segment_base(seg_idx); + internal_subscript(first_index); + } + } + + void clear() { + clear_segments(); + clear_table(); + my_size.store(0, std::memory_order_relaxed); + my_first_block.store(0, std::memory_order_relaxed); + } + + void clear_segments() { + segment_table_type current_segment_table = get_table(); + for (size_type i = number_of_segments(current_segment_table); i != 0; --i) { + if (current_segment_table[i - 1].load(std::memory_order_relaxed) != nullptr) { + // If the segment was enabled - disable and deallocate it + delete_segment(i - 1); + } + } + } + + void clear_table() { + segment_table_type current_segment_table = get_table(); + if (current_segment_table != my_embedded_table) { + // If the active table is not the embedded one - deallocate the active table + for (size_type i = 0; i != pointers_per_long_table; ++i) { + segment_table_allocator_traits::destroy(my_segment_table_allocator, ¤t_segment_table[i]); + } + + segment_table_allocator_traits::deallocate(my_segment_table_allocator, current_segment_table, pointers_per_long_table); + my_segment_table.store(my_embedded_table, std::memory_order_relaxed); + zero_table(my_embedded_table, pointers_per_embedded_table); + } + } + + void extend_table_if_necessary(segment_table_type& table, size_type start_index, size_type end_index) { + // extend_segment_table if an active table is an embedded table + // and the requested index is not in the embedded table + if (table == my_embedded_table && end_index > embedded_table_size) { + if (start_index <= embedded_table_size) { + try_call([&] { + table = self()->allocate_long_table(my_embedded_table, start_index); + // It is possible that the table was extended by the thread that allocated first_block. + // In this case it is necessary to re-read the current table. + + if (table) { + my_segment_table.store(table, std::memory_order_release); + } else { + table = my_segment_table.load(std::memory_order_acquire); + } + }).on_exception([&] { + my_segment_table_allocation_failed.store(true, std::memory_order_relaxed); + }); + } else { + atomic_backoff backoff; + do { + if (my_segment_table_allocation_failed.load(std::memory_order_relaxed)) { + throw_exception(exception_id::bad_alloc); + } + backoff.pause(); + table = my_segment_table.load(std::memory_order_acquire); + } while (table == my_embedded_table); + } + } + } + + // Return the segment where index is stored + static constexpr segment_index_type segment_index_of( size_type index ) { + return size_type(tbb::detail::log2(uintptr_t(index|1))); + } + + // Needed to calculate the offset in segment + static constexpr size_type segment_base( size_type index ) { + return size_type(1) << index & ~size_type(1); + } + + // Return size of the segment + static constexpr size_type segment_size( size_type index ) { + return index == 0 ? 2 : size_type(1) << index; + } + +private: + + derived_type* self() { + return static_cast(this); + } + + struct copy_segment_body_type { + void operator()( segment_index_type index, segment_type from, segment_type to ) const { + my_instance.self()->copy_segment(index, from, to); + } + segment_table& my_instance; + }; + + struct move_segment_body_type { + void operator()( segment_index_type index, segment_type from, segment_type to ) const { + my_instance.self()->move_segment(index, from, to); + } + segment_table& my_instance; + }; + + // Transgers all segments from the other table + template + void internal_transfer( const segment_table& other, TransferBody transfer_segment ) { + static_cast(this)->destroy_elements(); + + assign_first_block_if_necessary(other.my_first_block.load(std::memory_order_relaxed)); + my_size.store(other.my_size.load(std::memory_order_relaxed), std::memory_order_relaxed); + + segment_table_type other_table = other.get_table(); + size_type end_segment_size = segment_size(other.find_last_allocated_segment(other_table)); + + // If an exception occurred in other, then the size may be greater than the size of the end segment. + size_type other_size = end_segment_size < other.my_size.load(std::memory_order_relaxed) ? + other.my_size.load(std::memory_order_relaxed) : end_segment_size; + other_size = my_segment_table_allocation_failed ? embedded_table_size : other_size; + + for (segment_index_type i = 0; segment_base(i) < other_size; ++i) { + // If the segment in other table is enabled - transfer it + if (other_table[i].load(std::memory_order_relaxed) == segment_allocation_failure_tag) + { + my_size = segment_base(i); + break; + } else if (other_table[i].load(std::memory_order_relaxed) != nullptr) { + internal_subscript(segment_base(i)); + transfer_segment(i, other.get_table()[i].load(std::memory_order_relaxed) + segment_base(i), + get_table()[i].load(std::memory_order_relaxed) + segment_base(i)); + } + } + } + + // Moves the other segment table + // Only equal allocators are allowed + void internal_move( segment_table&& other ) { + // NOTE: allocators should be equal + clear(); + my_first_block.store(other.my_first_block.load(std::memory_order_relaxed), std::memory_order_relaxed); + my_size.store(other.my_size.load(std::memory_order_relaxed), std::memory_order_relaxed); + // If an active table in other is embedded - restore all of the embedded segments + if (other.get_table() == other.my_embedded_table) { + for ( size_type i = 0; i != pointers_per_embedded_table; ++i ) { + segment_type other_segment = other.my_embedded_table[i].load(std::memory_order_relaxed); + my_embedded_table[i].store(other_segment, std::memory_order_relaxed); + other.my_embedded_table[i].store(nullptr, std::memory_order_relaxed); + } + my_segment_table.store(my_embedded_table, std::memory_order_relaxed); + } else { + my_segment_table.store(other.my_segment_table, std::memory_order_relaxed); + other.my_segment_table.store(other.my_embedded_table, std::memory_order_relaxed); + zero_table(other.my_embedded_table, pointers_per_embedded_table); + } + other.my_size.store(0, std::memory_order_relaxed); + } + + // Move construct the segment table with the allocator object + // if any instances of allocator_type are always equal + void internal_move_construct_with_allocator( segment_table&& other, const allocator_type&, + /*is_always_equal = */ std::true_type ) { + internal_move(std::move(other)); + } + + // Move construct the segment table with the allocator object + // if any instances of allocator_type are always equal + void internal_move_construct_with_allocator( segment_table&& other, const allocator_type& alloc, + /*is_always_equal = */ std::false_type ) { + if (other.my_segment_table_allocator == alloc) { + // If allocators are equal - restore pointers + internal_move(std::move(other)); + } else { + // If allocators are not equal - perform per element move with reallocation + try_call( [&] { + internal_transfer(other, move_segment_body_type{*this}); + } ).on_exception( [&] { + clear(); + }); + } + } + + // Move assigns the segment table to other is any instances of allocator_type are always equal + // or propagate_on_container_move_assignment is true + void internal_move_assign( segment_table&& other, /*is_always_equal || POCMA = */ std::true_type ) { + internal_move(std::move(other)); + } + + // Move assigns the segment table to other is any instances of allocator_type are not always equal + // and propagate_on_container_move_assignment is false + void internal_move_assign( segment_table&& other, /*is_always_equal || POCMA = */ std::false_type ) { + if (my_segment_table_allocator == other.my_segment_table_allocator) { + // If allocators are equal - restore pointers + internal_move(std::move(other)); + } else { + // If allocators are not equal - perform per element move with reallocation + internal_transfer(other, move_segment_body_type{*this}); + } + } + + // Swaps two segment tables if any instances of allocator_type are always equal + // or propagate_on_container_swap is true + void internal_swap( segment_table& other, /*is_always_equal || POCS = */ std::true_type ) { + internal_swap_fields(other); + } + + // Swaps two segment tables if any instances of allocator_type are not always equal + // and propagate_on_container_swap is false + // According to the C++ standard, swapping of two containers with unequal allocators + // is an undefined behavior scenario + void internal_swap( segment_table& other, /*is_always_equal || POCS = */ std::false_type ) { + __TBB_ASSERT(my_segment_table_allocator == other.my_segment_table_allocator, + "Swapping with unequal allocators is not allowed"); + internal_swap_fields(other); + } + + void internal_swap_fields( segment_table& other ) { + // If an active table in either *this segment table or other is an embedded one - swaps the embedded tables + if (get_table() == my_embedded_table || + other.get_table() == other.my_embedded_table) { + + for (size_type i = 0; i != pointers_per_embedded_table; ++i) { + segment_type current_segment = my_embedded_table[i].load(std::memory_order_relaxed); + segment_type other_segment = other.my_embedded_table[i].load(std::memory_order_relaxed); + + my_embedded_table[i].store(other_segment, std::memory_order_relaxed); + other.my_embedded_table[i].store(current_segment, std::memory_order_relaxed); + } + } + + segment_table_type current_segment_table = get_table(); + segment_table_type other_segment_table = other.get_table(); + + // If an active table is an embedded one - + // store an active table in other to the embedded one from other + if (current_segment_table == my_embedded_table) { + other.my_segment_table.store(other.my_embedded_table, std::memory_order_relaxed); + } else { + // Otherwise - store it to the active segment table + other.my_segment_table.store(current_segment_table, std::memory_order_relaxed); + } + + // If an active table in other segment table is an embedded one - + // store an active table in other to the embedded one from *this + if (other_segment_table == other.my_embedded_table) { + my_segment_table.store(my_embedded_table, std::memory_order_relaxed); + } else { + // Otherwise - store it to the active segment table in other + my_segment_table.store(other_segment_table, std::memory_order_relaxed); + } + auto first_block = other.my_first_block.load(std::memory_order_relaxed); + other.my_first_block.store(my_first_block.load(std::memory_order_relaxed), std::memory_order_relaxed); + my_first_block.store(first_block, std::memory_order_relaxed); + + auto size = other.my_size.load(std::memory_order_relaxed); + other.my_size.store(my_size.load(std::memory_order_relaxed), std::memory_order_relaxed); + my_size.store(size, std::memory_order_relaxed); + } + +protected: + // A flag indicates that an exception was throws during segment allocations + const segment_type segment_allocation_failure_tag = reinterpret_cast(1); + static constexpr size_type embedded_table_size = segment_size(pointers_per_embedded_table); + + template + value_type& internal_subscript( size_type index ) { + segment_index_type seg_index = segment_index_of(index); + segment_table_type table = my_segment_table.load(std::memory_order_acquire); + segment_type segment = nullptr; + + if (allow_out_of_range_access) { + if (derived_type::allow_table_extending) { + extend_table_if_necessary(table, index, index + 1); + } + + segment = table[seg_index].load(std::memory_order_acquire); + // If the required segment is disabled - enable it + if (segment == nullptr) { + enable_segment(segment, table, seg_index, index); + } + // Check if an exception was thrown during segment allocation + if (segment == segment_allocation_failure_tag) { + throw_exception(exception_id::bad_alloc); + } + } else { + segment = table[seg_index].load(std::memory_order_acquire); + } + __TBB_ASSERT(segment != nullptr, nullptr); + + return segment[index]; + } + + void assign_first_block_if_necessary(segment_index_type index) { + size_type zero = 0; + if (this->my_first_block.load(std::memory_order_relaxed) == zero) { + this->my_first_block.compare_exchange_strong(zero, index); + } + } + + void zero_table( segment_table_type table, size_type count ) { + for (size_type i = 0; i != count; ++i) { + table[i].store(nullptr, std::memory_order_relaxed); + } + } + + segment_table_type get_table() const { + return my_segment_table.load(std::memory_order_acquire); + } + + segment_table_allocator_type my_segment_table_allocator; + std::atomic my_segment_table; + atomic_segment my_embedded_table[pointers_per_embedded_table]; + // Number of segments in first block + std::atomic my_first_block; + // Number of elements in table + std::atomic my_size; + // Flag to indicate failed extend table + std::atomic my_segment_table_allocation_failed; +}; // class segment_table + +} // namespace d1 +} // namespace detail +} // namespace tbb + +#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) +// #pragma warning(pop) // warning 4127 is back +#endif + +#endif // __TBB_detail__segment_table_H diff --git a/src/tbb/include/oneapi/tbb/detail/_small_object_pool.h b/src/tbb/include/oneapi/tbb/detail/_small_object_pool.h new file mode 100644 index 000000000..7485b31c7 --- /dev/null +++ b/src/tbb/include/oneapi/tbb/detail/_small_object_pool.h @@ -0,0 +1,108 @@ +/* + Copyright (c) 2020-2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB__small_object_pool_H +#define __TBB__small_object_pool_H + +#include "_config.h" +#include "_assert.h" + +#include "../profiling.h" +#include +#include +#include + +namespace tbb { +namespace detail { + +namespace d1 { +class small_object_pool { +protected: + small_object_pool() = default; +}; +struct execution_data; +} + +namespace r1 { +TBB_EXPORT void* __TBB_EXPORTED_FUNC allocate(d1::small_object_pool*& pool, std::size_t number_of_bytes, + const d1::execution_data& ed); +TBB_EXPORT void* __TBB_EXPORTED_FUNC allocate(d1::small_object_pool*& pool, std::size_t number_of_bytes); +TBB_EXPORT void __TBB_EXPORTED_FUNC deallocate(d1::small_object_pool& pool, void* ptr, std::size_t number_of_bytes, + const d1::execution_data& ed); +TBB_EXPORT void __TBB_EXPORTED_FUNC deallocate(d1::small_object_pool& pool, void* ptr, std::size_t number_of_bytes); +} + +namespace d1 { +class small_object_allocator { +public: + template + Type* new_object(execution_data& ed, Args&&... args) { + void* allocated_object = r1::allocate(m_pool, sizeof(Type), ed); + + auto constructed_object = new(allocated_object) Type(std::forward(args)...); + return constructed_object; + } + + template + Type* new_object(Args&&... args) { + void* allocated_object = r1::allocate(m_pool, sizeof(Type)); + + auto constructed_object = new(allocated_object) Type(std::forward(args)...); + return constructed_object; + } + + template + void delete_object(Type* object, const execution_data& ed) { + // Copy this since it can be a member of the passed object and + // unintentionally destroyed when Type destructor is called below + small_object_allocator alloc = *this; + object->~Type(); + alloc.deallocate(object, ed); + } + + template + void delete_object(Type* object) { + // Copy this since it can be a member of the passed object and + // unintentionally destroyed when Type destructor is called below + small_object_allocator alloc = *this; + object->~Type(); + alloc.deallocate(object); + } + + template + void deallocate(Type* ptr, const execution_data& ed) { + call_itt_task_notify(destroy, ptr); + + __TBB_ASSERT(m_pool != nullptr, "Pool must be valid for deallocate call"); + r1::deallocate(*m_pool, ptr, sizeof(Type), ed); + } + + template + void deallocate(Type* ptr) { + call_itt_task_notify(destroy, ptr); + + __TBB_ASSERT(m_pool != nullptr, "Pool must be valid for deallocate call"); + r1::deallocate(*m_pool, ptr, sizeof(Type)); + } +private: + small_object_pool* m_pool{}; +}; + +} // namespace d1 +} // namespace detail +} // namespace tbb + +#endif /* __TBB__small_object_pool_H */ diff --git a/inst/include/tbb/internal/_tbb_strings.h b/src/tbb/include/oneapi/tbb/detail/_string_resource.h similarity index 79% rename from inst/include/tbb/internal/_tbb_strings.h rename to src/tbb/include/oneapi/tbb/detail/_string_resource.h index 1aa532f3f..c06d5b5db 100644 --- a/inst/include/tbb/internal/_tbb_strings.h +++ b/src/tbb/include/oneapi/tbb/detail/_string_resource.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,12 +12,19 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ +TBB_STRING_RESOURCE(ALGORITHM, "tbb_algorithm") +TBB_STRING_RESOURCE(PARALLEL_FOR, "tbb_parallel_for") +TBB_STRING_RESOURCE(PARALLEL_FOR_EACH, "tbb_parallel_for_each") +TBB_STRING_RESOURCE(PARALLEL_INVOKE, "tbb_parallel_invoke") +TBB_STRING_RESOURCE(PARALLEL_REDUCE, "tbb_parallel_reduce") +TBB_STRING_RESOURCE(PARALLEL_SCAN, "tbb_parallel_scan") +TBB_STRING_RESOURCE(PARALLEL_SORT, "tbb_parallel_sort") +TBB_STRING_RESOURCE(PARALLEL_PIPELINE, "tbb_parallel_pipeline") +TBB_STRING_RESOURCE(CUSTOM_CTX, "tbb_custom") + +TBB_STRING_RESOURCE(FLOW_NULL, "null") TBB_STRING_RESOURCE(FLOW_BROADCAST_NODE, "broadcast_node") TBB_STRING_RESOURCE(FLOW_BUFFER_NODE, "buffer_node") TBB_STRING_RESOURCE(FLOW_CONTINUE_NODE, "continue_node") @@ -27,17 +34,16 @@ TBB_STRING_RESOURCE(FLOW_JOIN_NODE_RESERVING, "join_node (reserving)") TBB_STRING_RESOURCE(FLOW_JOIN_NODE_TAG_MATCHING, "join_node (tag_matching)") TBB_STRING_RESOURCE(FLOW_LIMITER_NODE, "limiter_node") TBB_STRING_RESOURCE(FLOW_MULTIFUNCTION_NODE, "multifunction_node") -TBB_STRING_RESOURCE(FLOW_OR_NODE, "or_node") //no longer in use, kept for backward compatibility TBB_STRING_RESOURCE(FLOW_OVERWRITE_NODE, "overwrite_node") TBB_STRING_RESOURCE(FLOW_PRIORITY_QUEUE_NODE, "priority_queue_node") TBB_STRING_RESOURCE(FLOW_QUEUE_NODE, "queue_node") TBB_STRING_RESOURCE(FLOW_SEQUENCER_NODE, "sequencer_node") -TBB_STRING_RESOURCE(FLOW_SOURCE_NODE, "source_node") +TBB_STRING_RESOURCE(FLOW_INPUT_NODE, "input_node") TBB_STRING_RESOURCE(FLOW_SPLIT_NODE, "split_node") TBB_STRING_RESOURCE(FLOW_WRITE_ONCE_NODE, "write_once_node") -TBB_STRING_RESOURCE(FLOW_BODY, "body") -TBB_STRING_RESOURCE(FLOW_GRAPH, "graph") -TBB_STRING_RESOURCE(FLOW_NODE, "node") +TBB_STRING_RESOURCE(FLOW_INDEXER_NODE, "indexer_node") +TBB_STRING_RESOURCE(FLOW_COMPOSITE_NODE, "composite_node") +TBB_STRING_RESOURCE(FLOW_ASYNC_NODE, "async_node") TBB_STRING_RESOURCE(FLOW_INPUT_PORT, "input_port") TBB_STRING_RESOURCE(FLOW_INPUT_PORT_0, "input_port_0") TBB_STRING_RESOURCE(FLOW_INPUT_PORT_1, "input_port_1") @@ -61,7 +67,12 @@ TBB_STRING_RESOURCE(FLOW_OUTPUT_PORT_7, "output_port_7") TBB_STRING_RESOURCE(FLOW_OUTPUT_PORT_8, "output_port_8") TBB_STRING_RESOURCE(FLOW_OUTPUT_PORT_9, "output_port_9") TBB_STRING_RESOURCE(FLOW_OBJECT_NAME, "object_name") -TBB_STRING_RESOURCE(FLOW_NULL, "null") -TBB_STRING_RESOURCE(FLOW_INDEXER_NODE, "indexer_node") -TBB_STRING_RESOURCE(FLOW_COMPOSITE_NODE, "composite_node") -TBB_STRING_RESOURCE(FLOW_ASYNC_NODE, "async_node") +TBB_STRING_RESOURCE(FLOW_BODY, "body") +TBB_STRING_RESOURCE(FLOW_GRAPH, "graph") +TBB_STRING_RESOURCE(FLOW_NODE, "node") +TBB_STRING_RESOURCE(FLOW_TASKS, "tbb_flow_graph") +TBB_STRING_RESOURCE(USER_EVENT, "user_event") + +#if __TBB_FLOW_TRACE_CODEPTR +TBB_STRING_RESOURCE(CODE_ADDRESS, "code_address") +#endif diff --git a/src/tbb/include/oneapi/tbb/detail/_task.h b/src/tbb/include/oneapi/tbb/detail/_task.h new file mode 100644 index 000000000..103a9868e --- /dev/null +++ b/src/tbb/include/oneapi/tbb/detail/_task.h @@ -0,0 +1,303 @@ +/* + Copyright (c) 2020-2024 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB__task_H +#define __TBB__task_H + +#include "_config.h" +#include "_assert.h" +#include "_template_helpers.h" +#include "_small_object_pool.h" + +#include "../profiling.h" + +#include +#include +#include +#include +#include +#include + +namespace tbb { +namespace detail { + +namespace d1 { +using slot_id = unsigned short; +constexpr slot_id no_slot = slot_id(~0); +constexpr slot_id any_slot = slot_id(~1); + +class task; +class wait_context; +class task_group_context; +struct execution_data; +class wait_tree_vertex_interface; +class task_arena_base; +} + +namespace d2 { +class task_group; +class task_group_base; +} + +namespace r1 { +//! Task spawn/wait entry points +TBB_EXPORT void __TBB_EXPORTED_FUNC spawn(d1::task& t, d1::task_group_context& ctx); +TBB_EXPORT void __TBB_EXPORTED_FUNC spawn(d1::task& t, d1::task_group_context& ctx, d1::slot_id id); +TBB_EXPORT void __TBB_EXPORTED_FUNC execute_and_wait(d1::task& t, d1::task_group_context& t_ctx, d1::wait_context&, d1::task_group_context& w_ctx); +TBB_EXPORT void __TBB_EXPORTED_FUNC wait(d1::wait_context&, d1::task_group_context& ctx); +TBB_EXPORT d1::slot_id __TBB_EXPORTED_FUNC execution_slot(const d1::execution_data*); +TBB_EXPORT d1::slot_id __TBB_EXPORTED_FUNC execution_slot(const d1::task_arena_base&); +TBB_EXPORT d1::task_group_context* __TBB_EXPORTED_FUNC current_context(); +TBB_EXPORT d1::wait_tree_vertex_interface* get_thread_reference_vertex(d1::wait_tree_vertex_interface* wc); + +// Do not place under __TBB_RESUMABLE_TASKS. It is a stub for unsupported platforms. +struct suspend_point_type; +using suspend_callback_type = void(*)(void*, suspend_point_type*); +//! The resumable tasks entry points +TBB_EXPORT void __TBB_EXPORTED_FUNC suspend(suspend_callback_type suspend_callback, void* user_callback); +TBB_EXPORT void __TBB_EXPORTED_FUNC resume(suspend_point_type* tag); +TBB_EXPORT suspend_point_type* __TBB_EXPORTED_FUNC current_suspend_point(); +TBB_EXPORT void __TBB_EXPORTED_FUNC notify_waiters(std::uintptr_t wait_ctx_addr); + +class thread_data; +class task_dispatcher; +class external_waiter; +struct task_accessor; +struct task_arena_impl; +} // namespace r1 + +namespace d1 { + +class task_arena; +using suspend_point = r1::suspend_point_type*; + +#if __TBB_RESUMABLE_TASKS +template +static void suspend_callback(void* user_callback, suspend_point sp) { + // Copy user function to a new stack after the context switch to avoid a race when the previous + // suspend point is resumed while the user_callback is being called. + F user_callback_copy = *static_cast(user_callback); + user_callback_copy(sp); +} + +template +void suspend(F f) { + r1::suspend(&suspend_callback, &f); +} + +inline void resume(suspend_point tag) { + r1::resume(tag); +} +#endif /* __TBB_RESUMABLE_TASKS */ + +// TODO align wait_context on cache lane +class wait_context { + static constexpr std::uint64_t overflow_mask = ~((1LLU << 32) - 1); + + std::uint64_t m_version_and_traits{1}; + std::atomic m_ref_count{}; + + void add_reference(std::int64_t delta) { + call_itt_task_notify(releasing, this); + std::uint64_t r = m_ref_count.fetch_add(static_cast(delta)) + static_cast(delta); + + __TBB_ASSERT_EX((r & overflow_mask) == 0, "Overflow is detected"); + + if (!r) { + // Some external waiters or coroutine waiters sleep in wait list + // Should to notify them that work is done + std::uintptr_t wait_ctx_addr = std::uintptr_t(this); + r1::notify_waiters(wait_ctx_addr); + } + } + +public: + bool continue_execution() const { + std::uint64_t r = m_ref_count.load(std::memory_order_acquire); + __TBB_ASSERT_EX((r & overflow_mask) == 0, "Overflow is detected"); + return r > 0; + } + +private: + friend class r1::thread_data; + friend class r1::task_dispatcher; + friend class r1::external_waiter; + friend class wait_context_vertex; + friend struct r1::task_arena_impl; + friend struct r1::suspend_point_type; +public: + // Despite the internal reference count is uin64_t we limit the user interface with uint32_t + // to preserve a part of the internal reference count for special needs. + wait_context(std::uint32_t ref_count) : m_ref_count{ref_count} { suppress_unused_warning(m_version_and_traits); } + wait_context(const wait_context&) = delete; + + ~wait_context() { + __TBB_ASSERT(!continue_execution(), nullptr); + } + + void reserve(std::uint32_t delta = 1) { + add_reference(delta); + } + + void release(std::uint32_t delta = 1) { + add_reference(-std::int64_t(delta)); + } +}; + +class wait_tree_vertex_interface { +public: + virtual void reserve(std::uint32_t delta = 1) = 0; + virtual void release(std::uint32_t delta = 1) = 0; + +protected: + virtual ~wait_tree_vertex_interface() = default; +}; + +class wait_context_vertex : public wait_tree_vertex_interface { +public: + wait_context_vertex(std::uint32_t ref = 0) : m_wait(ref) {} + + void reserve(std::uint32_t delta = 1) override { + m_wait.reserve(delta); + } + + void release(std::uint32_t delta = 1) override { + m_wait.release(delta); + } + + wait_context& get_context() { + return m_wait; + } +private: + friend class d2::task_group; + friend class d2::task_group_base; + + bool continue_execution() const { + return m_wait.continue_execution(); + } + + wait_context m_wait; +}; + +class reference_vertex : public wait_tree_vertex_interface { +public: + reference_vertex(wait_tree_vertex_interface* parent, std::uint32_t ref_count) : my_parent{parent}, m_ref_count{ref_count} + {} + + void reserve(std::uint32_t delta = 1) override { + if (m_ref_count.fetch_add(static_cast(delta)) == 0) { + my_parent->reserve(); + } + } + + void release(std::uint32_t delta = 1) override { + std::uint64_t ref = m_ref_count.fetch_sub(static_cast(delta)) - static_cast(delta); + if (ref == 0) { + my_parent->release(); + } + } + + std::uint32_t get_num_child() { + return static_cast(m_ref_count.load(std::memory_order_acquire)); + } +private: + wait_tree_vertex_interface* my_parent; + std::atomic m_ref_count; +}; + +struct execution_data { + task_group_context* context{}; + slot_id original_slot{}; + slot_id affinity_slot{}; +}; + +inline task_group_context* context(const execution_data& ed) { + return ed.context; +} + +inline slot_id original_slot(const execution_data& ed) { + return ed.original_slot; +} + +inline slot_id affinity_slot(const execution_data& ed) { + return ed.affinity_slot; +} + +inline slot_id execution_slot(const execution_data& ed) { + return r1::execution_slot(&ed); +} + +inline bool is_same_affinity(const execution_data& ed) { + return affinity_slot(ed) == no_slot || affinity_slot(ed) == execution_slot(ed); +} + +inline bool is_stolen(const execution_data& ed) { + return original_slot(ed) != execution_slot(ed); +} + +inline void spawn(task& t, task_group_context& ctx) { + call_itt_task_notify(releasing, &t); + r1::spawn(t, ctx); +} + +inline void spawn(task& t, task_group_context& ctx, slot_id id) { + call_itt_task_notify(releasing, &t); + r1::spawn(t, ctx, id); +} + +inline void execute_and_wait(task& t, task_group_context& t_ctx, wait_context& wait_ctx, task_group_context& w_ctx) { + r1::execute_and_wait(t, t_ctx, wait_ctx, w_ctx); + call_itt_task_notify(acquired, &wait_ctx); + call_itt_task_notify(destroy, &wait_ctx); +} + +inline void wait(wait_context& wait_ctx, task_group_context& ctx) { + r1::wait(wait_ctx, ctx); + call_itt_task_notify(acquired, &wait_ctx); + call_itt_task_notify(destroy, &wait_ctx); +} + +using r1::current_context; + +class task_traits { + std::uint64_t m_version_and_traits{}; + friend struct r1::task_accessor; +}; + +//! Alignment for a task object +static constexpr std::size_t task_alignment = 64; + +//! Base class for user-defined tasks. +/** @ingroup task_scheduling */ +class alignas(task_alignment) task : public task_traits { +protected: + virtual ~task() = default; + +public: + virtual task* execute(execution_data&) = 0; + virtual task* cancel(execution_data&) = 0; + +private: + std::uint64_t m_reserved[6]{}; + friend struct r1::task_accessor; +}; +static_assert(sizeof(task) == task_alignment, "task size is broken"); + +} // namespace d1 +} // namespace detail +} // namespace tbb + +#endif /* __TBB__task_H */ diff --git a/src/tbb/include/oneapi/tbb/detail/_task_handle.h b/src/tbb/include/oneapi/tbb/detail/_task_handle.h new file mode 100644 index 000000000..26212b462 --- /dev/null +++ b/src/tbb/include/oneapi/tbb/detail/_task_handle.h @@ -0,0 +1,123 @@ +/* + Copyright (c) 2020-2024 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + + +#ifndef __TBB_task_handle_H +#define __TBB_task_handle_H + +#include "_config.h" +#include "_task.h" +#include "_small_object_pool.h" +#include "_utils.h" +#include + +namespace tbb { +namespace detail { + +namespace d1 { class task_group_context; class wait_context; struct execution_data; } +namespace d2 { + +class task_handle; + +class task_handle_task : public d1::task { + std::uint64_t m_version_and_traits{}; + d1::wait_tree_vertex_interface* m_wait_tree_vertex; + d1::task_group_context& m_ctx; + d1::small_object_allocator m_allocator; +public: + void finalize(const d1::execution_data* ed = nullptr) { + if (ed) { + m_allocator.delete_object(this, *ed); + } else { + m_allocator.delete_object(this); + } + } + + task_handle_task(d1::wait_tree_vertex_interface* vertex, d1::task_group_context& ctx, d1::small_object_allocator& alloc) + : m_wait_tree_vertex(vertex) + , m_ctx(ctx) + , m_allocator(alloc) { + suppress_unused_warning(m_version_and_traits); + m_wait_tree_vertex->reserve(); + } + + ~task_handle_task() override { + m_wait_tree_vertex->release(); + } + + d1::task_group_context& ctx() const { return m_ctx; } +}; + + +class task_handle { + struct task_handle_task_finalizer_t{ + void operator()(task_handle_task* p){ p->finalize(); } + }; + using handle_impl_t = std::unique_ptr; + + handle_impl_t m_handle = {nullptr}; +public: + task_handle() = default; + task_handle(task_handle&&) = default; + task_handle& operator=(task_handle&&) = default; + + explicit operator bool() const noexcept { return static_cast(m_handle); } + + friend bool operator==(task_handle const& th, std::nullptr_t) noexcept; + friend bool operator==(std::nullptr_t, task_handle const& th) noexcept; + + friend bool operator!=(task_handle const& th, std::nullptr_t) noexcept; + friend bool operator!=(std::nullptr_t, task_handle const& th) noexcept; + +private: + friend struct task_handle_accessor; + + task_handle(task_handle_task* t) : m_handle {t}{}; + + d1::task* release() { + return m_handle.release(); + } +}; + +struct task_handle_accessor { +static task_handle construct(task_handle_task* t) { return {t}; } +static d1::task* release(task_handle& th) { return th.release(); } +static d1::task_group_context& ctx_of(task_handle& th) { + __TBB_ASSERT(th.m_handle, "ctx_of does not expect empty task_handle."); + return th.m_handle->ctx(); +} +}; + +inline bool operator==(task_handle const& th, std::nullptr_t) noexcept { + return th.m_handle == nullptr; +} +inline bool operator==(std::nullptr_t, task_handle const& th) noexcept { + return th.m_handle == nullptr; +} + +inline bool operator!=(task_handle const& th, std::nullptr_t) noexcept { + return th.m_handle != nullptr; +} + +inline bool operator!=(std::nullptr_t, task_handle const& th) noexcept { + return th.m_handle != nullptr; +} + +} // namespace d2 +} // namespace detail +} // namespace tbb + +#endif /* __TBB_task_handle_H */ diff --git a/src/tbb/include/oneapi/tbb/detail/_template_helpers.h b/src/tbb/include/oneapi/tbb/detail/_template_helpers.h new file mode 100644 index 000000000..50ce3d2d3 --- /dev/null +++ b/src/tbb/include/oneapi/tbb/detail/_template_helpers.h @@ -0,0 +1,403 @@ +/* + Copyright (c) 2005-2023 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_detail__template_helpers_H +#define __TBB_detail__template_helpers_H + +#include "_utils.h" +#include "_config.h" + +#include +#include +#include +#include +#include +#include + +namespace tbb { +namespace detail { +inline namespace d0 { + +// An internal implementation of void_t, which can be used in SFINAE contexts +template +struct void_impl { + using type = void; +}; // struct void_impl + +template +using void_t = typename void_impl::type; + +// Generic SFINAE helper for expression checks, based on the idea demonstrated in ISO C++ paper n4502 +template class... Checks> +struct supports_impl { + using type = std::false_type; +}; + +template class... Checks> +struct supports_impl...>, Checks...> { + using type = std::true_type; +}; + +template class... Checks> +using supports = typename supports_impl::type; + +//! A template to select either 32-bit or 64-bit constant as compile time, depending on machine word size. +template +struct select_size_t_constant { + // Explicit cast is needed to avoid compiler warnings about possible truncation. + // The value of the right size, which is selected by ?:, is anyway not truncated or promoted. + static const std::size_t value = static_cast((sizeof(std::size_t)==sizeof(u)) ? u : ull); +}; + +// TODO: do we really need it? +//! Cast between unrelated pointer types. +/** This method should be used sparingly as a last resort for dealing with + situations that inherently break strict ISO C++ aliasing rules. */ +// T is a pointer type because it will be explicitly provided by the programmer as a template argument; +// U is a referent type to enable the compiler to check that "ptr" is a pointer, deducing U in the process. +template +inline T punned_cast( U* ptr ) { + std::uintptr_t x = reinterpret_cast(ptr); + return reinterpret_cast(x); +} + +template +struct padded_base : T { + char pad[S - R]; +}; +template struct padded_base : T {}; + +//! Pads type T to fill out to a multiple of cache line size. +template +struct padded : padded_base {}; + +#if __TBB_CPP14_INTEGER_SEQUENCE_PRESENT + +using std::index_sequence; +using std::make_index_sequence; + +#else + +template class index_sequence {}; + +template +struct make_index_sequence_impl : make_index_sequence_impl < N - 1, N - 1, S... > {}; + +template +struct make_index_sequence_impl <0, S...> { + using type = index_sequence; +}; + +template +using make_index_sequence = typename make_index_sequence_impl::type; + +#endif /* __TBB_CPP14_INTEGER_SEQUENCE_PRESENT */ + +#if __TBB_CPP17_LOGICAL_OPERATIONS_PRESENT +using std::conjunction; +using std::disjunction; +#else // __TBB_CPP17_LOGICAL_OPERATIONS_PRESENT + +template +struct conjunction : std::true_type {}; + +template +struct conjunction + : std::conditional, First>::type {}; + +template +struct conjunction : T {}; + +template +struct disjunction : std::false_type {}; + +template +struct disjunction + : std::conditional>::type {}; + +template +struct disjunction : T {}; + +#endif // __TBB_CPP17_LOGICAL_OPERATIONS_PRESENT + +template +using iterator_value_t = typename std::iterator_traits::value_type; + +template +using iterator_key_t = typename std::remove_const::first_type>::type; + +template +using iterator_mapped_t = typename iterator_value_t::second_type; + +template +using iterator_alloc_pair_t = std::pair>::type, + iterator_mapped_t>; + +template using alloc_value_type = typename A::value_type; +template using alloc_ptr_t = typename std::allocator_traits::pointer; +template using has_allocate = decltype(std::declval&>() = std::declval().allocate(0)); +template using has_deallocate = decltype(std::declval().deallocate(std::declval>(), 0)); + +// alloc_value_type should be checked first, because it can be used in other checks +template +using is_allocator = supports; + +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT +template +inline constexpr bool is_allocator_v = is_allocator::value; +#endif + +// Template class in which the "type" determines the type of the element number N in pack Args +template +struct pack_element { + using type = void; +}; + +template +struct pack_element { + using type = typename pack_element::type; +}; + +template +struct pack_element<0, T, Args...> { + using type = T; +}; + +template +using pack_element_t = typename pack_element::type; + +template +class raii_guard { +public: + static_assert( + std::is_nothrow_copy_constructible::value && + std::is_nothrow_move_constructible::value, + "Throwing an exception during the Func copy or move construction cause an unexpected behavior." + ); + + raii_guard( Func f ) noexcept : my_func(f), is_active(true) {} + + raii_guard( raii_guard&& g ) noexcept : my_func(std::move(g.my_func)), is_active(g.is_active) { + g.is_active = false; + } + + ~raii_guard() { + if (is_active) { + my_func(); + } + } + + void dismiss() { + is_active = false; + } +private: + Func my_func; + bool is_active; +}; // class raii_guard + +template +raii_guard make_raii_guard( Func f ) { + return raii_guard(f); +} + +template +struct try_call_proxy { + try_call_proxy( Body b ) : body(b) {} + + template + void on_exception( OnExceptionBody on_exception_body ) { + auto guard = make_raii_guard(on_exception_body); + body(); + guard.dismiss(); + } + + template + void on_completion(OnCompletionBody on_completion_body) { + auto guard = make_raii_guard(on_completion_body); + body(); + } + + Body body; +}; // struct try_call_proxy + +// Template helper function for API +// try_call(lambda1).on_exception(lambda2) +// Executes lambda1 and if it throws an exception - executes lambda2 +template +try_call_proxy try_call( Body b ) { + return try_call_proxy(b); +} + +#if __TBB_CPP17_IS_SWAPPABLE_PRESENT +using std::is_nothrow_swappable; +using std::is_swappable; +#else // __TBB_CPP17_IS_SWAPPABLE_PRESENT +namespace is_swappable_detail { +using std::swap; + +template +using has_swap = decltype(swap(std::declval(), std::declval())); + +#if _MSC_VER && _MSC_VER <= 1900 && !__INTEL_COMPILER +// Workaround for VS2015: it fails to instantiate noexcept(...) inside std::integral_constant. +template +struct noexcept_wrapper { + static const bool value = noexcept(swap(std::declval(), std::declval())); +}; +template +struct is_nothrow_swappable_impl : std::integral_constant::value> {}; +#else +template +struct is_nothrow_swappable_impl : std::integral_constant(), std::declval()))> {}; +#endif +} + +template +struct is_swappable : supports {}; + +template +struct is_nothrow_swappable + : conjunction, is_swappable_detail::is_nothrow_swappable_impl> {}; +#endif // __TBB_CPP17_IS_SWAPPABLE_PRESENT + +//! Allows to store a function parameter pack as a variable and later pass it to another function +template< typename... Types > +struct stored_pack; + +template<> +struct stored_pack<> +{ + using pack_type = stored_pack<>; + stored_pack() {} + + // Friend front-end functions + template< typename F, typename Pack > friend void call(F&& f, Pack&& p); + template< typename Ret, typename F, typename Pack > friend Ret call_and_return(F&& f, Pack&& p); + +protected: + // Ideally, ref-qualified non-static methods would be used, + // but that would greatly reduce the set of compilers where it works. + template< typename Ret, typename F, typename... Preceding > + static Ret call(F&& f, const pack_type& /*pack*/, Preceding&&... params) { + return std::forward(f)(std::forward(params)...); + } + template< typename Ret, typename F, typename... Preceding > + static Ret call(F&& f, pack_type&& /*pack*/, Preceding&&... params) { + return std::forward(f)(std::forward(params)...); + } +}; + +template< typename T, typename... Types > +struct stored_pack : stored_pack +{ + using pack_type = stored_pack; + using pack_remainder = stored_pack; + + // Since lifetime of original values is out of control, copies should be made. + // Thus references should be stripped away from the deduced type. + typename std::decay::type leftmost_value; + + // Here rvalue references act in the same way as forwarding references, + // as long as class template parameters were deduced via forwarding references. + stored_pack(T&& t, Types&&... types) + : pack_remainder(std::forward(types)...), leftmost_value(std::forward(t)) {} + + // Friend front-end functions + template< typename F, typename Pack > friend void call(F&& f, Pack&& p); + template< typename Ret, typename F, typename Pack > friend Ret call_and_return(F&& f, Pack&& p); + +protected: + template< typename Ret, typename F, typename... Preceding > + static Ret call(F&& f, pack_type& pack, Preceding&&... params) { + return pack_remainder::template call( + std::forward(f), static_cast(pack), + std::forward(params)... , pack.leftmost_value + ); + } + + template< typename Ret, typename F, typename... Preceding > + static Ret call(F&& f, pack_type&& pack, Preceding&&... params) { + return pack_remainder::template call( + std::forward(f), static_cast(pack), + std::forward(params)... , std::move(pack.leftmost_value) + ); + } +}; + +//! Calls the given function with arguments taken from a stored_pack +template< typename F, typename Pack > +void call(F&& f, Pack&& p) { + std::decay::type::template call(std::forward(f), std::forward(p)); +} + +template< typename Ret, typename F, typename Pack > +Ret call_and_return(F&& f, Pack&& p) { + return std::decay::type::template call(std::forward(f), std::forward(p)); +} + +template< typename... Types > +stored_pack save_pack(Types&&... types) { + return stored_pack(std::forward(types)...); +} + +// A structure with the value which is equal to Trait::value +// but can be used in the immediate context due to parameter T +template +struct dependent_bool : std::integral_constant {}; + +template +struct body_arg_detector; + +template +struct body_arg_detector { + using arg_type = Arg; +}; + +template +struct body_arg_detector { + using arg_type = Arg; +}; + +template +struct argument_detector; + +template +struct argument_detector { + using type = typename body_arg_detector::arg_type; +}; + +template +struct argument_detector { + using type = Arg; +}; + +// Detects the argument type of callable, works for callable with one argument. +template +using argument_type_of = typename argument_detector::type>::type; + +template +struct type_identity { + using type = T; +}; + +template +using type_identity_t = typename type_identity::type; + +} // inline namespace d0 +} // namespace detail +} // namespace tbb + +#endif // __TBB_detail__template_helpers_H diff --git a/src/tbb/include/oneapi/tbb/detail/_utils.h b/src/tbb/include/oneapi/tbb/detail/_utils.h new file mode 100644 index 000000000..1f480702f --- /dev/null +++ b/src/tbb/include/oneapi/tbb/detail/_utils.h @@ -0,0 +1,393 @@ +/* + Copyright (c) 2005-2023 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_detail__utils_H +#define __TBB_detail__utils_H + +#include +#include +#include +#include + +#include "_config.h" +#include "_assert.h" +#include "_machine.h" + +namespace tbb { +namespace detail { +inline namespace d0 { + +//! Utility template function to prevent "unused" warnings by various compilers. +template void suppress_unused_warning(T&&...) {} + +//! Compile-time constant that is upper bound on cache line/sector size. +/** It should be used only in situations where having a compile-time upper + bound is more useful than a run-time exact answer. + @ingroup memory_allocation */ +constexpr size_t max_nfs_size = 128; +constexpr std::size_t max_nfs_size_exp = 7; +static_assert(1 << max_nfs_size_exp == max_nfs_size, "max_nfs_size_exp must be a log2(max_nfs_size)"); + +//! Class that implements exponential backoff. +class atomic_backoff { + //! Time delay, in units of "pause" instructions. + /** Should be equal to approximately the number of "pause" instructions + that take the same time as an context switch. Must be a power of two.*/ + static constexpr std::int32_t LOOPS_BEFORE_YIELD = 16; + std::int32_t count; + +public: + // In many cases, an object of this type is initialized eagerly on hot path, + // as in for(atomic_backoff b; ; b.pause()) { /*loop body*/ } + // For this reason, the construction cost must be very small! + atomic_backoff() : count(1) {} + // This constructor pauses immediately; do not use on hot paths! + atomic_backoff(bool) : count(1) { pause(); } + + //! No Copy + atomic_backoff(const atomic_backoff&) = delete; + atomic_backoff& operator=(const atomic_backoff&) = delete; + + //! Pause for a while. + void pause() { + if (count <= LOOPS_BEFORE_YIELD) { + machine_pause(count); + // Pause twice as long the next time. + count *= 2; + } else { + // Pause is so long that we might as well yield CPU to scheduler. + yield(); + } + } + + //! Pause for a few times and return false if saturated. + bool bounded_pause() { + machine_pause(count); + if (count < LOOPS_BEFORE_YIELD) { + // Pause twice as long the next time. + count *= 2; + return true; + } else { + return false; + } + } + + void reset() { + count = 1; + } +}; + +//! Spin WHILE the condition is true. +/** T and U should be comparable types. */ +template +T spin_wait_while(const std::atomic& location, C comp, std::memory_order order) { + atomic_backoff backoff; + T snapshot = location.load(order); + while (comp(snapshot)) { + backoff.pause(); + snapshot = location.load(order); + } + return snapshot; +} + +//! Spin WHILE the value of the variable is equal to a given value +/** T and U should be comparable types. */ +template +T spin_wait_while_eq(const std::atomic& location, const U value, std::memory_order order = std::memory_order_acquire) { + return spin_wait_while(location, [&value](T t) { return t == value; }, order); +} + +//! Spin UNTIL the value of the variable is equal to a given value +/** T and U should be comparable types. */ +template +T spin_wait_until_eq(const std::atomic& location, const U value, std::memory_order order = std::memory_order_acquire) { + return spin_wait_while(location, [&value](T t) { return t != value; }, order); +} + +//! Spin UNTIL the condition returns true or spinning time is up. +/** Returns what the passed functor returned last time it was invoked. */ +template +bool timed_spin_wait_until(Condition condition) { + // 32 pauses + 32 yields are meausered as balanced spin time before sleep. + bool finish = condition(); + for (int i = 1; !finish && i < 32; finish = condition(), i *= 2) { + machine_pause(i); + } + for (int i = 32; !finish && i < 64; finish = condition(), ++i) { + yield(); + } + return finish; +} + +template +T clamp(T value, T lower_bound, T upper_bound) { + __TBB_ASSERT(lower_bound <= upper_bound, "Incorrect bounds"); + return value > lower_bound ? (value > upper_bound ? upper_bound : value) : lower_bound; +} + +template +std::uintptr_t log2(T in) { + __TBB_ASSERT(in > 0, "The logarithm of a non-positive value is undefined."); + return machine_log2(in); +} + +template +T reverse_bits(T src) { + return machine_reverse_bits(src); +} + +template +T reverse_n_bits(T src, std::size_t n) { + __TBB_ASSERT(n != 0, "Reverse for 0 bits is undefined behavior."); + return reverse_bits(src) >> (number_of_bits() - n); +} + +// A function to check if passed integer is a power of two +template +constexpr bool is_power_of_two( IntegerType arg ) { + static_assert(std::is_integral::value, + "An argument for is_power_of_two should be integral type"); + return arg && (0 == (arg & (arg - 1))); +} + +// A function to determine if passed integer is a power of two +// at least as big as another power of two, i.e. for strictly positive i and j, +// with j being a power of two, determines whether i==j< +constexpr bool is_power_of_two_at_least(ArgIntegerType arg, DivisorIntegerType divisor) { + // Divisor should be a power of two + static_assert(std::is_integral::value, + "An argument for is_power_of_two_at_least should be integral type"); + return 0 == (arg & (arg - divisor)); +} + +// A function to compute arg modulo divisor where divisor is a power of 2. +template +inline ArgIntegerType modulo_power_of_two(ArgIntegerType arg, DivisorIntegerType divisor) { + __TBB_ASSERT( is_power_of_two(divisor), "Divisor should be a power of two" ); + return arg & (divisor - 1); +} + +//! A function to check if passed in pointer is aligned on a specific border +template +constexpr bool is_aligned(T* pointer, std::uintptr_t alignment) { + return 0 == (reinterpret_cast(pointer) & (alignment - 1)); +} + +#if TBB_USE_ASSERT +static void* const poisoned_ptr = reinterpret_cast(-1); + +//! Set p to invalid pointer value. +template +inline void poison_pointer( T* &p ) { p = reinterpret_cast(poisoned_ptr); } + +template +inline void poison_pointer(std::atomic& p) { p.store(reinterpret_cast(poisoned_ptr), std::memory_order_relaxed); } + +/** Expected to be used in assertions only, thus no empty form is defined. **/ +template +inline bool is_poisoned( T* p ) { return p == reinterpret_cast(poisoned_ptr); } + +template +inline bool is_poisoned(const std::atomic& p) { return is_poisoned(p.load(std::memory_order_relaxed)); } +#else +template +inline void poison_pointer(T&) {/*do nothing*/} +#endif /* !TBB_USE_ASSERT */ + +template +bool assert_pointer_valid(T* p, const char* comment = nullptr) { + suppress_unused_warning(p, comment); + __TBB_ASSERT(p != nullptr, comment); + __TBB_ASSERT(!is_poisoned(p), comment); +#if !(_MSC_VER && _MSC_VER <= 1900 && !__INTEL_COMPILER) + __TBB_ASSERT(is_aligned(p, alignment == 0 ? alignof(T) : alignment), comment); +#endif + // Returns something to simplify assert_pointers_valid implementation. + return true; +} + +template +void assert_pointers_valid(Args*... p) { + // suppress_unused_warning is used as an evaluation context for the variadic pack. + suppress_unused_warning(assert_pointer_valid(p)...); +} + +//! Base class for types that should not be assigned. +class no_assign { +public: + void operator=(const no_assign&) = delete; + no_assign(const no_assign&) = default; + no_assign() = default; +}; + +//! Base class for types that should not be copied or assigned. +class no_copy: no_assign { +public: + no_copy(const no_copy&) = delete; + no_copy() = default; +}; + +template +void swap_atomics_relaxed(std::atomic& lhs, std::atomic& rhs){ + T tmp = lhs.load(std::memory_order_relaxed); + lhs.store(rhs.load(std::memory_order_relaxed), std::memory_order_relaxed); + rhs.store(tmp, std::memory_order_relaxed); +} + +//! One-time initialization states +enum class do_once_state { + uninitialized = 0, ///< No execution attempts have been undertaken yet + pending, ///< A thread is executing associated do-once routine + executed, ///< Do-once routine has been executed + initialized = executed ///< Convenience alias +}; + +//! One-time initialization function +/** /param initializer Pointer to function without arguments + The variant that returns bool is used for cases when initialization can fail + and it is OK to continue execution, but the state should be reset so that + the initialization attempt was repeated the next time. + /param state Shared state associated with initializer that specifies its + initialization state. Must be initially set to #uninitialized value + (e.g. by means of default static zero initialization). **/ +template +void atomic_do_once( const F& initializer, std::atomic& state ) { + // The loop in the implementation is necessary to avoid race when thread T2 + // that arrived in the middle of initialization attempt by another thread T1 + // has just made initialization possible. + // In such a case T2 has to rely on T1 to initialize, but T1 may already be past + // the point where it can recognize the changed conditions. + do_once_state expected_state; + while ( state.load( std::memory_order_acquire ) != do_once_state::executed ) { + if( state.load( std::memory_order_relaxed ) == do_once_state::uninitialized ) { + expected_state = do_once_state::uninitialized; +#if defined(__INTEL_COMPILER) && __INTEL_COMPILER <= 1910 + using enum_type = typename std::underlying_type::type; + if( ((std::atomic&)state).compare_exchange_strong( (enum_type&)expected_state, (enum_type)do_once_state::pending ) ) { +#else + if( state.compare_exchange_strong( expected_state, do_once_state::pending ) ) { +#endif + run_initializer( initializer, state ); + break; + } + } + spin_wait_while_eq( state, do_once_state::pending ); + } +} + +// Run the initializer which can not fail +template +void run_initializer(const Functor& f, std::atomic& state ) { + f(); + state.store(do_once_state::executed, std::memory_order_release); +} + +#if __TBB_CPP20_CONCEPTS_PRESENT +template +concept boolean_testable_impl = std::convertible_to; + +template +concept boolean_testable = boolean_testable_impl && requires( T&& t ) { + { !std::forward(t) } -> boolean_testable_impl; + }; + +#if __TBB_CPP20_COMPARISONS_PRESENT +struct synthesized_three_way_comparator { + template + auto operator()( const T1& lhs, const T2& rhs ) const + requires requires { + { lhs < rhs } -> boolean_testable; + { rhs < lhs } -> boolean_testable; + } + { + if constexpr (std::three_way_comparable_with) { + return lhs <=> rhs; + } else { + if (lhs < rhs) { + return std::weak_ordering::less; + } + if (rhs < lhs) { + return std::weak_ordering::greater; + } + return std::weak_ordering::equivalent; + } + } +}; // struct synthesized_three_way_comparator + +template +using synthesized_three_way_result = decltype(synthesized_three_way_comparator{}(std::declval(), + std::declval())); + +#endif // __TBB_CPP20_COMPARISONS_PRESENT + +// Check if the type T is implicitly OR explicitly convertible to U +template +concept relaxed_convertible_to = std::constructible_from; + +template +concept adaptive_same_as = +#if __TBB_STRICT_CONSTRAINTS + std::same_as; +#else + std::convertible_to; +#endif +#endif // __TBB_CPP20_CONCEPTS_PRESENT + +template +auto invoke(F&& f, Args&&... args) +#if __TBB_CPP17_INVOKE_PRESENT + noexcept(std::is_nothrow_invocable_v) + -> std::invoke_result_t +{ + return std::invoke(std::forward(f), std::forward(args)...); +} +#else // __TBB_CPP17_INVOKE_PRESENT + noexcept(noexcept(std::forward(f)(std::forward(args)...))) + -> decltype(std::forward(f)(std::forward(args)...)) +{ + return std::forward(f)(std::forward(args)...); +} +#endif // __TBB_CPP17_INVOKE_PRESENT + +} // namespace d0 + +namespace d1 { + +class delegate_base { +public: + virtual bool operator()() const = 0; + virtual ~delegate_base() {} +}; + +template +class delegated_function : public delegate_base { +public: + delegated_function(FuncType& f) : my_func(f) {} + + bool operator()() const override { + return my_func(); + } + +private: + FuncType &my_func; +}; +} // namespace d1 + +} // namespace detail +} // namespace tbb + +#endif // __TBB_detail__utils_H diff --git a/src/tbb/include/oneapi/tbb/detail/_waitable_atomic.h b/src/tbb/include/oneapi/tbb/detail/_waitable_atomic.h new file mode 100644 index 000000000..1b18d11e5 --- /dev/null +++ b/src/tbb/include/oneapi/tbb/detail/_waitable_atomic.h @@ -0,0 +1,90 @@ +/* + Copyright (c) 2021-2023 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_detail__address_waiters_H +#define __TBB_detail__address_waiters_H + +#include "_utils.h" + +namespace tbb { +namespace detail { + +namespace r1 { +TBB_EXPORT void __TBB_EXPORTED_FUNC wait_on_address(void* address, d1::delegate_base& wakeup_condition, std::uintptr_t context); +TBB_EXPORT void __TBB_EXPORTED_FUNC notify_by_address(void* address, std::uintptr_t context); +TBB_EXPORT void __TBB_EXPORTED_FUNC notify_by_address_one(void* address); +TBB_EXPORT void __TBB_EXPORTED_FUNC notify_by_address_all(void* address); +} // namespace r1 + +namespace d1 { + +template +void adaptive_wait_on_address(void* address, Predicate wakeup_condition, std::uintptr_t context) { + if (!timed_spin_wait_until(wakeup_condition)) { + d1::delegated_function pred(wakeup_condition); + r1::wait_on_address(address, pred, context); + } +} + +template +class waitable_atomic { +public: + waitable_atomic() = default; + + explicit waitable_atomic(T value) : my_atomic(value) {} + + waitable_atomic(const waitable_atomic&) = delete; + waitable_atomic& operator=(const waitable_atomic&) = delete; + + T load(std::memory_order order) const noexcept { + return my_atomic.load(order); + } + + T exchange(T desired) noexcept { + return my_atomic.exchange(desired); + } + + void wait(T old, std::uintptr_t context, std::memory_order order) { + auto wakeup_condition = [&] { return my_atomic.load(order) != old; }; + if (!timed_spin_wait_until(wakeup_condition)) { + // We need to use while here, because notify_all() will wake up all threads + // But predicate for them might be false + d1::delegated_function pred(wakeup_condition); + do { + r1::wait_on_address(this, pred, context); + } while (!wakeup_condition()); + } + } + + void notify_one_relaxed() { + r1::notify_by_address_one(this); + } + + // TODO: consider adding following interfaces: + // store(desired, memory_order) + // notify_all_relaxed() + // wait_until(T, std::uintptr_t, std::memory_order) + // notify_relaxed(std::uintptr_t context) + +private: + std::atomic my_atomic{}; +}; + +} // namespace d1 +} // namespace detail +} // namespace tbb + +#endif // __TBB_detail__address_waiters_H diff --git a/src/tbb/include/oneapi/tbb/enumerable_thread_specific.h b/src/tbb/include/oneapi/tbb/enumerable_thread_specific.h new file mode 100644 index 000000000..caa53fa0d --- /dev/null +++ b/src/tbb/include/oneapi/tbb/enumerable_thread_specific.h @@ -0,0 +1,1121 @@ +/* + Copyright (c) 2005-2024 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_enumerable_thread_specific_H +#define __TBB_enumerable_thread_specific_H + +#include "detail/_config.h" +#include "detail/_namespace_injection.h" +#include "detail/_assert.h" +#include "detail/_template_helpers.h" +#include "detail/_aligned_space.h" + +#include "concurrent_vector.h" +#include "tbb_allocator.h" +#include "cache_aligned_allocator.h" +#include "profiling.h" + +#include +#include +#include // memcpy +#include // std::ptrdiff_t + +#include "task.h" // for task::suspend_point + +#if _WIN32 || _WIN64 +#ifndef NOMINMAX +#define NOMINMAX +#define __TBB_DEFINED_NOMINMAX 1 +#endif +#include +#if __TBB_DEFINED_NOMINMAX +#undef NOMINMAX +#undef __TBB_DEFINED_NOMINMAX +#endif +#else +#include +#endif + +namespace tbb { +namespace detail { +namespace d1 { + +//! enum for selecting between single key and key-per-instance versions +enum ets_key_usage_type { + ets_key_per_instance + , ets_no_key +#if __TBB_RESUMABLE_TASKS + , ets_suspend_aware +#endif +}; + +// Forward declaration to use in internal classes +template +class enumerable_thread_specific; + +template +struct internal_ets_key_selector { + using key_type = std::thread::id; + static key_type current_key() { + return std::this_thread::get_id(); + } +}; + +// Intel Compiler on OSX cannot create atomics objects that instantiated from non-fundamental types +#if __INTEL_COMPILER && __APPLE__ +template<> +struct internal_ets_key_selector { + using key_type = std::size_t; + static key_type current_key() { + auto id = std::this_thread::get_id(); + return reinterpret_cast(id); + } +}; +#endif + +template +struct ets_key_selector : internal_ets_key_selector {}; + +#if __TBB_RESUMABLE_TASKS +template <> +struct ets_key_selector { + using key_type = suspend_point; + static key_type current_key() { + return r1::current_suspend_point(); + } +}; +#endif + +template +class ets_base : detail::no_copy { +protected: + using key_type = typename ets_key_selector::key_type; + +public: + struct slot; + struct array { + array* next; + std::size_t lg_size; + slot& at( std::size_t k ) { + return (reinterpret_cast(reinterpret_cast(this+1)))[k]; + } + std::size_t size() const { return std::size_t(1) << lg_size; } + std::size_t mask() const { return size() - 1; } + std::size_t start( std::size_t h ) const { + return h >> (8 * sizeof(std::size_t) - lg_size); + } + }; + struct slot { + std::atomic key; + void* ptr; + bool empty() const { return key.load(std::memory_order_relaxed) == key_type(); } + bool match( key_type k ) const { return key.load(std::memory_order_relaxed) == k; } + bool claim( key_type k ) { + // TODO: maybe claim ptr, because key_type is not guaranteed to fit into word size + key_type expected = key_type(); + return key.compare_exchange_strong(expected, k); + } + }; + +protected: + //! Root of linked list of arrays of decreasing size. + /** nullptr if and only if my_count==0. + Each array in the list is half the size of its predecessor. */ + std::atomic my_root; + std::atomic my_count; + + virtual void* create_local() = 0; + virtual void* create_array(std::size_t _size) = 0; // _size in bytes + virtual void free_array(void* ptr, std::size_t _size) = 0; // _size in bytes + + array* allocate( std::size_t lg_size ) { + std::size_t n = std::size_t(1) << lg_size; + array* a = static_cast(create_array(sizeof(array) + n * sizeof(slot))); + a->lg_size = lg_size; + std::memset( a + 1, 0, n * sizeof(slot) ); + return a; + } + void deallocate(array* a) { + std::size_t n = std::size_t(1) << (a->lg_size); + free_array( static_cast(a), std::size_t(sizeof(array) + n * sizeof(slot)) ); + } + + ets_base() : my_root{nullptr}, my_count{0} {} + virtual ~ets_base(); // g++ complains if this is not virtual + + void* table_lookup( bool& exists ); + void table_clear(); + // The following functions are not used in concurrent context, + // so we don't need synchronization and ITT annotations there. + template + void table_elementwise_copy( const ets_base& other, + void*(*add_element)(ets_base&, void*) ) { + __TBB_ASSERT(!my_root.load(std::memory_order_relaxed), nullptr); + __TBB_ASSERT(!my_count.load(std::memory_order_relaxed), nullptr); + if( !other.my_root.load(std::memory_order_relaxed) ) return; + array* root = allocate(other.my_root.load(std::memory_order_relaxed)->lg_size); + my_root.store(root, std::memory_order_relaxed); + root->next = nullptr; + my_count.store(other.my_count.load(std::memory_order_relaxed), std::memory_order_relaxed); + std::size_t mask = root->mask(); + for( array* r = other.my_root.load(std::memory_order_relaxed); r; r = r->next ) { + for( std::size_t i = 0; i < r->size(); ++i ) { + slot& s1 = r->at(i); + if( !s1.empty() ) { + for( std::size_t j = root->start(std::hash{}(s1.key.load(std::memory_order_relaxed))); ; j = (j+1)&mask ) { + slot& s2 = root->at(j); + if( s2.empty() ) { + s2.ptr = add_element(static_cast&>(*this), s1.ptr); + s2.key.store(s1.key.load(std::memory_order_relaxed), std::memory_order_relaxed); + break; + } + else if( s2.match(s1.key.load(std::memory_order_relaxed)) ) + break; + } + } + } + } + } + void table_swap( ets_base& other ) { + __TBB_ASSERT(this!=&other, "Don't swap an instance with itself"); + swap_atomics_relaxed(my_root, other.my_root); + swap_atomics_relaxed(my_count, other.my_count); + } +}; + +template +ets_base::~ets_base() { + __TBB_ASSERT(!my_root.load(std::memory_order_relaxed), nullptr); +} + +template +void ets_base::table_clear() { + while ( array* r = my_root.load(std::memory_order_relaxed) ) { + my_root.store(r->next, std::memory_order_relaxed); + deallocate(r); + } + my_count.store(0, std::memory_order_relaxed); +} + +template +void* ets_base::table_lookup( bool& exists ) { + const key_type k = ets_key_selector::current_key(); + + __TBB_ASSERT(k != key_type(), nullptr); + void* found; + std::size_t h = std::hash{}(k); + for( array* r = my_root.load(std::memory_order_acquire); r; r = r->next ) { + call_itt_notify(acquired,r); + std::size_t mask=r->mask(); + for(std::size_t i = r->start(h); ;i=(i+1)&mask) { + slot& s = r->at(i); + if( s.empty() ) break; + if( s.match(k) ) { + if( r == my_root.load(std::memory_order_acquire) ) { + // Success at top level + exists = true; + return s.ptr; + } else { + // Success at some other level. Need to insert at top level. + exists = true; + found = s.ptr; + goto insert; + } + } + } + } + // Key does not yet exist. The density of slots in the table does not exceed 0.5, + // for if this will occur a new table is allocated with double the current table + // size, which is swapped in as the new root table. So an empty slot is guaranteed. + exists = false; + found = create_local(); + { + std::size_t c = ++my_count; + array* r = my_root.load(std::memory_order_acquire); + call_itt_notify(acquired,r); + if( !r || c > r->size()/2 ) { + std::size_t s = r ? r->lg_size : 2; + while( c > std::size_t(1)<<(s-1) ) ++s; + array* a = allocate(s); + for(;;) { + a->next = r; + call_itt_notify(releasing,a); + array* new_r = r; + if( my_root.compare_exchange_strong(new_r, a) ) break; + call_itt_notify(acquired, new_r); + __TBB_ASSERT(new_r != nullptr, nullptr); + if( new_r->lg_size >= s ) { + // Another thread inserted an equal or bigger array, so our array is superfluous. + deallocate(a); + break; + } + r = new_r; + } + } + } + insert: + // Whether a slot has been found in an older table, or if it has been inserted at this level, + // it has already been accounted for in the total. Guaranteed to be room for it, and it is + // not present, so search for empty slot and use it. + array* ir = my_root.load(std::memory_order_acquire); + call_itt_notify(acquired, ir); + std::size_t mask = ir->mask(); + for(std::size_t i = ir->start(h);; i = (i+1)&mask) { + slot& s = ir->at(i); + if( s.empty() ) { + if( s.claim(k) ) { + s.ptr = found; + return found; + } + } + } +} + +//! Specialization that exploits native TLS +template <> +class ets_base: public ets_base { + using super = ets_base; +#if _WIN32||_WIN64 +#if __TBB_WIN8UI_SUPPORT + using tls_key_t = DWORD; + void create_key() { my_key = FlsAlloc(nullptr); } + void destroy_key() { FlsFree(my_key); } + void set_tls(void * value) { FlsSetValue(my_key, (LPVOID)value); } + void* get_tls() { return (void *)FlsGetValue(my_key); } +#else + using tls_key_t = DWORD; + void create_key() { my_key = TlsAlloc(); } + void destroy_key() { TlsFree(my_key); } + void set_tls(void * value) { TlsSetValue(my_key, (LPVOID)value); } + void* get_tls() { return (void *)TlsGetValue(my_key); } +#endif +#else + using tls_key_t = pthread_key_t; + void create_key() { pthread_key_create(&my_key, nullptr); } + void destroy_key() { pthread_key_delete(my_key); } + void set_tls( void * value ) const { pthread_setspecific(my_key, value); } + void* get_tls() const { return pthread_getspecific(my_key); } +#endif + tls_key_t my_key; + virtual void* create_local() override = 0; + virtual void* create_array(std::size_t _size) override = 0; // _size in bytes + virtual void free_array(void* ptr, std::size_t _size) override = 0; // size in bytes +protected: + ets_base() {create_key();} + ~ets_base() {destroy_key();} + void* table_lookup( bool& exists ) { + void* found = get_tls(); + if( found ) { + exists=true; + } else { + found = super::table_lookup(exists); + set_tls(found); + } + return found; + } + void table_clear() { + destroy_key(); + create_key(); + super::table_clear(); + } + void table_swap( ets_base& other ) { + using std::swap; + __TBB_ASSERT(this!=&other, "Don't swap an instance with itself"); + swap(my_key, other.my_key); + super::table_swap(other); + } +}; + +//! Random access iterator for traversing the thread local copies. +template< typename Container, typename Value > +class enumerable_thread_specific_iterator +{ + //! current position in the concurrent_vector + + Container *my_container; + typename Container::size_type my_index; + mutable Value *my_value; + + template + friend bool operator==( const enumerable_thread_specific_iterator& i, + const enumerable_thread_specific_iterator& j ); + + template + friend bool operator<( const enumerable_thread_specific_iterator& i, + const enumerable_thread_specific_iterator& j ); + + template + friend std::ptrdiff_t operator-( const enumerable_thread_specific_iterator& i, + const enumerable_thread_specific_iterator& j ); + + template + friend class enumerable_thread_specific_iterator; + +public: + //! STL support + using difference_type = std::ptrdiff_t; + using value_type = Value; + using pointer = Value*; + using reference = Value&; + using iterator_category = std::random_access_iterator_tag; + + enumerable_thread_specific_iterator( const Container &container, typename Container::size_type index ) : + my_container(&const_cast(container)), my_index(index), my_value(nullptr) {} + + //! Default constructor + enumerable_thread_specific_iterator() : my_container(nullptr), my_index(0), my_value(nullptr) {} + + template + enumerable_thread_specific_iterator( const enumerable_thread_specific_iterator& other ) : + my_container( other.my_container ), my_index( other.my_index), my_value( const_cast(other.my_value) ) {} + + enumerable_thread_specific_iterator operator+( std::ptrdiff_t offset ) const { + return enumerable_thread_specific_iterator(*my_container, my_index + offset); + } + + friend enumerable_thread_specific_iterator operator+( std::ptrdiff_t offset, enumerable_thread_specific_iterator v ) { + return enumerable_thread_specific_iterator(*v.my_container, v.my_index + offset); + } + + enumerable_thread_specific_iterator &operator+=( std::ptrdiff_t offset ) { + my_index += offset; + my_value = nullptr; + return *this; + } + + enumerable_thread_specific_iterator operator-( std::ptrdiff_t offset ) const { + return enumerable_thread_specific_iterator( *my_container, my_index-offset ); + } + + enumerable_thread_specific_iterator &operator-=( std::ptrdiff_t offset ) { + my_index -= offset; + my_value = nullptr; + return *this; + } + + Value& operator*() const { + Value* value = my_value; + if( !value ) { + value = my_value = (*my_container)[my_index].value(); + } + __TBB_ASSERT( value==(*my_container)[my_index].value(), "corrupt cache" ); + return *value; + } + + Value& operator[]( std::ptrdiff_t k ) const { + return *(*my_container)[my_index + k].value(); + } + + Value* operator->() const {return &operator*();} + + enumerable_thread_specific_iterator& operator++() { + ++my_index; + my_value = nullptr; + return *this; + } + + enumerable_thread_specific_iterator& operator--() { + --my_index; + my_value = nullptr; + return *this; + } + + //! Post increment + enumerable_thread_specific_iterator operator++(int) { + enumerable_thread_specific_iterator result = *this; + ++my_index; + my_value = nullptr; + return result; + } + + //! Post decrement + enumerable_thread_specific_iterator operator--(int) { + enumerable_thread_specific_iterator result = *this; + --my_index; + my_value = nullptr; + return result; + } +}; + +template +bool operator==( const enumerable_thread_specific_iterator& i, + const enumerable_thread_specific_iterator& j ) { + return i.my_index == j.my_index && i.my_container == j.my_container; +} + +template +bool operator!=( const enumerable_thread_specific_iterator& i, + const enumerable_thread_specific_iterator& j ) { + return !(i==j); +} + +template +bool operator<( const enumerable_thread_specific_iterator& i, + const enumerable_thread_specific_iterator& j ) { + return i.my_index +bool operator>( const enumerable_thread_specific_iterator& i, + const enumerable_thread_specific_iterator& j ) { + return j +bool operator>=( const enumerable_thread_specific_iterator& i, + const enumerable_thread_specific_iterator& j ) { + return !(i +bool operator<=( const enumerable_thread_specific_iterator& i, + const enumerable_thread_specific_iterator& j ) { + return !(j +std::ptrdiff_t operator-( const enumerable_thread_specific_iterator& i, + const enumerable_thread_specific_iterator& j ) { + return i.my_index-j.my_index; +} + +template +class segmented_iterator +{ + template + friend bool operator==(const segmented_iterator& i, const segmented_iterator& j); + + template + friend bool operator!=(const segmented_iterator& i, const segmented_iterator& j); + + template + friend class segmented_iterator; + +public: + segmented_iterator() {my_segcont = nullptr;} + + segmented_iterator( const SegmentedContainer& _segmented_container ) : + my_segcont(const_cast(&_segmented_container)), + outer_iter(my_segcont->end()) { } + + ~segmented_iterator() {} + + using InnerContainer = typename SegmentedContainer::value_type; + using inner_iterator = typename InnerContainer::iterator; + using outer_iterator = typename SegmentedContainer::iterator; + + // STL support + // TODO: inherit all types from segmented container? + using difference_type = std::ptrdiff_t; + using value_type = Value; + using size_type = typename SegmentedContainer::size_type; + using pointer = Value*; + using reference = Value&; + using iterator_category = std::input_iterator_tag; + + // Copy Constructor + template + segmented_iterator(const segmented_iterator& other) : + my_segcont(other.my_segcont), + outer_iter(other.outer_iter), + // can we assign a default-constructed iterator to inner if we're at the end? + inner_iter(other.inner_iter) + {} + + // assignment + template + segmented_iterator& operator=( const segmented_iterator& other) { + my_segcont = other.my_segcont; + outer_iter = other.outer_iter; + if(outer_iter != my_segcont->end()) inner_iter = other.inner_iter; + return *this; + } + + // allow assignment of outer iterator to segmented iterator. Once it is + // assigned, move forward until a non-empty inner container is found or + // the end of the outer container is reached. + segmented_iterator& operator=(const outer_iterator& new_outer_iter) { + __TBB_ASSERT(my_segcont != nullptr, nullptr); + // check that this iterator points to something inside the segmented container + for(outer_iter = new_outer_iter ;outer_iter!=my_segcont->end(); ++outer_iter) { + if( !outer_iter->empty() ) { + inner_iter = outer_iter->begin(); + break; + } + } + return *this; + } + + // pre-increment + segmented_iterator& operator++() { + advance_me(); + return *this; + } + + // post-increment + segmented_iterator operator++(int) { + segmented_iterator tmp = *this; + operator++(); + return tmp; + } + + bool operator==(const outer_iterator& other_outer) const { + __TBB_ASSERT(my_segcont != nullptr, nullptr); + return (outer_iter == other_outer && + (outer_iter == my_segcont->end() || inner_iter == outer_iter->begin())); + } + + bool operator!=(const outer_iterator& other_outer) const { + return !operator==(other_outer); + + } + + // (i)* RHS + reference operator*() const { + __TBB_ASSERT(my_segcont != nullptr, nullptr); + __TBB_ASSERT(outer_iter != my_segcont->end(), "Dereferencing a pointer at end of container"); + __TBB_ASSERT(inner_iter != outer_iter->end(), nullptr); // should never happen + return *inner_iter; + } + + // i-> + pointer operator->() const { return &operator*();} + +private: + SegmentedContainer* my_segcont; + outer_iterator outer_iter; + inner_iterator inner_iter; + + void advance_me() { + __TBB_ASSERT(my_segcont != nullptr, nullptr); + __TBB_ASSERT(outer_iter != my_segcont->end(), nullptr); // not true if there are no inner containers + __TBB_ASSERT(inner_iter != outer_iter->end(), nullptr); // not true if the inner containers are all empty. + ++inner_iter; + while(inner_iter == outer_iter->end() && ++outer_iter != my_segcont->end()) { + inner_iter = outer_iter->begin(); + } + } +}; // segmented_iterator + +template +bool operator==( const segmented_iterator& i, + const segmented_iterator& j ) { + if(i.my_segcont != j.my_segcont) return false; + if(i.my_segcont == nullptr) return true; + if(i.outer_iter != j.outer_iter) return false; + if(i.outer_iter == i.my_segcont->end()) return true; + return i.inner_iter == j.inner_iter; +} + +// != +template +bool operator!=( const segmented_iterator& i, + const segmented_iterator& j ) { + return !(i==j); +} + +template +struct construct_by_default: no_assign { + void construct(void*where) {new(where) T();} // C++ note: the () in T() ensure zero initialization. + construct_by_default( int ) {} +}; + +template +struct construct_by_exemplar: no_assign { + const T exemplar; + void construct(void*where) {new(where) T(exemplar);} + construct_by_exemplar( const T& t ) : exemplar(t) {} + construct_by_exemplar( T&& t ) : exemplar(std::move(t)) {} +}; + +template +struct construct_by_finit: no_assign { + Finit f; + void construct(void* where) {new(where) T(f());} + construct_by_finit( Finit&& f_ ) : f(std::move(f_)) {} +}; + +template +struct construct_by_args: no_assign { + stored_pack pack; + void construct(void* where) { + call( [where](const typename std::decay

::type&... args ){ + new(where) T(args...); + }, pack ); + } + construct_by_args( P&& ... args ) : pack(std::forward

(args)...) {} +}; + +// storage for initialization function pointer +// TODO: consider removing the template parameter T here and in callback_leaf +class callback_base { +public: + // Clone *this + virtual callback_base* clone() const = 0; + // Destruct and free *this + virtual void destroy() = 0; + // Need virtual destructor to satisfy GCC compiler warning + virtual ~callback_base() { } + // Construct T at where + virtual void construct(void* where) = 0; +}; + +template +class callback_leaf: public callback_base, Constructor { + template callback_leaf( P&& ... params ) : Constructor(std::forward

(params)...) {} + // TODO: make the construction/destruction consistent (use allocator.construct/destroy) + using my_allocator_type = typename tbb::tbb_allocator; + + callback_base* clone() const override { + return make(*this); + } + + void destroy() override { + my_allocator_type alloc; + tbb::detail::allocator_traits::destroy(alloc, this); + tbb::detail::allocator_traits::deallocate(alloc, this, 1); + } + + void construct(void* where) override { + Constructor::construct(where); + } + +public: + template + static callback_base* make( P&& ... params ) { + void* where = my_allocator_type().allocate(1); + return new(where) callback_leaf( std::forward

(params)... ); + } +}; + +//! Template for recording construction of objects in table +/** All maintenance of the space will be done explicitly on push_back, + and all thread local copies must be destroyed before the concurrent + vector is deleted. + + The flag is_built is initialized to false. When the local is + successfully-constructed, set the flag to true or call value_committed(). + If the constructor throws, the flag will be false. +*/ +template +struct ets_element { + detail::aligned_space my_space; + bool is_built; + ets_element() { is_built = false; } // not currently-built + U* value() { return my_space.begin(); } + U* value_committed() { is_built = true; return my_space.begin(); } + ~ets_element() { + if(is_built) { + my_space.begin()->~U(); + is_built = false; + } + } +}; + +// A predicate that can be used for a compile-time compatibility check of ETS instances +// Ideally, it should have been declared inside the ETS class, but unfortunately +// in that case VS2013 does not enable the variadic constructor. +template struct is_compatible_ets : std::false_type {}; +template +struct is_compatible_ets< T, enumerable_thread_specific > : std::is_same {}; + +// A predicate that checks whether, for a variable 'foo' of type T, foo() is a valid expression +template using has_empty_braces_operator = decltype(std::declval()()); +template using is_callable_no_args = supports; + +//! The enumerable_thread_specific container +/** enumerable_thread_specific has the following properties: + - thread-local copies are lazily created, with default, exemplar or function initialization. + - thread-local copies do not move (during lifetime, and excepting clear()) so the address of a copy is invariant. + - the contained objects need not have operator=() defined if combine is not used. + - enumerable_thread_specific containers may be copy-constructed or assigned. + - thread-local copies can be managed by hash-table, or can be accessed via TLS storage for speed. + - outside of parallel contexts, the contents of all thread-local copies are accessible by iterator or using combine or combine_each methods + +@par Segmented iterator + When the thread-local objects are containers with input_iterators defined, a segmented iterator may + be used to iterate over all the elements of all thread-local copies. + +@par combine and combine_each + - Both methods are defined for enumerable_thread_specific. + - combine() requires the type T have operator=() defined. + - neither method modifies the contents of the object (though there is no guarantee that the applied methods do not modify the object.) + - Both are evaluated in serial context (the methods are assumed to be non-benign.) + +@ingroup containers */ +template , + ets_key_usage_type ETS_key_type=ets_no_key > +class enumerable_thread_specific: ets_base { + + template friend class enumerable_thread_specific; + + using padded_element = padded>; + + //! A generic range, used to create range objects from the iterators + template + class generic_range_type: public blocked_range { + public: + using value_type = T; + using reference = T&; + using const_reference = const T&; + using iterator = I; + using difference_type = std::ptrdiff_t; + + generic_range_type( I begin_, I end_, std::size_t grainsize_ = 1) : blocked_range(begin_,end_,grainsize_) {} + template + generic_range_type( const generic_range_type& r) : blocked_range(r.begin(),r.end(),r.grainsize()) {} + generic_range_type( generic_range_type& r, split ) : blocked_range(r,split()) {} + }; + + using allocator_traits_type = tbb::detail::allocator_traits; + + using padded_allocator_type = typename allocator_traits_type::template rebind_alloc; + using internal_collection_type = tbb::concurrent_vector< padded_element, padded_allocator_type >; + + callback_base *my_construct_callback; + + internal_collection_type my_locals; + + // TODO: consider unifying the callback mechanism for all create_local* methods below + // (likely non-compatible and requires interface version increase) + void* create_local() override { + padded_element& lref = *my_locals.grow_by(1); + my_construct_callback->construct(lref.value()); + return lref.value_committed(); + } + + static void* create_local_by_copy( ets_base& base, void* p ) { + enumerable_thread_specific& ets = static_cast(base); + padded_element& lref = *ets.my_locals.grow_by(1); + new(lref.value()) T(*static_cast(p)); + return lref.value_committed(); + } + + static void* create_local_by_move( ets_base& base, void* p ) { + enumerable_thread_specific& ets = static_cast(base); + padded_element& lref = *ets.my_locals.grow_by(1); + new(lref.value()) T(std::move(*static_cast(p))); + return lref.value_committed(); + } + + using array_allocator_type = typename allocator_traits_type::template rebind_alloc; + + // _size is in bytes + void* create_array(std::size_t _size) override { + std::size_t nelements = (_size + sizeof(uintptr_t) -1) / sizeof(uintptr_t); + return array_allocator_type().allocate(nelements); + } + + void free_array( void* _ptr, std::size_t _size) override { + std::size_t nelements = (_size + sizeof(uintptr_t) -1) / sizeof(uintptr_t); + array_allocator_type().deallocate( reinterpret_cast(_ptr),nelements); + } + +public: + + //! Basic types + using value_type = T; + using allocator_type = Allocator; + using size_type = typename internal_collection_type::size_type; + using difference_type = typename internal_collection_type::difference_type; + using reference = value_type&; + using const_reference = const value_type&; + + using pointer = typename allocator_traits_type::pointer; + using const_pointer = typename allocator_traits_type::const_pointer; + + // Iterator types + using iterator = enumerable_thread_specific_iterator; + using const_iterator = enumerable_thread_specific_iterator; + + // Parallel range types + using range_type = generic_range_type; + using const_range_type = generic_range_type; + + //! Default constructor. Each local instance of T is default constructed. + enumerable_thread_specific() : my_construct_callback( + callback_leaf >::make(/*dummy argument*/0) + ){} + + //! Constructor with initializer functor. Each local instance of T is constructed by T(finit()). + template ::type>::value>::type> + explicit enumerable_thread_specific( Finit finit ) : my_construct_callback( + callback_leaf >::make( std::move(finit) ) + ){} + + //! Constructor with exemplar. Each local instance of T is copy-constructed from the exemplar. + explicit enumerable_thread_specific( const T& exemplar ) : my_construct_callback( + callback_leaf >::make( exemplar ) + ){} + + explicit enumerable_thread_specific( T&& exemplar ) : my_construct_callback( + callback_leaf >::make( std::move(exemplar) ) + ){} + + //! Variadic constructor with initializer arguments. Each local instance of T is constructed by T(args...) + template ::type>::value + && !is_compatible_ets::type>::value + && !std::is_same::type>::value + >::type> + enumerable_thread_specific( P1&& arg1, P&& ... args ) : my_construct_callback( + callback_leaf >::make( std::forward(arg1), std::forward

(args)... ) + ){} + + //! Destructor + ~enumerable_thread_specific() { + if(my_construct_callback) my_construct_callback->destroy(); + // Deallocate the hash table before overridden free_array() becomes inaccessible + this->ets_base::table_clear(); + } + + //! returns reference to local, discarding exists + reference local() { + bool exists; + return local(exists); + } + + //! Returns reference to calling thread's local copy, creating one if necessary + reference local(bool& exists) { + void* ptr = this->table_lookup(exists); + return *(T*)ptr; + } + + //! Get the number of local copies + size_type size() const { return my_locals.size(); } + + //! true if there have been no local copies created + bool empty() const { return my_locals.empty(); } + + //! begin iterator + iterator begin() { return iterator( my_locals, 0 ); } + //! end iterator + iterator end() { return iterator(my_locals, my_locals.size() ); } + + //! begin const iterator + const_iterator begin() const { return const_iterator(my_locals, 0); } + + //! end const iterator + const_iterator end() const { return const_iterator(my_locals, my_locals.size()); } + + //! Get range for parallel algorithms + range_type range( std::size_t grainsize=1 ) { return range_type( begin(), end(), grainsize ); } + + //! Get const range for parallel algorithms + const_range_type range( std::size_t grainsize=1 ) const { return const_range_type( begin(), end(), grainsize ); } + + //! Destroys local copies + void clear() { + my_locals.clear(); + this->table_clear(); + // callback is not destroyed + } + +private: + template + void internal_copy(const enumerable_thread_specific& other) { + // this tests is_compatible_ets + static_assert( (is_compatible_ets::type>::value), "is_compatible_ets fails" ); + // Initialize my_construct_callback first, so that it is valid even if rest of this routine throws an exception. + my_construct_callback = other.my_construct_callback->clone(); + __TBB_ASSERT(my_locals.size()==0, nullptr); + my_locals.reserve(other.size()); + this->table_elementwise_copy( other, create_local_by_copy ); + } + + void internal_swap(enumerable_thread_specific& other) { + using std::swap; + __TBB_ASSERT( this!=&other, nullptr); + swap(my_construct_callback, other.my_construct_callback); + // concurrent_vector::swap() preserves storage space, + // so addresses to the vector kept in ETS hash table remain valid. + swap(my_locals, other.my_locals); + this->ets_base::table_swap(other); + } + + template + void internal_move(enumerable_thread_specific&& other) { + static_assert( (is_compatible_ets::type>::value), "is_compatible_ets fails" ); + my_construct_callback = other.my_construct_callback; + other.my_construct_callback = nullptr; + __TBB_ASSERT(my_locals.size()==0, nullptr); + my_locals.reserve(other.size()); + this->table_elementwise_copy( other, create_local_by_move ); + } + +public: + enumerable_thread_specific( const enumerable_thread_specific& other ) + : ets_base() /* prevents GCC warnings with -Wextra */ + { + internal_copy(other); + } + + template + enumerable_thread_specific( const enumerable_thread_specific& other ) + { + internal_copy(other); + } + + enumerable_thread_specific( enumerable_thread_specific&& other ) : my_construct_callback() + { + // TODO: use internal_move correctly here + internal_swap(other); + } + + template + enumerable_thread_specific( enumerable_thread_specific&& other ) : my_construct_callback() + { + internal_move(std::move(other)); + } + + enumerable_thread_specific& operator=( const enumerable_thread_specific& other ) + { + if( this != &other ) { + this->clear(); + my_construct_callback->destroy(); + internal_copy( other ); + } + return *this; + } + + template + enumerable_thread_specific& operator=( const enumerable_thread_specific& other ) + { + __TBB_ASSERT( static_cast(this)!=static_cast(&other), nullptr); // Objects of different types + this->clear(); + my_construct_callback->destroy(); + internal_copy(other); + return *this; + } + + enumerable_thread_specific& operator=( enumerable_thread_specific&& other ) + { + if( this != &other ) { + // TODO: use internal_move correctly here + internal_swap(other); + } + return *this; + } + + template + enumerable_thread_specific& operator=( enumerable_thread_specific&& other ) + { + __TBB_ASSERT( static_cast(this)!=static_cast(&other), nullptr); // Objects of different types + this->clear(); + my_construct_callback->destroy(); + internal_move(std::move(other)); + return *this; + } + + // CombineFunc has signature T(T,T) or T(const T&, const T&) + template + T combine(CombineFunc f_combine) { + if(begin() == end()) { + ets_element location; + my_construct_callback->construct(location.value()); + return *location.value_committed(); + } + const_iterator ci = begin(); + T my_result = *ci; + while(++ci != end()) + my_result = f_combine( my_result, *ci ); + return my_result; + } + + // combine_func_t takes T by value or by [const] reference, and returns nothing + template + void combine_each(CombineFunc f_combine) { + for(iterator ci = begin(); ci != end(); ++ci) { + f_combine( *ci ); + } + } + +}; // enumerable_thread_specific + +template< typename Container > +class flattened2d { + // This intermediate typedef is to address issues with VC7.1 compilers + using conval_type = typename Container::value_type; + +public: + //! Basic types + using size_type = typename conval_type::size_type; + using difference_type = typename conval_type::difference_type; + using allocator_type = typename conval_type::allocator_type; + using value_type = typename conval_type::value_type; + using reference = typename conval_type::reference; + using const_reference = typename conval_type::const_reference; + using pointer = typename conval_type::pointer; + using const_pointer = typename conval_type::const_pointer; + + using iterator = segmented_iterator; + using const_iterator = segmented_iterator; + + flattened2d( const Container &c, typename Container::const_iterator b, typename Container::const_iterator e ) : + my_container(const_cast(&c)), my_begin(b), my_end(e) { } + + explicit flattened2d( const Container &c ) : + my_container(const_cast(&c)), my_begin(c.begin()), my_end(c.end()) { } + + iterator begin() { return iterator(*my_container) = my_begin; } + iterator end() { return iterator(*my_container) = my_end; } + const_iterator begin() const { return const_iterator(*my_container) = my_begin; } + const_iterator end() const { return const_iterator(*my_container) = my_end; } + + size_type size() const { + size_type tot_size = 0; + for(typename Container::const_iterator i = my_begin; i != my_end; ++i) { + tot_size += i->size(); + } + return tot_size; + } + +private: + Container *my_container; + typename Container::const_iterator my_begin; + typename Container::const_iterator my_end; +}; + +template +flattened2d flatten2d(const Container &c, const typename Container::const_iterator b, const typename Container::const_iterator e) { + return flattened2d(c, b, e); +} + +template +flattened2d flatten2d(const Container &c) { + return flattened2d(c); +} + +} // namespace d1 +} // namespace detail + +inline namespace v1 { +using detail::d1::enumerable_thread_specific; +using detail::d1::flattened2d; +using detail::d1::flatten2d; +// ets enum keys +using detail::d1::ets_key_usage_type; +using detail::d1::ets_key_per_instance; +using detail::d1::ets_no_key; +#if __TBB_RESUMABLE_TASKS +using detail::d1::ets_suspend_aware; +#endif +} // inline namespace v1 + +} // namespace tbb + +#endif // __TBB_enumerable_thread_specific_H + diff --git a/src/tbb/include/oneapi/tbb/flow_graph.h b/src/tbb/include/oneapi/tbb/flow_graph.h new file mode 100644 index 000000000..035fef066 --- /dev/null +++ b/src/tbb/include/oneapi/tbb/flow_graph.h @@ -0,0 +1,3718 @@ +/* + Copyright (c) 2005-2024 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_flow_graph_H +#define __TBB_flow_graph_H + +#include +#include +#include + +#include "detail/_config.h" +#include "detail/_namespace_injection.h" +#include "spin_mutex.h" +#include "null_mutex.h" +#include "spin_rw_mutex.h" +#include "null_rw_mutex.h" +#include "detail/_pipeline_filters.h" +#include "detail/_task.h" +#include "detail/_small_object_pool.h" +#include "cache_aligned_allocator.h" +#include "detail/_exception.h" +#include "detail/_template_helpers.h" +#include "detail/_aggregator.h" +#include "detail/_allocator_traits.h" +#include "detail/_utils.h" +#include "profiling.h" +#include "task_arena.h" + +#if TBB_USE_PROFILING_TOOLS && ( __unix__ || __APPLE__ ) + #if __INTEL_COMPILER + // Disabled warning "routine is both inline and noinline" + // #pragma warning (push) + // #pragma warning( disable: 2196 ) + #endif + #define __TBB_NOINLINE_SYM __attribute__((noinline)) +#else + #define __TBB_NOINLINE_SYM +#endif + +#include +#include +#include +#include +#if __TBB_CPP20_CONCEPTS_PRESENT +#include +#endif + +/** @file + \brief The graph related classes and functions + + There are some applications that best express dependencies as messages + passed between nodes in a graph. These messages may contain data or + simply act as signals that a predecessors has completed. The graph + class and its associated node classes can be used to express such + applications. +*/ + +namespace tbb { +namespace detail { + +namespace d2 { + +//! An enumeration the provides the two most common concurrency levels: unlimited and serial +enum concurrency { unlimited = 0, serial = 1 }; + +//! A generic null type +struct null_type {}; + +//! An empty class used for messages that mean "I'm done" +class continue_msg {}; + +} // namespace d2 + +#if __TBB_CPP20_CONCEPTS_PRESENT +namespace d0 { + +template +concept node_body_return_type = std::same_as || + std::convertible_to; + +// TODO: consider using std::invocable here +template +concept continue_node_body = std::copy_constructible && + requires( Body& body, const tbb::detail::d2::continue_msg& v ) { + { body(v) } -> node_body_return_type; + }; + +template +concept function_node_body = std::copy_constructible && + std::invocable && + node_body_return_type, Output>; + +template +concept join_node_function_object = std::copy_constructible && + std::invocable && + std::convertible_to, Key>; + +template +concept input_node_body = std::copy_constructible && + requires( Body& body, tbb::detail::d1::flow_control& fc ) { + { body(fc) } -> adaptive_same_as; + }; + +template +concept multifunction_node_body = std::copy_constructible && + std::invocable; + +template +concept sequencer = std::copy_constructible && + std::invocable && + std::convertible_to, std::size_t>; + +template +concept async_node_body = std::copy_constructible && + std::invocable; + +} // namespace d0 +#endif // __TBB_CPP20_CONCEPTS_PRESENT + +namespace d2 { + +//! Forward declaration section +template< typename T > class sender; +template< typename T > class receiver; +class continue_receiver; + +template< typename T, typename U > class limiter_node; // needed for resetting decrementer + +template class successor_cache; +template class broadcast_cache; +template class round_robin_cache; +template class predecessor_cache; +template class reservable_predecessor_cache; + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET +namespace order { +struct following; +struct preceding; +} +template struct node_set; +#endif + + +} // namespace d2 +} // namespace detail +} // namespace tbb + +//! The graph class +#include "detail/_flow_graph_impl.h" + +namespace tbb { +namespace detail { +namespace d2 { + +static inline std::pair order_tasks(graph_task* first, graph_task* second) { + if (second->priority > first->priority) + return std::make_pair(second, first); + return std::make_pair(first, second); +} + +// submit task if necessary. Returns the non-enqueued task if there is one. +static inline graph_task* combine_tasks(graph& g, graph_task* left, graph_task* right) { + // if no RHS task, don't change left. + if (right == nullptr) return left; + // right != nullptr + if (left == nullptr) return right; + if (left == SUCCESSFULLY_ENQUEUED) return right; + // left contains a task + if (right != SUCCESSFULLY_ENQUEUED) { + // both are valid tasks + auto tasks_pair = order_tasks(left, right); + spawn_in_graph_arena(g, *tasks_pair.first); + return tasks_pair.second; + } + return left; +} + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT +class message_metainfo { +public: + using waiters_type = std::forward_list; + + message_metainfo() = default; + + message_metainfo(const waiters_type& waiters) : my_waiters(waiters) {} + message_metainfo(waiters_type&& waiters) : my_waiters(std::move(waiters)) {} + + const waiters_type& waiters() const & { return my_waiters; } + waiters_type&& waiters() && { return std::move(my_waiters); } + + bool empty() const { return my_waiters.empty(); } + + void merge(const message_metainfo& other) { + // TODO: should we avoid duplications on merging + my_waiters.insert_after(my_waiters.before_begin(), + other.waiters().begin(), + other.waiters().end()); + } +private: + waiters_type my_waiters; +}; // class message_metainfo + +#define __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo) , metainfo + +#else +#define __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo) +#endif // __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + +//! Pure virtual template class that defines a sender of messages of type T +template< typename T > +class sender { +public: + virtual ~sender() {} + + //! Request an item from the sender + virtual bool try_get( T & ) { return false; } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + virtual bool try_get( T &, message_metainfo& ) { return false; } +#endif + + //! Reserves an item in the sender + virtual bool try_reserve( T & ) { return false; } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + virtual bool try_reserve( T &, message_metainfo& ) { return false; } +#endif + + //! Releases the reserved item + virtual bool try_release( ) { return false; } + + //! Consumes the reserved item + virtual bool try_consume( ) { return false; } + +protected: + //! The output type of this sender + typedef T output_type; + + //! The successor type for this node + typedef receiver successor_type; + + //! Add a new successor to this node + virtual bool register_successor( successor_type &r ) = 0; + + //! Removes a successor from this node + virtual bool remove_successor( successor_type &r ) = 0; + + template + friend bool register_successor(sender& s, receiver& r); + + template + friend bool remove_successor (sender& s, receiver& r); +}; // class sender + +template +bool register_successor(sender& s, receiver& r) { + return s.register_successor(r); +} + +template +bool remove_successor(sender& s, receiver& r) { + return s.remove_successor(r); +} + +//! Pure virtual template class that defines a receiver of messages of type T +template< typename T > +class receiver { +private: + template + bool internal_try_put(const T& t, TryPutTaskArgs&&... args) { + graph_task* res = try_put_task(t, std::forward(args)...); + if (!res) return false; + if (res != SUCCESSFULLY_ENQUEUED) spawn_in_graph_arena(graph_reference(), *res); + return true; + } + +public: + //! Destructor + virtual ~receiver() {} + + //! Put an item to the receiver + bool try_put( const T& t ) { + return internal_try_put(t); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + //! Put an item to the receiver and wait for completion + bool try_put_and_wait( const T& t ) { + // Since try_put_and_wait is a blocking call, it is safe to create wait_context on stack + d1::wait_context_vertex msg_wait_vertex{}; + + bool res = internal_try_put(t, message_metainfo{message_metainfo::waiters_type{&msg_wait_vertex}}); + if (res) { + __TBB_ASSERT(graph_reference().my_context != nullptr, "No wait_context associated with the Flow Graph"); + wait(msg_wait_vertex.get_context(), *graph_reference().my_context); + } + return res; + } +#endif + + //! put item to successor; return task to run the successor if possible. +protected: + //! The input type of this receiver + typedef T input_type; + + //! The predecessor type for this node + typedef sender predecessor_type; + + template< typename R, typename B > friend class run_and_put_task; + template< typename X, typename Y > friend class broadcast_cache; + template< typename X, typename Y > friend class round_robin_cache; + virtual graph_task *try_put_task(const T& t) = 0; +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + virtual graph_task *try_put_task(const T& t, const message_metainfo&) = 0; +#endif + virtual graph& graph_reference() const = 0; + + template friend class successor_cache; + virtual bool is_continue_receiver() { return false; } + + // TODO revamp: reconsider the inheritance and move node priority out of receiver + virtual node_priority_t priority() const { return no_priority; } + + //! Add a predecessor to the node + virtual bool register_predecessor( predecessor_type & ) { return false; } + + //! Remove a predecessor from the node + virtual bool remove_predecessor( predecessor_type & ) { return false; } + + template + friend bool register_predecessor(receiver& r, sender& s); + template + friend bool remove_predecessor (receiver& r, sender& s); +}; // class receiver + +template +bool register_predecessor(receiver& r, sender& s) { + return r.register_predecessor(s); +} + +template +bool remove_predecessor(receiver& r, sender& s) { + return r.remove_predecessor(s); +} + +//! Base class for receivers of completion messages +/** These receivers automatically reset, but cannot be explicitly waited on */ +class continue_receiver : public receiver< continue_msg > { +protected: + + //! Constructor + explicit continue_receiver( int number_of_predecessors, node_priority_t a_priority ) { + my_predecessor_count = my_initial_predecessor_count = number_of_predecessors; + my_current_count = 0; + my_priority = a_priority; + } + + //! Copy constructor + continue_receiver( const continue_receiver& src ) : receiver() { + my_predecessor_count = my_initial_predecessor_count = src.my_initial_predecessor_count; + my_current_count = 0; + my_priority = src.my_priority; + } + + //! Increments the trigger threshold + bool register_predecessor( predecessor_type & ) override { + spin_mutex::scoped_lock l(my_mutex); + ++my_predecessor_count; + return true; + } + + //! Decrements the trigger threshold + /** Does not check to see if the removal of the predecessor now makes the current count + exceed the new threshold. So removing a predecessor while the graph is active can cause + unexpected results. */ + bool remove_predecessor( predecessor_type & ) override { + spin_mutex::scoped_lock l(my_mutex); + --my_predecessor_count; + return true; + } + + //! The input type + typedef continue_msg input_type; + + //! The predecessor type for this node + typedef receiver::predecessor_type predecessor_type; + + template< typename R, typename B > friend class run_and_put_task; + template friend class broadcast_cache; + template friend class round_robin_cache; + +private: + // execute body is supposed to be too small to create a task for. + graph_task* try_put_task_impl( const input_type& __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& metainfo) ) { +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + message_metainfo predecessor_metainfo; +#endif + { + spin_mutex::scoped_lock l(my_mutex); +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + // Prolong the wait and store the metainfo until receiving signals from all the predecessors + for (auto waiter : metainfo.waiters()) { + waiter->reserve(1); + } + my_current_metainfo.merge(metainfo); +#endif + if ( ++my_current_count < my_predecessor_count ) + return SUCCESSFULLY_ENQUEUED; + else { + my_current_count = 0; +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + predecessor_metainfo = my_current_metainfo; + my_current_metainfo = message_metainfo{}; +#endif + } + } +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + graph_task* res = execute(predecessor_metainfo); + for (auto waiter : predecessor_metainfo.waiters()) { + waiter->release(1); + } +#else + graph_task* res = execute(); +#endif + return res? res : SUCCESSFULLY_ENQUEUED; + } + +protected: + graph_task* try_put_task( const input_type& input ) override { + return try_put_task_impl(input __TBB_FLOW_GRAPH_METAINFO_ARG(message_metainfo{})); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + graph_task* try_put_task( const input_type& input, const message_metainfo& metainfo ) override { + return try_put_task_impl(input, metainfo); + } +#endif + + spin_mutex my_mutex; + int my_predecessor_count; + int my_current_count; + int my_initial_predecessor_count; +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + message_metainfo my_current_metainfo; +#endif + node_priority_t my_priority; + // the friend declaration in the base class did not eliminate the "protected class" + // error in gcc 4.1.2 + template friend class limiter_node; + + virtual void reset_receiver( reset_flags f ) { + my_current_count = 0; + if (f & rf_clear_edges) { + my_predecessor_count = my_initial_predecessor_count; + } + } + + //! Does whatever should happen when the threshold is reached + /** This should be very fast or else spawn a task. This is + called while the sender is blocked in the try_put(). */ +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + virtual graph_task* execute(const message_metainfo& metainfo) = 0; +#else + virtual graph_task* execute() = 0; +#endif + template friend class successor_cache; + bool is_continue_receiver() override { return true; } + + node_priority_t priority() const override { return my_priority; } +}; // class continue_receiver + +#if __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING + template + K key_from_message( const T &t ) { + return t.key(); + } +#endif /* __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING */ + +} // d1 +} // detail +} // tbb + +#include "detail/_flow_graph_trace_impl.h" +#include "detail/_hash_compare.h" + +namespace tbb { +namespace detail { +namespace d2 { + +#include "detail/_flow_graph_body_impl.h" +#include "detail/_flow_graph_cache_impl.h" +#include "detail/_flow_graph_types_impl.h" + +using namespace graph_policy_namespace; + +template +graph_iterator::graph_iterator(C *g, bool begin) : my_graph(g), current_node(nullptr) +{ + if (begin) current_node = my_graph->my_nodes; + //else it is an end iterator by default +} + +template +typename graph_iterator::reference graph_iterator::operator*() const { + __TBB_ASSERT(current_node, "graph_iterator at end"); + return *operator->(); +} + +template +typename graph_iterator::pointer graph_iterator::operator->() const { + return current_node; +} + +template +void graph_iterator::internal_forward() { + if (current_node) current_node = current_node->next; +} + +//! Constructs a graph with isolated task_group_context +inline graph::graph() : my_wait_context_vertex(0), my_nodes(nullptr), my_nodes_last(nullptr), my_task_arena(nullptr) { + prepare_task_arena(); + own_context = true; + cancelled = false; + caught_exception = false; + my_context = new (r1::cache_aligned_allocate(sizeof(task_group_context))) task_group_context(FLOW_TASKS); + fgt_graph(this); + my_is_active = true; +} + +inline graph::graph(task_group_context& use_this_context) : + my_wait_context_vertex(0), my_context(&use_this_context), my_nodes(nullptr), my_nodes_last(nullptr), my_task_arena(nullptr) { + prepare_task_arena(); + own_context = false; + cancelled = false; + caught_exception = false; + fgt_graph(this); + my_is_active = true; +} + +inline graph::~graph() { + wait_for_all(); + if (own_context) { + my_context->~task_group_context(); + r1::cache_aligned_deallocate(my_context); + } + delete my_task_arena; +} + +inline void graph::reserve_wait() { + my_wait_context_vertex.reserve(); + fgt_reserve_wait(this); +} + +inline void graph::release_wait() { + fgt_release_wait(this); + my_wait_context_vertex.release(); +} + +inline void graph::register_node(graph_node *n) { + n->next = nullptr; + { + spin_mutex::scoped_lock lock(nodelist_mutex); + n->prev = my_nodes_last; + if (my_nodes_last) my_nodes_last->next = n; + my_nodes_last = n; + if (!my_nodes) my_nodes = n; + } +} + +inline void graph::remove_node(graph_node *n) { + { + spin_mutex::scoped_lock lock(nodelist_mutex); + __TBB_ASSERT(my_nodes && my_nodes_last, "graph::remove_node: Error: no registered nodes"); + if (n->prev) n->prev->next = n->next; + if (n->next) n->next->prev = n->prev; + if (my_nodes_last == n) my_nodes_last = n->prev; + if (my_nodes == n) my_nodes = n->next; + } + n->prev = n->next = nullptr; +} + +inline void graph::reset( reset_flags f ) { + // reset context + deactivate_graph(*this); + + my_context->reset(); + cancelled = false; + caught_exception = false; + // reset all the nodes comprising the graph + for(iterator ii = begin(); ii != end(); ++ii) { + graph_node *my_p = &(*ii); + my_p->reset_node(f); + } + // Reattach the arena. Might be useful to run the graph in a particular task_arena + // while not limiting graph lifetime to a single task_arena::execute() call. + prepare_task_arena( /*reinit=*/true ); + activate_graph(*this); +} + +inline void graph::cancel() { + my_context->cancel_group_execution(); +} + +inline graph::iterator graph::begin() { return iterator(this, true); } + +inline graph::iterator graph::end() { return iterator(this, false); } + +inline graph::const_iterator graph::begin() const { return const_iterator(this, true); } + +inline graph::const_iterator graph::end() const { return const_iterator(this, false); } + +inline graph::const_iterator graph::cbegin() const { return const_iterator(this, true); } + +inline graph::const_iterator graph::cend() const { return const_iterator(this, false); } + +inline graph_node::graph_node(graph& g) : my_graph(g) { + my_graph.register_node(this); +} + +inline graph_node::~graph_node() { + my_graph.remove_node(this); +} + +#include "detail/_flow_graph_node_impl.h" + + +//! An executable node that acts as a source, i.e. it has no predecessors + +template < typename Output > + __TBB_requires(std::copyable) +class input_node : public graph_node, public sender< Output > { +public: + //! The type of the output message, which is complete + typedef Output output_type; + + //! The type of successors of this node + typedef typename sender::successor_type successor_type; + + // Input node has no input type + typedef null_type input_type; + + //! Constructor for a node with a successor + template< typename Body > + __TBB_requires(input_node_body) + __TBB_NOINLINE_SYM input_node( graph &g, Body body ) + : graph_node(g), my_active(false) + , my_body( new input_body_leaf< output_type, Body>(body) ) + , my_init_body( new input_body_leaf< output_type, Body>(body) ) + , my_successors(this), my_reserved(false), my_has_cached_item(false) + { + fgt_node_with_body(CODEPTR(), FLOW_INPUT_NODE, &this->my_graph, + static_cast *>(this), this->my_body); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + template + __TBB_requires(input_node_body) + input_node( const node_set& successors, Body body ) + : input_node(successors.graph_reference(), body) + { + make_edges(*this, successors); + } +#endif + + //! Copy constructor + __TBB_NOINLINE_SYM input_node( const input_node& src ) + : graph_node(src.my_graph), sender() + , my_active(false) + , my_body(src.my_init_body->clone()), my_init_body(src.my_init_body->clone()) + , my_successors(this), my_reserved(false), my_has_cached_item(false) + { + fgt_node_with_body(CODEPTR(), FLOW_INPUT_NODE, &this->my_graph, + static_cast *>(this), this->my_body); + } + + //! The destructor + ~input_node() { delete my_body; delete my_init_body; } + + //! Add a new successor to this node + bool register_successor( successor_type &r ) override { + spin_mutex::scoped_lock lock(my_mutex); + my_successors.register_successor(r); + if ( my_active ) + spawn_put(); + return true; + } + + //! Removes a successor from this node + bool remove_successor( successor_type &r ) override { + spin_mutex::scoped_lock lock(my_mutex); + my_successors.remove_successor(r); + return true; + } + + //! Request an item from the node + bool try_get( output_type &v ) override { + spin_mutex::scoped_lock lock(my_mutex); + if ( my_reserved ) + return false; + + if ( my_has_cached_item ) { + v = my_cached_item; + my_has_cached_item = false; + return true; + } + // we've been asked to provide an item, but we have none. enqueue a task to + // provide one. + if ( my_active ) + spawn_put(); + return false; + } + + //! Reserves an item. + bool try_reserve( output_type &v ) override { + spin_mutex::scoped_lock lock(my_mutex); + if ( my_reserved ) { + return false; + } + + if ( my_has_cached_item ) { + v = my_cached_item; + my_reserved = true; + return true; + } else { + return false; + } + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT +private: + bool try_reserve( output_type& v, message_metainfo& ) override { + return try_reserve(v); + } + + bool try_get( output_type& v, message_metainfo& ) override { + return try_get(v); + } +public: +#endif + + //! Release a reserved item. + /** true = item has been released and so remains in sender, dest must request or reserve future items */ + bool try_release( ) override { + spin_mutex::scoped_lock lock(my_mutex); + __TBB_ASSERT( my_reserved && my_has_cached_item, "releasing non-existent reservation" ); + my_reserved = false; + if(!my_successors.empty()) + spawn_put(); + return true; + } + + //! Consumes a reserved item + bool try_consume( ) override { + spin_mutex::scoped_lock lock(my_mutex); + __TBB_ASSERT( my_reserved && my_has_cached_item, "consuming non-existent reservation" ); + my_reserved = false; + my_has_cached_item = false; + if ( !my_successors.empty() ) { + spawn_put(); + } + return true; + } + + //! Activates a node that was created in the inactive state + void activate() { + spin_mutex::scoped_lock lock(my_mutex); + my_active = true; + if (!my_successors.empty()) + spawn_put(); + } + + template + Body copy_function_object() { + input_body &body_ref = *this->my_body; + return dynamic_cast< input_body_leaf & >(body_ref).get_body(); + } + +protected: + + //! resets the input_node to its initial state + void reset_node( reset_flags f) override { + my_active = false; + my_reserved = false; + my_has_cached_item = false; + + if(f & rf_clear_edges) my_successors.clear(); + if(f & rf_reset_bodies) { + input_body *tmp = my_init_body->clone(); + delete my_body; + my_body = tmp; + } + } + +private: + spin_mutex my_mutex; + bool my_active; + input_body *my_body; + input_body *my_init_body; + broadcast_cache< output_type > my_successors; + bool my_reserved; + bool my_has_cached_item; + output_type my_cached_item; + + // used by apply_body_bypass, can invoke body of node. + bool try_reserve_apply_body(output_type &v) { + spin_mutex::scoped_lock lock(my_mutex); + if ( my_reserved ) { + return false; + } + if ( !my_has_cached_item ) { + d1::flow_control control; + + fgt_begin_body( my_body ); + + my_cached_item = (*my_body)(control); + my_has_cached_item = !control.is_pipeline_stopped; + + fgt_end_body( my_body ); + } + if ( my_has_cached_item ) { + v = my_cached_item; + my_reserved = true; + return true; + } else { + return false; + } + } + + graph_task* create_put_task() { + d1::small_object_allocator allocator{}; + typedef input_node_task_bypass< input_node > task_type; + graph_task* t = allocator.new_object(my_graph, allocator, *this); + return t; + } + + //! Spawns a task that applies the body + void spawn_put( ) { + if(is_graph_active(this->my_graph)) { + spawn_in_graph_arena(this->my_graph, *create_put_task()); + } + } + + friend class input_node_task_bypass< input_node >; + //! Applies the body. Returning SUCCESSFULLY_ENQUEUED okay; forward_task_bypass will handle it. + graph_task* apply_body_bypass( ) { + output_type v; + if ( !try_reserve_apply_body(v) ) + return nullptr; + + graph_task *last_task = my_successors.try_put_task(v); + if ( last_task ) + try_consume(); + else + try_release(); + return last_task; + } +}; // class input_node + +//! Implements a function node that supports Input -> Output +template + __TBB_requires(std::default_initializable && + std::copy_constructible && + std::copy_constructible) +class function_node + : public graph_node + , public function_input< Input, Output, Policy, cache_aligned_allocator > + , public function_output +{ + typedef cache_aligned_allocator internals_allocator; + +public: + typedef Input input_type; + typedef Output output_type; + typedef function_input input_impl_type; + typedef function_input_queue input_queue_type; + typedef function_output fOutput_type; + typedef typename input_impl_type::predecessor_type predecessor_type; + typedef typename fOutput_type::successor_type successor_type; + + using input_impl_type::my_predecessors; + + //! Constructor + // input_queue_type is allocated here, but destroyed in the function_input_base. + // TODO: pass the graph_buffer_policy to the function_input_base so it can all + // be done in one place. This would be an interface-breaking change. + template< typename Body > + __TBB_requires(function_node_body) + __TBB_NOINLINE_SYM function_node( graph &g, size_t concurrency, + Body body, Policy = Policy(), node_priority_t a_priority = no_priority ) + : graph_node(g), input_impl_type(g, concurrency, body, a_priority), + fOutput_type(g) { + fgt_node_with_body( CODEPTR(), FLOW_FUNCTION_NODE, &this->my_graph, + static_cast *>(this), static_cast *>(this), this->my_body ); + } + + template + __TBB_requires(function_node_body) + function_node( graph& g, size_t concurrency, Body body, node_priority_t a_priority ) + : function_node(g, concurrency, body, Policy(), a_priority) {} + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + template + __TBB_requires(function_node_body) + function_node( const node_set& nodes, size_t concurrency, Body body, + Policy p = Policy(), node_priority_t a_priority = no_priority ) + : function_node(nodes.graph_reference(), concurrency, body, p, a_priority) { + make_edges_in_order(nodes, *this); + } + + template + __TBB_requires(function_node_body) + function_node( const node_set& nodes, size_t concurrency, Body body, node_priority_t a_priority ) + : function_node(nodes, concurrency, body, Policy(), a_priority) {} +#endif // __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + + //! Copy constructor + __TBB_NOINLINE_SYM function_node( const function_node& src ) : + graph_node(src.my_graph), + input_impl_type(src), + fOutput_type(src.my_graph) { + fgt_node_with_body( CODEPTR(), FLOW_FUNCTION_NODE, &this->my_graph, + static_cast *>(this), static_cast *>(this), this->my_body ); + } + +protected: + template< typename R, typename B > friend class run_and_put_task; + template friend class broadcast_cache; + template friend class round_robin_cache; + using input_impl_type::try_put_task; + + broadcast_cache &successors () override { return fOutput_type::my_successors; } + + void reset_node(reset_flags f) override { + input_impl_type::reset_function_input(f); + // TODO: use clear() instead. + if(f & rf_clear_edges) { + successors().clear(); + my_predecessors.clear(); + } + __TBB_ASSERT(!(f & rf_clear_edges) || successors().empty(), "function_node successors not empty"); + __TBB_ASSERT(this->my_predecessors.empty(), "function_node predecessors not empty"); + } + +}; // class function_node + +//! implements a function node that supports Input -> (set of outputs) +// Output is a tuple of output types. +template + __TBB_requires(std::default_initializable && + std::copy_constructible) +class multifunction_node : + public graph_node, + public multifunction_input + < + Input, + typename wrap_tuple_elements< + std::tuple_size::value, // #elements in tuple + multifunction_output, // wrap this around each element + Output // the tuple providing the types + >::type, + Policy, + cache_aligned_allocator + > +{ + typedef cache_aligned_allocator internals_allocator; + +protected: + static const int N = std::tuple_size::value; +public: + typedef Input input_type; + typedef null_type output_type; + typedef typename wrap_tuple_elements::type output_ports_type; + typedef multifunction_input< + input_type, output_ports_type, Policy, internals_allocator> input_impl_type; + typedef function_input_queue input_queue_type; +private: + using input_impl_type::my_predecessors; +public: + template + __TBB_requires(multifunction_node_body) + __TBB_NOINLINE_SYM multifunction_node( + graph &g, size_t concurrency, + Body body, Policy = Policy(), node_priority_t a_priority = no_priority + ) : graph_node(g), input_impl_type(g, concurrency, body, a_priority) { + fgt_multioutput_node_with_body( + CODEPTR(), FLOW_MULTIFUNCTION_NODE, + &this->my_graph, static_cast *>(this), + this->output_ports(), this->my_body + ); + } + + template + __TBB_requires(multifunction_node_body) + __TBB_NOINLINE_SYM multifunction_node(graph& g, size_t concurrency, Body body, node_priority_t a_priority) + : multifunction_node(g, concurrency, body, Policy(), a_priority) {} + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + template + __TBB_requires(multifunction_node_body) + __TBB_NOINLINE_SYM multifunction_node(const node_set& nodes, size_t concurrency, Body body, + Policy p = Policy(), node_priority_t a_priority = no_priority) + : multifunction_node(nodes.graph_reference(), concurrency, body, p, a_priority) { + make_edges_in_order(nodes, *this); + } + + template + __TBB_requires(multifunction_node_body) + __TBB_NOINLINE_SYM multifunction_node(const node_set& nodes, size_t concurrency, Body body, node_priority_t a_priority) + : multifunction_node(nodes, concurrency, body, Policy(), a_priority) {} +#endif // __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + + __TBB_NOINLINE_SYM multifunction_node( const multifunction_node &other) : + graph_node(other.my_graph), input_impl_type(other) { + fgt_multioutput_node_with_body( CODEPTR(), FLOW_MULTIFUNCTION_NODE, + &this->my_graph, static_cast *>(this), + this->output_ports(), this->my_body ); + } + + // all the guts are in multifunction_input... +protected: + void reset_node(reset_flags f) override { input_impl_type::reset(f); } +}; // multifunction_node + +//! split_node: accepts a tuple as input, forwards each element of the tuple to its +// successors. The node has unlimited concurrency, so it does not reject inputs. +template +class split_node : public graph_node, public receiver { + static const int N = std::tuple_size::value; + typedef receiver base_type; +public: + typedef TupleType input_type; + typedef typename wrap_tuple_elements< + N, // #elements in tuple + multifunction_output, // wrap this around each element + TupleType // the tuple providing the types + >::type output_ports_type; + + __TBB_NOINLINE_SYM explicit split_node(graph &g) + : graph_node(g), + my_output_ports(init_output_ports::call(g, my_output_ports)) + { + fgt_multioutput_node(CODEPTR(), FLOW_SPLIT_NODE, &this->my_graph, + static_cast *>(this), this->output_ports()); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + template + __TBB_NOINLINE_SYM split_node(const node_set& nodes) : split_node(nodes.graph_reference()) { + make_edges_in_order(nodes, *this); + } +#endif + + __TBB_NOINLINE_SYM split_node(const split_node& other) + : graph_node(other.my_graph), base_type(other), + my_output_ports(init_output_ports::call(other.my_graph, my_output_ports)) + { + fgt_multioutput_node(CODEPTR(), FLOW_SPLIT_NODE, &this->my_graph, + static_cast *>(this), this->output_ports()); + } + + output_ports_type &output_ports() { return my_output_ports; } + +protected: + graph_task *try_put_task(const TupleType& t) override { + // Sending split messages in parallel is not justified, as overheads would prevail. + // Also, we do not have successors here. So we just tell the task returned here is successful. + return emit_element::emit_this(this->my_graph, t, output_ports()); + } +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + graph_task* try_put_task(const TupleType& t, const message_metainfo& metainfo) override { + // Sending split messages in parallel is not justified, as overheads would prevail. + // Also, we do not have successors here. So we just tell the task returned here is successful. + return emit_element::emit_this(this->my_graph, t, output_ports(), metainfo); + } +#endif + + void reset_node(reset_flags f) override { + if (f & rf_clear_edges) + clear_element::clear_this(my_output_ports); + + __TBB_ASSERT(!(f & rf_clear_edges) || clear_element::this_empty(my_output_ports), "split_node reset failed"); + } + graph& graph_reference() const override { + return my_graph; + } + +private: + output_ports_type my_output_ports; +}; + +//! Implements an executable node that supports continue_msg -> Output +template > + __TBB_requires(std::copy_constructible) +class continue_node : public graph_node, public continue_input, + public function_output { +public: + typedef continue_msg input_type; + typedef Output output_type; + typedef continue_input input_impl_type; + typedef function_output fOutput_type; + typedef typename input_impl_type::predecessor_type predecessor_type; + typedef typename fOutput_type::successor_type successor_type; + + //! Constructor for executable node with continue_msg -> Output + template + __TBB_requires(continue_node_body) + __TBB_NOINLINE_SYM continue_node( + graph &g, + Body body, Policy = Policy(), node_priority_t a_priority = no_priority + ) : graph_node(g), input_impl_type( g, body, a_priority ), + fOutput_type(g) { + fgt_node_with_body( CODEPTR(), FLOW_CONTINUE_NODE, &this->my_graph, + + static_cast *>(this), + static_cast *>(this), this->my_body ); + } + + template + __TBB_requires(continue_node_body) + continue_node( graph& g, Body body, node_priority_t a_priority ) + : continue_node(g, body, Policy(), a_priority) {} + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + template + __TBB_requires(continue_node_body) + continue_node( const node_set& nodes, Body body, + Policy p = Policy(), node_priority_t a_priority = no_priority ) + : continue_node(nodes.graph_reference(), body, p, a_priority ) { + make_edges_in_order(nodes, *this); + } + template + __TBB_requires(continue_node_body) + continue_node( const node_set& nodes, Body body, node_priority_t a_priority) + : continue_node(nodes, body, Policy(), a_priority) {} +#endif // __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + + //! Constructor for executable node with continue_msg -> Output + template + __TBB_requires(continue_node_body) + __TBB_NOINLINE_SYM continue_node( + graph &g, int number_of_predecessors, + Body body, Policy = Policy(), node_priority_t a_priority = no_priority + ) : graph_node(g) + , input_impl_type(g, number_of_predecessors, body, a_priority), + fOutput_type(g) { + fgt_node_with_body( CODEPTR(), FLOW_CONTINUE_NODE, &this->my_graph, + static_cast *>(this), + static_cast *>(this), this->my_body ); + } + + template + __TBB_requires(continue_node_body) + continue_node( graph& g, int number_of_predecessors, Body body, node_priority_t a_priority) + : continue_node(g, number_of_predecessors, body, Policy(), a_priority) {} + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + template + __TBB_requires(continue_node_body) + continue_node( const node_set& nodes, int number_of_predecessors, + Body body, Policy p = Policy(), node_priority_t a_priority = no_priority ) + : continue_node(nodes.graph_reference(), number_of_predecessors, body, p, a_priority) { + make_edges_in_order(nodes, *this); + } + + template + __TBB_requires(continue_node_body) + continue_node( const node_set& nodes, int number_of_predecessors, + Body body, node_priority_t a_priority ) + : continue_node(nodes, number_of_predecessors, body, Policy(), a_priority) {} +#endif + + //! Copy constructor + __TBB_NOINLINE_SYM continue_node( const continue_node& src ) : + graph_node(src.my_graph), input_impl_type(src), + function_output(src.my_graph) { + fgt_node_with_body( CODEPTR(), FLOW_CONTINUE_NODE, &this->my_graph, + static_cast *>(this), + static_cast *>(this), this->my_body ); + } + +protected: + template< typename R, typename B > friend class run_and_put_task; + template friend class broadcast_cache; + template friend class round_robin_cache; + using input_impl_type::try_put_task; + broadcast_cache &successors () override { return fOutput_type::my_successors; } + + void reset_node(reset_flags f) override { + input_impl_type::reset_receiver(f); + if(f & rf_clear_edges)successors().clear(); + __TBB_ASSERT(!(f & rf_clear_edges) || successors().empty(), "continue_node not reset"); + } +}; // continue_node + +//! Forwards messages of type T to all successors +template +class broadcast_node : public graph_node, public receiver, public sender { +public: + typedef T input_type; + typedef T output_type; + typedef typename receiver::predecessor_type predecessor_type; + typedef typename sender::successor_type successor_type; +private: + broadcast_cache my_successors; +public: + + __TBB_NOINLINE_SYM explicit broadcast_node(graph& g) : graph_node(g), my_successors(this) { + fgt_node( CODEPTR(), FLOW_BROADCAST_NODE, &this->my_graph, + static_cast *>(this), static_cast *>(this) ); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + template + broadcast_node(const node_set& nodes) : broadcast_node(nodes.graph_reference()) { + make_edges_in_order(nodes, *this); + } +#endif + + // Copy constructor + __TBB_NOINLINE_SYM broadcast_node( const broadcast_node& src ) : broadcast_node(src.my_graph) {} + + //! Adds a successor + bool register_successor( successor_type &r ) override { + my_successors.register_successor( r ); + return true; + } + + //! Removes s as a successor + bool remove_successor( successor_type &r ) override { + my_successors.remove_successor( r ); + return true; + } + +private: + graph_task* try_put_task_impl(const T& t __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& metainfo)) { + graph_task* new_task = my_successors.try_put_task(t __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo)); + if (!new_task) new_task = SUCCESSFULLY_ENQUEUED; + return new_task; + } + +protected: + template< typename R, typename B > friend class run_and_put_task; + template friend class broadcast_cache; + template friend class round_robin_cache; + //! build a task to run the successor if possible. Default is old behavior. + graph_task* try_put_task(const T& t) override { + return try_put_task_impl(t __TBB_FLOW_GRAPH_METAINFO_ARG(message_metainfo{})); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + graph_task* try_put_task(const T& t, const message_metainfo& metainfo) override { + return try_put_task_impl(t, metainfo); + } +#endif + + graph& graph_reference() const override { + return my_graph; + } + + void reset_node(reset_flags f) override { + if (f&rf_clear_edges) { + my_successors.clear(); + } + __TBB_ASSERT(!(f & rf_clear_edges) || my_successors.empty(), "Error resetting broadcast_node"); + } +}; // broadcast_node + +//! Forwards messages in arbitrary order +template +class buffer_node + : public graph_node + , public reservable_item_buffer< T, cache_aligned_allocator > + , public receiver, public sender +{ + typedef cache_aligned_allocator internals_allocator; + +public: + typedef T input_type; + typedef T output_type; + typedef typename receiver::predecessor_type predecessor_type; + typedef typename sender::successor_type successor_type; + typedef buffer_node class_type; + +protected: + typedef size_t size_type; + round_robin_cache< T, null_rw_mutex > my_successors; + + friend class forward_task_bypass< class_type >; + + enum op_type {reg_succ, rem_succ, req_item, res_item, rel_res, con_res, put_item, try_fwd_task + }; + + // implements the aggregator_operation concept + class buffer_operation : public d1::aggregated_operation< buffer_operation > { + public: + char type; + T* elem; + graph_task* ltask; + successor_type *r; +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + message_metainfo* metainfo{ nullptr }; +#endif + + buffer_operation(const T& e, op_type t) : type(char(t)) + , elem(const_cast(&e)) , ltask(nullptr) + , r(nullptr) + {} + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + buffer_operation(const T& e, op_type t, const message_metainfo& info) + : type(char(t)), elem(const_cast(&e)), ltask(nullptr), r(nullptr) + , metainfo(const_cast(&info)) + {} + + buffer_operation(op_type t, message_metainfo& info) + : type(char(t)), elem(nullptr), ltask(nullptr), r(nullptr), metainfo(&info) {} +#endif + buffer_operation(op_type t) : type(char(t)), elem(nullptr), ltask(nullptr), r(nullptr) {} + }; + + bool forwarder_busy; + typedef d1::aggregating_functor handler_type; + friend class d1::aggregating_functor; + d1::aggregator< handler_type, buffer_operation> my_aggregator; + + virtual void handle_operations(buffer_operation *op_list) { + handle_operations_impl(op_list, this); + } + + template + void handle_operations_impl(buffer_operation *op_list, derived_type* derived) { + __TBB_ASSERT(static_cast(derived) == this, "'this' is not a base class for derived"); + + buffer_operation *tmp = nullptr; + bool try_forwarding = false; + while (op_list) { + tmp = op_list; + op_list = op_list->next; + switch (tmp->type) { + case reg_succ: internal_reg_succ(tmp); try_forwarding = true; break; + case rem_succ: internal_rem_succ(tmp); break; + case req_item: internal_pop(tmp); break; + case res_item: internal_reserve(tmp); break; + case rel_res: internal_release(tmp); try_forwarding = true; break; + case con_res: internal_consume(tmp); try_forwarding = true; break; + case put_item: try_forwarding = internal_push(tmp); break; + case try_fwd_task: internal_forward_task(tmp); break; + } + } + + derived->order(); + + if (try_forwarding && !forwarder_busy) { + if(is_graph_active(this->my_graph)) { + forwarder_busy = true; + typedef forward_task_bypass task_type; + d1::small_object_allocator allocator{}; + graph_task* new_task = allocator.new_object(graph_reference(), allocator, *this); + // tmp should point to the last item handled by the aggregator. This is the operation + // the handling thread enqueued. So modifying that record will be okay. + // TODO revamp: check that the issue is still present + // workaround for icc bug (at least 12.0 and 13.0) + // error: function "tbb::flow::interfaceX::combine_tasks" cannot be called with the given argument list + // argument types are: (graph, graph_task *, graph_task *) + graph_task *z = tmp->ltask; + graph &g = this->my_graph; + tmp->ltask = combine_tasks(g, z, new_task); // in case the op generated a task + } + } + } // handle_operations + + inline graph_task *grab_forwarding_task( buffer_operation &op_data) { + return op_data.ltask; + } + + inline bool enqueue_forwarding_task(buffer_operation &op_data) { + graph_task *ft = grab_forwarding_task(op_data); + if(ft) { + spawn_in_graph_arena(graph_reference(), *ft); + return true; + } + return false; + } + + //! This is executed by an enqueued task, the "forwarder" + virtual graph_task *forward_task() { + buffer_operation op_data(try_fwd_task); + graph_task *last_task = nullptr; + do { + op_data.status = WAIT; + op_data.ltask = nullptr; + my_aggregator.execute(&op_data); + + // workaround for icc bug + graph_task *xtask = op_data.ltask; + graph& g = this->my_graph; + last_task = combine_tasks(g, last_task, xtask); + } while (op_data.status ==SUCCEEDED); + return last_task; + } + + //! Register successor + virtual void internal_reg_succ(buffer_operation *op) { + __TBB_ASSERT(op->r, nullptr); + my_successors.register_successor(*(op->r)); + op->status.store(SUCCEEDED, std::memory_order_release); + } + + //! Remove successor + virtual void internal_rem_succ(buffer_operation *op) { + __TBB_ASSERT(op->r, nullptr); + my_successors.remove_successor(*(op->r)); + op->status.store(SUCCEEDED, std::memory_order_release); + } + +private: + void order() {} + + bool is_item_valid() { + return this->my_item_valid(this->my_tail - 1); + } + + void try_put_and_add_task(graph_task*& last_task) { + graph_task* new_task = my_successors.try_put_task(this->back() + __TBB_FLOW_GRAPH_METAINFO_ARG(this->back_metainfo())); + if (new_task) { + // workaround for icc bug + graph& g = this->my_graph; + last_task = combine_tasks(g, last_task, new_task); + this->destroy_back(); + } + } + +protected: + //! Tries to forward valid items to successors + virtual void internal_forward_task(buffer_operation *op) { + internal_forward_task_impl(op, this); + } + + template + void internal_forward_task_impl(buffer_operation *op, derived_type* derived) { + __TBB_ASSERT(static_cast(derived) == this, "'this' is not a base class for derived"); + + if (this->my_reserved || !derived->is_item_valid()) { + op->status.store(FAILED, std::memory_order_release); + this->forwarder_busy = false; + return; + } + // Try forwarding, giving each successor a chance + graph_task* last_task = nullptr; + size_type counter = my_successors.size(); + for (; counter > 0 && derived->is_item_valid(); --counter) + derived->try_put_and_add_task(last_task); + + op->ltask = last_task; // return task + if (last_task && !counter) { + op->status.store(SUCCEEDED, std::memory_order_release); + } + else { + op->status.store(FAILED, std::memory_order_release); + forwarder_busy = false; + } + } + + virtual bool internal_push(buffer_operation *op) { + __TBB_ASSERT(op->elem, nullptr); +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + __TBB_ASSERT(op->metainfo, nullptr); + this->push_back(*(op->elem), (*op->metainfo)); +#else + this->push_back(*(op->elem)); +#endif + op->status.store(SUCCEEDED, std::memory_order_release); + return true; + } + + virtual void internal_pop(buffer_operation *op) { + __TBB_ASSERT(op->elem, nullptr); +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + bool pop_result = op->metainfo ? this->pop_back(*(op->elem), *(op->metainfo)) + : this->pop_back(*(op->elem)); +#else + bool pop_result = this->pop_back(*(op->elem)); +#endif + if (pop_result) { + op->status.store(SUCCEEDED, std::memory_order_release); + } + else { + op->status.store(FAILED, std::memory_order_release); + } + } + + virtual void internal_reserve(buffer_operation *op) { + __TBB_ASSERT(op->elem, nullptr); +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + bool reserve_result = op->metainfo ? this->reserve_front(*(op->elem), *(op->metainfo)) + : this->reserve_front(*(op->elem)); +#else + bool reserve_result = this->reserve_front(*(op->elem)); +#endif + if (reserve_result) { + op->status.store(SUCCEEDED, std::memory_order_release); + } + else { + op->status.store(FAILED, std::memory_order_release); + } + } + + virtual void internal_consume(buffer_operation *op) { + this->consume_front(); + op->status.store(SUCCEEDED, std::memory_order_release); + } + + virtual void internal_release(buffer_operation *op) { + this->release_front(); + op->status.store(SUCCEEDED, std::memory_order_release); + } + +public: + //! Constructor + __TBB_NOINLINE_SYM explicit buffer_node( graph &g ) + : graph_node(g), reservable_item_buffer(), receiver(), + sender(), my_successors(this), forwarder_busy(false) + { + my_aggregator.initialize_handler(handler_type(this)); + fgt_node( CODEPTR(), FLOW_BUFFER_NODE, &this->my_graph, + static_cast *>(this), static_cast *>(this) ); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + template + buffer_node(const node_set& nodes) : buffer_node(nodes.graph_reference()) { + make_edges_in_order(nodes, *this); + } +#endif + + //! Copy constructor + __TBB_NOINLINE_SYM buffer_node( const buffer_node& src ) : buffer_node(src.my_graph) {} + + // + // message sender implementation + // + + //! Adds a new successor. + /** Adds successor r to the list of successors; may forward tasks. */ + bool register_successor( successor_type &r ) override { + buffer_operation op_data(reg_succ); + op_data.r = &r; + my_aggregator.execute(&op_data); + (void)enqueue_forwarding_task(op_data); + return true; + } + + //! Removes a successor. + /** Removes successor r from the list of successors. + It also calls r.remove_predecessor(*this) to remove this node as a predecessor. */ + bool remove_successor( successor_type &r ) override { + // TODO revamp: investigate why full qualification is necessary here + tbb::detail::d2::remove_predecessor(r, *this); + buffer_operation op_data(rem_succ); + op_data.r = &r; + my_aggregator.execute(&op_data); + // even though this operation does not cause a forward, if we are the handler, and + // a forward is scheduled, we may be the first to reach this point after the aggregator, + // and so should check for the task. + (void)enqueue_forwarding_task(op_data); + return true; + } + + //! Request an item from the buffer_node + /** true = v contains the returned item
+ false = no item has been returned */ + bool try_get( T &v ) override { + buffer_operation op_data(req_item); + op_data.elem = &v; + my_aggregator.execute(&op_data); + (void)enqueue_forwarding_task(op_data); + return (op_data.status==SUCCEEDED); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + bool try_get( T &v, message_metainfo& metainfo ) override { + buffer_operation op_data(req_item, metainfo); + op_data.elem = &v; + my_aggregator.execute(&op_data); + (void)enqueue_forwarding_task(op_data); + return (op_data.status==SUCCEEDED); + } +#endif + + //! Reserves an item. + /** false = no item can be reserved
+ true = an item is reserved */ + bool try_reserve( T &v ) override { + buffer_operation op_data(res_item); + op_data.elem = &v; + my_aggregator.execute(&op_data); + (void)enqueue_forwarding_task(op_data); + return (op_data.status==SUCCEEDED); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + bool try_reserve( output_type& v, message_metainfo& metainfo ) override { + buffer_operation op_data(res_item, metainfo); + op_data.elem = &v; + my_aggregator.execute(&op_data); + (void)enqueue_forwarding_task(op_data); + return op_data.status==SUCCEEDED; + } +#endif + + //! Release a reserved item. + /** true = item has been released and so remains in sender */ + bool try_release() override { + buffer_operation op_data(rel_res); + my_aggregator.execute(&op_data); + (void)enqueue_forwarding_task(op_data); + return true; + } + + //! Consumes a reserved item. + /** true = item is removed from sender and reservation removed */ + bool try_consume() override { + buffer_operation op_data(con_res); + my_aggregator.execute(&op_data); + (void)enqueue_forwarding_task(op_data); + return true; + } + +private: + graph_task* try_put_task_impl(const T& t __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& metainfo)) { + buffer_operation op_data(t, put_item __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo)); + my_aggregator.execute(&op_data); + graph_task *ft = grab_forwarding_task(op_data); + // sequencer_nodes can return failure (if an item has been previously inserted) + // We have to spawn the returned task if our own operation fails. + + if(ft && op_data.status ==FAILED) { + // we haven't succeeded queueing the item, but for some reason the + // call returned a task (if another request resulted in a successful + // forward this could happen.) Queue the task and reset the pointer. + spawn_in_graph_arena(graph_reference(), *ft); ft = nullptr; + } + else if(!ft && op_data.status ==SUCCEEDED) { + ft = SUCCESSFULLY_ENQUEUED; + } + return ft; + } + +protected: + + template< typename R, typename B > friend class run_and_put_task; + template friend class broadcast_cache; + template friend class round_robin_cache; + //! receive an item, return a task *if possible + graph_task *try_put_task(const T &t) override { + return try_put_task_impl(t __TBB_FLOW_GRAPH_METAINFO_ARG(message_metainfo{})); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + graph_task* try_put_task(const T& t, const message_metainfo& metainfo) override { + return try_put_task_impl(t, metainfo); + } +#endif + + graph& graph_reference() const override { + return my_graph; + } + +protected: + void reset_node( reset_flags f) override { + reservable_item_buffer::reset(); + // TODO: just clear structures + if (f&rf_clear_edges) { + my_successors.clear(); + } + forwarder_busy = false; + } +}; // buffer_node + +//! Forwards messages in FIFO order +template +class queue_node : public buffer_node { +protected: + typedef buffer_node base_type; + typedef typename base_type::size_type size_type; + typedef typename base_type::buffer_operation queue_operation; + typedef queue_node class_type; + +private: + template friend class buffer_node; + + bool is_item_valid() { + return this->my_item_valid(this->my_head); + } + + void try_put_and_add_task(graph_task*& last_task) { + graph_task* new_task = this->my_successors.try_put_task(this->front() + __TBB_FLOW_GRAPH_METAINFO_ARG(this->front_metainfo())); + + if (new_task) { + // workaround for icc bug + graph& graph_ref = this->graph_reference(); + last_task = combine_tasks(graph_ref, last_task, new_task); + this->destroy_front(); + } + } + +protected: + void internal_forward_task(queue_operation *op) override { + this->internal_forward_task_impl(op, this); + } + + void internal_pop(queue_operation *op) override { + if ( this->my_reserved || !this->my_item_valid(this->my_head)){ + op->status.store(FAILED, std::memory_order_release); + } + else { +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + if (op->metainfo) { + this->pop_front(*(op->elem), *(op->metainfo)); + } else +#endif + { + this->pop_front(*(op->elem)); + } + op->status.store(SUCCEEDED, std::memory_order_release); + } + } + void internal_reserve(queue_operation *op) override { + if (this->my_reserved || !this->my_item_valid(this->my_head)) { + op->status.store(FAILED, std::memory_order_release); + } + else { +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + if (op->metainfo) { + this->reserve_front(*(op->elem), *(op->metainfo)); + } + else +#endif + { + this->reserve_front(*(op->elem)); + } + op->status.store(SUCCEEDED, std::memory_order_release); + } + } + void internal_consume(queue_operation *op) override { + this->consume_front(); + op->status.store(SUCCEEDED, std::memory_order_release); + } + +public: + typedef T input_type; + typedef T output_type; + typedef typename receiver::predecessor_type predecessor_type; + typedef typename sender::successor_type successor_type; + + //! Constructor + __TBB_NOINLINE_SYM explicit queue_node( graph &g ) : base_type(g) { + fgt_node( CODEPTR(), FLOW_QUEUE_NODE, &(this->my_graph), + static_cast *>(this), + static_cast *>(this) ); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + template + queue_node( const node_set& nodes) : queue_node(nodes.graph_reference()) { + make_edges_in_order(nodes, *this); + } +#endif + + //! Copy constructor + __TBB_NOINLINE_SYM queue_node( const queue_node& src) : base_type(src) { + fgt_node( CODEPTR(), FLOW_QUEUE_NODE, &(this->my_graph), + static_cast *>(this), + static_cast *>(this) ); + } + + +protected: + void reset_node( reset_flags f) override { + base_type::reset_node(f); + } +}; // queue_node + +//! Forwards messages in sequence order +template + __TBB_requires(std::copyable) +class sequencer_node : public queue_node { + function_body< T, size_t > *my_sequencer; + // my_sequencer should be a benign function and must be callable + // from a parallel context. Does this mean it needn't be reset? +public: + typedef T input_type; + typedef T output_type; + typedef typename receiver::predecessor_type predecessor_type; + typedef typename sender::successor_type successor_type; + + //! Constructor + template< typename Sequencer > + __TBB_requires(sequencer) + __TBB_NOINLINE_SYM sequencer_node( graph &g, const Sequencer& s ) : queue_node(g), + my_sequencer(new function_body_leaf< T, size_t, Sequencer>(s) ) { + fgt_node( CODEPTR(), FLOW_SEQUENCER_NODE, &(this->my_graph), + static_cast *>(this), + static_cast *>(this) ); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + template + __TBB_requires(sequencer) + sequencer_node( const node_set& nodes, const Sequencer& s) + : sequencer_node(nodes.graph_reference(), s) { + make_edges_in_order(nodes, *this); + } +#endif + + //! Copy constructor + __TBB_NOINLINE_SYM sequencer_node( const sequencer_node& src ) : queue_node(src), + my_sequencer( src.my_sequencer->clone() ) { + fgt_node( CODEPTR(), FLOW_SEQUENCER_NODE, &(this->my_graph), + static_cast *>(this), + static_cast *>(this) ); + } + + //! Destructor + ~sequencer_node() { delete my_sequencer; } + +protected: + typedef typename buffer_node::size_type size_type; + typedef typename buffer_node::buffer_operation sequencer_operation; + +private: + bool internal_push(sequencer_operation *op) override { + size_type tag = (*my_sequencer)(*(op->elem)); +#if !TBB_DEPRECATED_SEQUENCER_DUPLICATES + if (tag < this->my_head) { + // have already emitted a message with this tag + op->status.store(FAILED, std::memory_order_release); + return false; + } +#endif + // cannot modify this->my_tail now; the buffer would be inconsistent. + size_t new_tail = (tag+1 > this->my_tail) ? tag+1 : this->my_tail; + + if (this->size(new_tail) > this->capacity()) { + this->grow_my_array(this->size(new_tail)); + } + this->my_tail = new_tail; + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + __TBB_ASSERT(op->metainfo, nullptr); + bool place_item_result = this->place_item(tag, *(op->elem), *(op->metainfo)); + const op_stat res = place_item_result ? SUCCEEDED : FAILED; +#else + const op_stat res = this->place_item(tag, *(op->elem)) ? SUCCEEDED : FAILED; +#endif + op->status.store(res, std::memory_order_release); + return res ==SUCCEEDED; + } +}; // sequencer_node + +//! Forwards messages in priority order +template> +class priority_queue_node : public buffer_node { +public: + typedef T input_type; + typedef T output_type; + typedef buffer_node base_type; + typedef priority_queue_node class_type; + typedef typename receiver::predecessor_type predecessor_type; + typedef typename sender::successor_type successor_type; + + //! Constructor + __TBB_NOINLINE_SYM explicit priority_queue_node( graph &g, const Compare& comp = Compare() ) + : buffer_node(g), compare(comp), mark(0) { + fgt_node( CODEPTR(), FLOW_PRIORITY_QUEUE_NODE, &(this->my_graph), + static_cast *>(this), + static_cast *>(this) ); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + template + priority_queue_node(const node_set& nodes, const Compare& comp = Compare()) + : priority_queue_node(nodes.graph_reference(), comp) { + make_edges_in_order(nodes, *this); + } +#endif + + //! Copy constructor + __TBB_NOINLINE_SYM priority_queue_node( const priority_queue_node &src ) + : buffer_node(src), mark(0) + { + fgt_node( CODEPTR(), FLOW_PRIORITY_QUEUE_NODE, &(this->my_graph), + static_cast *>(this), + static_cast *>(this) ); + } + +protected: + + void reset_node( reset_flags f) override { + mark = 0; + base_type::reset_node(f); + } + + typedef typename buffer_node::size_type size_type; + typedef typename buffer_node::item_type item_type; + typedef typename buffer_node::buffer_operation prio_operation; + + //! Tries to forward valid items to successors + void internal_forward_task(prio_operation *op) override { + this->internal_forward_task_impl(op, this); + } + + void handle_operations(prio_operation *op_list) override { + this->handle_operations_impl(op_list, this); + } + + bool internal_push(prio_operation *op) override { +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + __TBB_ASSERT(op->metainfo, nullptr); + prio_push(*(op->elem), *(op->metainfo)); +#else + prio_push(*(op->elem)); +#endif + op->status.store(SUCCEEDED, std::memory_order_release); + return true; + } + + void internal_pop(prio_operation *op) override { + // if empty or already reserved, don't pop + if ( this->my_reserved == true || this->my_tail == 0 ) { + op->status.store(FAILED, std::memory_order_release); + return; + } + + *(op->elem) = prio(); +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + if (op->metainfo) { + *(op->metainfo) = std::move(prio_metainfo()); + } +#endif + op->status.store(SUCCEEDED, std::memory_order_release); + prio_pop(); + + } + + // pops the highest-priority item, saves copy + void internal_reserve(prio_operation *op) override { + if (this->my_reserved == true || this->my_tail == 0) { + op->status.store(FAILED, std::memory_order_release); + return; + } + this->my_reserved = true; + *(op->elem) = prio(); +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + if (op->metainfo) { + *(op->metainfo) = std::move(prio_metainfo()); + reserved_metainfo = *(op->metainfo); + } +#endif + reserved_item = *(op->elem); + op->status.store(SUCCEEDED, std::memory_order_release); + prio_pop(); + } + + void internal_consume(prio_operation *op) override { + op->status.store(SUCCEEDED, std::memory_order_release); + this->my_reserved = false; + reserved_item = input_type(); +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + for (auto waiter : reserved_metainfo.waiters()) { + waiter->release(1); + } + + reserved_metainfo = message_metainfo{}; +#endif + } + + void internal_release(prio_operation *op) override { + op->status.store(SUCCEEDED, std::memory_order_release); + prio_push(reserved_item __TBB_FLOW_GRAPH_METAINFO_ARG(reserved_metainfo)); + this->my_reserved = false; + reserved_item = input_type(); +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + for (auto waiter : reserved_metainfo.waiters()) { + waiter->release(1); + } + + reserved_metainfo = message_metainfo{}; +#endif + } + +private: + template friend class buffer_node; + + void order() { + if (mark < this->my_tail) heapify(); + __TBB_ASSERT(mark == this->my_tail, "mark unequal after heapify"); + } + + bool is_item_valid() { + return this->my_tail > 0; + } + + void try_put_and_add_task(graph_task*& last_task) { + graph_task* new_task = this->my_successors.try_put_task(this->prio() + __TBB_FLOW_GRAPH_METAINFO_ARG(this->prio_metainfo())); + if (new_task) { + // workaround for icc bug + graph& graph_ref = this->graph_reference(); + last_task = combine_tasks(graph_ref, last_task, new_task); + prio_pop(); + } + } + +private: + Compare compare; + size_type mark; + + input_type reserved_item; +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + message_metainfo reserved_metainfo; +#endif + + // in case a reheap has not been done after a push, check if the mark item is higher than the 0'th item + bool prio_use_tail() { + __TBB_ASSERT(mark <= this->my_tail, "mark outside bounds before test"); + return mark < this->my_tail && compare(this->get_my_item(0), this->get_my_item(this->my_tail - 1)); + } + + // prio_push: checks that the item will fit, expand array if necessary, put at end + void prio_push(const T &src __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& metainfo)) { + if ( this->my_tail >= this->my_array_size ) + this->grow_my_array( this->my_tail + 1 ); + (void) this->place_item(this->my_tail, src __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo)); + ++(this->my_tail); + __TBB_ASSERT(mark < this->my_tail, "mark outside bounds after push"); + } + + // prio_pop: deletes highest priority item from the array, and if it is item + // 0, move last item to 0 and reheap. If end of array, just destroy and decrement tail + // and mark. Assumes the array has already been tested for emptiness; no failure. + void prio_pop() { + if (prio_use_tail()) { + // there are newly pushed elements; last one higher than top + // copy the data + this->destroy_item(this->my_tail-1); + --(this->my_tail); + __TBB_ASSERT(mark <= this->my_tail, "mark outside bounds after pop"); + return; + } + this->destroy_item(0); + if(this->my_tail > 1) { + // push the last element down heap + __TBB_ASSERT(this->my_item_valid(this->my_tail - 1), nullptr); + this->move_item(0,this->my_tail - 1); + } + --(this->my_tail); + if(mark > this->my_tail) --mark; + if (this->my_tail > 1) // don't reheap for heap of size 1 + reheap(); + __TBB_ASSERT(mark <= this->my_tail, "mark outside bounds after pop"); + } + + const T& prio() { + return this->get_my_item(prio_use_tail() ? this->my_tail-1 : 0); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + message_metainfo& prio_metainfo() { + return this->get_my_metainfo(prio_use_tail() ? this->my_tail-1 : 0); + } +#endif + + // turn array into heap + void heapify() { + if(this->my_tail == 0) { + mark = 0; + return; + } + if (!mark) mark = 1; + for (; markmy_tail; ++mark) { // for each unheaped element + size_type cur_pos = mark; + input_type to_place; +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + message_metainfo metainfo; +#endif + this->fetch_item(mark, to_place __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo)); + do { // push to_place up the heap + size_type parent = (cur_pos-1)>>1; + if (!compare(this->get_my_item(parent), to_place)) + break; + this->move_item(cur_pos, parent); + cur_pos = parent; + } while( cur_pos ); + this->place_item(cur_pos, to_place __TBB_FLOW_GRAPH_METAINFO_ARG(std::move(metainfo))); + } + } + + // otherwise heapified array with new root element; rearrange to heap + void reheap() { + size_type cur_pos=0, child=1; + while (child < mark) { + size_type target = child; + if (child+1get_my_item(child), + this->get_my_item(child+1))) + ++target; + // target now has the higher priority child + if (compare(this->get_my_item(target), + this->get_my_item(cur_pos))) + break; + // swap + this->swap_items(cur_pos, target); + cur_pos = target; + child = (cur_pos<<1)+1; + } + } +}; // priority_queue_node + +//! Forwards messages only if the threshold has not been reached +/** This node forwards items until its threshold is reached. + It contains no buffering. If the downstream node rejects, the + message is dropped. */ +template< typename T, typename DecrementType=continue_msg > +class limiter_node : public graph_node, public receiver< T >, public sender< T > { +public: + typedef T input_type; + typedef T output_type; + typedef typename receiver::predecessor_type predecessor_type; + typedef typename sender::successor_type successor_type; + //TODO: There is a lack of predefined types for its controlling "decrementer" port. It should be fixed later. + +private: + size_t my_threshold; + size_t my_count; // number of successful puts + size_t my_tries; // number of active put attempts + size_t my_future_decrement; // number of active decrement + reservable_predecessor_cache< T, spin_mutex > my_predecessors; + spin_mutex my_mutex; + broadcast_cache< T > my_successors; + + //! The internal receiver< DecrementType > that adjusts the count + threshold_regulator< limiter_node, DecrementType > decrement; + + graph_task* decrement_counter( long long delta ) { + if ( delta > 0 && size_t(delta) > my_threshold ) { + delta = my_threshold; + } + + { + spin_mutex::scoped_lock lock(my_mutex); + if ( delta > 0 && size_t(delta) > my_count ) { + if( my_tries > 0 ) { + my_future_decrement += (size_t(delta) - my_count); + } + my_count = 0; + } + else if ( delta < 0 && size_t(-delta) > my_threshold - my_count ) { + my_count = my_threshold; + } + else { + my_count -= size_t(delta); // absolute value of delta is sufficiently small + } + __TBB_ASSERT(my_count <= my_threshold, "counter values are truncated to be inside the [0, threshold] interval"); + } + return forward_task(); + } + + // Let threshold_regulator call decrement_counter() + friend class threshold_regulator< limiter_node, DecrementType >; + + friend class forward_task_bypass< limiter_node >; + + bool check_conditions() { // always called under lock + return ( my_count + my_tries < my_threshold && !my_predecessors.empty() && !my_successors.empty() ); + } + + // only returns a valid task pointer or nullptr, never SUCCESSFULLY_ENQUEUED + graph_task* forward_task() { + input_type v; + graph_task* rval = nullptr; + bool reserved = false; + + { + spin_mutex::scoped_lock lock(my_mutex); + if ( check_conditions() ) + ++my_tries; + else + return nullptr; + } + + //SUCCESS + // if we can reserve and can put, we consume the reservation + // we increment the count and decrement the tries +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + message_metainfo metainfo; +#endif + if ( (my_predecessors.try_reserve(v __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo))) == true ) { + reserved = true; + if ( (rval = my_successors.try_put_task(v __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo))) != nullptr ) { + { + spin_mutex::scoped_lock lock(my_mutex); + ++my_count; + if ( my_future_decrement ) { + if ( my_count > my_future_decrement ) { + my_count -= my_future_decrement; + my_future_decrement = 0; + } + else { + my_future_decrement -= my_count; + my_count = 0; + } + } + --my_tries; + my_predecessors.try_consume(); + if ( check_conditions() ) { + if ( is_graph_active(this->my_graph) ) { + typedef forward_task_bypass> task_type; + d1::small_object_allocator allocator{}; + graph_task* rtask = allocator.new_object( my_graph, allocator, *this ); + spawn_in_graph_arena(graph_reference(), *rtask); + } + } + } + return rval; + } + } + //FAILURE + //if we can't reserve, we decrement the tries + //if we can reserve but can't put, we decrement the tries and release the reservation + { + spin_mutex::scoped_lock lock(my_mutex); + --my_tries; + if (reserved) my_predecessors.try_release(); + if ( check_conditions() ) { + if ( is_graph_active(this->my_graph) ) { + d1::small_object_allocator allocator{}; + typedef forward_task_bypass> task_type; + graph_task* t = allocator.new_object(my_graph, allocator, *this); + __TBB_ASSERT(!rval, "Have two tasks to handle"); + return t; + } + } + return rval; + } + } + + void initialize() { + fgt_node( + CODEPTR(), FLOW_LIMITER_NODE, &this->my_graph, + static_cast *>(this), static_cast *>(&decrement), + static_cast *>(this) + ); + } + +public: + //! Constructor + limiter_node(graph &g, size_t threshold) + : graph_node(g), my_threshold(threshold), my_count(0), my_tries(0), my_future_decrement(0), + my_predecessors(this), my_successors(this), decrement(this) + { + initialize(); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + template + limiter_node(const node_set& nodes, size_t threshold) + : limiter_node(nodes.graph_reference(), threshold) { + make_edges_in_order(nodes, *this); + } +#endif + + //! Copy constructor + limiter_node( const limiter_node& src ) : limiter_node(src.my_graph, src.my_threshold) {} + + //! The interface for accessing internal receiver< DecrementType > that adjusts the count + receiver& decrementer() { return decrement; } + + //! Replace the current successor with this new successor + bool register_successor( successor_type &r ) override { + spin_mutex::scoped_lock lock(my_mutex); + bool was_empty = my_successors.empty(); + my_successors.register_successor(r); + //spawn a forward task if this is the only successor + if ( was_empty && !my_predecessors.empty() && my_count + my_tries < my_threshold ) { + if ( is_graph_active(this->my_graph) ) { + d1::small_object_allocator allocator{}; + typedef forward_task_bypass> task_type; + graph_task* t = allocator.new_object(my_graph, allocator, *this); + spawn_in_graph_arena(graph_reference(), *t); + } + } + return true; + } + + //! Removes a successor from this node + /** r.remove_predecessor(*this) is also called. */ + bool remove_successor( successor_type &r ) override { + // TODO revamp: investigate why qualification is needed for remove_predecessor() call + tbb::detail::d2::remove_predecessor(r, *this); + my_successors.remove_successor(r); + return true; + } + + //! Adds src to the list of cached predecessors. + bool register_predecessor( predecessor_type &src ) override { + spin_mutex::scoped_lock lock(my_mutex); + my_predecessors.add( src ); + if ( my_count + my_tries < my_threshold && !my_successors.empty() && is_graph_active(this->my_graph) ) { + d1::small_object_allocator allocator{}; + typedef forward_task_bypass> task_type; + graph_task* t = allocator.new_object(my_graph, allocator, *this); + spawn_in_graph_arena(graph_reference(), *t); + } + return true; + } + + //! Removes src from the list of cached predecessors. + bool remove_predecessor( predecessor_type &src ) override { + my_predecessors.remove( src ); + return true; + } + +protected: + + template< typename R, typename B > friend class run_and_put_task; + template friend class broadcast_cache; + template friend class round_robin_cache; + +private: + //! Puts an item to this receiver + graph_task* try_put_task_impl( const T &t __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& metainfo) ) { + { + spin_mutex::scoped_lock lock(my_mutex); + if ( my_count + my_tries >= my_threshold ) + return nullptr; + else + ++my_tries; + } + + graph_task* rtask = my_successors.try_put_task(t __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo)); + if ( !rtask ) { // try_put_task failed. + spin_mutex::scoped_lock lock(my_mutex); + --my_tries; + if (check_conditions() && is_graph_active(this->my_graph)) { + d1::small_object_allocator allocator{}; + typedef forward_task_bypass> task_type; + rtask = allocator.new_object(my_graph, allocator, *this); + } + } + else { + spin_mutex::scoped_lock lock(my_mutex); + ++my_count; + if ( my_future_decrement ) { + if ( my_count > my_future_decrement ) { + my_count -= my_future_decrement; + my_future_decrement = 0; + } + else { + my_future_decrement -= my_count; + my_count = 0; + } + } + --my_tries; + } + return rtask; + } + +protected: + graph_task* try_put_task(const T& t) override { + return try_put_task_impl(t __TBB_FLOW_GRAPH_METAINFO_ARG(message_metainfo{})); + } +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + graph_task* try_put_task(const T& t, const message_metainfo& metainfo) override { + return try_put_task_impl(t, metainfo); + } +#endif + + graph& graph_reference() const override { return my_graph; } + + void reset_node( reset_flags f ) override { + my_count = 0; + if ( f & rf_clear_edges ) { + my_predecessors.clear(); + my_successors.clear(); + } + else { + my_predecessors.reset(); + } + decrement.reset_receiver(f); + } +}; // limiter_node + +#include "detail/_flow_graph_join_impl.h" + +template class join_node; + +template +class join_node: public unfolded_join_node::value, reserving_port, OutputTuple, reserving> { +private: + static const int N = std::tuple_size::value; + typedef unfolded_join_node unfolded_type; +public: + typedef OutputTuple output_type; + typedef typename unfolded_type::input_ports_type input_ports_type; + __TBB_NOINLINE_SYM explicit join_node(graph &g) : unfolded_type(g) { + fgt_multiinput_node( CODEPTR(), FLOW_JOIN_NODE_RESERVING, &this->my_graph, + this->input_ports(), static_cast< sender< output_type > *>(this) ); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + template + __TBB_NOINLINE_SYM join_node(const node_set& nodes, reserving = reserving()) : join_node(nodes.graph_reference()) { + make_edges_in_order(nodes, *this); + } +#endif + + __TBB_NOINLINE_SYM join_node(const join_node &other) : unfolded_type(other) { + fgt_multiinput_node( CODEPTR(), FLOW_JOIN_NODE_RESERVING, &this->my_graph, + this->input_ports(), static_cast< sender< output_type > *>(this) ); + } + +}; + +template +class join_node: public unfolded_join_node::value, queueing_port, OutputTuple, queueing> { +private: + static const int N = std::tuple_size::value; + typedef unfolded_join_node unfolded_type; +public: + typedef OutputTuple output_type; + typedef typename unfolded_type::input_ports_type input_ports_type; + __TBB_NOINLINE_SYM explicit join_node(graph &g) : unfolded_type(g) { + fgt_multiinput_node( CODEPTR(), FLOW_JOIN_NODE_QUEUEING, &this->my_graph, + this->input_ports(), static_cast< sender< output_type > *>(this) ); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + template + __TBB_NOINLINE_SYM join_node(const node_set& nodes, queueing = queueing()) : join_node(nodes.graph_reference()) { + make_edges_in_order(nodes, *this); + } +#endif + + __TBB_NOINLINE_SYM join_node(const join_node &other) : unfolded_type(other) { + fgt_multiinput_node( CODEPTR(), FLOW_JOIN_NODE_QUEUEING, &this->my_graph, + this->input_ports(), static_cast< sender< output_type > *>(this) ); + } + +}; + +#if __TBB_CPP20_CONCEPTS_PRESENT +// Helper function which is well-formed only if all of the elements in OutputTuple +// satisfies join_node_function_object +template +void join_node_function_objects_helper( std::index_sequence ) + requires (std::tuple_size_v == sizeof...(Functions)) && + (... && join_node_function_object, K>); + +template +concept join_node_functions = requires { + join_node_function_objects_helper(std::make_index_sequence{}); +}; + +#endif + +// template for key_matching join_node +// tag_matching join_node is a specialization of key_matching, and is source-compatible. +template +class join_node > : public unfolded_join_node::value, + key_matching_port, OutputTuple, key_matching > { +private: + static const int N = std::tuple_size::value; + typedef unfolded_join_node > unfolded_type; +public: + typedef OutputTuple output_type; + typedef typename unfolded_type::input_ports_type input_ports_type; + +#if __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING + join_node(graph &g) : unfolded_type(g) {} +#endif /* __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING */ + + template + __TBB_requires(join_node_functions) + __TBB_NOINLINE_SYM join_node(graph &g, __TBB_B0 b0, __TBB_B1 b1) : unfolded_type(g, b0, b1) { + fgt_multiinput_node( CODEPTR(), FLOW_JOIN_NODE_TAG_MATCHING, &this->my_graph, + this->input_ports(), static_cast< sender< output_type > *>(this) ); + } + template + __TBB_requires(join_node_functions) + __TBB_NOINLINE_SYM join_node(graph &g, __TBB_B0 b0, __TBB_B1 b1, __TBB_B2 b2) : unfolded_type(g, b0, b1, b2) { + fgt_multiinput_node( CODEPTR(), FLOW_JOIN_NODE_TAG_MATCHING, &this->my_graph, + this->input_ports(), static_cast< sender< output_type > *>(this) ); + } + template + __TBB_requires(join_node_functions) + __TBB_NOINLINE_SYM join_node(graph &g, __TBB_B0 b0, __TBB_B1 b1, __TBB_B2 b2, __TBB_B3 b3) : unfolded_type(g, b0, b1, b2, b3) { + fgt_multiinput_node( CODEPTR(), FLOW_JOIN_NODE_TAG_MATCHING, &this->my_graph, + this->input_ports(), static_cast< sender< output_type > *>(this) ); + } + template + __TBB_requires(join_node_functions) + __TBB_NOINLINE_SYM join_node(graph &g, __TBB_B0 b0, __TBB_B1 b1, __TBB_B2 b2, __TBB_B3 b3, __TBB_B4 b4) : + unfolded_type(g, b0, b1, b2, b3, b4) { + fgt_multiinput_node( CODEPTR(), FLOW_JOIN_NODE_TAG_MATCHING, &this->my_graph, + this->input_ports(), static_cast< sender< output_type > *>(this) ); + } +#if __TBB_VARIADIC_MAX >= 6 + template + __TBB_requires(join_node_functions) + __TBB_NOINLINE_SYM join_node(graph &g, __TBB_B0 b0, __TBB_B1 b1, __TBB_B2 b2, __TBB_B3 b3, __TBB_B4 b4, __TBB_B5 b5) : + unfolded_type(g, b0, b1, b2, b3, b4, b5) { + fgt_multiinput_node( CODEPTR(), FLOW_JOIN_NODE_TAG_MATCHING, &this->my_graph, + this->input_ports(), static_cast< sender< output_type > *>(this) ); + } +#endif +#if __TBB_VARIADIC_MAX >= 7 + template + __TBB_requires(join_node_functions) + __TBB_NOINLINE_SYM join_node(graph &g, __TBB_B0 b0, __TBB_B1 b1, __TBB_B2 b2, __TBB_B3 b3, __TBB_B4 b4, __TBB_B5 b5, __TBB_B6 b6) : + unfolded_type(g, b0, b1, b2, b3, b4, b5, b6) { + fgt_multiinput_node( CODEPTR(), FLOW_JOIN_NODE_TAG_MATCHING, &this->my_graph, + this->input_ports(), static_cast< sender< output_type > *>(this) ); + } +#endif +#if __TBB_VARIADIC_MAX >= 8 + template + __TBB_requires(join_node_functions) + __TBB_NOINLINE_SYM join_node(graph &g, __TBB_B0 b0, __TBB_B1 b1, __TBB_B2 b2, __TBB_B3 b3, __TBB_B4 b4, __TBB_B5 b5, __TBB_B6 b6, + __TBB_B7 b7) : unfolded_type(g, b0, b1, b2, b3, b4, b5, b6, b7) { + fgt_multiinput_node( CODEPTR(), FLOW_JOIN_NODE_TAG_MATCHING, &this->my_graph, + this->input_ports(), static_cast< sender< output_type > *>(this) ); + } +#endif +#if __TBB_VARIADIC_MAX >= 9 + template + __TBB_requires(join_node_functions) + __TBB_NOINLINE_SYM join_node(graph &g, __TBB_B0 b0, __TBB_B1 b1, __TBB_B2 b2, __TBB_B3 b3, __TBB_B4 b4, __TBB_B5 b5, __TBB_B6 b6, + __TBB_B7 b7, __TBB_B8 b8) : unfolded_type(g, b0, b1, b2, b3, b4, b5, b6, b7, b8) { + fgt_multiinput_node( CODEPTR(), FLOW_JOIN_NODE_TAG_MATCHING, &this->my_graph, + this->input_ports(), static_cast< sender< output_type > *>(this) ); + } +#endif +#if __TBB_VARIADIC_MAX >= 10 + template + __TBB_requires(join_node_functions) + __TBB_NOINLINE_SYM join_node(graph &g, __TBB_B0 b0, __TBB_B1 b1, __TBB_B2 b2, __TBB_B3 b3, __TBB_B4 b4, __TBB_B5 b5, __TBB_B6 b6, + __TBB_B7 b7, __TBB_B8 b8, __TBB_B9 b9) : unfolded_type(g, b0, b1, b2, b3, b4, b5, b6, b7, b8, b9) { + fgt_multiinput_node( CODEPTR(), FLOW_JOIN_NODE_TAG_MATCHING, &this->my_graph, + this->input_ports(), static_cast< sender< output_type > *>(this) ); + } +#endif + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + template < +#if (__clang_major__ == 3 && __clang_minor__ == 4) + // clang 3.4 misdeduces 'Args...' for 'node_set' while it can cope with template template parameter. + template class node_set, +#endif + typename... Args, typename... Bodies + > + __TBB_requires((sizeof...(Bodies) == 0) || join_node_functions) + __TBB_NOINLINE_SYM join_node(const node_set& nodes, Bodies... bodies) + : join_node(nodes.graph_reference(), bodies...) { + make_edges_in_order(nodes, *this); + } +#endif // __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + + __TBB_NOINLINE_SYM join_node(const join_node &other) : unfolded_type(other) { + fgt_multiinput_node( CODEPTR(), FLOW_JOIN_NODE_TAG_MATCHING, &this->my_graph, + this->input_ports(), static_cast< sender< output_type > *>(this) ); + } + +}; + +// indexer node +#include "detail/_flow_graph_indexer_impl.h" + +// TODO: Implement interface with variadic template or tuple +template class indexer_node; + +//indexer node specializations +template +class indexer_node : public unfolded_indexer_node > { +private: + static const int N = 1; +public: + typedef std::tuple InputTuple; + typedef tagged_msg output_type; + typedef unfolded_indexer_node unfolded_type; + __TBB_NOINLINE_SYM indexer_node(graph& g) : unfolded_type(g) { + fgt_multiinput_node( CODEPTR(), FLOW_INDEXER_NODE, &this->my_graph, + this->input_ports(), static_cast< sender< output_type > *>(this) ); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + template + indexer_node(const node_set& nodes) : indexer_node(nodes.graph_reference()) { + make_edges_in_order(nodes, *this); + } +#endif + + // Copy constructor + __TBB_NOINLINE_SYM indexer_node( const indexer_node& other ) : unfolded_type(other) { + fgt_multiinput_node( CODEPTR(), FLOW_INDEXER_NODE, &this->my_graph, + this->input_ports(), static_cast< sender< output_type > *>(this) ); + } +}; + +template +class indexer_node : public unfolded_indexer_node > { +private: + static const int N = 2; +public: + typedef std::tuple InputTuple; + typedef tagged_msg output_type; + typedef unfolded_indexer_node unfolded_type; + __TBB_NOINLINE_SYM indexer_node(graph& g) : unfolded_type(g) { + fgt_multiinput_node( CODEPTR(), FLOW_INDEXER_NODE, &this->my_graph, + this->input_ports(), static_cast< sender< output_type > *>(this) ); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + template + indexer_node(const node_set& nodes) : indexer_node(nodes.graph_reference()) { + make_edges_in_order(nodes, *this); + } +#endif + + // Copy constructor + __TBB_NOINLINE_SYM indexer_node( const indexer_node& other ) : unfolded_type(other) { + fgt_multiinput_node( CODEPTR(), FLOW_INDEXER_NODE, &this->my_graph, + this->input_ports(), static_cast< sender< output_type > *>(this) ); + } + +}; + +template +class indexer_node : public unfolded_indexer_node > { +private: + static const int N = 3; +public: + typedef std::tuple InputTuple; + typedef tagged_msg output_type; + typedef unfolded_indexer_node unfolded_type; + __TBB_NOINLINE_SYM indexer_node(graph& g) : unfolded_type(g) { + fgt_multiinput_node( CODEPTR(), FLOW_INDEXER_NODE, &this->my_graph, + this->input_ports(), static_cast< sender< output_type > *>(this) ); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + template + indexer_node(const node_set& nodes) : indexer_node(nodes.graph_reference()) { + make_edges_in_order(nodes, *this); + } +#endif + + // Copy constructor + __TBB_NOINLINE_SYM indexer_node( const indexer_node& other ) : unfolded_type(other) { + fgt_multiinput_node( CODEPTR(), FLOW_INDEXER_NODE, &this->my_graph, + this->input_ports(), static_cast< sender< output_type > *>(this) ); + } + +}; + +template +class indexer_node : public unfolded_indexer_node > { +private: + static const int N = 4; +public: + typedef std::tuple InputTuple; + typedef tagged_msg output_type; + typedef unfolded_indexer_node unfolded_type; + __TBB_NOINLINE_SYM indexer_node(graph& g) : unfolded_type(g) { + fgt_multiinput_node( CODEPTR(), FLOW_INDEXER_NODE, &this->my_graph, + this->input_ports(), static_cast< sender< output_type > *>(this) ); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + template + indexer_node(const node_set& nodes) : indexer_node(nodes.graph_reference()) { + make_edges_in_order(nodes, *this); + } +#endif + + // Copy constructor + __TBB_NOINLINE_SYM indexer_node( const indexer_node& other ) : unfolded_type(other) { + fgt_multiinput_node( CODEPTR(), FLOW_INDEXER_NODE, &this->my_graph, + this->input_ports(), static_cast< sender< output_type > *>(this) ); + } + +}; + +template +class indexer_node : public unfolded_indexer_node > { +private: + static const int N = 5; +public: + typedef std::tuple InputTuple; + typedef tagged_msg output_type; + typedef unfolded_indexer_node unfolded_type; + __TBB_NOINLINE_SYM indexer_node(graph& g) : unfolded_type(g) { + fgt_multiinput_node( CODEPTR(), FLOW_INDEXER_NODE, &this->my_graph, + this->input_ports(), static_cast< sender< output_type > *>(this) ); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + template + indexer_node(const node_set& nodes) : indexer_node(nodes.graph_reference()) { + make_edges_in_order(nodes, *this); + } +#endif + + // Copy constructor + __TBB_NOINLINE_SYM indexer_node( const indexer_node& other ) : unfolded_type(other) { + fgt_multiinput_node( CODEPTR(), FLOW_INDEXER_NODE, &this->my_graph, + this->input_ports(), static_cast< sender< output_type > *>(this) ); + } + +}; + +#if __TBB_VARIADIC_MAX >= 6 +template +class indexer_node : public unfolded_indexer_node > { +private: + static const int N = 6; +public: + typedef std::tuple InputTuple; + typedef tagged_msg output_type; + typedef unfolded_indexer_node unfolded_type; + __TBB_NOINLINE_SYM indexer_node(graph& g) : unfolded_type(g) { + fgt_multiinput_node( CODEPTR(), FLOW_INDEXER_NODE, &this->my_graph, + this->input_ports(), static_cast< sender< output_type > *>(this) ); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + template + indexer_node(const node_set& nodes) : indexer_node(nodes.graph_reference()) { + make_edges_in_order(nodes, *this); + } +#endif + + // Copy constructor + __TBB_NOINLINE_SYM indexer_node( const indexer_node& other ) : unfolded_type(other) { + fgt_multiinput_node( CODEPTR(), FLOW_INDEXER_NODE, &this->my_graph, + this->input_ports(), static_cast< sender< output_type > *>(this) ); + } + +}; +#endif //variadic max 6 + +#if __TBB_VARIADIC_MAX >= 7 +template +class indexer_node : public unfolded_indexer_node > { +private: + static const int N = 7; +public: + typedef std::tuple InputTuple; + typedef tagged_msg output_type; + typedef unfolded_indexer_node unfolded_type; + __TBB_NOINLINE_SYM indexer_node(graph& g) : unfolded_type(g) { + fgt_multiinput_node( CODEPTR(), FLOW_INDEXER_NODE, &this->my_graph, + this->input_ports(), static_cast< sender< output_type > *>(this) ); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + template + indexer_node(const node_set& nodes) : indexer_node(nodes.graph_reference()) { + make_edges_in_order(nodes, *this); + } +#endif + + // Copy constructor + __TBB_NOINLINE_SYM indexer_node( const indexer_node& other ) : unfolded_type(other) { + fgt_multiinput_node( CODEPTR(), FLOW_INDEXER_NODE, &this->my_graph, + this->input_ports(), static_cast< sender< output_type > *>(this) ); + } + +}; +#endif //variadic max 7 + +#if __TBB_VARIADIC_MAX >= 8 +template +class indexer_node : public unfolded_indexer_node > { +private: + static const int N = 8; +public: + typedef std::tuple InputTuple; + typedef tagged_msg output_type; + typedef unfolded_indexer_node unfolded_type; + indexer_node(graph& g) : unfolded_type(g) { + fgt_multiinput_node( CODEPTR(), FLOW_INDEXER_NODE, &this->my_graph, + this->input_ports(), static_cast< sender< output_type > *>(this) ); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + template + indexer_node(const node_set& nodes) : indexer_node(nodes.graph_reference()) { + make_edges_in_order(nodes, *this); + } +#endif + + // Copy constructor + indexer_node( const indexer_node& other ) : unfolded_type(other) { + fgt_multiinput_node( CODEPTR(), FLOW_INDEXER_NODE, &this->my_graph, + this->input_ports(), static_cast< sender< output_type > *>(this) ); + } + +}; +#endif //variadic max 8 + +#if __TBB_VARIADIC_MAX >= 9 +template +class indexer_node : public unfolded_indexer_node > { +private: + static const int N = 9; +public: + typedef std::tuple InputTuple; + typedef tagged_msg output_type; + typedef unfolded_indexer_node unfolded_type; + __TBB_NOINLINE_SYM indexer_node(graph& g) : unfolded_type(g) { + fgt_multiinput_node( CODEPTR(), FLOW_INDEXER_NODE, &this->my_graph, + this->input_ports(), static_cast< sender< output_type > *>(this) ); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + template + indexer_node(const node_set& nodes) : indexer_node(nodes.graph_reference()) { + make_edges_in_order(nodes, *this); + } +#endif + + // Copy constructor + __TBB_NOINLINE_SYM indexer_node( const indexer_node& other ) : unfolded_type(other) { + fgt_multiinput_node( CODEPTR(), FLOW_INDEXER_NODE, &this->my_graph, + this->input_ports(), static_cast< sender< output_type > *>(this) ); + } + +}; +#endif //variadic max 9 + +#if __TBB_VARIADIC_MAX >= 10 +template +class indexer_node/*default*/ : public unfolded_indexer_node > { +private: + static const int N = 10; +public: + typedef std::tuple InputTuple; + typedef tagged_msg output_type; + typedef unfolded_indexer_node unfolded_type; + __TBB_NOINLINE_SYM indexer_node(graph& g) : unfolded_type(g) { + fgt_multiinput_node( CODEPTR(), FLOW_INDEXER_NODE, &this->my_graph, + this->input_ports(), static_cast< sender< output_type > *>(this) ); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + template + indexer_node(const node_set& nodes) : indexer_node(nodes.graph_reference()) { + make_edges_in_order(nodes, *this); + } +#endif + + // Copy constructor + __TBB_NOINLINE_SYM indexer_node( const indexer_node& other ) : unfolded_type(other) { + fgt_multiinput_node( CODEPTR(), FLOW_INDEXER_NODE, &this->my_graph, + this->input_ports(), static_cast< sender< output_type > *>(this) ); + } + +}; +#endif //variadic max 10 + +template< typename T > +inline void internal_make_edge( sender &p, receiver &s ) { + register_successor(p, s); + fgt_make_edge( &p, &s ); +} + +//! Makes an edge between a single predecessor and a single successor +template< typename T > +inline void make_edge( sender &p, receiver &s ) { + internal_make_edge( p, s ); +} + +//Makes an edge from port 0 of a multi-output predecessor to port 0 of a multi-input successor. +template< typename T, typename V, + typename = typename T::output_ports_type, typename = typename V::input_ports_type > +inline void make_edge( T& output, V& input) { + make_edge(std::get<0>(output.output_ports()), std::get<0>(input.input_ports())); +} + +//Makes an edge from port 0 of a multi-output predecessor to a receiver. +template< typename T, typename R, + typename = typename T::output_ports_type > +inline void make_edge( T& output, receiver& input) { + make_edge(std::get<0>(output.output_ports()), input); +} + +//Makes an edge from a sender to port 0 of a multi-input successor. +template< typename S, typename V, + typename = typename V::input_ports_type > +inline void make_edge( sender& output, V& input) { + make_edge(output, std::get<0>(input.input_ports())); +} + +template< typename T > +inline void internal_remove_edge( sender &p, receiver &s ) { + remove_successor( p, s ); + fgt_remove_edge( &p, &s ); +} + +//! Removes an edge between a single predecessor and a single successor +template< typename T > +inline void remove_edge( sender &p, receiver &s ) { + internal_remove_edge( p, s ); +} + +//Removes an edge between port 0 of a multi-output predecessor and port 0 of a multi-input successor. +template< typename T, typename V, + typename = typename T::output_ports_type, typename = typename V::input_ports_type > +inline void remove_edge( T& output, V& input) { + remove_edge(std::get<0>(output.output_ports()), std::get<0>(input.input_ports())); +} + +//Removes an edge between port 0 of a multi-output predecessor and a receiver. +template< typename T, typename R, + typename = typename T::output_ports_type > +inline void remove_edge( T& output, receiver& input) { + remove_edge(std::get<0>(output.output_ports()), input); +} +//Removes an edge between a sender and port 0 of a multi-input successor. +template< typename S, typename V, + typename = typename V::input_ports_type > +inline void remove_edge( sender& output, V& input) { + remove_edge(output, std::get<0>(input.input_ports())); +} + +//! Returns a copy of the body from a function or continue node +template< typename Body, typename Node > +Body copy_body( Node &n ) { + return n.template copy_function_object(); +} + +//composite_node +template< typename InputTuple, typename OutputTuple > class composite_node; + +template< typename... InputTypes, typename... OutputTypes> +class composite_node , std::tuple > : public graph_node { + +public: + typedef std::tuple< receiver&... > input_ports_type; + typedef std::tuple< sender&... > output_ports_type; + +private: + std::unique_ptr my_input_ports; + std::unique_ptr my_output_ports; + + static const size_t NUM_INPUTS = sizeof...(InputTypes); + static const size_t NUM_OUTPUTS = sizeof...(OutputTypes); + +protected: + void reset_node(reset_flags) override {} + +public: + composite_node( graph &g ) : graph_node(g) { + fgt_multiinput_multioutput_node( CODEPTR(), FLOW_COMPOSITE_NODE, this, &this->my_graph ); + } + + template + void set_external_ports(T1&& input_ports_tuple, T2&& output_ports_tuple) { + static_assert(NUM_INPUTS == std::tuple_size::value, "number of arguments does not match number of input ports"); + static_assert(NUM_OUTPUTS == std::tuple_size::value, "number of arguments does not match number of output ports"); + + fgt_internal_input_alias_helper::alias_port( this, input_ports_tuple); + fgt_internal_output_alias_helper::alias_port( this, output_ports_tuple); + + my_input_ports.reset( new input_ports_type(std::forward(input_ports_tuple)) ); + my_output_ports.reset( new output_ports_type(std::forward(output_ports_tuple)) ); + } + + template< typename... NodeTypes > + void add_visible_nodes(const NodeTypes&... n) { add_nodes_impl(this, true, n...); } + + template< typename... NodeTypes > + void add_nodes(const NodeTypes&... n) { add_nodes_impl(this, false, n...); } + + + input_ports_type& input_ports() { + __TBB_ASSERT(my_input_ports, "input ports not set, call set_external_ports to set input ports"); + return *my_input_ports; + } + + output_ports_type& output_ports() { + __TBB_ASSERT(my_output_ports, "output ports not set, call set_external_ports to set output ports"); + return *my_output_ports; + } +}; // class composite_node + +//composite_node with only input ports +template< typename... InputTypes> +class composite_node , std::tuple<> > : public graph_node { +public: + typedef std::tuple< receiver&... > input_ports_type; + +private: + std::unique_ptr my_input_ports; + static const size_t NUM_INPUTS = sizeof...(InputTypes); + +protected: + void reset_node(reset_flags) override {} + +public: + composite_node( graph &g ) : graph_node(g) { + fgt_composite( CODEPTR(), this, &g ); + } + + template + void set_external_ports(T&& input_ports_tuple) { + static_assert(NUM_INPUTS == std::tuple_size::value, "number of arguments does not match number of input ports"); + + fgt_internal_input_alias_helper::alias_port( this, input_ports_tuple); + + my_input_ports.reset( new input_ports_type(std::forward(input_ports_tuple)) ); + } + + template< typename... NodeTypes > + void add_visible_nodes(const NodeTypes&... n) { add_nodes_impl(this, true, n...); } + + template< typename... NodeTypes > + void add_nodes( const NodeTypes&... n) { add_nodes_impl(this, false, n...); } + + + input_ports_type& input_ports() { + __TBB_ASSERT(my_input_ports, "input ports not set, call set_external_ports to set input ports"); + return *my_input_ports; + } + +}; // class composite_node + +//composite_nodes with only output_ports +template +class composite_node , std::tuple > : public graph_node { +public: + typedef std::tuple< sender&... > output_ports_type; + +private: + std::unique_ptr my_output_ports; + static const size_t NUM_OUTPUTS = sizeof...(OutputTypes); + +protected: + void reset_node(reset_flags) override {} + +public: + __TBB_NOINLINE_SYM composite_node( graph &g ) : graph_node(g) { + fgt_composite( CODEPTR(), this, &g ); + } + + template + void set_external_ports(T&& output_ports_tuple) { + static_assert(NUM_OUTPUTS == std::tuple_size::value, "number of arguments does not match number of output ports"); + + fgt_internal_output_alias_helper::alias_port( this, output_ports_tuple); + + my_output_ports.reset( new output_ports_type(std::forward(output_ports_tuple)) ); + } + + template + void add_visible_nodes(const NodeTypes&... n) { add_nodes_impl(this, true, n...); } + + template + void add_nodes(const NodeTypes&... n) { add_nodes_impl(this, false, n...); } + + + output_ports_type& output_ports() { + __TBB_ASSERT(my_output_ports, "output ports not set, call set_external_ports to set output ports"); + return *my_output_ports; + } + +}; // class composite_node + +template +class async_body_base: no_assign { +public: + typedef Gateway gateway_type; + + async_body_base(gateway_type *gateway): my_gateway(gateway) { } + void set_gateway(gateway_type *gateway) { + my_gateway = gateway; + } + +protected: + gateway_type *my_gateway; +}; + +template +class async_body: public async_body_base { +private: + Body my_body; + +public: + typedef async_body_base base_type; + typedef Gateway gateway_type; + + async_body(const Body &body, gateway_type *gateway) + : base_type(gateway), my_body(body) { } + + void operator()( const Input &v, Ports & ) noexcept(noexcept(tbb::detail::invoke(my_body, v, std::declval()))) { + tbb::detail::invoke(my_body, v, *this->my_gateway); + } + + Body get_body() { return my_body; } +}; + +//! Implements async node +template < typename Input, typename Output, + typename Policy = queueing_lightweight > + __TBB_requires(std::default_initializable && std::copy_constructible) +class async_node + : public multifunction_node< Input, std::tuple< Output >, Policy >, public sender< Output > +{ + typedef multifunction_node< Input, std::tuple< Output >, Policy > base_type; + typedef multifunction_input< + Input, typename base_type::output_ports_type, Policy, cache_aligned_allocator> mfn_input_type; + +public: + typedef Input input_type; + typedef Output output_type; + typedef receiver receiver_type; + typedef receiver successor_type; + typedef sender predecessor_type; + typedef receiver_gateway gateway_type; + typedef async_body_base async_body_base_type; + typedef typename base_type::output_ports_type output_ports_type; + +private: + class receiver_gateway_impl: public receiver_gateway { + public: + receiver_gateway_impl(async_node* node): my_node(node) {} + void reserve_wait() override { + fgt_async_reserve(static_cast(my_node), &my_node->my_graph); + my_node->my_graph.reserve_wait(); + } + + void release_wait() override { + async_node* n = my_node; + graph* g = &n->my_graph; + g->release_wait(); + fgt_async_commit(static_cast(n), g); + } + + //! Implements gateway_type::try_put for an external activity to submit a message to FG + bool try_put(const Output &i) override { + return my_node->try_put_impl(i); + } + + private: + async_node* my_node; + } my_gateway; + + //The substitute of 'this' for member construction, to prevent compiler warnings + async_node* self() { return this; } + + //! Implements gateway_type::try_put for an external activity to submit a message to FG + bool try_put_impl(const Output &i) { + multifunction_output &port_0 = output_port<0>(*this); + broadcast_cache& port_successors = port_0.successors(); + fgt_async_try_put_begin(this, &port_0); + // TODO revamp: change to std::list + graph_task_list tasks; + bool is_at_least_one_put_successful = port_successors.gather_successful_try_puts(i, tasks); + __TBB_ASSERT( is_at_least_one_put_successful || tasks.empty(), + "Return status is inconsistent with the method operation." ); + + while( !tasks.empty() ) { + enqueue_in_graph_arena(this->my_graph, tasks.pop_front()); + } + fgt_async_try_put_end(this, &port_0); + return is_at_least_one_put_successful; + } + +public: + template + __TBB_requires(async_node_body) + __TBB_NOINLINE_SYM async_node( + graph &g, size_t concurrency, + Body body, Policy = Policy(), node_priority_t a_priority = no_priority + ) : base_type( + g, concurrency, + async_body + (body, &my_gateway), a_priority ), my_gateway(self()) { + fgt_multioutput_node_with_body<1>( + CODEPTR(), FLOW_ASYNC_NODE, + &this->my_graph, static_cast *>(this), + this->output_ports(), this->my_body + ); + } + + template + __TBB_requires(async_node_body) + __TBB_NOINLINE_SYM async_node(graph& g, size_t concurrency, Body body, node_priority_t a_priority) + : async_node(g, concurrency, body, Policy(), a_priority) {} + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + template + __TBB_requires(async_node_body) + __TBB_NOINLINE_SYM async_node( + const node_set& nodes, size_t concurrency, Body body, + Policy = Policy(), node_priority_t a_priority = no_priority ) + : async_node(nodes.graph_reference(), concurrency, body, a_priority) { + make_edges_in_order(nodes, *this); + } + + template + __TBB_requires(async_node_body) + __TBB_NOINLINE_SYM async_node(const node_set& nodes, size_t concurrency, Body body, node_priority_t a_priority) + : async_node(nodes, concurrency, body, Policy(), a_priority) {} +#endif // __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + + __TBB_NOINLINE_SYM async_node( const async_node &other ) : base_type(other), sender(), my_gateway(self()) { + static_cast(this->my_body->get_body_ptr())->set_gateway(&my_gateway); + static_cast(this->my_init_body->get_body_ptr())->set_gateway(&my_gateway); + + fgt_multioutput_node_with_body<1>( CODEPTR(), FLOW_ASYNC_NODE, + &this->my_graph, static_cast *>(this), + this->output_ports(), this->my_body ); + } + + gateway_type& gateway() { + return my_gateway; + } + + // Define sender< Output > + + //! Add a new successor to this node + bool register_successor(successor_type&) override { + __TBB_ASSERT(false, "Successors must be registered only via ports"); + return false; + } + + //! Removes a successor from this node + bool remove_successor(successor_type&) override { + __TBB_ASSERT(false, "Successors must be removed only via ports"); + return false; + } + + template + Body copy_function_object() { + typedef multifunction_body mfn_body_type; + typedef async_body async_body_type; + mfn_body_type &body_ref = *this->my_body; + async_body_type ab = *static_cast(dynamic_cast< multifunction_body_leaf & >(body_ref).get_body_ptr()); + return ab.get_body(); + } + +protected: + + void reset_node( reset_flags f) override { + base_type::reset_node(f); + } +}; + +#include "detail/_flow_graph_node_set_impl.h" + +template< typename T > +class overwrite_node : public graph_node, public receiver, public sender { +public: + typedef T input_type; + typedef T output_type; + typedef typename receiver::predecessor_type predecessor_type; + typedef typename sender::successor_type successor_type; + + __TBB_NOINLINE_SYM explicit overwrite_node(graph &g) + : graph_node(g), my_successors(this), my_buffer_is_valid(false) + { + fgt_node( CODEPTR(), FLOW_OVERWRITE_NODE, &this->my_graph, + static_cast *>(this), static_cast *>(this) ); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + template + overwrite_node(const node_set& nodes) : overwrite_node(nodes.graph_reference()) { + make_edges_in_order(nodes, *this); + } +#endif + + //! Copy constructor; doesn't take anything from src; default won't work + __TBB_NOINLINE_SYM overwrite_node( const overwrite_node& src ) : overwrite_node(src.my_graph) {} + + ~overwrite_node() {} + + bool register_successor( successor_type &s ) override { + spin_mutex::scoped_lock l( my_mutex ); + if (my_buffer_is_valid && is_graph_active( my_graph )) { + // We have a valid value that must be forwarded immediately. + bool ret = s.try_put( my_buffer ); + if ( ret ) { + // We add the successor that accepted our put + my_successors.register_successor( s ); + } else { + // In case of reservation a race between the moment of reservation and register_successor can appear, + // because failed reserve does not mean that register_successor is not ready to put a message immediately. + // We have some sort of infinite loop: reserving node tries to set pull state for the edge, + // but overwrite_node tries to return push state back. That is why we have to break this loop with task creation. + d1::small_object_allocator allocator{}; + typedef register_predecessor_task task_type; + graph_task* t = allocator.new_object(graph_reference(), allocator, *this, s); + spawn_in_graph_arena( my_graph, *t ); + } + } else { + // No valid value yet, just add as successor + my_successors.register_successor( s ); + } + return true; + } + + bool remove_successor( successor_type &s ) override { + spin_mutex::scoped_lock l( my_mutex ); + my_successors.remove_successor(s); + return true; + } + + bool try_get( input_type &v ) override { + spin_mutex::scoped_lock l( my_mutex ); + if ( my_buffer_is_valid ) { + v = my_buffer; + return true; + } + return false; + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + bool try_get( input_type &v, message_metainfo& metainfo ) override { + spin_mutex::scoped_lock l( my_mutex ); + if (my_buffer_is_valid) { + v = my_buffer; + metainfo = my_buffered_metainfo; + + // Since the successor of the node will use move semantics while wrapping the metainfo + // that is designed to transfer the ownership of the value from single-push buffer to the task + // It is required to reserve one more reference here because the value keeps in the buffer + // and the ownership is not transferred + for (auto msg_waiter : metainfo.waiters()) { + msg_waiter->reserve(1); + } + return true; + } + return false; + } +#endif + + //! Reserves an item + bool try_reserve( T &v ) override { + return try_get(v); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT +private: + bool try_reserve(T& v, message_metainfo& metainfo) override { + spin_mutex::scoped_lock l( my_mutex ); + if (my_buffer_is_valid) { + v = my_buffer; + metainfo = my_buffered_metainfo; + return true; + } + return false; + } +public: +#endif + + //! Releases the reserved item + bool try_release() override { return true; } + + //! Consumes the reserved item + bool try_consume() override { return true; } + + bool is_valid() { + spin_mutex::scoped_lock l( my_mutex ); + return my_buffer_is_valid; + } + + void clear() { + spin_mutex::scoped_lock l( my_mutex ); + my_buffer_is_valid = false; +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + for (auto msg_waiter : my_buffered_metainfo.waiters()) { + msg_waiter->release(1); + } + my_buffered_metainfo = message_metainfo{}; +#endif + } + +protected: + + template< typename R, typename B > friend class run_and_put_task; + template friend class broadcast_cache; + template friend class round_robin_cache; + graph_task* try_put_task( const input_type &v ) override { + spin_mutex::scoped_lock l( my_mutex ); + return try_put_task_impl(v __TBB_FLOW_GRAPH_METAINFO_ARG(message_metainfo{})); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + graph_task* try_put_task(const input_type& v, const message_metainfo& metainfo) override { + spin_mutex::scoped_lock l( my_mutex ); + return try_put_task_impl(v, metainfo); + } +#endif + + graph_task * try_put_task_impl(const input_type &v __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& metainfo)) { + my_buffer = v; + my_buffer_is_valid = true; +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + // Since the new item is pushed to the buffer - reserving the waiters + for (auto msg_waiter : metainfo.waiters()) { + msg_waiter->reserve(1); + } + + // Since the item is taken out from the buffer - releasing the stored waiters + for (auto msg_waiter : my_buffered_metainfo.waiters()) { + msg_waiter->release(1); + } + + my_buffered_metainfo = metainfo; +#endif + graph_task* rtask = my_successors.try_put_task(v __TBB_FLOW_GRAPH_METAINFO_ARG(my_buffered_metainfo) ); + if (!rtask) rtask = SUCCESSFULLY_ENQUEUED; + return rtask; + } + + graph& graph_reference() const override { + return my_graph; + } + + //! Breaks an infinite loop between the node reservation and register_successor call + struct register_predecessor_task : public graph_task { + register_predecessor_task( + graph& g, d1::small_object_allocator& allocator, predecessor_type& owner, successor_type& succ) + : graph_task(g, allocator), o(owner), s(succ) {}; + + d1::task* execute(d1::execution_data& ed) override { + // TODO revamp: investigate why qualification is needed for register_successor() call + using tbb::detail::d2::register_predecessor; + using tbb::detail::d2::register_successor; + if ( !register_predecessor(s, o) ) { + register_successor(o, s); + } + finalize(ed); + return nullptr; + } + + d1::task* cancel(d1::execution_data& ed) override { + finalize(ed); + return nullptr; + } + + predecessor_type& o; + successor_type& s; + }; + + spin_mutex my_mutex; + broadcast_cache< input_type, null_rw_mutex > my_successors; + input_type my_buffer; +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + message_metainfo my_buffered_metainfo; +#endif + bool my_buffer_is_valid; + + void reset_node( reset_flags f) override { + my_buffer_is_valid = false; + if (f&rf_clear_edges) { + my_successors.clear(); + } + } +}; // overwrite_node + +template< typename T > +class write_once_node : public overwrite_node { +public: + typedef T input_type; + typedef T output_type; + typedef overwrite_node base_type; + typedef typename receiver::predecessor_type predecessor_type; + typedef typename sender::successor_type successor_type; + + //! Constructor + __TBB_NOINLINE_SYM explicit write_once_node(graph& g) : base_type(g) { + fgt_node( CODEPTR(), FLOW_WRITE_ONCE_NODE, &(this->my_graph), + static_cast *>(this), + static_cast *>(this) ); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + template + write_once_node(const node_set& nodes) : write_once_node(nodes.graph_reference()) { + make_edges_in_order(nodes, *this); + } +#endif + + //! Copy constructor: call base class copy constructor + __TBB_NOINLINE_SYM write_once_node( const write_once_node& src ) : base_type(src) { + fgt_node( CODEPTR(), FLOW_WRITE_ONCE_NODE, &(this->my_graph), + static_cast *>(this), + static_cast *>(this) ); + } + +protected: + template< typename R, typename B > friend class run_and_put_task; + template friend class broadcast_cache; + template friend class round_robin_cache; + graph_task *try_put_task( const T &v ) override { + spin_mutex::scoped_lock l( this->my_mutex ); + return this->my_buffer_is_valid ? nullptr : this->try_put_task_impl(v __TBB_FLOW_GRAPH_METAINFO_ARG(message_metainfo{})); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + graph_task* try_put_task(const T& v, const message_metainfo& metainfo) override { + spin_mutex::scoped_lock l( this->my_mutex ); + return this->my_buffer_is_valid ? nullptr : this->try_put_task_impl(v, metainfo); + } +#endif +}; // write_once_node + +inline void set_name(const graph& g, const char *name) { + fgt_graph_desc(&g, name); +} + +template +inline void set_name(const input_node& node, const char *name) { + fgt_node_desc(&node, name); +} + +template +inline void set_name(const function_node& node, const char *name) { + fgt_node_desc(&node, name); +} + +template +inline void set_name(const continue_node& node, const char *name) { + fgt_node_desc(&node, name); +} + +template +inline void set_name(const broadcast_node& node, const char *name) { + fgt_node_desc(&node, name); +} + +template +inline void set_name(const buffer_node& node, const char *name) { + fgt_node_desc(&node, name); +} + +template +inline void set_name(const queue_node& node, const char *name) { + fgt_node_desc(&node, name); +} + +template +inline void set_name(const sequencer_node& node, const char *name) { + fgt_node_desc(&node, name); +} + +template +inline void set_name(const priority_queue_node& node, const char *name) { + fgt_node_desc(&node, name); +} + +template +inline void set_name(const limiter_node& node, const char *name) { + fgt_node_desc(&node, name); +} + +template +inline void set_name(const join_node& node, const char *name) { + fgt_node_desc(&node, name); +} + +template +inline void set_name(const indexer_node& node, const char *name) { + fgt_node_desc(&node, name); +} + +template +inline void set_name(const overwrite_node& node, const char *name) { + fgt_node_desc(&node, name); +} + +template +inline void set_name(const write_once_node& node, const char *name) { + fgt_node_desc(&node, name); +} + +template +inline void set_name(const multifunction_node& node, const char *name) { + fgt_multioutput_node_desc(&node, name); +} + +template +inline void set_name(const split_node& node, const char *name) { + fgt_multioutput_node_desc(&node, name); +} + +template< typename InputTuple, typename OutputTuple > +inline void set_name(const composite_node& node, const char *name) { + fgt_multiinput_multioutput_node_desc(&node, name); +} + +template +inline void set_name(const async_node& node, const char *name) +{ + fgt_multioutput_node_desc(&node, name); +} +} // d2 +} // detail +} // tbb + + +// Include deduction guides for node classes +#include "detail/_flow_graph_nodes_deduction.h" + +namespace tbb { +namespace flow { +inline namespace v1 { + using detail::d2::receiver; + using detail::d2::sender; + + using detail::d2::serial; + using detail::d2::unlimited; + + using detail::d2::reset_flags; + using detail::d2::rf_reset_protocol; + using detail::d2::rf_reset_bodies; + using detail::d2::rf_clear_edges; + + using detail::d2::graph; + using detail::d2::graph_node; + using detail::d2::continue_msg; + + using detail::d2::input_node; + using detail::d2::function_node; + using detail::d2::multifunction_node; + using detail::d2::split_node; + using detail::d2::output_port; + using detail::d2::indexer_node; + using detail::d2::tagged_msg; + using detail::d2::cast_to; + using detail::d2::is_a; + using detail::d2::continue_node; + using detail::d2::overwrite_node; + using detail::d2::write_once_node; + using detail::d2::broadcast_node; + using detail::d2::buffer_node; + using detail::d2::queue_node; + using detail::d2::sequencer_node; + using detail::d2::priority_queue_node; + using detail::d2::limiter_node; + using namespace detail::d2::graph_policy_namespace; + using detail::d2::join_node; + using detail::d2::input_port; + using detail::d2::copy_body; + using detail::d2::make_edge; + using detail::d2::remove_edge; + using detail::d2::tag_value; + using detail::d2::composite_node; + using detail::d2::async_node; + using detail::d2::node_priority_t; + using detail::d2::no_priority; + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + using detail::d2::follows; + using detail::d2::precedes; + using detail::d2::make_node_set; + using detail::d2::make_edges; +#endif + +} // v1 +} // flow + + using detail::d1::flow_control; + +namespace profiling { + using detail::d2::set_name; +} // profiling + +} // tbb + + +#if TBB_USE_PROFILING_TOOLS && ( __unix__ || __APPLE__ ) + // We don't do pragma pop here, since it still gives warning on the USER side + #undef __TBB_NOINLINE_SYM +#endif + +#endif // __TBB_flow_graph_H diff --git a/inst/include/tbb/flow_graph_abstractions.h b/src/tbb/include/oneapi/tbb/flow_graph_abstractions.h similarity index 87% rename from inst/include/tbb/flow_graph_abstractions.h rename to src/tbb/include/oneapi/tbb/flow_graph_abstractions.h index f6eb3fb3a..329e75c43 100644 --- a/inst/include/tbb/flow_graph_abstractions.h +++ b/src/tbb/include/oneapi/tbb/flow_graph_abstractions.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2024 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,18 +12,14 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_flow_graph_abstractions_H #define __TBB_flow_graph_abstractions_H namespace tbb { -namespace flow { -namespace interface9 { +namespace detail { +namespace d2 { //! Pure virtual template classes that define interfaces for async communication class graph_proxy { @@ -47,11 +43,9 @@ class receiver_gateway : public graph_proxy { virtual bool try_put(const input_type&) = 0; }; -} //interface9 +} // d2 -using interface9::graph_proxy; -using interface9::receiver_gateway; -} //flow -} //tbb +} // detail +} // tbb #endif diff --git a/src/tbb/include/oneapi/tbb/global_control.h b/src/tbb/include/oneapi/tbb/global_control.h new file mode 100644 index 000000000..57f3b9dbc --- /dev/null +++ b/src/tbb/include/oneapi/tbb/global_control.h @@ -0,0 +1,200 @@ +/* + Copyright (c) 2005-2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_global_control_H +#define __TBB_global_control_H + +#include "detail/_config.h" + +#include "detail/_assert.h" +#include "detail/_attach.h" +#include "detail/_exception.h" +#include "detail/_namespace_injection.h" +#include "detail/_template_helpers.h" + +#include +#include // std::nothrow_t + +namespace tbb { +namespace detail { + +namespace d1 { +class global_control; +class task_scheduler_handle; +} + +namespace r1 { +TBB_EXPORT void __TBB_EXPORTED_FUNC create(d1::global_control&); +TBB_EXPORT void __TBB_EXPORTED_FUNC destroy(d1::global_control&); +TBB_EXPORT std::size_t __TBB_EXPORTED_FUNC global_control_active_value(int); +struct global_control_impl; +struct control_storage_comparator; +void release_impl(d1::task_scheduler_handle& handle); +bool finalize_impl(d1::task_scheduler_handle& handle); +TBB_EXPORT void __TBB_EXPORTED_FUNC get(d1::task_scheduler_handle&); +TBB_EXPORT bool __TBB_EXPORTED_FUNC finalize(d1::task_scheduler_handle&, std::intptr_t mode); +} + +namespace d1 { + +class global_control { +public: + enum parameter { + max_allowed_parallelism, + thread_stack_size, + terminate_on_exception, + scheduler_handle, // not a public parameter + parameter_max // insert new parameters above this point + }; + + global_control(parameter p, std::size_t value) : + my_value(value), my_reserved(), my_param(p) { + suppress_unused_warning(my_reserved); + __TBB_ASSERT(my_param < parameter_max, "Invalid parameter"); +#if __TBB_WIN8UI_SUPPORT && (_WIN32_WINNT < 0x0A00) + // For Windows 8 Store* apps it's impossible to set stack size + if (p==thread_stack_size) + return; +#elif __TBB_x86_64 && (_WIN32 || _WIN64) + if (p==thread_stack_size) + __TBB_ASSERT_RELEASE((unsigned)value == value, "Stack size is limited to unsigned int range"); +#endif + if (my_param==max_allowed_parallelism) + __TBB_ASSERT_RELEASE(my_value>0, "max_allowed_parallelism cannot be 0."); + r1::create(*this); + } + + ~global_control() { + __TBB_ASSERT(my_param < parameter_max, "Invalid parameter"); +#if __TBB_WIN8UI_SUPPORT && (_WIN32_WINNT < 0x0A00) + // For Windows 8 Store* apps it's impossible to set stack size + if (my_param==thread_stack_size) + return; +#endif + r1::destroy(*this); + } + + static std::size_t active_value(parameter p) { + __TBB_ASSERT(p < parameter_max, "Invalid parameter"); + return r1::global_control_active_value((int)p); + } + +private: + std::size_t my_value; + std::intptr_t my_reserved; // TODO: substitution of global_control* not to break backward compatibility + parameter my_param; + + friend struct r1::global_control_impl; + friend struct r1::control_storage_comparator; +}; + +//! Finalization options. +//! Outside of the class to avoid extensive friendship. +static constexpr std::intptr_t release_nothrowing = 0; +static constexpr std::intptr_t finalize_nothrowing = 1; +static constexpr std::intptr_t finalize_throwing = 2; + +//! User side wrapper for a task scheduler lifetime control object +class task_scheduler_handle { +public: + //! Creates an empty task_scheduler_handle + task_scheduler_handle() = default; + + //! Creates an attached instance of task_scheduler_handle + task_scheduler_handle(attach) { + r1::get(*this); + } + + //! Release a reference if any + ~task_scheduler_handle() { + release(); + } + + //! No copy + task_scheduler_handle(const task_scheduler_handle& other) = delete; + task_scheduler_handle& operator=(const task_scheduler_handle& other) = delete; + + //! Move only + task_scheduler_handle(task_scheduler_handle&& other) noexcept { + std::swap(m_ctl, other.m_ctl); + } + task_scheduler_handle& operator=(task_scheduler_handle&& other) noexcept { + std::swap(m_ctl, other.m_ctl); + return *this; + }; + + //! Checks if the task_scheduler_handle is empty + explicit operator bool() const noexcept { + return m_ctl != nullptr; + } + + //! Release the reference and deactivate handle + void release() { + if (m_ctl != nullptr) { + r1::finalize(*this, release_nothrowing); + m_ctl = nullptr; + } + } + +private: + friend void r1::release_impl(task_scheduler_handle& handle); + friend bool r1::finalize_impl(task_scheduler_handle& handle); + friend void __TBB_EXPORTED_FUNC r1::get(task_scheduler_handle&); + + friend void finalize(task_scheduler_handle&); + friend bool finalize(task_scheduler_handle&, const std::nothrow_t&) noexcept; + + global_control* m_ctl{nullptr}; +}; + +#if TBB_USE_EXCEPTIONS +//! Waits for worker threads termination. Throws exception on error. +inline void finalize(task_scheduler_handle& handle) { + try_call([&] { + if (handle.m_ctl != nullptr) { + bool finalized = r1::finalize(handle, finalize_throwing); + __TBB_ASSERT_EX(finalized, "r1::finalize did not respect finalize_throwing ?"); + + } + }).on_completion([&] { + __TBB_ASSERT(!handle, "The handle should be empty after finalize"); + }); +} +#endif +//! Waits for worker threads termination. Returns false on error. +inline bool finalize(task_scheduler_handle& handle, const std::nothrow_t&) noexcept { + bool finalized = true; + if (handle.m_ctl != nullptr) { + finalized = r1::finalize(handle, finalize_nothrowing); + } + __TBB_ASSERT(!handle, "The handle should be empty after finalize"); + return finalized; +} + +} // namespace d1 +} // namespace detail + +inline namespace v1 { +using detail::d1::global_control; +using detail::d1::attach; +using detail::d1::finalize; +using detail::d1::task_scheduler_handle; +using detail::r1::unsafe_wait; +} // namespace v1 + +} // namespace tbb + +#endif // __TBB_global_control_H diff --git a/src/tbb/include/oneapi/tbb/info.h b/src/tbb/include/oneapi/tbb/info.h new file mode 100644 index 000000000..dfcfcc031 --- /dev/null +++ b/src/tbb/include/oneapi/tbb/info.h @@ -0,0 +1,125 @@ +/* + Copyright (c) 2019-2022 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_info_H +#define __TBB_info_H + +#include "detail/_config.h" +#include "detail/_namespace_injection.h" + +#if __TBB_ARENA_BINDING +#include +#include + +namespace tbb { +namespace detail { + +namespace d1{ + +using numa_node_id = int; +using core_type_id = int; + +// TODO: consider version approach to resolve backward compatibility potential issues. +struct constraints { +#if !__TBB_CPP20_PRESENT + constraints(numa_node_id id = -1, int maximal_concurrency = -1) + : numa_id(id) + , max_concurrency(maximal_concurrency) + {} +#endif /*!__TBB_CPP20_PRESENT*/ + + constraints& set_numa_id(numa_node_id id) { + numa_id = id; + return *this; + } + constraints& set_max_concurrency(int maximal_concurrency) { + max_concurrency = maximal_concurrency; + return *this; + } + constraints& set_core_type(core_type_id id) { + core_type = id; + return *this; + } + constraints& set_max_threads_per_core(int threads_number) { + max_threads_per_core = threads_number; + return *this; + } + + numa_node_id numa_id = -1; + int max_concurrency = -1; + core_type_id core_type = -1; + int max_threads_per_core = -1; +}; + +} // namespace d1 + +namespace r1 { +TBB_EXPORT unsigned __TBB_EXPORTED_FUNC numa_node_count(); +TBB_EXPORT void __TBB_EXPORTED_FUNC fill_numa_indices(int* index_array); +TBB_EXPORT int __TBB_EXPORTED_FUNC numa_default_concurrency(int numa_id); + +// Reserved fields are required to save binary backward compatibility in case of future changes. +// They must be defined to 0 at this moment. +TBB_EXPORT unsigned __TBB_EXPORTED_FUNC core_type_count(intptr_t reserved = 0); +TBB_EXPORT void __TBB_EXPORTED_FUNC fill_core_type_indices(int* index_array, intptr_t reserved = 0); + +TBB_EXPORT int __TBB_EXPORTED_FUNC constraints_default_concurrency(const d1::constraints& c, intptr_t reserved = 0); +TBB_EXPORT int __TBB_EXPORTED_FUNC constraints_threads_per_core(const d1::constraints& c, intptr_t reserved = 0); +} // namespace r1 + +namespace d1 { + +inline std::vector numa_nodes() { + std::vector node_indices(r1::numa_node_count()); + r1::fill_numa_indices(node_indices.data()); + return node_indices; +} + +inline int default_concurrency(numa_node_id id = -1) { + return r1::numa_default_concurrency(id); +} + +inline std::vector core_types() { + std::vector core_type_indexes(r1::core_type_count()); + r1::fill_core_type_indices(core_type_indexes.data()); + return core_type_indexes; +} + +inline int default_concurrency(constraints c) { + if (c.max_concurrency > 0) { return c.max_concurrency; } + return r1::constraints_default_concurrency(c); +} + +} // namespace d1 +} // namespace detail + +inline namespace v1 { +using detail::d1::numa_node_id; +using detail::d1::core_type_id; + +namespace info { +using detail::d1::numa_nodes; +using detail::d1::core_types; + +using detail::d1::default_concurrency; +} // namespace info +} // namespace v1 + +} // namespace tbb + +#endif /*__TBB_ARENA_BINDING*/ + +#endif /*__TBB_info_H*/ diff --git a/inst/include/tbb/memory_pool.h b/src/tbb/include/oneapi/tbb/memory_pool.h similarity index 77% rename from inst/include/tbb/memory_pool.h rename to src/tbb/include/oneapi/tbb/memory_pool.h index b3bba397a..de01466e9 100644 --- a/inst/include/tbb/memory_pool.h +++ b/src/tbb/include/oneapi/tbb/memory_pool.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2024 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_memory_pool_H @@ -27,13 +23,11 @@ /** @file */ #include "scalable_allocator.h" + #include // std::bad_alloc #include // std::runtime_error, std::invalid_argument -// required in C++03 to construct std::runtime_error and std::invalid_argument -#include -#if __TBB_ALLOCATOR_CONSTRUCT_VARIADIC #include // std::forward -#endif + #if __TBB_EXTRA_DEBUG #define __TBBMALLOC_ASSERT ASSERT @@ -42,12 +36,11 @@ #endif namespace tbb { -namespace interface6 { -//! @cond INTERNAL -namespace internal { +namespace detail { +namespace d1 { //! Base of thread-safe pool allocator for variable-size requests -class pool_base : tbb::internal::no_copy { +class pool_base : no_copy { // Pool interface is separate from standard allocator classes because it has // to maintain internal state, no copy or assignment. Move and swap are possible. public: @@ -73,18 +66,15 @@ class pool_base : tbb::internal::no_copy { rml::MemoryPool *my_pool; }; -} // namespace internal -//! @endcond - #if _MSC_VER && !defined(__INTEL_COMPILER) // Workaround for erroneous "unreferenced parameter" warning in method destroy. - #pragma warning (push) - #pragma warning (disable: 4100) + // #pragma warning (push) + // #pragma warning (disable: 4100) #endif //! Meets "allocator" requirements of ISO C++ Standard, Section 20.1.5 /** @ingroup memory_allocation */ -template +template class memory_pool_allocator { protected: typedef P pool_type; @@ -96,7 +86,7 @@ class memory_pool_allocator { template friend bool operator!=( const memory_pool_allocator& a, const memory_pool_allocator& b); public: - typedef typename tbb::internal::allocator_type::value_type value_type; + typedef T value_type; typedef value_type* pointer; typedef const value_type* const_pointer; typedef value_type& reference; @@ -107,19 +97,19 @@ class memory_pool_allocator { typedef memory_pool_allocator other; }; - explicit memory_pool_allocator(pool_type &pool) throw() : my_pool(&pool) {} - memory_pool_allocator(const memory_pool_allocator& src) throw() : my_pool(src.my_pool) {} + explicit memory_pool_allocator(pool_type &pool) noexcept : my_pool(&pool) {} + memory_pool_allocator(const memory_pool_allocator& src) noexcept : my_pool(src.my_pool) {} template - memory_pool_allocator(const memory_pool_allocator& src) throw() : my_pool(src.my_pool) {} + memory_pool_allocator(const memory_pool_allocator& src) noexcept : my_pool(src.my_pool) {} pointer address(reference x) const { return &x; } const_pointer address(const_reference x) const { return &x; } //! Allocate space for n objects. - pointer allocate( size_type n, const void* /*hint*/ = 0) { + pointer allocate( size_type n, const void* /*hint*/ = nullptr) { pointer p = static_cast( my_pool->malloc( n*sizeof(value_type) ) ); if (!p) - tbb::internal::throw_exception(std::bad_alloc()); + throw_exception(std::bad_alloc()); return p; } //! Free previously allocated block of memory. @@ -127,21 +117,15 @@ class memory_pool_allocator { my_pool->free(p); } //! Largest value for which method allocate might succeed. - size_type max_size() const throw() { + size_type max_size() const noexcept { size_type max = static_cast(-1) / sizeof (value_type); return (max > 0 ? max : 1); } //! Copy-construct value at location pointed to by p. -#if __TBB_ALLOCATOR_CONSTRUCT_VARIADIC + template void construct(U *p, Args&&... args) { ::new((void *)p) U(std::forward(args)...); } -#else // __TBB_ALLOCATOR_CONSTRUCT_VARIADIC -#if __TBB_CPP11_RVALUE_REF_PRESENT - void construct( pointer p, value_type&& value ) {::new((void*)(p)) value_type(std::move(value));} -#endif - void construct( pointer p, const value_type& value ) { ::new((void*)(p)) value_type(value); } -#endif // __TBB_ALLOCATOR_CONSTRUCT_VARIADIC //! Destroy value at location pointed to by p. void destroy( pointer p ) { p->~value_type(); } @@ -149,7 +133,7 @@ class memory_pool_allocator { }; #if _MSC_VER && !defined(__INTEL_COMPILER) - #pragma warning (pop) + // #pragma warning (pop) #endif // warning 4100 is back //! Analogous to std::allocator, as defined in ISO C++ Standard, Section 20.4.1 @@ -165,10 +149,10 @@ class memory_pool_allocator { typedef memory_pool_allocator other; }; - explicit memory_pool_allocator( pool_type &pool) throw() : my_pool(&pool) {} - memory_pool_allocator( const memory_pool_allocator& src) throw() : my_pool(src.my_pool) {} + explicit memory_pool_allocator( pool_type &pool) noexcept : my_pool(&pool) {} + memory_pool_allocator( const memory_pool_allocator& src) noexcept : my_pool(src.my_pool) {} template - memory_pool_allocator(const memory_pool_allocator& src) throw() : my_pool(src.my_pool) {} + memory_pool_allocator(const memory_pool_allocator& src) noexcept : my_pool(src.my_pool) {} protected: pool_type *my_pool; @@ -186,10 +170,9 @@ inline bool operator==( const memory_pool_allocator& a, const memory_pool_a template inline bool operator!=( const memory_pool_allocator& a, const memory_pool_allocator& b) {return a.my_pool!=b.my_pool;} - //! Thread-safe growable pool allocator for variable-size requests template -class memory_pool : public internal::pool_base { +class memory_pool : public pool_base { Alloc my_alloc; // TODO: base-class optimization static void *allocate_request(intptr_t pool_id, size_t & bytes); static int deallocate_request(intptr_t pool_id, void*, size_t raw_bytes); @@ -200,10 +183,9 @@ class memory_pool : public internal::pool_base { //! destroy pool ~memory_pool() { destroy(); } // call the callbacks first and destroy my_alloc latter - }; -class fixed_pool : public internal::pool_base { +class fixed_pool : public pool_base { void *my_buffer; size_t my_size; inline static void *allocate_request(intptr_t pool_id, size_t & bytes); @@ -223,43 +205,50 @@ memory_pool::memory_pool(const Alloc &src) : my_alloc(src) { sizeof(typename Alloc::value_type)); rml::MemPoolError res = rml::pool_create_v1(intptr_t(this), &args, &my_pool); if (res!=rml::POOL_OK) - tbb::internal::throw_exception(std::runtime_error("Can't create pool")); + throw_exception(std::runtime_error("Can't create pool")); } template void *memory_pool::allocate_request(intptr_t pool_id, size_t & bytes) { memory_pool &self = *reinterpret_cast*>(pool_id); const size_t unit_size = sizeof(typename Alloc::value_type); - __TBBMALLOC_ASSERT( 0 == bytes%unit_size, NULL); + __TBBMALLOC_ASSERT( 0 == bytes%unit_size, nullptr); void *ptr; - __TBB_TRY { ptr = self.my_alloc.allocate( bytes/unit_size ); } - __TBB_CATCH(...) { return 0; } +#if TBB_USE_EXCEPTIONS + try { +#endif + ptr = self.my_alloc.allocate( bytes/unit_size ); +#if TBB_USE_EXCEPTIONS + } catch(...) { + return nullptr; + } +#endif return ptr; } #if __TBB_MSVC_UNREACHABLE_CODE_IGNORED // Workaround for erroneous "unreachable code" warning in the template below. // Specific for VC++ 17-18 compiler - #pragma warning (push) - #pragma warning (disable: 4702) + // #pragma warning (push) + // #pragma warning (disable: 4702) #endif template int memory_pool::deallocate_request(intptr_t pool_id, void* raw_ptr, size_t raw_bytes) { memory_pool &self = *reinterpret_cast*>(pool_id); const size_t unit_size = sizeof(typename Alloc::value_type); - __TBBMALLOC_ASSERT( 0 == raw_bytes%unit_size, NULL); + __TBBMALLOC_ASSERT( 0 == raw_bytes%unit_size, nullptr); self.my_alloc.deallocate( static_cast(raw_ptr), raw_bytes/unit_size ); return 0; } #if __TBB_MSVC_UNREACHABLE_CODE_IGNORED - #pragma warning (pop) + // #pragma warning (pop) #endif inline fixed_pool::fixed_pool(void *buf, size_t size) : my_buffer(buf), my_size(size) { if (!buf || !size) // TODO: improve support for mode with exceptions disabled - tbb::internal::throw_exception(std::invalid_argument("Zero in parameter is invalid")); - rml::MemPoolPolicy args(allocate_request, 0, size, /*fixedPool=*/true); + throw_exception(std::invalid_argument("Zero in parameter is invalid")); + rml::MemPoolPolicy args(allocate_request, nullptr, size, /*fixedPool=*/true); rml::MemPoolError res = rml::pool_create_v1(intptr_t(this), &args, &my_pool); if (res!=rml::POOL_OK) - tbb::internal::throw_exception(std::runtime_error("Can't create pool")); + throw_exception(std::runtime_error("Can't create pool")); } inline void *fixed_pool::allocate_request(intptr_t pool_id, size_t & bytes) { fixed_pool &self = *reinterpret_cast(pool_id); @@ -269,11 +258,15 @@ inline void *fixed_pool::allocate_request(intptr_t pool_id, size_t & bytes) { return self.my_buffer; } -} //namespace interface6 -using interface6::memory_pool_allocator; -using interface6::memory_pool; -using interface6::fixed_pool; -} //namespace tbb +} // namespace d1 +} // namespace detail + +inline namespace v1 { +using detail::d1::memory_pool_allocator; +using detail::d1::memory_pool; +using detail::d1::fixed_pool; +} // inline namepspace v1 +} // namespace tbb #undef __TBBMALLOC_ASSERT #endif// __TBB_memory_pool_H diff --git a/src/tbb/include/oneapi/tbb/mutex.h b/src/tbb/include/oneapi/tbb/mutex.h new file mode 100644 index 000000000..169b7a3ca --- /dev/null +++ b/src/tbb/include/oneapi/tbb/mutex.h @@ -0,0 +1,93 @@ +/* + Copyright (c) 2021-2023 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_mutex_H +#define __TBB_mutex_H + +#include "detail/_namespace_injection.h" +#include "detail/_utils.h" +#include "detail/_scoped_lock.h" +#include "detail/_waitable_atomic.h" +#include "detail/_mutex_common.h" +#include "profiling.h" + +namespace tbb { +namespace detail { +namespace d1 { + +class mutex { +public: + //! Constructors + mutex() { + create_itt_sync(this, "tbb::mutex", ""); + }; + + //! Destructor + ~mutex() = default; + + //! No Copy + mutex(const mutex&) = delete; + mutex& operator=(const mutex&) = delete; + + using scoped_lock = unique_scoped_lock; + + //! Mutex traits + static constexpr bool is_rw_mutex = false; + static constexpr bool is_recursive_mutex = false; + static constexpr bool is_fair_mutex = false; + + //! Acquire lock + /** Spin if the lock is taken */ + void lock() { + call_itt_notify(prepare, this); + while (!try_lock()) { + my_flag.wait(true, /* context = */ 0, std::memory_order_relaxed); + } + } + + //! Try acquiring lock (non-blocking) + /** Return true if lock acquired; false otherwise. */ + bool try_lock() { + bool result = !my_flag.load(std::memory_order_relaxed) && !my_flag.exchange(true); + if (result) { + call_itt_notify(acquired, this); + } + return result; + } + + //! Release lock + void unlock() { + call_itt_notify(releasing, this); + // We need Write Read memory barrier before notify that reads the waiter list. + // In C++ only full fence covers this type of barrier. + my_flag.exchange(false); + my_flag.notify_one_relaxed(); + } + +private: + waitable_atomic my_flag{0}; +}; // class mutex + +} // namespace d1 +} // namespace detail + +inline namespace v1 { +using detail::d1::mutex; +} // namespace v1 + +} // namespace tbb + +#endif // __TBB_mutex_H diff --git a/src/tbb/include/oneapi/tbb/null_mutex.h b/src/tbb/include/oneapi/tbb/null_mutex.h new file mode 100644 index 000000000..1797e35bb --- /dev/null +++ b/src/tbb/include/oneapi/tbb/null_mutex.h @@ -0,0 +1,80 @@ +/* + Copyright (c) 2005-2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_null_mutex_H +#define __TBB_null_mutex_H + +#include "detail/_config.h" +#include "detail/_namespace_injection.h" +#include "detail/_mutex_common.h" + +namespace tbb { +namespace detail { +namespace d1 { + +//! A mutex which does nothing +/** A null_mutex does no operation and simulates success. + @ingroup synchronization */ +class null_mutex { +public: + //! Constructors + constexpr null_mutex() noexcept = default; + + //! Destructor + ~null_mutex() = default; + + //! No Copy + null_mutex(const null_mutex&) = delete; + null_mutex& operator=(const null_mutex&) = delete; + + //! Represents acquisition of a mutex. + class scoped_lock { + public: + //! Constructors + constexpr scoped_lock() noexcept = default; + scoped_lock(null_mutex&) {} + + //! Destructor + ~scoped_lock() = default; + + //! No Copy + scoped_lock(const scoped_lock&) = delete; + scoped_lock& operator=(const scoped_lock&) = delete; + + void acquire(null_mutex&) {} + bool try_acquire(null_mutex&) { return true; } + void release() {} + }; + + //! Mutex traits + static constexpr bool is_rw_mutex = false; + static constexpr bool is_recursive_mutex = true; + static constexpr bool is_fair_mutex = true; + + void lock() {} + bool try_lock() { return true; } + void unlock() {} +}; // class null_mutex + +} // namespace d1 +} // namespace detail + +inline namespace v1 { +using detail::d1::null_mutex; +} // namespace v1 +} // namespace tbb + +#endif /* __TBB_null_mutex_H */ diff --git a/src/tbb/include/oneapi/tbb/null_rw_mutex.h b/src/tbb/include/oneapi/tbb/null_rw_mutex.h new file mode 100644 index 000000000..43983a005 --- /dev/null +++ b/src/tbb/include/oneapi/tbb/null_rw_mutex.h @@ -0,0 +1,87 @@ +/* + Copyright (c) 2005-2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_null_rw_mutex_H +#define __TBB_null_rw_mutex_H + +#include "detail/_config.h" +#include "detail/_namespace_injection.h" +#include "detail/_mutex_common.h" + +namespace tbb { +namespace detail { +namespace d1 { + +//! A rw mutex which does nothing +/** A null_rw_mutex is a rw mutex that does nothing and simulates successful operation. + @ingroup synchronization */ +class null_rw_mutex { +public: + //! Constructors + constexpr null_rw_mutex() noexcept = default; + + //! Destructor + ~null_rw_mutex() = default; + + //! No Copy + null_rw_mutex(const null_rw_mutex&) = delete; + null_rw_mutex& operator=(const null_rw_mutex&) = delete; + + //! Represents acquisition of a mutex. + class scoped_lock { + public: + //! Constructors + constexpr scoped_lock() noexcept = default; + scoped_lock(null_rw_mutex&, bool = true) {} + + //! Destructor + ~scoped_lock() = default; + + //! No Copy + scoped_lock(const scoped_lock&) = delete; + scoped_lock& operator=(const scoped_lock&) = delete; + + void acquire(null_rw_mutex&, bool = true) {} + bool try_acquire(null_rw_mutex&, bool = true) { return true; } + void release() {} + bool upgrade_to_writer() { return true; } + bool downgrade_to_reader() { return true; } + + bool is_writer() const { return true; } + }; + + //! Mutex traits + static constexpr bool is_rw_mutex = true; + static constexpr bool is_recursive_mutex = true; + static constexpr bool is_fair_mutex = true; + + void lock() {} + bool try_lock() { return true; } + void unlock() {} + void lock_shared() {} + bool try_lock_shared() { return true; } + void unlock_shared() {} +}; // class null_rw_mutex + +} // namespace d1 +} // namespace detail + +inline namespace v1 { +using detail::d1::null_rw_mutex; +} // namespace v1 +} // namespace tbb + +#endif /* __TBB_null_rw_mutex_H */ diff --git a/src/tbb/include/oneapi/tbb/parallel_for.h b/src/tbb/include/oneapi/tbb/parallel_for.h new file mode 100644 index 000000000..37a261350 --- /dev/null +++ b/src/tbb/include/oneapi/tbb/parallel_for.h @@ -0,0 +1,469 @@ +/* + Copyright (c) 2005-2024 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_parallel_for_H +#define __TBB_parallel_for_H + +#include "detail/_config.h" +#include "detail/_namespace_injection.h" +#include "detail/_exception.h" +#include "detail/_task.h" +#include "detail/_small_object_pool.h" +#include "profiling.h" + +#include "partitioner.h" +#include "blocked_range.h" +#include "task_group.h" + +#include +#include + +namespace tbb { +namespace detail { +#if __TBB_CPP20_CONCEPTS_PRESENT +inline namespace d0 { + +template +concept parallel_for_body = std::copy_constructible && std::invocable&, Range&>; + +template +concept parallel_for_index = std::constructible_from && + std::copyable && + requires( const std::remove_reference_t& lhs, const std::remove_reference_t& rhs ) { + { lhs < rhs } -> adaptive_same_as; + { lhs - rhs } -> std::convertible_to; + { lhs + (rhs - lhs) } -> std::convertible_to; + }; + +template +concept parallel_for_function = std::invocable&, Index>; + +} // namespace d0 +#endif // __TBB_CPP20_CONCEPTS_PRESENT +namespace d1 { + +//! Task type used in parallel_for +/** @ingroup algorithms */ +template +struct start_for : public task { + Range my_range; + const Body my_body; + node* my_parent; + + typename Partitioner::task_partition_type my_partition; + small_object_allocator my_allocator; + + task* execute(execution_data&) override; + task* cancel(execution_data&) override; + void finalize(const execution_data&); + + //! Constructor for root task. + start_for( const Range& range, const Body& body, Partitioner& partitioner, small_object_allocator& alloc ) : + my_range(range), + my_body(body), + my_parent(nullptr), + my_partition(partitioner), + my_allocator(alloc) {} + //! Splitting constructor used to generate children. + /** parent_ becomes left child. Newly constructed object is right child. */ + start_for( start_for& parent_, typename Partitioner::split_type& split_obj, small_object_allocator& alloc ) : + my_range(parent_.my_range, get_range_split_object(split_obj)), + my_body(parent_.my_body), + my_parent(nullptr), + my_partition(parent_.my_partition, split_obj), + my_allocator(alloc) {} + //! Construct right child from the given range as response to the demand. + /** parent_ remains left child. Newly constructed object is right child. */ + start_for( start_for& parent_, const Range& r, depth_t d, small_object_allocator& alloc ) : + my_range(r), + my_body(parent_.my_body), + my_parent(nullptr), + my_partition(parent_.my_partition, split()), + my_allocator(alloc) + { + my_partition.align_depth( d ); + } + static void run(const Range& range, const Body& body, Partitioner& partitioner) { + task_group_context context(PARALLEL_FOR); + run(range, body, partitioner, context); + } + + static void run(const Range& range, const Body& body, Partitioner& partitioner, task_group_context& context) { + if ( !range.empty() ) { + small_object_allocator alloc{}; + start_for& for_task = *alloc.new_object(range, body, partitioner, alloc); + + // defer creation of the wait node until task allocation succeeds + wait_node wn; + for_task.my_parent = &wn; + execute_and_wait(for_task, context, wn.m_wait, context); + } + } + //! Run body for range, serves as callback for partitioner + void run_body( Range &r ) { + tbb::detail::invoke(my_body, r); + } + + //! spawn right task, serves as callback for partitioner + void offer_work(typename Partitioner::split_type& split_obj, execution_data& ed) { + offer_work_impl(ed, *this, split_obj); + } + + //! spawn right task, serves as callback for partitioner + void offer_work(const Range& r, depth_t d, execution_data& ed) { + offer_work_impl(ed, *this, r, d); + } + +private: + template + void offer_work_impl(execution_data& ed, Args&&... constructor_args) { + // New right child + small_object_allocator alloc{}; + start_for& right_child = *alloc.new_object(ed, std::forward(constructor_args)..., alloc); + + // New root node as a continuation and ref count. Left and right child attach to the new parent. + right_child.my_parent = my_parent = alloc.new_object(ed, my_parent, 2, alloc); + // Spawn the right sibling + right_child.spawn_self(ed); + } + + void spawn_self(execution_data& ed) { + my_partition.spawn_task(*this, *context(ed)); + } +}; + +//! fold the tree and deallocate the task +template +void start_for::finalize(const execution_data& ed) { + // Get the current parent and allocator an object destruction + node* parent = my_parent; + auto allocator = my_allocator; + // Task execution finished - destroy it + this->~start_for(); + // Unwind the tree decrementing the parent`s reference count + + fold_tree(parent, ed); + allocator.deallocate(this, ed); + +} + +//! execute task for parallel_for +template +task* start_for::execute(execution_data& ed) { + if (!is_same_affinity(ed)) { + my_partition.note_affinity(execution_slot(ed)); + } + my_partition.check_being_stolen(*this, ed); + my_partition.execute(*this, my_range, ed); + finalize(ed); + return nullptr; +} + +//! cancel task for parallel_for +template +task* start_for::cancel(execution_data& ed) { + finalize(ed); + return nullptr; +} + +//! Calls the function with values from range [begin, end) with a step provided +template +class parallel_for_body_wrapper : detail::no_assign { + const Function &my_func; + const Index my_begin; + const Index my_step; +public: + parallel_for_body_wrapper( const Function& _func, Index& _begin, Index& _step ) + : my_func(_func), my_begin(_begin), my_step(_step) {} + + void operator()( const blocked_range& r ) const { + // A set of local variables to help the compiler with vectorization of the following loop. + Index b = r.begin(); + Index e = r.end(); + Index ms = my_step; + Index k = my_begin + b*ms; + +#if __INTEL_COMPILER +#pragma ivdep +#if __TBB_ASSERT_ON_VECTORIZATION_FAILURE +#pragma vector always assert +#endif +#endif + for ( Index i = b; i < e; ++i, k += ms ) { + tbb::detail::invoke(my_func, k); + } + } +}; + +// Requirements on Range concept are documented in blocked_range.h + +/** \page parallel_for_body_req Requirements on parallel_for body + Class \c Body implementing the concept of parallel_for body must define: + - \code Body::Body( const Body& ); \endcode Copy constructor + - \code Body::~Body(); \endcode Destructor + - \code void Body::operator()( Range& r ) const; \endcode Function call operator applying the body to range \c r. +**/ + +/** \name parallel_for + See also requirements on \ref range_req "Range" and \ref parallel_for_body_req "parallel_for Body". **/ +//@{ + +//! Parallel iteration over range with default partitioner. +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_for_body) +void parallel_for( const Range& range, const Body& body ) { + start_for::run(range,body,__TBB_DEFAULT_PARTITIONER()); +} + +//! Parallel iteration over range with simple partitioner. +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_for_body) +void parallel_for( const Range& range, const Body& body, const simple_partitioner& partitioner ) { + start_for::run(range,body,partitioner); +} + +//! Parallel iteration over range with auto_partitioner. +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_for_body) +void parallel_for( const Range& range, const Body& body, const auto_partitioner& partitioner ) { + start_for::run(range,body,partitioner); +} + +//! Parallel iteration over range with static_partitioner. +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_for_body) +void parallel_for( const Range& range, const Body& body, const static_partitioner& partitioner ) { + start_for::run(range,body,partitioner); +} + +//! Parallel iteration over range with affinity_partitioner. +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_for_body) +void parallel_for( const Range& range, const Body& body, affinity_partitioner& partitioner ) { + start_for::run(range,body,partitioner); +} + +//! Parallel iteration over range with default partitioner and user-supplied context. +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_for_body) +void parallel_for( const Range& range, const Body& body, task_group_context& context ) { + start_for::run(range, body, __TBB_DEFAULT_PARTITIONER(), context); +} + +//! Parallel iteration over range with simple partitioner and user-supplied context. +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_for_body) +void parallel_for( const Range& range, const Body& body, const simple_partitioner& partitioner, task_group_context& context ) { + start_for::run(range, body, partitioner, context); +} + +//! Parallel iteration over range with auto_partitioner and user-supplied context. +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_for_body) +void parallel_for( const Range& range, const Body& body, const auto_partitioner& partitioner, task_group_context& context ) { + start_for::run(range, body, partitioner, context); +} + +//! Parallel iteration over range with static_partitioner and user-supplied context. +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_for_body) +void parallel_for( const Range& range, const Body& body, const static_partitioner& partitioner, task_group_context& context ) { + start_for::run(range, body, partitioner, context); +} + +//! Parallel iteration over range with affinity_partitioner and user-supplied context. +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_for_body) +void parallel_for( const Range& range, const Body& body, affinity_partitioner& partitioner, task_group_context& context ) { + start_for::run(range,body,partitioner, context); +} + +//! Implementation of parallel iteration over stepped range of integers with explicit step and partitioner +template +void parallel_for_impl(Index first, Index last, Index step, const Function& f, Partitioner& partitioner) { + if (step <= 0 ) + throw_exception(exception_id::nonpositive_step); // throws std::invalid_argument + else if (first < last) { + // Above "else" avoids "potential divide by zero" warning on some platforms + Index end = Index(last - first - 1ul) / step + Index(1); + blocked_range range(static_cast(0), end); + parallel_for_body_wrapper body(f, first, step); + parallel_for(range, body, partitioner); + } +} + +//! Parallel iteration over a range of integers with a step provided and default partitioner +template + __TBB_requires(parallel_for_index && parallel_for_function) +void parallel_for(Index first, Index last, Index step, const Function& f) { + parallel_for_impl(first, last, step, f, __TBB_DEFAULT_PARTITIONER()); +} +//! Parallel iteration over a range of integers with a step provided and simple partitioner +template + __TBB_requires(parallel_for_index && parallel_for_function) +void parallel_for(Index first, Index last, Index step, const Function& f, const simple_partitioner& partitioner) { + parallel_for_impl(first, last, step, f, partitioner); +} +//! Parallel iteration over a range of integers with a step provided and auto partitioner +template + __TBB_requires(parallel_for_index && parallel_for_function) +void parallel_for(Index first, Index last, Index step, const Function& f, const auto_partitioner& partitioner) { + parallel_for_impl(first, last, step, f, partitioner); +} +//! Parallel iteration over a range of integers with a step provided and static partitioner +template + __TBB_requires(parallel_for_index && parallel_for_function) +void parallel_for(Index first, Index last, Index step, const Function& f, const static_partitioner& partitioner) { + parallel_for_impl(first, last, step, f, partitioner); +} +//! Parallel iteration over a range of integers with a step provided and affinity partitioner +template + __TBB_requires(parallel_for_index && parallel_for_function) +void parallel_for(Index first, Index last, Index step, const Function& f, affinity_partitioner& partitioner) { + parallel_for_impl(first, last, step, f, partitioner); +} + +//! Parallel iteration over a range of integers with a default step value and default partitioner +template + __TBB_requires(parallel_for_index && parallel_for_function) +void parallel_for(Index first, Index last, const Function& f) { + parallel_for_impl(first, last, static_cast(1), f, __TBB_DEFAULT_PARTITIONER()); +} +//! Parallel iteration over a range of integers with a default step value and simple partitioner +template + __TBB_requires(parallel_for_index && parallel_for_function) +void parallel_for(Index first, Index last, const Function& f, const simple_partitioner& partitioner) { + parallel_for_impl(first, last, static_cast(1), f, partitioner); +} +//! Parallel iteration over a range of integers with a default step value and auto partitioner +template + __TBB_requires(parallel_for_index && parallel_for_function) +void parallel_for(Index first, Index last, const Function& f, const auto_partitioner& partitioner) { + parallel_for_impl(first, last, static_cast(1), f, partitioner); +} +//! Parallel iteration over a range of integers with a default step value and static partitioner +template + __TBB_requires(parallel_for_index && parallel_for_function) +void parallel_for(Index first, Index last, const Function& f, const static_partitioner& partitioner) { + parallel_for_impl(first, last, static_cast(1), f, partitioner); +} +//! Parallel iteration over a range of integers with a default step value and affinity partitioner +template + __TBB_requires(parallel_for_index && parallel_for_function) +void parallel_for(Index first, Index last, const Function& f, affinity_partitioner& partitioner) { + parallel_for_impl(first, last, static_cast(1), f, partitioner); +} + +//! Implementation of parallel iteration over stepped range of integers with explicit step, task group context, and partitioner +template +void parallel_for_impl(Index first, Index last, Index step, const Function& f, Partitioner& partitioner, task_group_context &context) { + if (step <= 0 ) + throw_exception(exception_id::nonpositive_step); // throws std::invalid_argument + else if (first < last) { + // Above "else" avoids "potential divide by zero" warning on some platforms + Index end = (last - first - Index(1)) / step + Index(1); + blocked_range range(static_cast(0), end); + parallel_for_body_wrapper body(f, first, step); + parallel_for(range, body, partitioner, context); + } +} + +//! Parallel iteration over a range of integers with explicit step, task group context, and default partitioner +template + __TBB_requires(parallel_for_index && parallel_for_function) +void parallel_for(Index first, Index last, Index step, const Function& f, task_group_context &context) { + parallel_for_impl(first, last, step, f, __TBB_DEFAULT_PARTITIONER(), context); +} +//! Parallel iteration over a range of integers with explicit step, task group context, and simple partitioner +template + __TBB_requires(parallel_for_index && parallel_for_function) +void parallel_for(Index first, Index last, Index step, const Function& f, const simple_partitioner& partitioner, task_group_context &context) { + parallel_for_impl(first, last, step, f, partitioner, context); +} +//! Parallel iteration over a range of integers with explicit step, task group context, and auto partitioner +template + __TBB_requires(parallel_for_index && parallel_for_function) +void parallel_for(Index first, Index last, Index step, const Function& f, const auto_partitioner& partitioner, task_group_context &context) { + parallel_for_impl(first, last, step, f, partitioner, context); +} +//! Parallel iteration over a range of integers with explicit step, task group context, and static partitioner +template + __TBB_requires(parallel_for_index && parallel_for_function) +void parallel_for(Index first, Index last, Index step, const Function& f, const static_partitioner& partitioner, task_group_context &context) { + parallel_for_impl(first, last, step, f, partitioner, context); +} +//! Parallel iteration over a range of integers with explicit step, task group context, and affinity partitioner +template + __TBB_requires(parallel_for_index && parallel_for_function) +void parallel_for(Index first, Index last, Index step, const Function& f, affinity_partitioner& partitioner, task_group_context &context) { + parallel_for_impl(first, last, step, f, partitioner, context); +} + +//! Parallel iteration over a range of integers with a default step value, explicit task group context, and default partitioner +template + __TBB_requires(parallel_for_index && parallel_for_function) +void parallel_for(Index first, Index last, const Function& f, task_group_context &context) { + parallel_for_impl(first, last, static_cast(1), f, __TBB_DEFAULT_PARTITIONER(), context); +} +//! Parallel iteration over a range of integers with a default step value, explicit task group context, and simple partitioner +template + __TBB_requires(parallel_for_index && parallel_for_function) +void parallel_for(Index first, Index last, const Function& f, const simple_partitioner& partitioner, task_group_context &context) { + parallel_for_impl(first, last, static_cast(1), f, partitioner, context); +} +//! Parallel iteration over a range of integers with a default step value, explicit task group context, and auto partitioner +template + __TBB_requires(parallel_for_index && parallel_for_function) +void parallel_for(Index first, Index last, const Function& f, const auto_partitioner& partitioner, task_group_context &context) { + parallel_for_impl(first, last, static_cast(1), f, partitioner, context); +} +//! Parallel iteration over a range of integers with a default step value, explicit task group context, and static partitioner +template + __TBB_requires(parallel_for_index && parallel_for_function) +void parallel_for(Index first, Index last, const Function& f, const static_partitioner& partitioner, task_group_context &context) { + parallel_for_impl(first, last, static_cast(1), f, partitioner, context); +} +//! Parallel iteration over a range of integers with a default step value, explicit task group context, and affinity_partitioner +template + __TBB_requires(parallel_for_index && parallel_for_function) +void parallel_for(Index first, Index last, const Function& f, affinity_partitioner& partitioner, task_group_context &context) { + parallel_for_impl(first, last, static_cast(1), f, partitioner, context); +} +// @} + +} // namespace d1 +} // namespace detail + +inline namespace v1 { +using detail::d1::parallel_for; +// Split types +using detail::split; +using detail::proportional_split; +} // namespace v1 + +} // namespace tbb + +#endif /* __TBB_parallel_for_H */ diff --git a/src/tbb/include/oneapi/tbb/parallel_for_each.h b/src/tbb/include/oneapi/tbb/parallel_for_each.h new file mode 100644 index 000000000..ce68a7ff9 --- /dev/null +++ b/src/tbb/include/oneapi/tbb/parallel_for_each.h @@ -0,0 +1,712 @@ +/* + Copyright (c) 2005-2024 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_parallel_for_each_H +#define __TBB_parallel_for_each_H + +#include "detail/_config.h" +#include "detail/_namespace_injection.h" +#include "detail/_exception.h" +#include "detail/_task.h" +#include "detail/_aligned_space.h" +#include "detail/_small_object_pool.h" +#include "detail/_utils.h" + +#include "parallel_for.h" +#include "task_group.h" // task_group_context + +#include +#include + +namespace tbb { +namespace detail { +#if __TBB_CPP20_CONCEPTS_PRESENT +namespace d1 { +template +class feeder; + +} // namespace d1 +inline namespace d0 { + +template +concept parallel_for_each_body = std::invocable&, ItemType&&> || + std::invocable&, ItemType&&, tbb::detail::d1::feeder&>; + +} // namespace d0 +#endif // __TBB_CPP20_CONCEPTS_PRESENT +namespace d2 { +template class feeder_impl; +} // namespace d2 + +namespace d1 { +//! Class the user supplied algorithm body uses to add new tasks +template +class feeder { + feeder() {} + feeder(const feeder&) = delete; + void operator=( const feeder&) = delete; + + virtual ~feeder () {} + virtual void internal_add_copy(const Item& item) = 0; + virtual void internal_add_move(Item&& item) = 0; + + template friend class d2::feeder_impl; +public: + //! Add a work item to a running parallel_for_each. + void add(const Item& item) {internal_add_copy(item);} + void add(Item&& item) {internal_add_move(std::move(item));} +}; + +} // namespace d1 + +namespace d2 { +using namespace tbb::detail::d1; +/** Selects one of the two possible forms of function call member operator. + @ingroup algorithms **/ +template +struct parallel_for_each_operator_selector { +public: + template + static auto call(const Body& body, ItemArg&& item, FeederArg*) + -> decltype(tbb::detail::invoke(body, std::forward(item)), void()) { + #if defined(_MSC_VER) && !defined(__INTEL_COMPILER) + // Suppression of Microsoft non-standard extension warnings + // #pragma warning (push) + // #pragma warning (disable: 4239) + #endif + + tbb::detail::invoke(body, std::forward(item)); + + #if defined(_MSC_VER) && !defined(__INTEL_COMPILER) + // #pragma warning (pop) + #endif + } + + template + static auto call(const Body& body, ItemArg&& item, FeederArg* feeder) + -> decltype(tbb::detail::invoke(body, std::forward(item), *feeder), void()) { + #if defined(_MSC_VER) && !defined(__INTEL_COMPILER) + // Suppression of Microsoft non-standard extension warnings + // #pragma warning (push) + // #pragma warning (disable: 4239) + #endif + __TBB_ASSERT(feeder, "Feeder was not created but should be"); + + tbb::detail::invoke(body, std::forward(item), *feeder); + + #if defined(_MSC_VER) && !defined(__INTEL_COMPILER) + // #pragma warning (pop) + #endif + } +}; + +template +struct feeder_item_task: public task { + using feeder_type = feeder_impl; + + template + feeder_item_task(ItemType&& input_item, feeder_type& feeder, small_object_allocator& alloc, wait_tree_vertex_interface& wait_vertex) : + item(std::forward(input_item)), + my_feeder(feeder), + my_allocator(alloc), + m_wait_tree_vertex(r1::get_thread_reference_vertex(&wait_vertex)) + { + m_wait_tree_vertex->reserve(); + } + + void finalize(const execution_data& ed) { + m_wait_tree_vertex->release(); + my_allocator.delete_object(this, ed); + } + + //! Hack for resolve ambiguity between calls to the body with and without moving the stored copy + //! Executing body with moving the copy should have higher priority + using first_priority = int; + using second_priority = double; + + template + static auto call(const BodyType& call_body, ItemType& call_item, FeederType& call_feeder, first_priority) + -> decltype(parallel_for_each_operator_selector::call(call_body, std::move(call_item), &call_feeder), void()) + { + parallel_for_each_operator_selector::call(call_body, std::move(call_item), &call_feeder); + } + + template + static void call(const BodyType& call_body, ItemType& call_item, FeederType& call_feeder, second_priority) { + parallel_for_each_operator_selector::call(call_body, call_item, &call_feeder); + } + + task* execute(execution_data& ed) override { + call(my_feeder.my_body, item, my_feeder, first_priority{}); + finalize(ed); + return nullptr; + } + + task* cancel(execution_data& ed) override { + finalize(ed); + return nullptr; + } + + Item item; + feeder_type& my_feeder; + small_object_allocator my_allocator; + wait_tree_vertex_interface* m_wait_tree_vertex; +}; // class feeder_item_task + +/** Implements new task adding procedure. + @ingroup algorithms **/ +template +class feeder_impl : public feeder { + // Avoiding use of copy constructor in a virtual method if the type does not support it + void internal_add_copy_impl(std::true_type, const Item& item) { + using feeder_task = feeder_item_task; + small_object_allocator alloc; + auto task = alloc.new_object(item, *this, alloc, my_wait_context); + + spawn(*task, my_execution_context); + } + + void internal_add_copy_impl(std::false_type, const Item&) { + __TBB_ASSERT(false, "Overloading for r-value reference doesn't work or it's not movable and not copyable object"); + } + + void internal_add_copy(const Item& item) override { + internal_add_copy_impl(typename std::is_copy_constructible::type(), item); + } + + void internal_add_move(Item&& item) override { + using feeder_task = feeder_item_task; + small_object_allocator alloc{}; + auto task = alloc.new_object(std::move(item), *this, alloc, my_wait_context); + + spawn(*task, my_execution_context); + } +public: + feeder_impl(const Body& body, wait_context_vertex& w_context, task_group_context &context) + : my_body(body), + my_wait_context(w_context) + , my_execution_context(context) + {} + + const Body& my_body; + wait_context_vertex& my_wait_context; + task_group_context& my_execution_context; +}; // class feeder_impl + +/** Execute computation under one element of the range + @ingroup algorithms **/ +template +struct for_each_iteration_task: public task { + using feeder_type = feeder_impl; + + for_each_iteration_task(Iterator input_item_ptr, const Body& body, feeder_impl* feeder_ptr, wait_context& wait_context) : + item_ptr(input_item_ptr), my_body(body), my_feeder_ptr(feeder_ptr), parent_wait_context(wait_context) + {} + + void finalize() { + parent_wait_context.release(); + } + + task* execute(execution_data&) override { + parallel_for_each_operator_selector::call(my_body, *item_ptr, my_feeder_ptr); + finalize(); + return nullptr; + } + + task* cancel(execution_data&) override { + finalize(); + return nullptr; + } + + Iterator item_ptr; + const Body& my_body; + feeder_impl* my_feeder_ptr; + wait_context& parent_wait_context; +}; // class for_each_iteration_task + +// Helper to get the type of the iterator to the internal sequence of copies +// If the element can be passed to the body as an rvalue - this iterator should be move_iterator +template +struct input_iteration_task_iterator_helper { + // For input iterators we pass const lvalue reference to the body + // It is prohibited to take non-constant lvalue references for input iterators + using type = const Item*; +}; + +template +struct input_iteration_task_iterator_helper::call(std::declval(), + std::declval(), + std::declval*>()))>> +{ + using type = std::move_iterator; +}; + +/** Split one block task to several(max_block_size) iteration tasks for input iterators + @ingroup algorithms **/ +template +struct input_block_handling_task : public task { + static constexpr size_t max_block_size = 4; + + using feeder_type = feeder_impl; + using iteration_task_iterator_type = typename input_iteration_task_iterator_helper::type; + using iteration_task = for_each_iteration_task; + + input_block_handling_task(wait_context_vertex& root_wait_context, task_group_context& e_context, + const Body& body, feeder_impl* feeder_ptr, small_object_allocator& alloc) + :my_size(0), my_wait_context(0), my_root_wait_context(root_wait_context), + my_execution_context(e_context), my_allocator(alloc) + { + auto item_it = block_iteration_space.begin(); + for (auto* it = task_pool.begin(); it != task_pool.end(); ++it) { + new (it) iteration_task(iteration_task_iterator_type(item_it++), body, feeder_ptr, my_wait_context); + } + } + + void finalize(const execution_data& ed) { + my_root_wait_context.release(); + my_allocator.delete_object(this, ed); + } + + task* execute(execution_data& ed) override { + __TBB_ASSERT( my_size > 0, "Negative size was passed to task"); + for (std::size_t counter = 1; counter < my_size; ++counter) { + my_wait_context.reserve(); + spawn(*(task_pool.begin() + counter), my_execution_context); + } + my_wait_context.reserve(); + execute_and_wait(*task_pool.begin(), my_execution_context, + my_wait_context, my_execution_context); + + // deallocate current task after children execution + finalize(ed); + return nullptr; + } + + task* cancel(execution_data& ed) override { + finalize(ed); + return nullptr; + } + + ~input_block_handling_task() { + for(std::size_t counter = 0; counter < max_block_size; ++counter) { + (task_pool.begin() + counter)->~iteration_task(); + if (counter < my_size) { + (block_iteration_space.begin() + counter)->~Item(); + } + } + } + + aligned_space block_iteration_space; + aligned_space task_pool; + std::size_t my_size; + wait_context my_wait_context; + wait_context_vertex& my_root_wait_context; + task_group_context& my_execution_context; + small_object_allocator my_allocator; +}; // class input_block_handling_task + +/** Split one block task to several(max_block_size) iteration tasks for forward iterators + @ingroup algorithms **/ +template +struct forward_block_handling_task : public task { + static constexpr size_t max_block_size = 4; + + using iteration_task = for_each_iteration_task; + + forward_block_handling_task(Iterator first, std::size_t size, + wait_context_vertex& w_context, task_group_context& e_context, + const Body& body, feeder_impl* feeder_ptr, + small_object_allocator& alloc) + : my_size(size), my_wait_context(0), my_root_wait_context(w_context), + my_execution_context(e_context), my_allocator(alloc) + { + auto* task_it = task_pool.begin(); + for (std::size_t i = 0; i < size; i++) { + new (task_it++) iteration_task(first, body, feeder_ptr, my_wait_context); + ++first; + } + } + + void finalize(const execution_data& ed) { + my_root_wait_context.release(); + my_allocator.delete_object(this, ed); + } + + task* execute(execution_data& ed) override { + __TBB_ASSERT( my_size > 0, "Negative size was passed to task"); + for(std::size_t counter = 1; counter < my_size; ++counter) { + my_wait_context.reserve(); + spawn(*(task_pool.begin() + counter), my_execution_context); + } + my_wait_context.reserve(); + execute_and_wait(*task_pool.begin(), my_execution_context, + my_wait_context, my_execution_context); + + // deallocate current task after children execution + finalize(ed); + return nullptr; + } + + task* cancel(execution_data& ed) override { + finalize(ed); + return nullptr; + } + + ~forward_block_handling_task() { + for(std::size_t counter = 0; counter < my_size; ++counter) { + (task_pool.begin() + counter)->~iteration_task(); + } + } + + aligned_space task_pool; + std::size_t my_size; + wait_context my_wait_context; + wait_context_vertex& my_root_wait_context; + task_group_context& my_execution_context; + small_object_allocator my_allocator; +}; // class forward_block_handling_task + +/** Body for parallel_for algorithm. + * Allows to redirect operations under random access iterators range to the parallel_for algorithm. + @ingroup algorithms **/ +template +class parallel_for_body_wrapper { + Iterator my_first; + const Body& my_body; + feeder_impl* my_feeder_ptr; +public: + parallel_for_body_wrapper(Iterator first, const Body& body, feeder_impl* feeder_ptr) + : my_first(first), my_body(body), my_feeder_ptr(feeder_ptr) {} + + void operator()(tbb::blocked_range range) const { +#if __INTEL_COMPILER +#pragma ivdep +#endif + for (std::size_t count = range.begin(); count != range.end(); count++) { + parallel_for_each_operator_selector::call(my_body, *(my_first + count), + my_feeder_ptr); + } + } +}; // class parallel_for_body_wrapper + + +/** Helper for getting iterators tag including inherited custom tags + @ingroup algorithms */ +template +using tag = typename std::iterator_traits::iterator_category; + +#if __TBB_CPP20_PRESENT +template +struct move_iterator_dispatch_helper { + using type = It; +}; + +// Until C++23, std::move_iterator::iterator_concept always defines +// to std::input_iterator_tag and hence std::forward_iterator concept +// always evaluates to false, so std::move_iterator dispatch should be +// made according to the base iterator type. +template +struct move_iterator_dispatch_helper> { + using type = It; +}; + +template +using iterator_tag_dispatch_impl = + std::conditional_t, + std::random_access_iterator_tag, + std::conditional_t, + std::forward_iterator_tag, + std::input_iterator_tag>>; + +template +using iterator_tag_dispatch = + iterator_tag_dispatch_impl::type>; + +#else +template +using iterator_tag_dispatch = typename + std::conditional< + std::is_base_of>::value, + std::random_access_iterator_tag, + typename std::conditional< + std::is_base_of>::value, + std::forward_iterator_tag, + std::input_iterator_tag + >::type + >::type; +#endif // __TBB_CPP20_PRESENT + +template +using feeder_is_required = tbb::detail::void_t(), + std::declval::reference>(), + std::declval&>()))>; + +// Creates feeder object only if the body can accept it +template +struct feeder_holder { + feeder_holder( wait_context_vertex&, task_group_context&, const Body& ) {} + + feeder_impl* feeder_ptr() { return nullptr; } +}; // class feeder_holder + +template +class feeder_holder> { +public: + feeder_holder( wait_context_vertex& w_context, task_group_context& context, const Body& body ) + : my_feeder(body, w_context, context) {} + + feeder_impl* feeder_ptr() { return &my_feeder; } +private: + feeder_impl my_feeder; +}; // class feeder_holder + +template +class for_each_root_task_base : public task { +public: + for_each_root_task_base(Iterator first, Iterator last, const Body& body, wait_context_vertex& w_context, task_group_context& e_context) + : my_first(first), my_last(last), my_wait_context(w_context), my_execution_context(e_context), + my_body(body), my_feeder_holder(my_wait_context, my_execution_context, my_body) + { + my_wait_context.reserve(); + } +private: + task* cancel(execution_data&) override { + this->my_wait_context.release(); + return nullptr; + } +protected: + Iterator my_first; + Iterator my_last; + wait_context_vertex& my_wait_context; + task_group_context& my_execution_context; + const Body& my_body; + feeder_holder my_feeder_holder; +}; // class for_each_root_task_base + +/** parallel_for_each algorithm root task - most generic version + * Splits input range to blocks + @ingroup algorithms **/ +template > +class for_each_root_task : public for_each_root_task_base +{ + using base_type = for_each_root_task_base; +public: + using base_type::base_type; +private: + task* execute(execution_data& ed) override { + using block_handling_type = input_block_handling_task; + + if (this->my_first == this->my_last) { + this->my_wait_context.release(); + return nullptr; + } + + this->my_wait_context.reserve(); + small_object_allocator alloc{}; + auto block_handling_task = alloc.new_object(ed, this->my_wait_context, this->my_execution_context, + this->my_body, this->my_feeder_holder.feeder_ptr(), + alloc); + + auto* block_iterator = block_handling_task->block_iteration_space.begin(); + for (; !(this->my_first == this->my_last) && block_handling_task->my_size < block_handling_type::max_block_size; ++this->my_first) { + // Move semantics are automatically used when supported by the iterator + new (block_iterator++) Item(*this->my_first); + ++block_handling_task->my_size; + } + + // Do not access this after spawn to avoid races + spawn(*this, this->my_execution_context); + return block_handling_task; + } +}; // class for_each_root_task - most generic implementation + +/** parallel_for_each algorithm root task - forward iterator based specialization + * Splits input range to blocks + @ingroup algorithms **/ +template +class for_each_root_task + : public for_each_root_task_base +{ + using base_type = for_each_root_task_base; +public: + using base_type::base_type; +private: + task* execute(execution_data& ed) override { + using block_handling_type = forward_block_handling_task; + if (this->my_first == this->my_last) { + this->my_wait_context.release(); + return nullptr; + } + + std::size_t block_size{0}; + Iterator first_block_element = this->my_first; + for (; !(this->my_first == this->my_last) && block_size < block_handling_type::max_block_size; ++this->my_first) { + ++block_size; + } + + this->my_wait_context.reserve(); + small_object_allocator alloc{}; + auto block_handling_task = alloc.new_object(ed, first_block_element, block_size, + this->my_wait_context, this->my_execution_context, + this->my_body, this->my_feeder_holder.feeder_ptr(), alloc); + + // Do not access this after spawn to avoid races + spawn(*this, this->my_execution_context); + return block_handling_task; + } +}; // class for_each_root_task - forward iterator based specialization + +/** parallel_for_each algorithm root task - random access iterator based specialization + * Splits input range to blocks + @ingroup algorithms **/ +template +class for_each_root_task + : public for_each_root_task_base +{ + using base_type = for_each_root_task_base; +public: + using base_type::base_type; +private: + task* execute(execution_data&) override { + tbb::parallel_for( + tbb::blocked_range(0, std::distance(this->my_first, this->my_last)), + parallel_for_body_wrapper(this->my_first, this->my_body, this->my_feeder_holder.feeder_ptr()) + , this->my_execution_context + ); + + this->my_wait_context.release(); + return nullptr; + } +}; // class for_each_root_task - random access iterator based specialization + +/** Helper for getting item type. If item type can be deduced from feeder - got it from feeder, + if feeder is generic - got item type from range. + @ingroup algorithms */ +template +auto feeder_argument_parser(void (Body::*)(Item, feeder&) const) -> FeederArg; + +template +decltype(feeder_argument_parser(&Body::operator())) get_item_type_impl(int); // for (T, feeder) +template Item get_item_type_impl(...); // stub + +template +using get_item_type = decltype(get_item_type_impl(0)); + +#if __TBB_CPP20_CONCEPTS_PRESENT +template +using feeder_item_type = std::remove_cvref_t>; + +template +concept parallel_for_each_iterator_body = + parallel_for_each_body, feeder_item_type>>; + +template +concept parallel_for_each_range_body = + parallel_for_each_body, feeder_item_type>>; +#endif + +/** Implements parallel iteration over a range. + @ingroup algorithms */ +template +void run_parallel_for_each( Iterator first, Iterator last, const Body& body, task_group_context& context) +{ + if (!(first == last)) { + using ItemType = get_item_type::value_type>; + wait_context_vertex w_context(0); + + for_each_root_task root_task(first, last, body, w_context, context); + + execute_and_wait(root_task, context, w_context.get_context(), context); + } +} + +/** \page parallel_for_each_body_req Requirements on parallel_for_each body + Class \c Body implementing the concept of parallel_for_each body must define: + - \code + B::operator()( + cv_item_type item, + feeder& feeder + ) const + + OR + + B::operator()( cv_item_type& item ) const + \endcode Process item. + May be invoked concurrently for the same \c this but different \c item. + + - \code item_type( const item_type& ) \endcode + Copy a work item. + - \code ~item_type() \endcode Destroy a work item +**/ + +/** \name parallel_for_each + See also requirements on \ref parallel_for_each_body_req "parallel_for_each Body". **/ +//@{ +//! Parallel iteration over a range, with optional addition of more work. +/** @ingroup algorithms */ +template + __TBB_requires(std::input_iterator && parallel_for_each_iterator_body) +void parallel_for_each(Iterator first, Iterator last, const Body& body) { + task_group_context context(PARALLEL_FOR_EACH); + run_parallel_for_each(first, last, body, context); +} + +template + __TBB_requires(container_based_sequence && parallel_for_each_range_body) +void parallel_for_each(Range& rng, const Body& body) { + parallel_for_each(std::begin(rng), std::end(rng), body); +} + +template + __TBB_requires(container_based_sequence && parallel_for_each_range_body) +void parallel_for_each(const Range& rng, const Body& body) { + parallel_for_each(std::begin(rng), std::end(rng), body); +} + +//! Parallel iteration over a range, with optional addition of more work and user-supplied context +/** @ingroup algorithms */ +template + __TBB_requires(std::input_iterator && parallel_for_each_iterator_body) +void parallel_for_each(Iterator first, Iterator last, const Body& body, task_group_context& context) { + run_parallel_for_each(first, last, body, context); +} + +template + __TBB_requires(container_based_sequence && parallel_for_each_range_body) +void parallel_for_each(Range& rng, const Body& body, task_group_context& context) { + parallel_for_each(std::begin(rng), std::end(rng), body, context); +} + +template + __TBB_requires(container_based_sequence && parallel_for_each_range_body) +void parallel_for_each(const Range& rng, const Body& body, task_group_context& context) { + parallel_for_each(std::begin(rng), std::end(rng), body, context); +} + +} // namespace d2 +} // namespace detail +//! @endcond +//@} + +inline namespace v1 { +using detail::d2::parallel_for_each; +using detail::d1::feeder; +} // namespace v1 + +} // namespace tbb + +#endif /* __TBB_parallel_for_each_H */ diff --git a/src/tbb/include/oneapi/tbb/parallel_invoke.h b/src/tbb/include/oneapi/tbb/parallel_invoke.h new file mode 100644 index 000000000..4bc5d8533 --- /dev/null +++ b/src/tbb/include/oneapi/tbb/parallel_invoke.h @@ -0,0 +1,227 @@ +/* + Copyright (c) 2005-2023 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_parallel_invoke_H +#define __TBB_parallel_invoke_H + +#include "detail/_config.h" +#include "detail/_namespace_injection.h" +#include "detail/_exception.h" +#include "detail/_task.h" +#include "detail/_template_helpers.h" +#include "detail/_small_object_pool.h" + +#include "task_group.h" + +#include +#include +#include + +namespace tbb { +namespace detail { +namespace d1 { + +//! Simple task object, executing user method +template +struct function_invoker : public task { + function_invoker(const Function& function, WaitObject& wait_ctx) : + my_function(function), + parent_wait_ctx(wait_ctx) + {} + + task* execute(execution_data& ed) override { + my_function(); + parent_wait_ctx.release(ed); + call_itt_task_notify(destroy, this); + return nullptr; + } + + task* cancel(execution_data& ed) override { + parent_wait_ctx.release(ed); + return nullptr; + } + + const Function& my_function; + WaitObject& parent_wait_ctx; +}; // struct function_invoker + +//! Task object for managing subroots in trinary task trees. +// Endowed with additional synchronization logic (compatible with wait object interfaces) to support +// continuation passing execution. This task spawns 2 function_invoker tasks with first and second functors +// and then executes first functor by itself. But only the last executed functor must destruct and deallocate +// the subroot task. +template +struct invoke_subroot_task : public task { + wait_context& root_wait_ctx; + std::atomic ref_count{0}; + bool child_spawned = false; + + const F1& self_invoked_functor; + function_invoker> f2_invoker; + function_invoker> f3_invoker; + + task_group_context& my_execution_context; + small_object_allocator my_allocator; + + invoke_subroot_task(const F1& f1, const F2& f2, const F3& f3, wait_context& wait_ctx, task_group_context& context, + small_object_allocator& alloc) : + root_wait_ctx(wait_ctx), + self_invoked_functor(f1), + f2_invoker(f2, *this), + f3_invoker(f3, *this), + my_execution_context(context), + my_allocator(alloc) + { + root_wait_ctx.reserve(); + } + + void finalize(const execution_data& ed) { + root_wait_ctx.release(); + + my_allocator.delete_object(this, ed); + } + + void release(const execution_data& ed) { + __TBB_ASSERT(ref_count > 0, nullptr); + call_itt_task_notify(releasing, this); + if( --ref_count == 0 ) { + call_itt_task_notify(acquired, this); + finalize(ed); + } + } + + task* execute(execution_data& ed) override { + ref_count.fetch_add(3, std::memory_order_relaxed); + spawn(f3_invoker, my_execution_context); + spawn(f2_invoker, my_execution_context); + self_invoked_functor(); + + release(ed); + return nullptr; + } + + task* cancel(execution_data& ed) override { + if( ref_count > 0 ) { // detect children spawn + release(ed); + } else { + finalize(ed); + } + return nullptr; + } +}; // struct subroot_task + +class invoke_root_task { +public: + invoke_root_task(wait_context& wc) : my_wait_context(wc) {} + void release(const execution_data&) { + my_wait_context.release(); + } +private: + wait_context& my_wait_context; +}; + +template +void invoke_recursive_separation(wait_context& root_wait_ctx, task_group_context& context, const F1& f1) { + root_wait_ctx.reserve(1); + invoke_root_task root(root_wait_ctx); + function_invoker invoker1(f1, root); + + execute_and_wait(invoker1, context, root_wait_ctx, context); +} + +template +void invoke_recursive_separation(wait_context& root_wait_ctx, task_group_context& context, const F1& f1, const F2& f2) { + root_wait_ctx.reserve(2); + invoke_root_task root(root_wait_ctx); + function_invoker invoker1(f1, root); + function_invoker invoker2(f2, root); + + spawn(invoker1, context); + execute_and_wait(invoker2, context, root_wait_ctx, context); +} + +template +void invoke_recursive_separation(wait_context& root_wait_ctx, task_group_context& context, const F1& f1, const F2& f2, const F3& f3) { + root_wait_ctx.reserve(3); + invoke_root_task root(root_wait_ctx); + function_invoker invoker1(f1, root); + function_invoker invoker2(f2, root); + function_invoker invoker3(f3, root); + + //TODO: implement sub root for two tasks (measure performance) + spawn(invoker1, context); + spawn(invoker2, context); + execute_and_wait(invoker3, context, root_wait_ctx, context); +} + +template +void invoke_recursive_separation(wait_context& root_wait_ctx, task_group_context& context, + const F1& f1, const F2& f2, const F3& f3, const Fs&... fs) { + small_object_allocator alloc{}; + auto sub_root = alloc.new_object>(f1, f2, f3, root_wait_ctx, context, alloc); + spawn(*sub_root, context); + + invoke_recursive_separation(root_wait_ctx, context, fs...); +} + +template +void parallel_invoke_impl(task_group_context& context, const Fs&... fs) { + static_assert(sizeof...(Fs) >= 2, "Parallel invoke may be called with at least two callable"); + wait_context root_wait_ctx{0}; + + invoke_recursive_separation(root_wait_ctx, context, fs...); +} + +template +void parallel_invoke_impl(const F1& f1, const Fs&... fs) { + static_assert(sizeof...(Fs) >= 1, "Parallel invoke may be called with at least two callable"); + task_group_context context(PARALLEL_INVOKE); + wait_context root_wait_ctx{0}; + + invoke_recursive_separation(root_wait_ctx, context, fs..., f1); +} + +//! Passes last argument of variadic pack as first for handling user provided task_group_context +template +struct invoke_helper; + +template +struct invoke_helper, T, Fs...> : invoke_helper, Fs...> {}; + +template +struct invoke_helper, T> { + void operator()(Fs&&... args, T&& t) { + parallel_invoke_impl(std::forward(t), std::forward(args)...); + } +}; + +//! Parallel execution of several function objects +// We need to pass parameter pack through forwarding reference, +// since this pack may contain task_group_context that must be passed via lvalue non-const reference +template +void parallel_invoke(Fs&&... fs) { + invoke_helper, Fs...>()(std::forward(fs)...); +} + +} // namespace d1 +} // namespace detail + +inline namespace v1 { +using detail::d1::parallel_invoke; +} // namespace v1 + +} // namespace tbb +#endif /* __TBB_parallel_invoke_H */ diff --git a/src/tbb/include/oneapi/tbb/parallel_pipeline.h b/src/tbb/include/oneapi/tbb/parallel_pipeline.h new file mode 100644 index 000000000..a204b9c4c --- /dev/null +++ b/src/tbb/include/oneapi/tbb/parallel_pipeline.h @@ -0,0 +1,153 @@ +/* + Copyright (c) 2005-2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_parallel_pipeline_H +#define __TBB_parallel_pipeline_H + +#include "detail/_pipeline_filters.h" +#include "detail/_config.h" +#include "detail/_namespace_injection.h" +#include "task_group.h" + +#include +#include +#include + +namespace tbb { +namespace detail { + +namespace r1 { +TBB_EXPORT void __TBB_EXPORTED_FUNC parallel_pipeline(task_group_context&, std::size_t, const d1::filter_node&); +} + +namespace d1 { + +enum class filter_mode : unsigned int +{ + //! processes multiple items in parallel and in no particular order + parallel = base_filter::filter_is_out_of_order, + //! processes items one at a time; all such filters process items in the same order + serial_in_order = base_filter::filter_is_serial, + //! processes items one at a time and in no particular order + serial_out_of_order = base_filter::filter_is_serial | base_filter::filter_is_out_of_order +}; +//! Class representing a chain of type-safe pipeline filters +/** @ingroup algorithms */ +template +class filter { + filter_node_ptr my_root; + filter( filter_node_ptr root ) : my_root(root) {} + friend void parallel_pipeline( size_t, const filter&, task_group_context& ); + template + friend filter make_filter( filter_mode, const Body& ); + template + friend filter operator&( const filter&, const filter& ); +public: + filter() = default; + filter( const filter& rhs ) : my_root(rhs.my_root) {} + filter( filter&& rhs ) : my_root(std::move(rhs.my_root)) {} + + void operator=(const filter& rhs) { + my_root = rhs.my_root; + } + void operator=( filter&& rhs ) { + my_root = std::move(rhs.my_root); + } + + template + filter( filter_mode mode, const Body& body ) : + my_root( new(r1::allocate_memory(sizeof(filter_node_leaf))) + filter_node_leaf(static_cast(mode), body) ) { + } + + filter& operator&=( const filter& right ) { + *this = *this & right; + return *this; + } + + void clear() { + // Like operator= with filter() on right side. + my_root = nullptr; + } +}; + +//! Create a filter to participate in parallel_pipeline +/** @ingroup algorithms */ +template +filter make_filter( filter_mode mode, const Body& body ) { + return filter_node_ptr( new(r1::allocate_memory(sizeof(filter_node_leaf))) + filter_node_leaf(static_cast(mode), body) ); +} + +//! Create a filter to participate in parallel_pipeline +/** @ingroup algorithms */ +template +filter, filter_output> make_filter( filter_mode mode, const Body& body ) { + return make_filter, filter_output>(mode, body); +} + +//! Composition of filters left and right. +/** @ingroup algorithms */ +template +filter operator&( const filter& left, const filter& right ) { + __TBB_ASSERT(left.my_root,"cannot use default-constructed filter as left argument of '&'"); + __TBB_ASSERT(right.my_root,"cannot use default-constructed filter as right argument of '&'"); + return filter_node_ptr( new (r1::allocate_memory(sizeof(filter_node))) filter_node(left.my_root,right.my_root) ); +} + +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT +template +filter(filter_mode, Body) +->filter, filter_output>; +#endif // __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +//! Parallel pipeline over chain of filters with user-supplied context. +/** @ingroup algorithms **/ +inline void parallel_pipeline(size_t max_number_of_live_tokens, const filter& filter_chain, task_group_context& context) { + r1::parallel_pipeline(context, max_number_of_live_tokens, *filter_chain.my_root); +} + +//! Parallel pipeline over chain of filters. +/** @ingroup algorithms **/ +inline void parallel_pipeline(size_t max_number_of_live_tokens, const filter& filter_chain) { + task_group_context context; + parallel_pipeline(max_number_of_live_tokens, filter_chain, context); +} + +//! Parallel pipeline over sequence of filters. +/** @ingroup algorithms **/ +template +void parallel_pipeline(size_t max_number_of_live_tokens, + const F1& filter1, + const F2& filter2, + FiltersContext&&... filters) { + parallel_pipeline(max_number_of_live_tokens, filter1 & filter2, std::forward(filters)...); +} + +} // namespace d1 +} // namespace detail + +inline namespace v1 +{ +using detail::d1::parallel_pipeline; +using detail::d1::filter; +using detail::d1::make_filter; +using detail::d1::filter_mode; +using detail::d1::flow_control; +} +} // tbb + +#endif /* __TBB_parallel_pipeline_H */ diff --git a/src/tbb/include/oneapi/tbb/parallel_reduce.h b/src/tbb/include/oneapi/tbb/parallel_reduce.h new file mode 100644 index 000000000..205c97ef9 --- /dev/null +++ b/src/tbb/include/oneapi/tbb/parallel_reduce.h @@ -0,0 +1,772 @@ +/* + Copyright (c) 2005-2024 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_parallel_reduce_H +#define __TBB_parallel_reduce_H + +#include +#include "detail/_namespace_injection.h" +#include "detail/_task.h" +#include "detail/_aligned_space.h" +#include "detail/_small_object_pool.h" +#include "detail/_range_common.h" + +#include "task_group.h" // task_group_context +#include "partitioner.h" +#include "profiling.h" + +namespace tbb { +namespace detail { +#if __TBB_CPP20_CONCEPTS_PRESENT +inline namespace d0 { + +template +concept parallel_reduce_body = splittable && + requires( Body& body, const Range& range, Body& rhs ) { + body(range); + body.join(rhs); + }; + +template +concept parallel_reduce_function = std::invocable&, + const Range&, Value&&> && + std::convertible_to&, + const Range&, Value&&>, + Value>; + +template +concept parallel_reduce_combine = std::invocable&, + Value&&, Value&&> && + std::convertible_to&, + Value&&, Value&&>, + Value>; + +} // namespace d0 +#endif // __TBB_CPP20_CONCEPTS_PRESENT +namespace d1 { + +//! Tree node type for parallel_reduce. +/** @ingroup algorithms */ +//TODO: consider folding tree via bypass execution(instead of manual folding) +// for better cancellation and critical tasks handling (performance measurements required). +template +struct reduction_tree_node : public tree_node { + tbb::detail::aligned_space zombie_space; + Body& left_body; + bool has_right_zombie{false}; + + reduction_tree_node(node* parent, int ref_count, Body& input_left_body, small_object_allocator& alloc) : + tree_node{parent, ref_count, alloc}, + left_body(input_left_body) /* gcc4.8 bug - braced-initialization doesn't work for class members of reference type */ + {} + + void join(task_group_context* context) { + if (has_right_zombie && !context->is_group_execution_cancelled()) + left_body.join(*zombie_space.begin()); + } + + ~reduction_tree_node() { + if( has_right_zombie ) zombie_space.begin()->~Body(); + } +}; + +//! Task type used to split the work of parallel_reduce. +/** @ingroup algorithms */ +template +struct start_reduce : public task { + Range my_range; + Body* my_body; + node* my_parent; + + typename Partitioner::task_partition_type my_partition; + small_object_allocator my_allocator; + bool is_right_child; + + task* execute(execution_data&) override; + task* cancel(execution_data&) override; + void finalize(const execution_data&); + + using tree_node_type = reduction_tree_node; + + //! Constructor reduce root task. + start_reduce( const Range& range, Body& body, Partitioner& partitioner, small_object_allocator& alloc ) : + my_range(range), + my_body(&body), + my_parent(nullptr), + my_partition(partitioner), + my_allocator(alloc), + is_right_child(false) {} + //! Splitting constructor used to generate children. + /** parent_ becomes left child. Newly constructed object is right child. */ + start_reduce( start_reduce& parent_, typename Partitioner::split_type& split_obj, small_object_allocator& alloc ) : + my_range(parent_.my_range, get_range_split_object(split_obj)), + my_body(parent_.my_body), + my_parent(nullptr), + my_partition(parent_.my_partition, split_obj), + my_allocator(alloc), + is_right_child(true) + { + parent_.is_right_child = false; + } + //! Construct right child from the given range as response to the demand. + /** parent_ remains left child. Newly constructed object is right child. */ + start_reduce( start_reduce& parent_, const Range& r, depth_t d, small_object_allocator& alloc ) : + my_range(r), + my_body(parent_.my_body), + my_parent(nullptr), + my_partition(parent_.my_partition, split()), + my_allocator(alloc), + is_right_child(true) + { + my_partition.align_depth( d ); + parent_.is_right_child = false; + } + static void run(const Range& range, Body& body, Partitioner& partitioner, task_group_context& context) { + if ( !range.empty() ) { + wait_node wn; + small_object_allocator alloc{}; + auto reduce_task = alloc.new_object(range, body, partitioner, alloc); + reduce_task->my_parent = &wn; + execute_and_wait(*reduce_task, context, wn.m_wait, context); + } + } + static void run(const Range& range, Body& body, Partitioner& partitioner) { + // Bound context prevents exceptions from body to affect nesting or sibling algorithms, + // and allows users to handle exceptions safely by wrapping parallel_reduce in the try-block. + task_group_context context(PARALLEL_REDUCE); + run(range, body, partitioner, context); + } + //! Run body for range, serves as callback for partitioner + void run_body( Range &r ) { + tbb::detail::invoke(*my_body, r); + } + + //! spawn right task, serves as callback for partitioner + void offer_work(typename Partitioner::split_type& split_obj, execution_data& ed) { + offer_work_impl(ed, *this, split_obj); + } + //! spawn right task, serves as callback for partitioner + void offer_work(const Range& r, depth_t d, execution_data& ed) { + offer_work_impl(ed, *this, r, d); + } + +private: + template + void offer_work_impl(execution_data& ed, Args&&... args) { + small_object_allocator alloc{}; + // New right child + auto right_child = alloc.new_object(ed, std::forward(args)..., alloc); + + // New root node as a continuation and ref count. Left and right child attach to the new parent. + right_child->my_parent = my_parent = alloc.new_object(ed, my_parent, 2, *my_body, alloc); + + // Spawn the right sibling + right_child->spawn_self(ed); + } + + void spawn_self(execution_data& ed) { + my_partition.spawn_task(*this, *context(ed)); + } +}; + +//! fold the tree and deallocate the task +template +void start_reduce::finalize(const execution_data& ed) { + // Get the current parent and wait object before an object destruction + node* parent = my_parent; + auto allocator = my_allocator; + // Task execution finished - destroy it + this->~start_reduce(); + // Unwind the tree decrementing the parent`s reference count + fold_tree(parent, ed); + allocator.deallocate(this, ed); +} + +//! Execute parallel_reduce task +template +task* start_reduce::execute(execution_data& ed) { + if (!is_same_affinity(ed)) { + my_partition.note_affinity(execution_slot(ed)); + } + my_partition.check_being_stolen(*this, ed); + + // The acquire barrier synchronizes the data pointed with my_body if the left + // task has already finished. + __TBB_ASSERT(my_parent, nullptr); + if( is_right_child && my_parent->m_ref_count.load(std::memory_order_acquire) == 2 ) { + tree_node_type* parent_ptr = static_cast(my_parent); + my_body = static_cast(new( parent_ptr->zombie_space.begin() ) Body(*my_body, split())); + parent_ptr->has_right_zombie = true; + } + __TBB_ASSERT(my_body != nullptr, "Incorrect body value"); + + my_partition.execute(*this, my_range, ed); + + finalize(ed); + return nullptr; +} + +//! Cancel parallel_reduce task +template +task* start_reduce::cancel(execution_data& ed) { + finalize(ed); + return nullptr; +} + +//! Tree node type for parallel_deterministic_reduce. +/** @ingroup algorithms */ +template +struct deterministic_reduction_tree_node : public tree_node { + Body right_body; + Body& left_body; + + deterministic_reduction_tree_node(node* parent, int ref_count, Body& input_left_body, small_object_allocator& alloc) : + tree_node{parent, ref_count, alloc}, + right_body{input_left_body, detail::split()}, + left_body(input_left_body) + {} + + void join(task_group_context* context) { + if (!context->is_group_execution_cancelled()) + left_body.join(right_body); + } +}; + +//! Task type used to split the work of parallel_deterministic_reduce. +/** @ingroup algorithms */ +template +struct start_deterministic_reduce : public task { + Range my_range; + Body& my_body; + node* my_parent; + + typename Partitioner::task_partition_type my_partition; + small_object_allocator my_allocator; + + task* execute(execution_data&) override; + task* cancel(execution_data&) override; + void finalize(const execution_data&); + + using tree_node_type = deterministic_reduction_tree_node; + + //! Constructor deterministic_reduce root task. + start_deterministic_reduce( const Range& range, Partitioner& partitioner, Body& body, small_object_allocator& alloc ) : + my_range(range), + my_body(body), + my_parent(nullptr), + my_partition(partitioner), + my_allocator(alloc) {} + //! Splitting constructor used to generate children. + /** parent_ becomes left child. Newly constructed object is right child. */ + start_deterministic_reduce( start_deterministic_reduce& parent_, typename Partitioner::split_type& split_obj, Body& body, + small_object_allocator& alloc ) : + my_range(parent_.my_range, get_range_split_object(split_obj)), + my_body(body), + my_parent(nullptr), + my_partition(parent_.my_partition, split_obj), + my_allocator(alloc) {} + static void run(const Range& range, Body& body, Partitioner& partitioner, task_group_context& context) { + if ( !range.empty() ) { + wait_node wn; + small_object_allocator alloc{}; + auto deterministic_reduce_task = + alloc.new_object(range, partitioner, body, alloc); + deterministic_reduce_task->my_parent = &wn; + execute_and_wait(*deterministic_reduce_task, context, wn.m_wait, context); + } + } + static void run(const Range& range, Body& body, Partitioner& partitioner) { + // Bound context prevents exceptions from body to affect nesting or sibling algorithms, + // and allows users to handle exceptions safely by wrapping parallel_deterministic_reduce + // in the try-block. + task_group_context context(PARALLEL_REDUCE); + run(range, body, partitioner, context); + } + //! Run body for range, serves as callback for partitioner + void run_body( Range &r ) { + tbb::detail::invoke(my_body, r); + } + //! Spawn right task, serves as callback for partitioner + void offer_work(typename Partitioner::split_type& split_obj, execution_data& ed) { + offer_work_impl(ed, *this, split_obj); + } +private: + template + void offer_work_impl(execution_data& ed, Args&&... args) { + small_object_allocator alloc{}; + // New root node as a continuation and ref count. Left and right child attach to the new parent. Split the body. + auto new_tree_node = alloc.new_object(ed, my_parent, 2, my_body, alloc); + + // New right child + auto right_child = alloc.new_object(ed, std::forward(args)..., new_tree_node->right_body, alloc); + + right_child->my_parent = my_parent = new_tree_node; + + // Spawn the right sibling + right_child->spawn_self(ed); + } + + void spawn_self(execution_data& ed) { + my_partition.spawn_task(*this, *context(ed)); + } +}; + +//! Fold the tree and deallocate the task +template +void start_deterministic_reduce::finalize(const execution_data& ed) { + // Get the current parent and wait object before an object destruction + node* parent = my_parent; + + auto allocator = my_allocator; + // Task execution finished - destroy it + this->~start_deterministic_reduce(); + // Unwind the tree decrementing the parent`s reference count + fold_tree(parent, ed); + allocator.deallocate(this, ed); +} + +//! Execute parallel_deterministic_reduce task +template +task* start_deterministic_reduce::execute(execution_data& ed) { + if (!is_same_affinity(ed)) { + my_partition.note_affinity(execution_slot(ed)); + } + my_partition.check_being_stolen(*this, ed); + + my_partition.execute(*this, my_range, ed); + + finalize(ed); + return nullptr; +} + +//! Cancel parallel_deterministic_reduce task +template +task* start_deterministic_reduce::cancel(execution_data& ed) { + finalize(ed); + return nullptr; +} + + +//! Auxiliary class for parallel_reduce; for internal use only. +/** The adaptor class that implements \ref parallel_reduce_body_req "parallel_reduce Body" + using given \ref parallel_reduce_lambda_req "anonymous function objects". + **/ +/** @ingroup algorithms */ +template +class lambda_reduce_body { +//TODO: decide if my_real_body, my_reduction, and my_identity_element should be copied or referenced +// (might require some performance measurements) + + const Value& my_identity_element; + const RealBody& my_real_body; + const Reduction& my_reduction; + Value my_value; + lambda_reduce_body& operator= ( const lambda_reduce_body& other ); +public: + lambda_reduce_body( const Value& identity, const RealBody& body, const Reduction& reduction ) + : my_identity_element(identity) + , my_real_body(body) + , my_reduction(reduction) + , my_value(identity) + { } + lambda_reduce_body( const lambda_reduce_body& other ) = default; + lambda_reduce_body( lambda_reduce_body& other, tbb::split ) + : my_identity_element(other.my_identity_element) + , my_real_body(other.my_real_body) + , my_reduction(other.my_reduction) + , my_value(other.my_identity_element) + { } + void operator()(Range& range) { + my_value = tbb::detail::invoke(my_real_body, range, std::move(my_value)); + } + + void join( lambda_reduce_body& rhs ) { + my_value = tbb::detail::invoke(my_reduction, std::move(my_value), std::move(rhs.my_value)); + } + + __TBB_nodiscard Value&& result() && noexcept { + return std::move(my_value); + } +}; + + +// Requirements on Range concept are documented in blocked_range.h + +/** \page parallel_reduce_body_req Requirements on parallel_reduce body + Class \c Body implementing the concept of parallel_reduce body must define: + - \code Body::Body( Body&, split ); \endcode Splitting constructor. + Must be able to run concurrently with operator() and method \c join + - \code Body::~Body(); \endcode Destructor + - \code void Body::operator()( Range& r ); \endcode Function call operator applying body to range \c r + and accumulating the result + - \code void Body::join( Body& b ); \endcode Join results. + The result in \c b should be merged into the result of \c this +**/ + +/** \page parallel_reduce_lambda_req Requirements on parallel_reduce anonymous function objects (lambda functions) + TO BE DOCUMENTED +**/ + +/** \name parallel_reduce + See also requirements on \ref range_req "Range" and \ref parallel_reduce_body_req "parallel_reduce Body". **/ +//@{ + +//! Parallel iteration with reduction and default partitioner. +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_reduce_body) +void parallel_reduce( const Range& range, Body& body ) { + start_reduce::run( range, body, __TBB_DEFAULT_PARTITIONER() ); +} + +//! Parallel iteration with reduction and simple_partitioner +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_reduce_body) +void parallel_reduce( const Range& range, Body& body, const simple_partitioner& partitioner ) { + start_reduce::run( range, body, partitioner ); +} + +//! Parallel iteration with reduction and auto_partitioner +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_reduce_body) +void parallel_reduce( const Range& range, Body& body, const auto_partitioner& partitioner ) { + start_reduce::run( range, body, partitioner ); +} + +//! Parallel iteration with reduction and static_partitioner +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_reduce_body) +void parallel_reduce( const Range& range, Body& body, const static_partitioner& partitioner ) { + start_reduce::run( range, body, partitioner ); +} + +//! Parallel iteration with reduction and affinity_partitioner +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_reduce_body) +void parallel_reduce( const Range& range, Body& body, affinity_partitioner& partitioner ) { + start_reduce::run( range, body, partitioner ); +} + +//! Parallel iteration with reduction, default partitioner and user-supplied context. +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_reduce_body) +void parallel_reduce( const Range& range, Body& body, task_group_context& context ) { + start_reduce::run( range, body, __TBB_DEFAULT_PARTITIONER(), context ); +} + +//! Parallel iteration with reduction, simple partitioner and user-supplied context. +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_reduce_body) +void parallel_reduce( const Range& range, Body& body, const simple_partitioner& partitioner, task_group_context& context ) { + start_reduce::run( range, body, partitioner, context ); +} + +//! Parallel iteration with reduction, auto_partitioner and user-supplied context +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_reduce_body) +void parallel_reduce( const Range& range, Body& body, const auto_partitioner& partitioner, task_group_context& context ) { + start_reduce::run( range, body, partitioner, context ); +} + +//! Parallel iteration with reduction, static_partitioner and user-supplied context +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_reduce_body) +void parallel_reduce( const Range& range, Body& body, const static_partitioner& partitioner, task_group_context& context ) { + start_reduce::run( range, body, partitioner, context ); +} + +//! Parallel iteration with reduction, affinity_partitioner and user-supplied context +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_reduce_body) +void parallel_reduce( const Range& range, Body& body, affinity_partitioner& partitioner, task_group_context& context ) { + start_reduce::run( range, body, partitioner, context ); +} +/** parallel_reduce overloads that work with anonymous function objects + (see also \ref parallel_reduce_lambda_req "requirements on parallel_reduce anonymous function objects"). **/ + +//! Parallel iteration with reduction and default partitioner. +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_reduce_function && + parallel_reduce_combine) +Value parallel_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction ) { + lambda_reduce_body body(identity, real_body, reduction); + start_reduce,const __TBB_DEFAULT_PARTITIONER> + ::run(range, body, __TBB_DEFAULT_PARTITIONER() ); + return std::move(body).result(); +} + +//! Parallel iteration with reduction and simple_partitioner. +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_reduce_function && + parallel_reduce_combine) +Value parallel_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, + const simple_partitioner& partitioner ) { + lambda_reduce_body body(identity, real_body, reduction); + start_reduce,const simple_partitioner> + ::run(range, body, partitioner ); + return std::move(body).result(); +} + +//! Parallel iteration with reduction and auto_partitioner +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_reduce_function && + parallel_reduce_combine) +Value parallel_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, + const auto_partitioner& partitioner ) { + lambda_reduce_body body(identity, real_body, reduction); + start_reduce,const auto_partitioner> + ::run( range, body, partitioner ); + return std::move(body).result(); +} + +//! Parallel iteration with reduction and static_partitioner +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_reduce_function && + parallel_reduce_combine) +Value parallel_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, + const static_partitioner& partitioner ) { + lambda_reduce_body body(identity, real_body, reduction); + start_reduce,const static_partitioner> + ::run( range, body, partitioner ); + return std::move(body).result(); +} + +//! Parallel iteration with reduction and affinity_partitioner +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_reduce_function && + parallel_reduce_combine) +Value parallel_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, + affinity_partitioner& partitioner ) { + lambda_reduce_body body(identity, real_body, reduction); + start_reduce,affinity_partitioner> + ::run( range, body, partitioner ); + return std::move(body).result(); +} + +//! Parallel iteration with reduction, default partitioner and user-supplied context. +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_reduce_function && + parallel_reduce_combine) +Value parallel_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, + task_group_context& context ) { + lambda_reduce_body body(identity, real_body, reduction); + start_reduce,const __TBB_DEFAULT_PARTITIONER> + ::run( range, body, __TBB_DEFAULT_PARTITIONER(), context ); + return std::move(body).result(); +} + +//! Parallel iteration with reduction, simple partitioner and user-supplied context. +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_reduce_function && + parallel_reduce_combine) +Value parallel_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, + const simple_partitioner& partitioner, task_group_context& context ) { + lambda_reduce_body body(identity, real_body, reduction); + start_reduce,const simple_partitioner> + ::run( range, body, partitioner, context ); + return std::move(body).result(); +} + +//! Parallel iteration with reduction, auto_partitioner and user-supplied context +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_reduce_function && + parallel_reduce_combine) +Value parallel_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, + const auto_partitioner& partitioner, task_group_context& context ) { + lambda_reduce_body body(identity, real_body, reduction); + start_reduce,const auto_partitioner> + ::run( range, body, partitioner, context ); + return std::move(body).result(); +} + +//! Parallel iteration with reduction, static_partitioner and user-supplied context +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_reduce_function && + parallel_reduce_combine) +Value parallel_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, + const static_partitioner& partitioner, task_group_context& context ) { + lambda_reduce_body body(identity, real_body, reduction); + start_reduce,const static_partitioner> + ::run( range, body, partitioner, context ); + return std::move(body).result(); +} + +//! Parallel iteration with reduction, affinity_partitioner and user-supplied context +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_reduce_function && + parallel_reduce_combine) +Value parallel_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, + affinity_partitioner& partitioner, task_group_context& context ) { + lambda_reduce_body body(identity, real_body, reduction); + start_reduce,affinity_partitioner> + ::run( range, body, partitioner, context ); + return std::move(body).result(); +} + +//! Parallel iteration with deterministic reduction and default simple partitioner. +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_reduce_body) +void parallel_deterministic_reduce( const Range& range, Body& body ) { + start_deterministic_reduce::run(range, body, simple_partitioner()); +} + +//! Parallel iteration with deterministic reduction and simple partitioner. +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_reduce_body) +void parallel_deterministic_reduce( const Range& range, Body& body, const simple_partitioner& partitioner ) { + start_deterministic_reduce::run(range, body, partitioner); +} + +//! Parallel iteration with deterministic reduction and static partitioner. +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_reduce_body) +void parallel_deterministic_reduce( const Range& range, Body& body, const static_partitioner& partitioner ) { + start_deterministic_reduce::run(range, body, partitioner); +} + +//! Parallel iteration with deterministic reduction, default simple partitioner and user-supplied context. +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_reduce_body) +void parallel_deterministic_reduce( const Range& range, Body& body, task_group_context& context ) { + start_deterministic_reduce::run( range, body, simple_partitioner(), context ); +} + +//! Parallel iteration with deterministic reduction, simple partitioner and user-supplied context. +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_reduce_body) +void parallel_deterministic_reduce( const Range& range, Body& body, const simple_partitioner& partitioner, task_group_context& context ) { + start_deterministic_reduce::run(range, body, partitioner, context); +} + +//! Parallel iteration with deterministic reduction, static partitioner and user-supplied context. +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_reduce_body) +void parallel_deterministic_reduce( const Range& range, Body& body, const static_partitioner& partitioner, task_group_context& context ) { + start_deterministic_reduce::run(range, body, partitioner, context); +} + +/** parallel_reduce overloads that work with anonymous function objects + (see also \ref parallel_reduce_lambda_req "requirements on parallel_reduce anonymous function objects"). **/ + +//! Parallel iteration with deterministic reduction and default simple partitioner. +// TODO: consider making static_partitioner the default +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_reduce_function && + parallel_reduce_combine) +Value parallel_deterministic_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction ) { + return parallel_deterministic_reduce(range, identity, real_body, reduction, simple_partitioner()); +} + +//! Parallel iteration with deterministic reduction and simple partitioner. +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_reduce_function && + parallel_reduce_combine) +Value parallel_deterministic_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, const simple_partitioner& partitioner ) { + lambda_reduce_body body(identity, real_body, reduction); + start_deterministic_reduce, const simple_partitioner> + ::run(range, body, partitioner); + return std::move(body).result(); +} + +//! Parallel iteration with deterministic reduction and static partitioner. +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_reduce_function && + parallel_reduce_combine) +Value parallel_deterministic_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, const static_partitioner& partitioner ) { + lambda_reduce_body body(identity, real_body, reduction); + start_deterministic_reduce, const static_partitioner> + ::run(range, body, partitioner); + return std::move(body).result(); +} + +//! Parallel iteration with deterministic reduction, default simple partitioner and user-supplied context. +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_reduce_function && + parallel_reduce_combine) +Value parallel_deterministic_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, + task_group_context& context ) { + return parallel_deterministic_reduce(range, identity, real_body, reduction, simple_partitioner(), context); +} + +//! Parallel iteration with deterministic reduction, simple partitioner and user-supplied context. +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_reduce_function && + parallel_reduce_combine) +Value parallel_deterministic_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, + const simple_partitioner& partitioner, task_group_context& context ) { + lambda_reduce_body body(identity, real_body, reduction); + start_deterministic_reduce, const simple_partitioner> + ::run(range, body, partitioner, context); + return std::move(body).result(); +} + +//! Parallel iteration with deterministic reduction, static partitioner and user-supplied context. +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_reduce_function && + parallel_reduce_combine) +Value parallel_deterministic_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, + const static_partitioner& partitioner, task_group_context& context ) { + lambda_reduce_body body(identity, real_body, reduction); + start_deterministic_reduce, const static_partitioner> + ::run(range, body, partitioner, context); + return std::move(body).result(); +} +//@} + +} // namespace d1 +} // namespace detail + +inline namespace v1 { +using detail::d1::parallel_reduce; +using detail::d1::parallel_deterministic_reduce; +// Split types +using detail::split; +using detail::proportional_split; +} // namespace v1 + +} // namespace tbb +#endif /* __TBB_parallel_reduce_H */ diff --git a/src/tbb/include/oneapi/tbb/parallel_scan.h b/src/tbb/include/oneapi/tbb/parallel_scan.h new file mode 100644 index 000000000..d624f7ebd --- /dev/null +++ b/src/tbb/include/oneapi/tbb/parallel_scan.h @@ -0,0 +1,630 @@ +/* + Copyright (c) 2005-2024 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_parallel_scan_H +#define __TBB_parallel_scan_H + +#include + +#include "detail/_config.h" +#include "detail/_namespace_injection.h" +#include "detail/_exception.h" +#include "detail/_task.h" + +#include "profiling.h" +#include "partitioner.h" +#include "blocked_range.h" +#include "task_group.h" + +namespace tbb { +namespace detail { +namespace d1 { + +//! Used to indicate that the initial scan is being performed. +/** @ingroup algorithms */ +struct pre_scan_tag { + static bool is_final_scan() {return false;} + operator bool() {return is_final_scan();} +}; + +//! Used to indicate that the final scan is being performed. +/** @ingroup algorithms */ +struct final_scan_tag { + static bool is_final_scan() {return true;} + operator bool() {return is_final_scan();} +}; + +template +struct sum_node; + +#if __TBB_CPP20_CONCEPTS_PRESENT +} // namespace d1 +namespace d0 { + +template +concept parallel_scan_body = splittable && + requires( Body& body, const Range& range, Body& other ) { + body(range, tbb::detail::d1::pre_scan_tag{}); + body(range, tbb::detail::d1::final_scan_tag{}); + body.reverse_join(other); + body.assign(other); + }; + +template +concept parallel_scan_function = std::invocable&, + const Range&, const Value&, bool> && + std::convertible_to&, + const Range&, const Value&, bool>, + Value>; + +template +concept parallel_scan_combine = std::invocable&, + const Value&, const Value&> && + std::convertible_to&, + const Value&, const Value&>, + Value>; + +} // namespace d0 +namespace d1 { +#endif // __TBB_CPP20_CONCEPTS_PRESENT + +//! Performs final scan for a leaf +/** @ingroup algorithms */ +template +struct final_sum : public task { +private: + using sum_node_type = sum_node; + Body m_body; + aligned_space m_range; + //! Where to put result of last subrange, or nullptr if not last subrange. + Body* m_stuff_last; + + wait_context& m_wait_context; + sum_node_type* m_parent = nullptr; +public: + small_object_allocator m_allocator; + final_sum( Body& body, wait_context& w_o, small_object_allocator& alloc ) : + m_body(body, split()), m_wait_context(w_o), m_allocator(alloc) { + poison_pointer(m_stuff_last); + } + + final_sum( final_sum& sum, small_object_allocator& alloc ) : + m_body(sum.m_body, split()), m_wait_context(sum.m_wait_context), m_allocator(alloc) { + poison_pointer(m_stuff_last); + } + + ~final_sum() { + m_range.begin()->~Range(); + } + void finish_construction( sum_node_type* parent, const Range& range, Body* stuff_last ) { + __TBB_ASSERT( m_parent == nullptr, nullptr ); + m_parent = parent; + new( m_range.begin() ) Range(range); + m_stuff_last = stuff_last; + } +private: + sum_node_type* release_parent() { + call_itt_task_notify(releasing, m_parent); + if (m_parent) { + auto parent = m_parent; + m_parent = nullptr; + if (parent->ref_count.fetch_sub(1) == 1) { + return parent; + } + } + else + m_wait_context.release(); + return nullptr; + } + sum_node_type* finalize(const execution_data& ed){ + sum_node_type* next_task = release_parent(); + m_allocator.delete_object(this, ed); + return next_task; + } + +public: + task* execute(execution_data& ed) override { + m_body( *m_range.begin(), final_scan_tag() ); + if( m_stuff_last ) + m_stuff_last->assign(m_body); + + return finalize(ed); + } + task* cancel(execution_data& ed) override { + return finalize(ed); + } + template + void operator()( const Range& r, Tag tag ) { + m_body( r, tag ); + } + void reverse_join( final_sum& a ) { + m_body.reverse_join(a.m_body); + } + void reverse_join( Body& body ) { + m_body.reverse_join(body); + } + void assign_to( Body& body ) { + body.assign(m_body); + } + void self_destroy(const execution_data& ed) { + m_allocator.delete_object(this, ed); + } +}; + +//! Split work to be done in the scan. +/** @ingroup algorithms */ +template +struct sum_node : public task { +private: + using final_sum_type = final_sum; +public: + final_sum_type *m_incoming; + final_sum_type *m_body; + Body *m_stuff_last; +private: + final_sum_type *m_left_sum; + sum_node *m_left; + sum_node *m_right; + bool m_left_is_final; + Range m_range; + wait_context& m_wait_context; + sum_node* m_parent; + small_object_allocator m_allocator; +public: + std::atomic ref_count{0}; + sum_node( const Range range, bool left_is_final_, sum_node* parent, wait_context& w_o, small_object_allocator& alloc ) : + m_stuff_last(nullptr), + m_left_sum(nullptr), + m_left(nullptr), + m_right(nullptr), + m_left_is_final(left_is_final_), + m_range(range), + m_wait_context(w_o), + m_parent(parent), + m_allocator(alloc) + { + if( m_parent ) + m_parent->ref_count.fetch_add(1); + // Poison fields that will be set by second pass. + poison_pointer(m_body); + poison_pointer(m_incoming); + } + + ~sum_node() { + if (m_parent) + m_parent->ref_count.fetch_sub(1); + } +private: + sum_node* release_parent() { + call_itt_task_notify(releasing, m_parent); + if (m_parent) { + auto parent = m_parent; + m_parent = nullptr; + if (parent->ref_count.fetch_sub(1) == 1) { + return parent; + } + } + else + m_wait_context.release(); + return nullptr; + } + task* create_child( const Range& range, final_sum_type& body, sum_node* child, final_sum_type* incoming, Body* stuff_last ) { + if( child ) { + __TBB_ASSERT( is_poisoned(child->m_body) && is_poisoned(child->m_incoming), nullptr ); + child->prepare_for_execution(body, incoming, stuff_last); + return child; + } else { + body.finish_construction(this, range, stuff_last); + return &body; + } + } + + sum_node* finalize(const execution_data& ed) { + sum_node* next_task = release_parent(); + m_allocator.delete_object(this, ed); + return next_task; + } + +public: + void prepare_for_execution(final_sum_type& body, final_sum_type* incoming, Body *stuff_last) { + this->m_body = &body; + this->m_incoming = incoming; + this->m_stuff_last = stuff_last; + } + task* execute(execution_data& ed) override { + if( m_body ) { + if( m_incoming ) + m_left_sum->reverse_join( *m_incoming ); + task* right_child = this->create_child(Range(m_range,split()), *m_left_sum, m_right, m_left_sum, m_stuff_last); + task* left_child = m_left_is_final ? nullptr : this->create_child(m_range, *m_body, m_left, m_incoming, nullptr); + ref_count = (left_child != nullptr) + (right_child != nullptr); + m_body = nullptr; + if( left_child ) { + spawn(*right_child, *ed.context); + return left_child; + } else { + return right_child; + } + } else { + return finalize(ed); + } + } + task* cancel(execution_data& ed) override { + return finalize(ed); + } + void self_destroy(const execution_data& ed) { + m_allocator.delete_object(this, ed); + } + template + friend struct start_scan; + + template + friend struct finish_scan; +}; + +//! Combine partial results +/** @ingroup algorithms */ +template +struct finish_scan : public task { +private: + using sum_node_type = sum_node; + using final_sum_type = final_sum; + final_sum_type** const m_sum_slot; + sum_node_type*& m_return_slot; + small_object_allocator m_allocator; +public: + std::atomic m_right_zombie; + sum_node_type& m_result; + std::atomic ref_count{2}; + finish_scan* m_parent; + wait_context& m_wait_context; + task* execute(execution_data& ed) override { + __TBB_ASSERT( m_result.ref_count.load() == static_cast((m_result.m_left!=nullptr)+(m_result.m_right!=nullptr)), nullptr ); + if( m_result.m_left ) + m_result.m_left_is_final = false; + final_sum_type* right_zombie = m_right_zombie.load(std::memory_order_acquire); + if( right_zombie && m_sum_slot ) + (*m_sum_slot)->reverse_join(*m_result.m_left_sum); + __TBB_ASSERT( !m_return_slot, nullptr ); + if( right_zombie || m_result.m_right ) { + m_return_slot = &m_result; + } else { + m_result.self_destroy(ed); + } + if( right_zombie && !m_sum_slot && !m_result.m_right ) { + right_zombie->self_destroy(ed); + m_right_zombie.store(nullptr, std::memory_order_relaxed); + } + return finalize(ed); + } + task* cancel(execution_data& ed) override { + return finalize(ed); + } + finish_scan(sum_node_type*& return_slot, final_sum_type** sum, sum_node_type& result_, finish_scan* parent, wait_context& w_o, small_object_allocator& alloc) : + m_sum_slot(sum), + m_return_slot(return_slot), + m_allocator(alloc), + m_right_zombie(nullptr), + m_result(result_), + m_parent(parent), + m_wait_context(w_o) + { + __TBB_ASSERT( !m_return_slot, nullptr ); + } +private: + finish_scan* release_parent() { + call_itt_task_notify(releasing, m_parent); + if (m_parent) { + auto parent = m_parent; + m_parent = nullptr; + if (parent->ref_count.fetch_sub(1) == 1) { + return parent; + } + } + else + m_wait_context.release(); + return nullptr; + } + finish_scan* finalize(const execution_data& ed) { + finish_scan* next_task = release_parent(); + m_allocator.delete_object(this, ed); + return next_task; + } +}; + +//! Initial task to split the work +/** @ingroup algorithms */ +template +struct start_scan : public task { +private: + using sum_node_type = sum_node; + using final_sum_type = final_sum; + using finish_pass1_type = finish_scan; + std::reference_wrapper m_return_slot; + Range m_range; + std::reference_wrapper m_body; + typename Partitioner::partition_type m_partition; + /** Non-null if caller is requesting total. */ + final_sum_type** m_sum_slot; + bool m_is_final; + bool m_is_right_child; + + finish_pass1_type* m_parent; + small_object_allocator m_allocator; + wait_context& m_wait_context; + + finish_pass1_type* release_parent() { + call_itt_task_notify(releasing, m_parent); + if (m_parent) { + auto parent = m_parent; + m_parent = nullptr; + if (parent->ref_count.fetch_sub(1) == 1) { + return parent; + } + } + else + m_wait_context.release(); + return nullptr; + } + + finish_pass1_type* finalize( const execution_data& ed ) { + finish_pass1_type* next_task = release_parent(); + m_allocator.delete_object(this, ed); + return next_task; + } + +public: + task* execute( execution_data& ) override; + task* cancel( execution_data& ed ) override { + return finalize(ed); + } + start_scan( sum_node_type*& return_slot, start_scan& parent, small_object_allocator& alloc ) : + m_return_slot(return_slot), + m_range(parent.m_range,split()), + m_body(parent.m_body), + m_partition(parent.m_partition,split()), + m_sum_slot(parent.m_sum_slot), + m_is_final(parent.m_is_final), + m_is_right_child(true), + m_parent(parent.m_parent), + m_allocator(alloc), + m_wait_context(parent.m_wait_context) + { + __TBB_ASSERT( !m_return_slot, nullptr ); + parent.m_is_right_child = false; + } + + start_scan( sum_node_type*& return_slot, const Range& range, final_sum_type& body, const Partitioner& partitioner, wait_context& w_o, small_object_allocator& alloc ) : + m_return_slot(return_slot), + m_range(range), + m_body(body), + m_partition(partitioner), + m_sum_slot(nullptr), + m_is_final(true), + m_is_right_child(false), + m_parent(nullptr), + m_allocator(alloc), + m_wait_context(w_o) + { + __TBB_ASSERT( !m_return_slot, nullptr ); + } + + static void run( const Range& range, Body& body, const Partitioner& partitioner ) { + if( !range.empty() ) { + task_group_context context(PARALLEL_SCAN); + + using start_pass1_type = start_scan; + sum_node_type* root = nullptr; + wait_context w_ctx{1}; + small_object_allocator alloc{}; + + auto& temp_body = *alloc.new_object(body, w_ctx, alloc); + temp_body.reverse_join(body); + + auto& pass1 = *alloc.new_object(/*m_return_slot=*/root, range, temp_body, partitioner, w_ctx, alloc); + + execute_and_wait(pass1, context, w_ctx, context); + if( root ) { + root->prepare_for_execution(temp_body, nullptr, &body); + w_ctx.reserve(); + execute_and_wait(*root, context, w_ctx, context); + } else { + temp_body.assign_to(body); + temp_body.finish_construction(nullptr, range, nullptr); + alloc.delete_object(&temp_body); + } + } + } +}; + +template +task* start_scan::execute( execution_data& ed ) { + // Inspecting m_parent->result.left_sum would ordinarily be a race condition. + // But we inspect it only if we are not a stolen task, in which case we + // know that task assigning to m_parent->result.left_sum has completed. + __TBB_ASSERT(!m_is_right_child || m_parent, "right child is never an orphan"); + bool treat_as_stolen = m_is_right_child && (is_stolen(ed) || &m_body.get()!=m_parent->m_result.m_left_sum); + if( treat_as_stolen ) { + // Invocation is for right child that has been really stolen or needs to be virtually stolen + small_object_allocator alloc{}; + final_sum_type* right_zombie = alloc.new_object(m_body, alloc); + m_parent->m_right_zombie.store(right_zombie, std::memory_order_release); + m_body = *right_zombie; + m_is_final = false; + } + task* next_task = nullptr; + if( (m_is_right_child && !treat_as_stolen) || !m_range.is_divisible() || m_partition.should_execute_range(ed) ) { + if( m_is_final ) + m_body(m_range, final_scan_tag()); + else if( m_sum_slot ) + m_body(m_range, pre_scan_tag()); + if( m_sum_slot ) + *m_sum_slot = &m_body.get(); + __TBB_ASSERT( !m_return_slot, nullptr ); + + next_task = finalize(ed); + } else { + small_object_allocator alloc{}; + auto result = alloc.new_object(m_range,/*m_left_is_final=*/m_is_final, m_parent? &m_parent->m_result: nullptr, m_wait_context, alloc); + + auto new_parent = alloc.new_object(m_return_slot, m_sum_slot, *result, m_parent, m_wait_context, alloc); + m_parent = new_parent; + + // Split off right child + auto& right_child = *alloc.new_object(/*m_return_slot=*/result->m_right, *this, alloc); + + spawn(right_child, *ed.context); + + m_sum_slot = &result->m_left_sum; + m_return_slot = result->m_left; + + __TBB_ASSERT( !m_return_slot, nullptr ); + next_task = this; + } + return next_task; +} + +template +class lambda_scan_body { + Value m_sum_slot; + const Value& identity_element; + const Scan& m_scan; + const ReverseJoin& m_reverse_join; +public: + void operator=(const lambda_scan_body&) = delete; + lambda_scan_body(const lambda_scan_body&) = default; + + lambda_scan_body( const Value& identity, const Scan& scan, const ReverseJoin& rev_join ) + : m_sum_slot(identity) + , identity_element(identity) + , m_scan(scan) + , m_reverse_join(rev_join) {} + + lambda_scan_body( lambda_scan_body& b, split ) + : m_sum_slot(b.identity_element) + , identity_element(b.identity_element) + , m_scan(b.m_scan) + , m_reverse_join(b.m_reverse_join) {} + + template + void operator()( const Range& r, Tag tag ) { + m_sum_slot = tbb::detail::invoke(m_scan, r, m_sum_slot, tag); + } + + void reverse_join( lambda_scan_body& a ) { + m_sum_slot = tbb::detail::invoke(m_reverse_join, a.m_sum_slot, m_sum_slot); + } + + void assign( lambda_scan_body& b ) { + m_sum_slot = b.m_sum_slot; + } + + Value result() const { + return m_sum_slot; + } +}; + +// Requirements on Range concept are documented in blocked_range.h + +/** \page parallel_scan_body_req Requirements on parallel_scan body + Class \c Body implementing the concept of parallel_scan body must define: + - \code Body::Body( Body&, split ); \endcode Splitting constructor. + Split \c b so that \c this and \c b can accumulate separately + - \code Body::~Body(); \endcode Destructor + - \code void Body::operator()( const Range& r, pre_scan_tag ); \endcode + Preprocess iterations for range \c r + - \code void Body::operator()( const Range& r, final_scan_tag ); \endcode + Do final processing for iterations of range \c r + - \code void Body::reverse_join( Body& a ); \endcode + Merge preprocessing state of \c a into \c this, where \c a was + created earlier from \c b by b's splitting constructor +**/ + +/** \name parallel_scan + See also requirements on \ref range_req "Range" and \ref parallel_scan_body_req "parallel_scan Body". **/ +//@{ + +//! Parallel prefix with default partitioner +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_scan_body) +void parallel_scan( const Range& range, Body& body ) { + start_scan::run(range,body,__TBB_DEFAULT_PARTITIONER()); +} + +//! Parallel prefix with simple_partitioner +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_scan_body) +void parallel_scan( const Range& range, Body& body, const simple_partitioner& partitioner ) { + start_scan::run(range, body, partitioner); +} + +//! Parallel prefix with auto_partitioner +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_scan_body) +void parallel_scan( const Range& range, Body& body, const auto_partitioner& partitioner ) { + start_scan::run(range, body, partitioner); +} + +//! Parallel prefix with default partitioner +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_scan_function && + parallel_scan_combine) +Value parallel_scan( const Range& range, const Value& identity, const Scan& scan, const ReverseJoin& reverse_join ) { + lambda_scan_body body(identity, scan, reverse_join); + parallel_scan(range, body, __TBB_DEFAULT_PARTITIONER()); + return body.result(); +} + +//! Parallel prefix with simple_partitioner +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_scan_function && + parallel_scan_combine) +Value parallel_scan( const Range& range, const Value& identity, const Scan& scan, const ReverseJoin& reverse_join, + const simple_partitioner& partitioner ) { + lambda_scan_body body(identity, scan, reverse_join); + parallel_scan(range, body, partitioner); + return body.result(); +} + +//! Parallel prefix with auto_partitioner +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_scan_function && + parallel_scan_combine) +Value parallel_scan( const Range& range, const Value& identity, const Scan& scan, const ReverseJoin& reverse_join, + const auto_partitioner& partitioner ) { + lambda_scan_body body(identity, scan, reverse_join); + parallel_scan(range, body, partitioner); + return body.result(); +} + +} // namespace d1 +} // namespace detail + +inline namespace v1 { + using detail::d1::parallel_scan; + using detail::d1::pre_scan_tag; + using detail::d1::final_scan_tag; +} // namespace v1 + +} // namespace tbb + +#endif /* __TBB_parallel_scan_H */ diff --git a/src/tbb/include/oneapi/tbb/parallel_sort.h b/src/tbb/include/oneapi/tbb/parallel_sort.h new file mode 100644 index 000000000..4c0578eb0 --- /dev/null +++ b/src/tbb/include/oneapi/tbb/parallel_sort.h @@ -0,0 +1,288 @@ +/* + Copyright (c) 2005-2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_parallel_sort_H +#define __TBB_parallel_sort_H + +#include "detail/_namespace_injection.h" +#include "parallel_for.h" +#include "blocked_range.h" +#include "profiling.h" + +#include +#include +#include +#include + +namespace tbb { +namespace detail { +#if __TBB_CPP20_CONCEPTS_PRESENT +inline namespace d0 { + +// TODO: consider using std::strict_weak_order concept +template +concept compare = requires( const std::remove_reference_t& comp, typename std::iterator_traits::reference value ) { + // Forward via iterator_traits::reference + { comp(typename std::iterator_traits::reference(value), + typename std::iterator_traits::reference(value)) } -> std::convertible_to; +}; + +// Inspired by std::__PartiallyOrderedWith exposition only concept +template +concept less_than_comparable = requires( const std::remove_reference_t& lhs, + const std::remove_reference_t& rhs ) { + { lhs < rhs } -> boolean_testable; +}; + +} // namespace d0 +#endif // __TBB_CPP20_CONCEPTS_PRESENT +namespace d1 { + +//! Range used in quicksort to split elements into subranges based on a value. +/** The split operation selects a splitter and places all elements less than or equal + to the value in the first range and the remaining elements in the second range. + @ingroup algorithms */ +template +class quick_sort_range { + std::size_t median_of_three( const RandomAccessIterator& array, std::size_t l, std::size_t m, std::size_t r ) const { + return comp(array[l], array[m]) ? ( comp(array[m], array[r]) ? m : ( comp(array[l], array[r]) ? r : l ) ) + : ( comp(array[r], array[m]) ? m : ( comp(array[r], array[l]) ? r : l ) ); + } + + std::size_t pseudo_median_of_nine( const RandomAccessIterator& array, const quick_sort_range& range ) const { + std::size_t offset = range.size / 8u; + return median_of_three(array, + median_of_three(array, 0 , offset, offset * 2), + median_of_three(array, offset * 3, offset * 4, offset * 5), + median_of_three(array, offset * 6, offset * 7, range.size - 1)); + + } + + std::size_t split_range( quick_sort_range& range ) { + RandomAccessIterator array = range.begin; + RandomAccessIterator first_element = range.begin; + std::size_t m = pseudo_median_of_nine(array, range); + if( m != 0 ) std::iter_swap(array, array + m); + + std::size_t i = 0; + std::size_t j = range.size; + // Partition interval [i + 1,j - 1] with key *first_element. + for(;;) { + __TBB_ASSERT( i < j, nullptr ); + // Loop must terminate since array[l] == *first_element. + do { + --j; + __TBB_ASSERT( i <= j, "bad ordering relation?" ); + } while( comp(*first_element, array[j]) ); + do { + __TBB_ASSERT( i <= j, nullptr ); + if( i == j ) goto partition; + ++i; + } while( comp(array[i], *first_element) ); + if( i == j ) goto partition; + std::iter_swap(array + i, array + j); + } +partition: + // Put the partition key were it belongs + std::iter_swap(array + j, first_element); + // array[l..j) is less or equal to key. + // array(j..r) is greater or equal to key. + // array[j] is equal to key + i = j + 1; + std::size_t new_range_size = range.size - i; + range.size = j; + return new_range_size; + } + +public: + quick_sort_range() = default; + quick_sort_range( const quick_sort_range& ) = default; + void operator=( const quick_sort_range& ) = delete; + + static constexpr std::size_t grainsize = 500; + const Compare& comp; + std::size_t size; + RandomAccessIterator begin; + + quick_sort_range( RandomAccessIterator begin_, std::size_t size_, const Compare& comp_ ) : + comp(comp_), size(size_), begin(begin_) {} + + bool empty() const { return size == 0; } + bool is_divisible() const { return size >= grainsize; } + + quick_sort_range( quick_sort_range& range, split ) + : comp(range.comp) + , size(split_range(range)) + // +1 accounts for the pivot element, which is at its correct place + // already and, therefore, is not included into subranges. + , begin(range.begin + range.size + 1) {} +}; + +//! Body class used to test if elements in a range are presorted +/** @ingroup algorithms */ +template +class quick_sort_pretest_body { + const Compare& comp; + task_group_context& context; + +public: + quick_sort_pretest_body() = default; + quick_sort_pretest_body( const quick_sort_pretest_body& ) = default; + void operator=( const quick_sort_pretest_body& ) = delete; + + quick_sort_pretest_body( const Compare& _comp, task_group_context& _context ) : comp(_comp), context(_context) {} + + void operator()( const blocked_range& range ) const { + RandomAccessIterator my_end = range.end(); + + int i = 0; + //TODO: consider using std::is_sorted() for each 64 iterations (requires performance measurements) + for( RandomAccessIterator k = range.begin(); k != my_end; ++k, ++i ) { + if( i % 64 == 0 && context.is_group_execution_cancelled() ) break; + + // The k - 1 is never out-of-range because the first chunk starts at begin+serial_cutoff+1 + if( comp(*(k), *(k - 1)) ) { + context.cancel_group_execution(); + break; + } + } + } +}; + +//! Body class used to sort elements in a range that is smaller than the grainsize. +/** @ingroup algorithms */ +template +struct quick_sort_body { + void operator()( const quick_sort_range& range ) const { + std::sort(range.begin, range.begin + range.size, range.comp); + } +}; + +//! Method to perform parallel_for based quick sort. +/** @ingroup algorithms */ +template +void do_parallel_quick_sort( RandomAccessIterator begin, RandomAccessIterator end, const Compare& comp ) { + parallel_for(quick_sort_range(begin, end - begin, comp), + quick_sort_body(), + auto_partitioner()); +} + +//! Wrapper method to initiate the sort by calling parallel_for. +/** @ingroup algorithms */ +template +void parallel_quick_sort( RandomAccessIterator begin, RandomAccessIterator end, const Compare& comp ) { + task_group_context my_context(PARALLEL_SORT); + constexpr int serial_cutoff = 9; + + __TBB_ASSERT( begin + serial_cutoff < end, "min_parallel_size is smaller than serial cutoff?" ); + RandomAccessIterator k = begin; + for( ; k != begin + serial_cutoff; ++k ) { + if( comp(*(k + 1), *k) ) { + do_parallel_quick_sort(begin, end, comp); + return; + } + } + + // Check is input range already sorted + parallel_for(blocked_range(k + 1, end), + quick_sort_pretest_body(comp, my_context), + auto_partitioner(), + my_context); + + if( my_context.is_group_execution_cancelled() ) + do_parallel_quick_sort(begin, end, comp); +} + +/** \page parallel_sort_iter_req Requirements on iterators for parallel_sort + Requirements on the iterator type \c It and its value type \c T for \c parallel_sort: + + - \code void iter_swap( It a, It b ) \endcode Swaps the values of the elements the given + iterators \c a and \c b are pointing to. \c It should be a random access iterator. + + - \code bool Compare::operator()( const T& x, const T& y ) \endcode True if x comes before y; +**/ + +/** \name parallel_sort + See also requirements on \ref parallel_sort_iter_req "iterators for parallel_sort". **/ +//@{ + +#if __TBB_CPP20_CONCEPTS_PRESENT +template +using iter_value_type = typename std::iterator_traits::value_type; + +template +using range_value_type = typename std::iterator_traits>::value_type; +#endif + +//! Sorts the data in [begin,end) using the given comparator +/** The compare function object is used for all comparisons between elements during sorting. + The compare object must define a bool operator() function. + @ingroup algorithms **/ +template + __TBB_requires(std::random_access_iterator && + compare && + std::movable>) +void parallel_sort( RandomAccessIterator begin, RandomAccessIterator end, const Compare& comp ) { + constexpr int min_parallel_size = 500; + if( end > begin ) { + if( end - begin < min_parallel_size ) { + std::sort(begin, end, comp); + } else { + parallel_quick_sort(begin, end, comp); + } + } +} + +//! Sorts the data in [begin,end) with a default comparator \c std::less +/** @ingroup algorithms **/ +template + __TBB_requires(std::random_access_iterator && + less_than_comparable> && + std::movable>) +void parallel_sort( RandomAccessIterator begin, RandomAccessIterator end ) { + parallel_sort(begin, end, std::less::value_type>()); +} + +//! Sorts the data in rng using the given comparator +/** @ingroup algorithms **/ +template + __TBB_requires(container_based_sequence && + compare> && + std::movable>) +void parallel_sort( Range&& rng, const Compare& comp ) { + parallel_sort(std::begin(rng), std::end(rng), comp); +} + +//! Sorts the data in rng with a default comparator \c std::less +/** @ingroup algorithms **/ +template + __TBB_requires(container_based_sequence && + less_than_comparable> && + std::movable>) +void parallel_sort( Range&& rng ) { + parallel_sort(std::begin(rng), std::end(rng)); +} +//@} + +} // namespace d1 +} // namespace detail + +inline namespace v1 { + using detail::d1::parallel_sort; +} // namespace v1 +} // namespace tbb + +#endif /*__TBB_parallel_sort_H*/ diff --git a/inst/include/tbb/partitioner.h b/src/tbb/include/oneapi/tbb/partitioner.h similarity index 51% rename from inst/include/tbb/partitioner.h rename to src/tbb/include/oneapi/tbb/partitioner.h index 96a0b757a..58ff726fa 100644 --- a/inst/include/tbb/partitioner.h +++ b/src/tbb/include/oneapi/tbb/partitioner.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2023 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_partitioner_H @@ -37,112 +33,154 @@ // when imbalance is found range splits this value times more #define __TBB_DEMAND_DEPTH_ADD 1 #endif -#ifndef __TBB_STATIC_THRESHOLD -// necessary number of clocks for the work to be distributed among all tasks -#define __TBB_STATIC_THRESHOLD 40000 -#endif -#if __TBB_DEFINE_MIC -#define __TBB_NONUNIFORM_TASK_CREATION 1 -#ifdef __TBB_time_stamp -#define __TBB_USE_MACHINE_TIME_STAMPS 1 -#define __TBB_task_duration() __TBB_STATIC_THRESHOLD -#endif // __TBB_machine_time_stamp -#endif // __TBB_DEFINE_MIC - -#include "task.h" -#include "aligned_space.h" -#include "atomic.h" -#include "internal/_template_helpers.h" + +#include "detail/_config.h" +#include "detail/_namespace_injection.h" +#include "detail/_aligned_space.h" +#include "detail/_utils.h" +#include "detail/_template_helpers.h" +#include "detail/_range_common.h" +#include "detail/_task.h" +#include "detail/_small_object_pool.h" + +#include "cache_aligned_allocator.h" +#include "task_group.h" // task_group_context +#include "task_arena.h" + +#include +#include +#include #if defined(_MSC_VER) && !defined(__INTEL_COMPILER) // Workaround for overzealous compiler warnings - #pragma warning (push) - #pragma warning (disable: 4244) + // #pragma warning (push) + // #pragma warning (disable: 4244) #endif namespace tbb { +namespace detail { +namespace d1 { class auto_partitioner; class simple_partitioner; class static_partitioner; class affinity_partitioner; +class affinity_partition_type; +class affinity_partitioner_base; -namespace interface9 { - namespace internal { - class affinity_partition_type; - } +inline std::size_t get_initial_auto_partitioner_divisor() { + const std::size_t factor = 4; + return factor * static_cast(max_concurrency()); } -namespace internal { //< @cond INTERNAL -size_t __TBB_EXPORTED_FUNC get_initial_auto_partitioner_divisor(); - -//! Defines entry point for affinity partitioner into tbb run-time library. -class affinity_partitioner_base_v3: no_copy { - friend class tbb::affinity_partitioner; - friend class tbb::interface9::internal::affinity_partition_type; +//! Defines entry point for affinity partitioner into oneTBB run-time library. +class affinity_partitioner_base: no_copy { + friend class affinity_partitioner; + friend class affinity_partition_type; //! Array that remembers affinities of tree positions to affinity_id. - /** NULL if my_size==0. */ - affinity_id* my_array; + /** nullptr if my_size==0. */ + slot_id* my_array; //! Number of elements in my_array. - size_t my_size; + std::size_t my_size; //! Zeros the fields. - affinity_partitioner_base_v3() : my_array(NULL), my_size(0) {} + affinity_partitioner_base() : my_array(nullptr), my_size(0) {} //! Deallocates my_array. - ~affinity_partitioner_base_v3() {resize(0);} + ~affinity_partitioner_base() { resize(0); } //! Resize my_array. /** Retains values if resulting size is the same. */ - void __TBB_EXPORTED_METHOD resize( unsigned factor ); + void resize(unsigned factor) { + // Check factor to avoid asking for number of workers while there might be no arena. + unsigned max_threads_in_arena = static_cast(max_concurrency()); + std::size_t new_size = factor ? factor * max_threads_in_arena : 0; + if (new_size != my_size) { + if (my_array) { + r1::cache_aligned_deallocate(my_array); + // Following two assignments must be done here for sake of exception safety. + my_array = nullptr; + my_size = 0; + } + if (new_size) { + my_array = static_cast(r1::cache_aligned_allocate(new_size * sizeof(slot_id))); + std::fill_n(my_array, new_size, no_slot); + my_size = new_size; + } + } + } }; -//! Provides backward-compatible methods for partition objects without affinity. -class partition_type_base { -public: - void set_affinity( task & ) {} - void note_affinity( task::affinity_id ) {} - task* continue_after_execute_range() {return NULL;} - bool decide_whether_to_delay() {return false;} - void spawn_or_delay( bool, task& b ) { - task::spawn(b); +template struct start_for; +template struct start_scan; +template struct start_reduce; +template struct start_deterministic_reduce; + +struct node { + node* my_parent{}; + std::atomic m_ref_count{}; + + node() = default; + node(node* parent, int ref_count) : + my_parent{parent}, m_ref_count{ref_count} { + __TBB_ASSERT(ref_count > 0, "The ref count must be positive"); } }; -template class start_scan; +struct wait_node : node { + wait_node() : node{ nullptr, 1 } {} + wait_context m_wait{1}; +}; -} //< namespace internal @endcond +//! Join task node that contains shared flag for stealing feedback +struct tree_node : public node { + small_object_allocator m_allocator; + std::atomic m_child_stolen{false}; -namespace serial { -namespace interface9 { -template class start_for; -} -} + tree_node(node* parent, int ref_count, small_object_allocator& alloc) + : node{parent, ref_count} + , m_allocator{alloc} {} -namespace interface9 { -//! @cond INTERNAL -namespace internal { -using namespace tbb::internal; -template class start_for; -template class start_reduce; + void join(task_group_context*) {/*dummy, required only for reduction algorithms*/}; -//! Join task node that contains shared flag for stealing feedback -class flag_task: public task { -public: - tbb::atomic my_child_stolen; - flag_task() { my_child_stolen = false; } - task* execute() __TBB_override { return NULL; } - static void mark_task_stolen(task &t) { - tbb::atomic &flag = static_cast(t.parent())->my_child_stolen; -#if TBB_USE_THREADING_TOOLS + template + static void mark_task_stolen(Task &t) { + std::atomic &flag = static_cast(t.my_parent)->m_child_stolen; +#if TBB_USE_PROFILING_TOOLS // Threading tools respect lock prefix but report false-positive data-race via plain store - flag.fetch_and_store(true); + flag.exchange(true); #else - flag = true; -#endif //TBB_USE_THREADING_TOOLS + flag.store(true, std::memory_order_relaxed); +#endif // TBB_USE_PROFILING_TOOLS } - static bool is_peer_stolen(task &t) { - return static_cast(t.parent())->my_child_stolen; + template + static bool is_peer_stolen(Task &t) { + return static_cast(t.my_parent)->m_child_stolen.load(std::memory_order_relaxed); } }; +// Context used to check cancellation state during reduction join process +template +void fold_tree(node* n, const execution_data& ed) { + for (;;) { + __TBB_ASSERT(n, nullptr); + __TBB_ASSERT(n->m_ref_count.load(std::memory_order_relaxed) > 0, "The refcount must be positive."); + call_itt_task_notify(releasing, n); + if (--n->m_ref_count > 0) { + return; + } + node* parent = n->my_parent; + if (!parent) { + break; + }; + + call_itt_task_notify(acquired, n); + TreeNodeType* self = static_cast(n); + self->join(ed.context); + self->m_allocator.delete_object(self, ed); + n = parent; + } + // Finish parallel for execution when the root (last node) is reached + static_cast(n)->m_wait.release(); +} + //! Depth is a relative depth of recursive division inside a range pool. Relative depth allows //! infinite absolute depth of the recursion for heavily unbalanced workloads with range represented //! by a number that cannot fit into machine word. @@ -155,7 +193,7 @@ class range_vector { depth_t my_tail; depth_t my_size; depth_t my_depth[MaxCapacity]; // relative depths of stored ranges - tbb::aligned_space my_pool; + tbb::detail::aligned_space my_pool; public: //! initialize via first range in pool @@ -176,7 +214,7 @@ class range_vector { my_head = (my_head + 1) % MaxCapacity; new(my_pool.begin()+my_head) T(my_pool.begin()[prev]); // copy TODO: std::move? my_pool.begin()[prev].~T(); // instead of assignment - new(my_pool.begin()+prev) T(my_pool.begin()[my_head], split()); // do 'inverse' split + new(my_pool.begin()+prev) T(my_pool.begin()[my_head], detail::split()); // do 'inverse' split my_depth[my_head] = ++my_depth[prev]; my_size++; } @@ -218,25 +256,21 @@ class range_vector { //! Provides default methods for partition objects and common algorithm blocks. template struct partition_type_base { - typedef split split_type; + typedef detail::split split_type; // decision makers - void set_affinity( task & ) {} - void note_affinity( task::affinity_id ) {} - bool check_being_stolen(task &) { return false; } // part of old should_execute_range() - bool check_for_demand(task &) { return false; } - bool is_divisible() { return true; } // part of old should_execute_range() - depth_t max_depth() { return 0; } - void align_depth(depth_t) { } + void note_affinity( slot_id ) {} + template + bool check_being_stolen(Task&, const execution_data&) { return false; } // part of old should_execute_range() template split_type get_split() { return split(); } Partition& self() { return *static_cast(this); } // CRTP helper template - void work_balance(StartType &start, Range &range) { - start.run_body( range ); // simple partitioner goes always here + void work_balance(StartType &start, Range &range, const execution_data&) { + start.run_body( range ); // static partitioner goes here } template - void execute(StartType &start, Range &range) { + void execute(StartType &start, Range &range, execution_data& ed) { // The algorithm in a few words ([]-denotes calls to decision methods of partitioner): // [If this task is stolen, adjust depth and divisions if necessary, set flag]. // If range is divisible { @@ -248,11 +282,11 @@ struct partition_type_base { if ( self().is_divisible() ) { do { // split until is divisible typename Partition::split_type split_obj = self().template get_split(); - start.offer_work( split_obj ); + start.offer_work( split_obj, ed ); } while ( range.is_divisible() && self().is_divisible() ); } } - self().work_balance(start, range); + self().work_balance(start, range, ed); } }; @@ -260,118 +294,98 @@ struct partition_type_base { template struct adaptive_mode : partition_type_base { typedef Partition my_partition; - using partition_type_base::self; // CRTP helper to get access to derived classes - size_t my_divisor; + std::size_t my_divisor; // For affinity_partitioner, my_divisor indicates the number of affinity array indices the task reserves. // A task which has only one index must produce the right split without reserved index in order to avoid // it to be overwritten in note_affinity() of the created (right) task. // I.e. a task created deeper than the affinity array can remember must not save its affinity (LIFO order) static const unsigned factor = 1; - adaptive_mode() : my_divisor(tbb::internal::get_initial_auto_partitioner_divisor() / 4 * my_partition::factor) {} + adaptive_mode() : my_divisor(get_initial_auto_partitioner_divisor() / 4 * my_partition::factor) {} adaptive_mode(adaptive_mode &src, split) : my_divisor(do_split(src, split())) {} - adaptive_mode(adaptive_mode &src, const proportional_split& split_obj) : my_divisor(do_split(src, split_obj)) {} + adaptive_mode(adaptive_mode&, const proportional_split&) : my_divisor(0) + { + // left blank as my_divisor gets overridden in the successors' constructors + } /*! Override do_split methods in order to specify splitting strategy */ - size_t do_split(adaptive_mode &src, split) { + std::size_t do_split(adaptive_mode &src, split) { return src.my_divisor /= 2u; } - size_t do_split(adaptive_mode &src, const proportional_split& split_obj) { -#if __TBB_ENABLE_RANGE_FEEDBACK - size_t portion = size_t(float(src.my_divisor) * float(split_obj.right()) - / float(split_obj.left() + split_obj.right()) + 0.5f); -#else - size_t portion = split_obj.right() * my_partition::factor; -#endif +}; + + +//! Provides proportional splitting strategy for partition objects +template +struct proportional_mode : adaptive_mode { + typedef Partition my_partition; + using partition_type_base::self; // CRTP helper to get access to derived classes + + proportional_mode() : adaptive_mode() {} + proportional_mode(proportional_mode &src, split) : adaptive_mode(src, split()) {} + proportional_mode(proportional_mode &src, const proportional_split& split_obj) + : adaptive_mode(src, split_obj) + { + self().my_divisor = do_split(src, split_obj); + } + std::size_t do_split(proportional_mode &src, const proportional_split& split_obj) { + std::size_t portion = split_obj.right() * my_partition::factor; portion = (portion + my_partition::factor/2) & (0ul - my_partition::factor); -#if __TBB_ENABLE_RANGE_FEEDBACK - /** Corner case handling */ - if (!portion) - portion = my_partition::factor; - else if (portion == src.my_divisor) - portion = src.my_divisor - my_partition::factor; -#endif src.my_divisor -= portion; return portion; } bool is_divisible() { // part of old should_execute_range() - return my_divisor > my_partition::factor; + return self().my_divisor > my_partition::factor; } -}; + template + proportional_split get_split() { + // Create the proportion from partitioner internal resources (threads) that would be used: + // - into proportional_mode constructor to split the partitioner + // - if Range supports the proportional_split constructor it would use proposed proportion, + // otherwise, the tbb::proportional_split object will be implicitly (for Range implementer) + // casted to tbb::split -//! Provides default linear indexing of partitioner's sequence -template -struct linear_affinity_mode : adaptive_mode { - using adaptive_mode::my_divisor; - size_t my_head; - using adaptive_mode::self; - linear_affinity_mode() : adaptive_mode(), my_head(0) {} - linear_affinity_mode(linear_affinity_mode &src, split) : adaptive_mode(src, split()) - , my_head(src.my_head + src.my_divisor) {} - linear_affinity_mode(linear_affinity_mode &src, const proportional_split& split_obj) : adaptive_mode(src, split_obj) - , my_head(src.my_head + src.my_divisor) {} - void set_affinity( task &t ) { - if( my_divisor ) - t.set_affinity( affinity_id(my_head) + 1 ); + std::size_t n = self().my_divisor / my_partition::factor; + std::size_t right = n / 2; + std::size_t left = n - right; + return proportional_split(left, right); } }; -//! Class determines whether template parameter has static boolean constant -//! 'is_splittable_in_proportion' initialized with value of 'true' or not. -/** If template parameter has such field that has been initialized with non-zero - * value then class field will be set to 'true', otherwise - 'false' - */ -template -class is_splittable_in_proportion { -private: - typedef char yes[1]; - typedef char no [2]; - - template static yes& decide(typename enable_if::type *); - template static no& decide(...); -public: - // equals to 'true' if and only if static const variable 'is_splittable_in_proportion' of template parameter - // initialized with the value of 'true' - static const bool value = (sizeof(decide(0)) == sizeof(yes)); -}; +static std::size_t get_initial_partition_head() { + int current_index = tbb::this_task_arena::current_thread_index(); + if (current_index == tbb::task_arena::not_initialized) + current_index = 0; + return size_t(current_index); +} -//! Provides default methods for non-balancing partition objects. -template -struct unbalancing_partition_type : Mode { - using Mode::self; - unbalancing_partition_type() : Mode() {} - unbalancing_partition_type(unbalancing_partition_type& p, split) : Mode(p, split()) {} - unbalancing_partition_type(unbalancing_partition_type& p, const proportional_split& split_obj) : Mode(p, split_obj) {} -#if _MSC_VER && !defined(__INTEL_COMPILER) - // Suppress "conditional expression is constant" warning. - #pragma warning( push ) - #pragma warning( disable: 4127 ) -#endif - template - proportional_split get_split() { - if (is_splittable_in_proportion::value) { - size_t size = self().my_divisor / Mode::my_partition::factor; -#if __TBB_NONUNIFORM_TASK_CREATION - size_t right = (size + 2) / 3; -#else - size_t right = size / 2; -#endif - size_t left = size - right; - return proportional_split(left, right); +//! Provides default linear indexing of partitioner's sequence +template +struct linear_affinity_mode : proportional_mode { + std::size_t my_head; + std::size_t my_max_affinity; + using proportional_mode::self; + linear_affinity_mode() : proportional_mode(), my_head(get_initial_partition_head()), + my_max_affinity(self().my_divisor) {} + linear_affinity_mode(linear_affinity_mode &src, split) : proportional_mode(src, split()) + , my_head((src.my_head + src.my_divisor) % src.my_max_affinity), my_max_affinity(src.my_max_affinity) {} + linear_affinity_mode(linear_affinity_mode &src, const proportional_split& split_obj) : proportional_mode(src, split_obj) + , my_head((src.my_head + src.my_divisor) % src.my_max_affinity), my_max_affinity(src.my_max_affinity) {} + void spawn_task(task& t, task_group_context& ctx) { + if (self().my_divisor) { + spawn(t, ctx, slot_id(my_head)); } else { - return proportional_split(1, 1); + spawn(t, ctx); } } -#if _MSC_VER && !defined(__INTEL_COMPILER) - #pragma warning( pop ) -#endif // warning 4127 is back }; +static bool is_stolen_task(const execution_data& ed) { + return execution_slot(ed) != original_slot(ed); +} + /*! Determine work-balance phase implementing splitting & stealing actions */ template -struct balancing_partition_type : unbalancing_partition_type { +struct dynamic_grainsize_mode : Mode { using Mode::self; -#ifdef __TBB_USE_MACHINE_TIME_STAMPS - tbb::internal::machine_tsc_t my_dst_tsc; -#endif enum { begin = 0, run, @@ -379,38 +393,30 @@ struct balancing_partition_type : unbalancing_partition_type { } my_delay; depth_t my_max_depth; static const unsigned range_pool_size = __TBB_RANGE_POOL_CAPACITY; - balancing_partition_type(): unbalancing_partition_type() -#ifdef __TBB_USE_MACHINE_TIME_STAMPS - , my_dst_tsc(0) -#endif + dynamic_grainsize_mode(): Mode() , my_delay(begin) , my_max_depth(__TBB_INIT_DEPTH) {} - balancing_partition_type(balancing_partition_type& p, split) - : unbalancing_partition_type(p, split()) -#ifdef __TBB_USE_MACHINE_TIME_STAMPS - , my_dst_tsc(0) -#endif + dynamic_grainsize_mode(dynamic_grainsize_mode& p, split) + : Mode(p, split()) , my_delay(pass) , my_max_depth(p.my_max_depth) {} - balancing_partition_type(balancing_partition_type& p, const proportional_split& split_obj) - : unbalancing_partition_type(p, split_obj) -#ifdef __TBB_USE_MACHINE_TIME_STAMPS - , my_dst_tsc(0) -#endif + dynamic_grainsize_mode(dynamic_grainsize_mode& p, const proportional_split& split_obj) + : Mode(p, split_obj) , my_delay(begin) , my_max_depth(p.my_max_depth) {} - bool check_being_stolen( task &t) { // part of old should_execute_range() + template + bool check_being_stolen(Task &t, const execution_data& ed) { // part of old should_execute_range() if( !(self().my_divisor / Mode::my_partition::factor) ) { // if not from the top P tasks of binary tree self().my_divisor = 1; // TODO: replace by on-stack flag (partition_state's member)? - if( t.is_stolen_task() && t.parent()->ref_count() >= 2 ) { // runs concurrently with the left task + if( is_stolen_task(ed) && t.my_parent->m_ref_count >= 2 ) { // runs concurrently with the left task #if __TBB_USE_OPTIONAL_RTTI // RTTI is available, check whether the cast is valid - __TBB_ASSERT(dynamic_cast(t.parent()), 0); + // TODO: TBB_REVAMP_TODO __TBB_ASSERT(dynamic_cast(t.m_parent), 0); // correctness of the cast relies on avoiding the root task for which: // - initial value of my_divisor != 0 (protected by separate assertion) // - is_stolen_task() always returns false for the root task. #endif - flag_task::mark_task_stolen(t); + tree_node::mark_task_stolen(t); if( !my_max_depth ) my_max_depth++; my_max_depth += __TBB_DEMAND_DEPTH_ADD; return true; @@ -420,21 +426,21 @@ struct balancing_partition_type : unbalancing_partition_type { } depth_t max_depth() { return my_max_depth; } void align_depth(depth_t base) { - __TBB_ASSERT(base <= my_max_depth, 0); + __TBB_ASSERT(base <= my_max_depth, nullptr); my_max_depth -= base; } template - void work_balance(StartType &start, Range &range) { + void work_balance(StartType &start, Range &range, execution_data& ed) { if( !range.is_divisible() || !self().max_depth() ) { - start.run_body( range ); // simple partitioner goes always here + start.run_body( range ); } else { // do range pool - internal::range_vector range_pool(range); + range_vector range_pool(range); do { range_pool.split_to_fill(self().max_depth()); // fill range pool if( self().check_for_demand( start ) ) { if( range_pool.size() > 1 ) { - start.offer_work( range_pool.front(), range_pool.front_depth() ); + start.offer_work( range_pool.front(), range_pool.front_depth(), ed ); range_pool.pop_front(); continue; } @@ -443,49 +449,36 @@ struct balancing_partition_type : unbalancing_partition_type { } start.run_body( range_pool.back() ); range_pool.pop_back(); - } while( !range_pool.empty() && !start.is_cancelled() ); + } while( !range_pool.empty() && !ed.context->is_group_execution_cancelled() ); } } - bool check_for_demand( task &t ) { - if( pass == my_delay ) { - if( self().my_divisor > 1 ) // produce affinitized tasks while they have slot in array + template + bool check_for_demand(Task& t) { + if ( pass == my_delay ) { + if ( self().my_divisor > 1 ) // produce affinitized tasks while they have slot in array return true; // do not do my_max_depth++ here, but be sure range_pool is splittable once more - else if( self().my_divisor && my_max_depth ) { // make balancing task + else if ( self().my_divisor && my_max_depth ) { // make balancing task self().my_divisor = 0; // once for each task; depth will be decreased in align_depth() return true; } - else if( flag_task::is_peer_stolen(t) ) { + else if ( tree_node::is_peer_stolen(t) ) { my_max_depth += __TBB_DEMAND_DEPTH_ADD; return true; } } else if( begin == my_delay ) { -#ifndef __TBB_USE_MACHINE_TIME_STAMPS - my_delay = pass; -#else - my_dst_tsc = __TBB_time_stamp() + __TBB_task_duration(); - my_delay = run; - } else if( run == my_delay ) { - if( __TBB_time_stamp() < my_dst_tsc ) { - __TBB_ASSERT(my_max_depth > 0, NULL); - my_max_depth--; // increase granularity since tasks seem having too small work - return false; - } my_delay = pass; - return true; -#endif // __TBB_USE_MACHINE_TIME_STAMPS } return false; } }; -class auto_partition_type: public balancing_partition_type > { +class auto_partition_type: public dynamic_grainsize_mode > { public: - auto_partition_type( const auto_partitioner& ) - : balancing_partition_type >() { + auto_partition_type( const auto_partitioner& ) { my_divisor *= __TBB_INITIAL_CHUNKS; } auto_partition_type( auto_partition_type& src, split) - : balancing_partition_type >(src, split()) {} + : dynamic_grainsize_mode >(src, split()) {} bool is_divisible() { // part of old should_execute_range() if( my_divisor > 1 ) return true; if( my_divisor && my_max_depth ) { // can split the task. TODO: on-stack flag instead @@ -495,12 +488,16 @@ class auto_partition_type: public balancing_partition_type + bool check_for_demand(Task& t) { + if (tree_node::is_peer_stolen(t)) { my_max_depth += __TBB_DEMAND_DEPTH_ADD; return true; } else return false; } + void spawn_task(task& t, task_group_context& ctx) { + spawn(t, ctx); + } }; class simple_partition_type: public partition_type_base { @@ -509,83 +506,62 @@ class simple_partition_type: public partition_type_base { simple_partition_type( const simple_partition_type&, split ) {} //! simplified algorithm template - void execute(StartType &start, Range &range) { + void execute(StartType &start, Range &range, execution_data& ed) { split_type split_obj = split(); // start.offer_work accepts split_type as reference while( range.is_divisible() ) - start.offer_work( split_obj ); + start.offer_work( split_obj, ed ); start.run_body( range ); } + void spawn_task(task& t, task_group_context& ctx) { + spawn(t, ctx); + } }; -class static_partition_type : public unbalancing_partition_type > { +class static_partition_type : public linear_affinity_mode { public: - typedef proportional_split split_type; - static_partition_type( const static_partitioner& ) - : unbalancing_partition_type >() {} - static_partition_type( static_partition_type& p, split ) - : unbalancing_partition_type >(p, split()) {} + typedef detail::proportional_split split_type; + static_partition_type( const static_partitioner& ) {} static_partition_type( static_partition_type& p, const proportional_split& split_obj ) - : unbalancing_partition_type >(p, split_obj) {} + : linear_affinity_mode(p, split_obj) {} }; -class affinity_partition_type : public balancing_partition_type > { +class affinity_partition_type : public dynamic_grainsize_mode > { static const unsigned factor_power = 4; // TODO: get a unified formula based on number of computing units - tbb::internal::affinity_id* my_array; + slot_id* my_array; public: static const unsigned factor = 1 << factor_power; // number of slots in affinity array per task - typedef proportional_split split_type; - affinity_partition_type( tbb::internal::affinity_partitioner_base_v3& ap ) - : balancing_partition_type >() { + typedef detail::proportional_split split_type; + affinity_partition_type( affinity_partitioner_base& ap ) { __TBB_ASSERT( (factor&(factor-1))==0, "factor must be power of two" ); ap.resize(factor); my_array = ap.my_array; my_max_depth = factor_power + 1; - __TBB_ASSERT( my_max_depth < __TBB_RANGE_POOL_CAPACITY, 0 ); + __TBB_ASSERT( my_max_depth < __TBB_RANGE_POOL_CAPACITY, nullptr ); } affinity_partition_type(affinity_partition_type& p, split) - : balancing_partition_type >(p, split()) + : dynamic_grainsize_mode >(p, split()) , my_array(p.my_array) {} affinity_partition_type(affinity_partition_type& p, const proportional_split& split_obj) - : balancing_partition_type >(p, split_obj) + : dynamic_grainsize_mode >(p, split_obj) , my_array(p.my_array) {} - void set_affinity( task &t ) { - if( my_divisor ) { - if( !my_array[my_head] ) - // TODO: consider new ideas with my_array for both affinity and static partitioner's, then code reuse - t.set_affinity( affinity_id(my_head / factor + 1) ); - else - t.set_affinity( my_array[my_head] ); - } - } - void note_affinity( task::affinity_id id ) { + void note_affinity(slot_id id) { if( my_divisor ) my_array[my_head] = id; } -}; - -//! Backward-compatible partition for auto and affinity partition objects. -class old_auto_partition_type: public tbb::internal::partition_type_base { - size_t num_chunks; - static const size_t VICTIM_CHUNKS = 4; -public: - bool should_execute_range(const task &t) { - if( num_chunks friend class serial::interface9::start_for; - template friend class interface9::internal::start_for; - template friend class interface9::internal::start_reduce; - template friend class internal::start_scan; - // backward compatibility - class partition_type: public internal::partition_type_base { + template friend struct start_for; + template friend struct start_reduce; + template friend struct start_deterministic_reduce; + template friend struct start_scan; + // new implementation just extends existing interface + typedef simple_partition_type task_partition_type; + // TODO: consider to make split_type public + typedef simple_partition_type::split_type split_type; + + // for parallel_scan only + class partition_type { public: - bool should_execute_range(const task& ) {return false;} + bool should_execute_range(const execution_data& ) {return false;} partition_type( const simple_partitioner& ) {} partition_type( const partition_type&, split ) {} }; - // new implementation just extends existing interface - typedef interface9::internal::simple_partition_type task_partition_type; - - // TODO: consider to make split_type public - typedef interface9::internal::simple_partition_type::split_type split_type; }; //! An auto partitioner @@ -620,17 +596,31 @@ class auto_partitioner { auto_partitioner() {} private: - template friend class serial::interface9::start_for; - template friend class interface9::internal::start_for; - template friend class interface9::internal::start_reduce; - template friend class internal::start_scan; - // backward compatibility - typedef interface9::internal::old_auto_partition_type partition_type; + template friend struct start_for; + template friend struct start_reduce; + template friend struct start_deterministic_reduce; + template friend struct start_scan; // new implementation just extends existing interface - typedef interface9::internal::auto_partition_type task_partition_type; - + typedef auto_partition_type task_partition_type; // TODO: consider to make split_type public - typedef interface9::internal::auto_partition_type::split_type split_type; + typedef auto_partition_type::split_type split_type; + + //! Backward-compatible partition for auto and affinity partition objects. + class partition_type { + size_t num_chunks; + static const size_t VICTIM_CHUNKS = 4; + public: + bool should_execute_range(const execution_data& ed) { + if( num_chunks friend class serial::interface9::start_for; - template friend class interface9::internal::start_for; - template friend class interface9::internal::start_reduce; - template friend class internal::start_scan; - // backward compatibility - typedef interface9::internal::old_auto_partition_type partition_type; + template friend struct start_for; + template friend struct start_reduce; + template friend struct start_deterministic_reduce; + template friend struct start_scan; // new implementation just extends existing interface - typedef interface9::internal::static_partition_type task_partition_type; - + typedef static_partition_type task_partition_type; // TODO: consider to make split_type public - typedef interface9::internal::static_partition_type::split_type split_type; + typedef static_partition_type::split_type split_type; }; //! An affinity partitioner -class affinity_partitioner: internal::affinity_partitioner_base_v3 { +class affinity_partitioner : affinity_partitioner_base { public: affinity_partitioner() {} private: - template friend class serial::interface9::start_for; - template friend class interface9::internal::start_for; - template friend class interface9::internal::start_reduce; - template friend class internal::start_scan; - // backward compatibility - for parallel_scan only - typedef interface9::internal::old_auto_partition_type partition_type; + template friend struct start_for; + template friend struct start_reduce; + template friend struct start_deterministic_reduce; + template friend struct start_scan; // new implementation just extends existing interface - typedef interface9::internal::affinity_partition_type task_partition_type; - + typedef affinity_partition_type task_partition_type; // TODO: consider to make split_type public - typedef interface9::internal::affinity_partition_type::split_type split_type; + typedef affinity_partition_type::split_type split_type; }; +} // namespace d1 +} // namespace detail + +inline namespace v1 { +// Partitioners +using detail::d1::auto_partitioner; +using detail::d1::simple_partitioner; +using detail::d1::static_partitioner; +using detail::d1::affinity_partitioner; +// Split types +using detail::split; +using detail::proportional_split; +} // namespace v1 + } // namespace tbb #if defined(_MSC_VER) && !defined(__INTEL_COMPILER) - #pragma warning (pop) + // #pragma warning (pop) #endif // warning 4244 is back + #undef __TBB_INITIAL_CHUNKS #undef __TBB_RANGE_POOL_CAPACITY #undef __TBB_INIT_DEPTH + #endif /* __TBB_partitioner_H */ diff --git a/src/tbb/include/oneapi/tbb/profiling.h b/src/tbb/include/oneapi/tbb/profiling.h new file mode 100644 index 000000000..412b5a35e --- /dev/null +++ b/src/tbb/include/oneapi/tbb/profiling.h @@ -0,0 +1,243 @@ +/* + Copyright (c) 2005-2023 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_profiling_H +#define __TBB_profiling_H + +#include "detail/_config.h" +#include + +#include + +namespace tbb { +namespace detail { +inline namespace d0 { + // include list of index names + #define TBB_STRING_RESOURCE(index_name,str) index_name, + enum string_resource_index : std::uintptr_t { + #include "detail/_string_resource.h" + NUM_STRINGS + }; + #undef TBB_STRING_RESOURCE + + enum itt_relation + { + __itt_relation_is_unknown = 0, + __itt_relation_is_dependent_on, /**< "A is dependent on B" means that A cannot start until B completes */ + __itt_relation_is_sibling_of, /**< "A is sibling of B" means that A and B were created as a group */ + __itt_relation_is_parent_of, /**< "A is parent of B" means that A created B */ + __itt_relation_is_continuation_of, /**< "A is continuation of B" means that A assumes the dependencies of B */ + __itt_relation_is_child_of, /**< "A is child of B" means that A was created by B (inverse of is_parent_of) */ + __itt_relation_is_continued_by, /**< "A is continued by B" means that B assumes the dependencies of A (inverse of is_continuation_of) */ + __itt_relation_is_predecessor_to /**< "A is predecessor to B" means that B cannot start until A completes (inverse of is_dependent_on) */ + }; + +//! Unicode support +#if (_WIN32||_WIN64) + //! Unicode character type. Always wchar_t on Windows. + using tchar = wchar_t; +#else /* !WIN */ + using tchar = char; +#endif /* !WIN */ + +} // namespace d0 +} // namespace detail +} // namespace tbb + +#include +#if _WIN32||_WIN64 +#include /* mbstowcs_s */ +#endif +// Need these to work regardless of tools support +namespace tbb { +namespace detail { +namespace d1 { + enum notify_type {prepare=0, cancel, acquired, releasing, destroy}; + enum itt_domain_enum { ITT_DOMAIN_FLOW=0, ITT_DOMAIN_MAIN=1, ITT_DOMAIN_ALGO=2, ITT_NUM_DOMAINS }; +} // namespace d1 + +namespace r1 { + TBB_EXPORT void __TBB_EXPORTED_FUNC call_itt_notify(int t, void* ptr); + TBB_EXPORT void __TBB_EXPORTED_FUNC create_itt_sync(void* ptr, const tchar* objtype, const tchar* objname); + TBB_EXPORT void __TBB_EXPORTED_FUNC itt_make_task_group(d1::itt_domain_enum domain, void* group, unsigned long long group_extra, + void* parent, unsigned long long parent_extra, string_resource_index name_index); + TBB_EXPORT void __TBB_EXPORTED_FUNC itt_task_begin(d1::itt_domain_enum domain, void* task, unsigned long long task_extra, + void* parent, unsigned long long parent_extra, string_resource_index name_index); + TBB_EXPORT void __TBB_EXPORTED_FUNC itt_task_end(d1::itt_domain_enum domain); + TBB_EXPORT void __TBB_EXPORTED_FUNC itt_set_sync_name(void* obj, const tchar* name); + TBB_EXPORT void __TBB_EXPORTED_FUNC itt_metadata_str_add(d1::itt_domain_enum domain, void* addr, unsigned long long addr_extra, + string_resource_index key, const char* value); + TBB_EXPORT void __TBB_EXPORTED_FUNC itt_metadata_ptr_add(d1::itt_domain_enum domain, void* addr, unsigned long long addr_extra, + string_resource_index key, void* value); + TBB_EXPORT void __TBB_EXPORTED_FUNC itt_relation_add(d1::itt_domain_enum domain, void* addr0, unsigned long long addr0_extra, + itt_relation relation, void* addr1, unsigned long long addr1_extra); + TBB_EXPORT void __TBB_EXPORTED_FUNC itt_region_begin(d1::itt_domain_enum domain, void* region, unsigned long long region_extra, + void* parent, unsigned long long parent_extra, string_resource_index /* name_index */); + TBB_EXPORT void __TBB_EXPORTED_FUNC itt_region_end(d1::itt_domain_enum domain, void* region, unsigned long long region_extra); +} // namespace r1 + +namespace d1 { +#if TBB_USE_PROFILING_TOOLS && (_WIN32||_WIN64) + inline std::size_t multibyte_to_widechar(wchar_t* wcs, const char* mbs, std::size_t bufsize) { + std::size_t len; + mbstowcs_s(&len, wcs, bufsize, mbs, _TRUNCATE); + return len; // mbstowcs_s counts null terminator + } +#endif + +#if TBB_USE_PROFILING_TOOLS + inline void create_itt_sync(void *ptr, const char *objtype, const char *objname) { +#if (_WIN32||_WIN64) + std::size_t len_type = multibyte_to_widechar(nullptr, objtype, 0); + wchar_t *type = new wchar_t[len_type]; + multibyte_to_widechar(type, objtype, len_type); + std::size_t len_name = multibyte_to_widechar(nullptr, objname, 0); + wchar_t *name = new wchar_t[len_name]; + multibyte_to_widechar(name, objname, len_name); +#else // WIN + const char *type = objtype; + const char *name = objname; +#endif + r1::create_itt_sync(ptr, type, name); + +#if (_WIN32||_WIN64) + delete[] type; + delete[] name; +#endif // WIN + } + +// Distinguish notifications on task for reducing overheads +#if TBB_USE_PROFILING_TOOLS == 2 + inline void call_itt_task_notify(d1::notify_type t, void *ptr) { + r1::call_itt_notify(static_cast(t), ptr); + } +#else + inline void call_itt_task_notify(d1::notify_type, void *) {} +#endif // TBB_USE_PROFILING_TOOLS + + inline void call_itt_notify(d1::notify_type t, void *ptr) { + r1::call_itt_notify(static_cast(t), ptr); + } + +#if (_WIN32||_WIN64) && !__MINGW32__ + inline void itt_set_sync_name(void* obj, const wchar_t* name) { + r1::itt_set_sync_name(obj, name); + } + inline void itt_set_sync_name(void* obj, const char* name) { + std::size_t len_name = multibyte_to_widechar(nullptr, name, 0); + wchar_t *obj_name = new wchar_t[len_name]; + multibyte_to_widechar(obj_name, name, len_name); + r1::itt_set_sync_name(obj, obj_name); + delete[] obj_name; + } +#else + inline void itt_set_sync_name( void* obj, const char* name) { + r1::itt_set_sync_name(obj, name); + } +#endif //WIN + + inline void itt_make_task_group(itt_domain_enum domain, void* group, unsigned long long group_extra, + void* parent, unsigned long long parent_extra, string_resource_index name_index) { + r1::itt_make_task_group(domain, group, group_extra, parent, parent_extra, name_index); + } + + inline void itt_metadata_str_add( itt_domain_enum domain, void *addr, unsigned long long addr_extra, + string_resource_index key, const char *value ) { + r1::itt_metadata_str_add( domain, addr, addr_extra, key, value ); + } + + inline void register_node_addr(itt_domain_enum domain, void *addr, unsigned long long addr_extra, + string_resource_index key, void *value) { + r1::itt_metadata_ptr_add(domain, addr, addr_extra, key, value); + } + + inline void itt_relation_add( itt_domain_enum domain, void *addr0, unsigned long long addr0_extra, + itt_relation relation, void *addr1, unsigned long long addr1_extra ) { + r1::itt_relation_add( domain, addr0, addr0_extra, relation, addr1, addr1_extra ); + } + + inline void itt_task_begin( itt_domain_enum domain, void *task, unsigned long long task_extra, + void *parent, unsigned long long parent_extra, string_resource_index name_index ) { + r1::itt_task_begin( domain, task, task_extra, parent, parent_extra, name_index ); + } + + inline void itt_task_end( itt_domain_enum domain ) { + r1::itt_task_end( domain ); + } + + inline void itt_region_begin( itt_domain_enum domain, void *region, unsigned long long region_extra, + void *parent, unsigned long long parent_extra, string_resource_index name_index ) { + r1::itt_region_begin( domain, region, region_extra, parent, parent_extra, name_index ); + } + + inline void itt_region_end( itt_domain_enum domain, void *region, unsigned long long region_extra ) { + r1::itt_region_end( domain, region, region_extra ); + } +#else + inline void create_itt_sync(void* /*ptr*/, const char* /*objtype*/, const char* /*objname*/) {} + + inline void call_itt_notify(notify_type /*t*/, void* /*ptr*/) {} + + inline void call_itt_task_notify(notify_type /*t*/, void* /*ptr*/) {} +#endif // TBB_USE_PROFILING_TOOLS + +#if TBB_USE_PROFILING_TOOLS && !(TBB_USE_PROFILING_TOOLS == 2) +class event { +/** This class supports user event traces through itt. + Common use-case is tagging data flow graph tasks (data-id) + and visualization by Intel Advisor Flow Graph Analyzer (FGA) **/ +// TODO: Replace implementation by itt user event api. + + const std::string my_name; + + static void emit_trace(const std::string &input) { + itt_metadata_str_add( ITT_DOMAIN_FLOW, nullptr, FLOW_NULL, USER_EVENT, ( "FGA::DATAID::" + input ).c_str() ); + } + +public: + event(const std::string &input) + : my_name( input ) + { } + + void emit() { + emit_trace(my_name); + } + + static void emit(const std::string &description) { + emit_trace(description); + } + +}; +#else // TBB_USE_PROFILING_TOOLS && !(TBB_USE_PROFILING_TOOLS == 2) +// Using empty struct if user event tracing is disabled: +struct event { + event(const std::string &) { } + + void emit() { } + + static void emit(const std::string &) { } +}; +#endif // TBB_USE_PROFILING_TOOLS && !(TBB_USE_PROFILING_TOOLS == 2) +} // namespace d1 +} // namespace detail + +namespace profiling { + using detail::d1::event; +} +} // namespace tbb + + +#endif /* __TBB_profiling_H */ diff --git a/src/tbb/include/oneapi/tbb/queuing_mutex.h b/src/tbb/include/oneapi/tbb/queuing_mutex.h new file mode 100644 index 000000000..f7ee5a94d --- /dev/null +++ b/src/tbb/include/oneapi/tbb/queuing_mutex.h @@ -0,0 +1,192 @@ +/* + Copyright (c) 2005-2022 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_queuing_mutex_H +#define __TBB_queuing_mutex_H + +#include "detail/_namespace_injection.h" +#include "detail/_assert.h" +#include "detail/_utils.h" +#include "detail/_mutex_common.h" + +#include "profiling.h" + +#include + +namespace tbb { +namespace detail { +namespace d1 { + +//! Queuing mutex with local-only spinning. +/** @ingroup synchronization */ +class queuing_mutex { +public: + //! Construct unacquired mutex. + queuing_mutex() noexcept { + create_itt_sync(this, "tbb::queuing_mutex", ""); + }; + + queuing_mutex(const queuing_mutex&) = delete; + queuing_mutex& operator=(const queuing_mutex&) = delete; + + //! The scoped locking pattern + /** It helps to avoid the common problem of forgetting to release lock. + It also nicely provides the "node" for queuing locks. */ + class scoped_lock { + //! Reset fields to mean "no lock held". + void reset() { + m_mutex = nullptr; + } + + public: + //! Construct lock that has not acquired a mutex. + /** Equivalent to zero-initialization of *this. */ + scoped_lock() = default; + + //! Acquire lock on given mutex. + scoped_lock(queuing_mutex& m) { + acquire(m); + } + + //! Release lock (if lock is held). + ~scoped_lock() { + if (m_mutex) release(); + } + + //! No Copy + scoped_lock( const scoped_lock& ) = delete; + scoped_lock& operator=( const scoped_lock& ) = delete; + + //! Acquire lock on given mutex. + void acquire( queuing_mutex& m ) { + __TBB_ASSERT(!m_mutex, "scoped_lock is already holding a mutex"); + + // Must set all fields before the exchange, because once the + // exchange executes, *this becomes accessible to other threads. + m_mutex = &m; + m_next.store(nullptr, std::memory_order_relaxed); + m_going.store(0U, std::memory_order_relaxed); + + // x86 compare exchange operation always has a strong fence + // "sending" the fields initialized above to other processors. + scoped_lock* pred = m.q_tail.exchange(this); + if (pred) { + call_itt_notify(prepare, &m); + __TBB_ASSERT(pred->m_next.load(std::memory_order_relaxed) == nullptr, "the predecessor has another successor!"); + + pred->m_next.store(this, std::memory_order_release); + spin_wait_while_eq(m_going, 0U); + } + call_itt_notify(acquired, &m); + + } + + //! Acquire lock on given mutex if free (i.e. non-blocking) + bool try_acquire( queuing_mutex& m ) { + __TBB_ASSERT(!m_mutex, "scoped_lock is already holding a mutex"); + + // Must set all fields before the compare_exchange_strong, because once the + // compare_exchange_strong executes, *this becomes accessible to other threads. + m_next.store(nullptr, std::memory_order_relaxed); + m_going.store(0U, std::memory_order_relaxed); + + scoped_lock* expected = nullptr; + // The compare_exchange_strong must have release semantics, because we are + // "sending" the fields initialized above to other processors. + // x86 compare exchange operation always has a strong fence + if (!m.q_tail.compare_exchange_strong(expected, this, std::memory_order_acq_rel)) + return false; + + m_mutex = &m; + + call_itt_notify(acquired, &m); + return true; + } + + //! Release lock. + void release() + { + __TBB_ASSERT(this->m_mutex, "no lock acquired"); + + call_itt_notify(releasing, this->m_mutex); + + if (m_next.load(std::memory_order_relaxed) == nullptr) { + scoped_lock* expected = this; + if (m_mutex->q_tail.compare_exchange_strong(expected, nullptr)) { + // this was the only item in the queue, and the queue is now empty. + reset(); + return; + } + // Someone in the queue + spin_wait_while_eq(m_next, nullptr); + } + m_next.load(std::memory_order_acquire)->m_going.store(1U, std::memory_order_release); + + reset(); + } + + private: + //! The pointer to the mutex owned, or nullptr if not holding a mutex. + queuing_mutex* m_mutex{nullptr}; + + //! The pointer to the next competitor for a mutex + std::atomic m_next{nullptr}; + + //! The local spin-wait variable + /** Inverted (0 - blocked, 1 - acquired the mutex) for the sake of + zero-initialization. Defining it as an entire word instead of + a byte seems to help performance slightly. */ + std::atomic m_going{0U}; + }; + + // Mutex traits + static constexpr bool is_rw_mutex = false; + static constexpr bool is_recursive_mutex = false; + static constexpr bool is_fair_mutex = true; + +private: + //! The last competitor requesting the lock + std::atomic q_tail{nullptr}; + +}; + +#if TBB_USE_PROFILING_TOOLS +inline void set_name(queuing_mutex& obj, const char* name) { + itt_set_sync_name(&obj, name); +} +#if (_WIN32||_WIN64) +inline void set_name(queuing_mutex& obj, const wchar_t* name) { + itt_set_sync_name(&obj, name); +} +#endif //WIN +#else +inline void set_name(queuing_mutex&, const char*) {} +#if (_WIN32||_WIN64) +inline void set_name(queuing_mutex&, const wchar_t*) {} +#endif //WIN +#endif +} // namespace d1 +} // namespace detail + +inline namespace v1 { +using detail::d1::queuing_mutex; +} // namespace v1 +namespace profiling { + using detail::d1::set_name; +} +} // namespace tbb + +#endif /* __TBB_queuing_mutex_H */ diff --git a/src/tbb/include/oneapi/tbb/queuing_rw_mutex.h b/src/tbb/include/oneapi/tbb/queuing_rw_mutex.h new file mode 100644 index 000000000..f8325dfd9 --- /dev/null +++ b/src/tbb/include/oneapi/tbb/queuing_rw_mutex.h @@ -0,0 +1,207 @@ +/* + Copyright (c) 2005-2022 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_queuing_rw_mutex_H +#define __TBB_queuing_rw_mutex_H + +#include "detail/_config.h" +#include "detail/_namespace_injection.h" +#include "detail/_assert.h" +#include "detail/_mutex_common.h" + +#include "profiling.h" + +#include +#include + +namespace tbb { +namespace detail { +namespace r1 { +struct queuing_rw_mutex_impl; +} +namespace d1 { + +//! Queuing reader-writer mutex with local-only spinning. +/** Adapted from Krieger, Stumm, et al. pseudocode at + https://www.researchgate.net/publication/221083709_A_Fair_Fast_Scalable_Reader-Writer_Lock + @ingroup synchronization */ +class queuing_rw_mutex { + friend r1::queuing_rw_mutex_impl; +public: + //! Construct unacquired mutex. + queuing_rw_mutex() noexcept { + create_itt_sync(this, "tbb::queuing_rw_mutex", ""); + } + + //! Destructor asserts if the mutex is acquired, i.e. q_tail is non-null + ~queuing_rw_mutex() { + __TBB_ASSERT(q_tail.load(std::memory_order_relaxed) == nullptr, "destruction of an acquired mutex"); + } + + //! No Copy + queuing_rw_mutex(const queuing_rw_mutex&) = delete; + queuing_rw_mutex& operator=(const queuing_rw_mutex&) = delete; + + //! The scoped locking pattern + /** It helps to avoid the common problem of forgetting to release lock. + It also nicely provides the "node" for queuing locks. */ + class scoped_lock { + friend r1::queuing_rw_mutex_impl; + //! Initialize fields to mean "no lock held". + void initialize() { + my_mutex = nullptr; + my_internal_lock.store(0, std::memory_order_relaxed); + my_going.store(0, std::memory_order_relaxed); +#if TBB_USE_ASSERT + my_state = 0xFF; // Set to invalid state + my_next.store(reinterpret_cast(reinterpret_cast(-1)), std::memory_order_relaxed); + my_prev.store(reinterpret_cast(reinterpret_cast(-1)), std::memory_order_relaxed); +#endif /* TBB_USE_ASSERT */ + } + + public: + //! Construct lock that has not acquired a mutex. + /** Equivalent to zero-initialization of *this. */ + scoped_lock() {initialize();} + + //! Acquire lock on given mutex. + scoped_lock( queuing_rw_mutex& m, bool write=true ) { + initialize(); + acquire(m,write); + } + + //! Release lock (if lock is held). + ~scoped_lock() { + if( my_mutex ) release(); + } + + //! No Copy + scoped_lock(const scoped_lock&) = delete; + scoped_lock& operator=(const scoped_lock&) = delete; + + //! Acquire lock on given mutex. + void acquire( queuing_rw_mutex& m, bool write=true ); + + //! Acquire lock on given mutex if free (i.e. non-blocking) + bool try_acquire( queuing_rw_mutex& m, bool write=true ); + + //! Release lock. + void release(); + + //! Upgrade reader to become a writer. + /** Returns whether the upgrade happened without releasing and re-acquiring the lock */ + bool upgrade_to_writer(); + + //! Downgrade writer to become a reader. + bool downgrade_to_reader(); + + bool is_writer() const; + + private: + //! The pointer to the mutex owned, or nullptr if not holding a mutex. + queuing_rw_mutex* my_mutex; + + //! The 'pointer' to the previous and next competitors for a mutex + std::atomic my_prev; + std::atomic my_next; + + using state_t = unsigned char ; + + //! State of the request: reader, writer, active reader, other service states + std::atomic my_state; + + //! The local spin-wait variable + /** Corresponds to "spin" in the pseudocode but inverted for the sake of zero-initialization */ + std::atomic my_going; + + //! A tiny internal lock + std::atomic my_internal_lock; + }; + + // Mutex traits + static constexpr bool is_rw_mutex = true; + static constexpr bool is_recursive_mutex = false; + static constexpr bool is_fair_mutex = true; + +private: + //! The last competitor requesting the lock + std::atomic q_tail{nullptr}; +}; +#if TBB_USE_PROFILING_TOOLS +inline void set_name(queuing_rw_mutex& obj, const char* name) { + itt_set_sync_name(&obj, name); +} +#if (_WIN32||_WIN64) +inline void set_name(queuing_rw_mutex& obj, const wchar_t* name) { + itt_set_sync_name(&obj, name); +} +#endif //WIN +#else +inline void set_name(queuing_rw_mutex&, const char*) {} +#if (_WIN32||_WIN64) +inline void set_name(queuing_rw_mutex&, const wchar_t*) {} +#endif //WIN +#endif +} // namespace d1 + +namespace r1 { +TBB_EXPORT void acquire(d1::queuing_rw_mutex&, d1::queuing_rw_mutex::scoped_lock&, bool); +TBB_EXPORT bool try_acquire(d1::queuing_rw_mutex&, d1::queuing_rw_mutex::scoped_lock&, bool); +TBB_EXPORT void release(d1::queuing_rw_mutex::scoped_lock&); +TBB_EXPORT bool upgrade_to_writer(d1::queuing_rw_mutex::scoped_lock&); +TBB_EXPORT bool downgrade_to_reader(d1::queuing_rw_mutex::scoped_lock&); +TBB_EXPORT bool is_writer(const d1::queuing_rw_mutex::scoped_lock&); +} // namespace r1 + +namespace d1 { + + +inline void queuing_rw_mutex::scoped_lock::acquire(queuing_rw_mutex& m,bool write) { + r1::acquire(m, *this, write); +} + +inline bool queuing_rw_mutex::scoped_lock::try_acquire(queuing_rw_mutex& m, bool write) { + return r1::try_acquire(m, *this, write); +} + +inline void queuing_rw_mutex::scoped_lock::release() { + r1::release(*this); +} + +inline bool queuing_rw_mutex::scoped_lock::upgrade_to_writer() { + return r1::upgrade_to_writer(*this); +} + +inline bool queuing_rw_mutex::scoped_lock::downgrade_to_reader() { + return r1::downgrade_to_reader(*this); +} + +inline bool queuing_rw_mutex::scoped_lock::is_writer() const { + return r1::is_writer(*this); +} +} // namespace d1 + +} // namespace detail + +inline namespace v1 { +using detail::d1::queuing_rw_mutex; +} // namespace v1 +namespace profiling { + using detail::d1::set_name; +} +} // namespace tbb + +#endif /* __TBB_queuing_rw_mutex_H */ diff --git a/src/tbb/include/oneapi/tbb/rw_mutex.h b/src/tbb/include/oneapi/tbb/rw_mutex.h new file mode 100644 index 000000000..c3fbaf657 --- /dev/null +++ b/src/tbb/include/oneapi/tbb/rw_mutex.h @@ -0,0 +1,216 @@ +/* + Copyright (c) 2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_rw_mutex_H +#define __TBB_rw_mutex_H + +#include "detail/_namespace_injection.h" +#include "detail/_utils.h" +#include "detail/_waitable_atomic.h" +#include "detail/_scoped_lock.h" +#include "detail/_mutex_common.h" +#include "profiling.h" + +namespace tbb { +namespace detail { +namespace d1 { + +class rw_mutex { +public: + //! Constructors + rw_mutex() noexcept : m_state(0) { + create_itt_sync(this, "tbb::rw_mutex", ""); + } + + //! Destructor + ~rw_mutex() { + __TBB_ASSERT(!m_state.load(std::memory_order_relaxed), "destruction of an acquired mutex"); + } + + //! No Copy + rw_mutex(const rw_mutex&) = delete; + rw_mutex& operator=(const rw_mutex&) = delete; + + using scoped_lock = rw_scoped_lock; + + //! Mutex traits + static constexpr bool is_rw_mutex = true; + static constexpr bool is_recursive_mutex = false; + static constexpr bool is_fair_mutex = false; + + //! Acquire lock + void lock() { + call_itt_notify(prepare, this); + while (!try_lock()) { + if (!(m_state.load(std::memory_order_relaxed) & WRITER_PENDING)) { // no pending writers + m_state |= WRITER_PENDING; + } + + auto wakeup_condition = [&] { return !(m_state.load(std::memory_order_relaxed) & BUSY); }; + adaptive_wait_on_address(this, wakeup_condition, WRITER_CONTEXT); + } + + call_itt_notify(acquired, this); + } + + //! Try acquiring lock (non-blocking) + /** Return true if lock acquired; false otherwise. */ + bool try_lock() { + // for a writer: only possible to acquire if no active readers or writers + // Use relaxed memory fence is OK here because + // Acquire memory fence guaranteed by compare_exchange_strong() + state_type s = m_state.load(std::memory_order_relaxed); + if (!(s & BUSY)) { // no readers, no writers; mask is 1..1101 + if (m_state.compare_exchange_strong(s, WRITER)) { + call_itt_notify(acquired, this); + return true; // successfully stored writer flag + } + } + return false; + } + + //! Release lock + void unlock() { + call_itt_notify(releasing, this); + state_type curr_state = (m_state &= READERS | WRITER_PENDING); // Returns current state + + if (curr_state & WRITER_PENDING) { + r1::notify_by_address(this, WRITER_CONTEXT); + } else { + // It's possible that WRITER sleeps without WRITER_PENDING, + // because other thread might clear this bit at upgrade() + r1::notify_by_address_all(this); + } + } + + //! Lock shared ownership mutex + void lock_shared() { + call_itt_notify(prepare, this); + while (!try_lock_shared()) { + state_type has_writer = WRITER | WRITER_PENDING; + auto wakeup_condition = [&] { return !(m_state.load(std::memory_order_relaxed) & has_writer); }; + adaptive_wait_on_address(this, wakeup_condition, READER_CONTEXT); + } + __TBB_ASSERT(m_state.load(std::memory_order_relaxed) & READERS, "invalid state of a read lock: no readers"); + } + + //! Try lock shared ownership mutex + bool try_lock_shared() { + // for a reader: acquire if no active or waiting writers + // Use relaxed memory fence is OK here because + // Acquire memory fence guaranteed by fetch_add() + state_type has_writer = WRITER | WRITER_PENDING; + if (!(m_state.load(std::memory_order_relaxed) & has_writer)) { + if (m_state.fetch_add(ONE_READER) & has_writer) { + m_state -= ONE_READER; + r1::notify_by_address(this, WRITER_CONTEXT); + } else { + call_itt_notify(acquired, this); + return true; // successfully stored increased number of readers + } + } + return false; + } + + //! Unlock shared ownership mutex + void unlock_shared() { + __TBB_ASSERT(m_state.load(std::memory_order_relaxed) & READERS, "invalid state of a read lock: no readers"); + call_itt_notify(releasing, this); + + state_type curr_state = (m_state -= ONE_READER); // Returns current state + + if (curr_state & (WRITER_PENDING)) { + r1::notify_by_address(this, WRITER_CONTEXT); + } else { + // It's possible that WRITER sleeps without WRITER_PENDING, + // because other thread might clear this bit at upgrade() + r1::notify_by_address_all(this); + } + } + +private: + /** Internal non ISO C++ standard API **/ + //! This API is used through the scoped_lock class + + //! Upgrade reader to become a writer. + /** Returns whether the upgrade happened without releasing and re-acquiring the lock */ + bool upgrade() { + state_type s = m_state.load(std::memory_order_relaxed); + __TBB_ASSERT(s & READERS, "invalid state before upgrade: no readers "); + // Check and set writer-pending flag. + // Required conditions: either no pending writers, or we are the only reader + // (with multiple readers and pending writer, another upgrade could have been requested) + while ((s & READERS) == ONE_READER || !(s & WRITER_PENDING)) { + if (m_state.compare_exchange_strong(s, s | WRITER | WRITER_PENDING)) { + auto wakeup_condition = [&] { return (m_state.load(std::memory_order_relaxed) & READERS) == ONE_READER; }; + while ((m_state.load(std::memory_order_relaxed) & READERS) != ONE_READER) { + adaptive_wait_on_address(this, wakeup_condition, WRITER_CONTEXT); + } + + __TBB_ASSERT((m_state.load(std::memory_order_relaxed) & (WRITER_PENDING|WRITER)) == (WRITER_PENDING | WRITER), + "invalid state when upgrading to writer"); + // Both new readers and writers are blocked at this time + m_state -= (ONE_READER + WRITER_PENDING); + return true; // successfully upgraded + } + } + // Slow reacquire + unlock_shared(); + lock(); + return false; + } + + //! Downgrade writer to a reader + void downgrade() { + __TBB_ASSERT(m_state.load(std::memory_order_relaxed) & WRITER, nullptr), + call_itt_notify(releasing, this); + m_state += (ONE_READER - WRITER); + + if (!(m_state & WRITER_PENDING)) { + r1::notify_by_address(this, READER_CONTEXT); + } + + __TBB_ASSERT(m_state.load(std::memory_order_relaxed) & READERS, "invalid state after downgrade: no readers"); + } + + using state_type = std::intptr_t; + static constexpr state_type WRITER = 1; + static constexpr state_type WRITER_PENDING = 2; + static constexpr state_type READERS = ~(WRITER | WRITER_PENDING); + static constexpr state_type ONE_READER = 4; + static constexpr state_type BUSY = WRITER | READERS; + + using context_type = std::uintptr_t; + static constexpr context_type WRITER_CONTEXT = 0; + static constexpr context_type READER_CONTEXT = 1; + friend scoped_lock; + //! State of lock + /** Bit 0 = writer is holding lock + Bit 1 = request by a writer to acquire lock (hint to readers to wait) + Bit 2..N = number of readers holding lock */ + std::atomic m_state; +}; // class rw_mutex + +} // namespace d1 +} // namespace detail + +inline namespace v1 { +using detail::d1::rw_mutex; +} // namespace v1 + +} // namespace tbb + +#endif // __TBB_rw_mutex_H diff --git a/src/tbb/include/oneapi/tbb/scalable_allocator.h b/src/tbb/include/oneapi/tbb/scalable_allocator.h new file mode 100644 index 000000000..31650a0a7 --- /dev/null +++ b/src/tbb/include/oneapi/tbb/scalable_allocator.h @@ -0,0 +1,335 @@ +/* + Copyright (c) 2005-2023 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_scalable_allocator_H +#define __TBB_scalable_allocator_H + +#ifdef __cplusplus +#include "oneapi/tbb/detail/_config.h" +#include "oneapi/tbb/detail/_utils.h" +#include "oneapi/tbb/detail/_namespace_injection.h" +#include +#include +#include /* std::bad_alloc() */ +#else +#include "oneapi/tbb/detail/_export.h" +#include /* Need ptrdiff_t and size_t from here. */ +#if !defined(_MSC_VER) || defined(__clang__) +#include /* Need intptr_t from here. */ +#endif +#endif + +#if __TBB_CPP17_MEMORY_RESOURCE_PRESENT +#include +#endif + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +#if _MSC_VER + #define __TBB_EXPORTED_FUNC __cdecl +#else + #define __TBB_EXPORTED_FUNC +#endif + +/** The "malloc" analogue to allocate block of memory of size bytes. + * @ingroup memory_allocation */ +TBBMALLOC_EXPORT void* __TBB_EXPORTED_FUNC scalable_malloc(size_t size); + +/** The "free" analogue to discard a previously allocated piece of memory. + @ingroup memory_allocation */ +TBBMALLOC_EXPORT void __TBB_EXPORTED_FUNC scalable_free(void* ptr); + +/** The "realloc" analogue complementing scalable_malloc. + @ingroup memory_allocation */ +TBBMALLOC_EXPORT void* __TBB_EXPORTED_FUNC scalable_realloc(void* ptr, size_t size); + +/** The "calloc" analogue complementing scalable_malloc. + @ingroup memory_allocation */ +TBBMALLOC_EXPORT void* __TBB_EXPORTED_FUNC scalable_calloc(size_t nobj, size_t size); + +/** The "posix_memalign" analogue. + @ingroup memory_allocation */ +TBBMALLOC_EXPORT int __TBB_EXPORTED_FUNC scalable_posix_memalign(void** memptr, size_t alignment, size_t size); + +/** The "_aligned_malloc" analogue. + @ingroup memory_allocation */ +TBBMALLOC_EXPORT void* __TBB_EXPORTED_FUNC scalable_aligned_malloc(size_t size, size_t alignment); + +/** The "_aligned_realloc" analogue. + @ingroup memory_allocation */ +TBBMALLOC_EXPORT void* __TBB_EXPORTED_FUNC scalable_aligned_realloc(void* ptr, size_t size, size_t alignment); + +/** The "_aligned_free" analogue. + @ingroup memory_allocation */ +TBBMALLOC_EXPORT void __TBB_EXPORTED_FUNC scalable_aligned_free(void* ptr); + +/** The analogue of _msize/malloc_size/malloc_usable_size. + Returns the usable size of a memory block previously allocated by scalable_*, + or 0 (zero) if ptr does not point to such a block. + @ingroup memory_allocation */ +TBBMALLOC_EXPORT size_t __TBB_EXPORTED_FUNC scalable_msize(void* ptr); + +/* Results for scalable_allocation_* functions */ +typedef enum { + TBBMALLOC_OK, + TBBMALLOC_INVALID_PARAM, + TBBMALLOC_UNSUPPORTED, + TBBMALLOC_NO_MEMORY, + TBBMALLOC_NO_EFFECT +} ScalableAllocationResult; + +/* Setting TBB_MALLOC_USE_HUGE_PAGES environment variable to 1 enables huge pages. + scalable_allocation_mode call has priority over environment variable. */ +typedef enum { + TBBMALLOC_USE_HUGE_PAGES, /* value turns using huge pages on and off */ + /* deprecated, kept for backward compatibility only */ + USE_HUGE_PAGES = TBBMALLOC_USE_HUGE_PAGES, + /* try to limit memory consumption value (Bytes), clean internal buffers + if limit is exceeded, but not prevents from requesting memory from OS */ + TBBMALLOC_SET_SOFT_HEAP_LIMIT, + /* Lower bound for the size (Bytes), that is interpreted as huge + * and not released during regular cleanup operations. */ + TBBMALLOC_SET_HUGE_SIZE_THRESHOLD +} AllocationModeParam; + +/** Set TBB allocator-specific allocation modes. + @ingroup memory_allocation */ +TBBMALLOC_EXPORT int __TBB_EXPORTED_FUNC scalable_allocation_mode(int param, intptr_t value); + +typedef enum { + /* Clean internal allocator buffers for all threads. + Returns TBBMALLOC_NO_EFFECT if no buffers cleaned, + TBBMALLOC_OK if some memory released from buffers. */ + TBBMALLOC_CLEAN_ALL_BUFFERS, + /* Clean internal allocator buffer for current thread only. + Return values same as for TBBMALLOC_CLEAN_ALL_BUFFERS. */ + TBBMALLOC_CLEAN_THREAD_BUFFERS +} ScalableAllocationCmd; + +/** Call TBB allocator-specific commands. + @ingroup memory_allocation */ +TBBMALLOC_EXPORT int __TBB_EXPORTED_FUNC scalable_allocation_command(int cmd, void *param); + +#ifdef __cplusplus +} /* extern "C" */ +#endif /* __cplusplus */ + +#ifdef __cplusplus + +//! The namespace rml contains components of low-level memory pool interface. +namespace rml { +class MemoryPool; + +typedef void *(*rawAllocType)(std::intptr_t pool_id, std::size_t &bytes); +// returns non-zero in case of error +typedef int (*rawFreeType)(std::intptr_t pool_id, void* raw_ptr, std::size_t raw_bytes); + +struct MemPoolPolicy { + enum { + TBBMALLOC_POOL_VERSION = 1 + }; + + rawAllocType pAlloc; + rawFreeType pFree; + // granularity of pAlloc allocations. 0 means default used. + std::size_t granularity; + int version; + // all memory consumed at 1st pAlloc call and never returned, + // no more pAlloc calls after 1st + unsigned fixedPool : 1, + // memory consumed but returned only at pool termination + keepAllMemory : 1, + reserved : 30; + + MemPoolPolicy(rawAllocType pAlloc_, rawFreeType pFree_, + std::size_t granularity_ = 0, bool fixedPool_ = false, + bool keepAllMemory_ = false) : + pAlloc(pAlloc_), pFree(pFree_), granularity(granularity_), version(TBBMALLOC_POOL_VERSION), + fixedPool(fixedPool_), keepAllMemory(keepAllMemory_), + reserved(0) {} +}; + +// enums have same values as appropriate enums from ScalableAllocationResult +// TODO: use ScalableAllocationResult in pool_create directly +enum MemPoolError { + // pool created successfully + POOL_OK = TBBMALLOC_OK, + // invalid policy parameters found + INVALID_POLICY = TBBMALLOC_INVALID_PARAM, + // requested pool policy is not supported by allocator library + UNSUPPORTED_POLICY = TBBMALLOC_UNSUPPORTED, + // lack of memory during pool creation + NO_MEMORY = TBBMALLOC_NO_MEMORY, + // action takes no effect + NO_EFFECT = TBBMALLOC_NO_EFFECT +}; + +TBBMALLOC_EXPORT MemPoolError pool_create_v1(std::intptr_t pool_id, const MemPoolPolicy *policy, + rml::MemoryPool **pool); + +TBBMALLOC_EXPORT bool pool_destroy(MemoryPool* memPool); +TBBMALLOC_EXPORT void *pool_malloc(MemoryPool* memPool, std::size_t size); +TBBMALLOC_EXPORT void *pool_realloc(MemoryPool* memPool, void *object, std::size_t size); +TBBMALLOC_EXPORT void *pool_aligned_malloc(MemoryPool* mPool, std::size_t size, std::size_t alignment); +TBBMALLOC_EXPORT void *pool_aligned_realloc(MemoryPool* mPool, void *ptr, std::size_t size, std::size_t alignment); +TBBMALLOC_EXPORT bool pool_reset(MemoryPool* memPool); +TBBMALLOC_EXPORT bool pool_free(MemoryPool *memPool, void *object); +TBBMALLOC_EXPORT MemoryPool *pool_identify(void *object); +TBBMALLOC_EXPORT std::size_t pool_msize(MemoryPool *memPool, void *object); + +} // namespace rml + +namespace tbb { +namespace detail { +namespace d1 { + +// keep throw in a separate function to prevent code bloat +template +void throw_exception(const E &e) { +#if TBB_USE_EXCEPTIONS + throw e; +#else + suppress_unused_warning(e); +#endif +} + +template +class scalable_allocator { +public: + using value_type = T; + using propagate_on_container_move_assignment = std::true_type; + + //! Always defined for TBB containers + using is_always_equal = std::true_type; + + scalable_allocator() = default; + template scalable_allocator(const scalable_allocator&) noexcept {} + + //! Allocate space for n objects. + __TBB_nodiscard T* allocate(std::size_t n) { + T* p = static_cast(scalable_malloc(n * sizeof(value_type))); + if (!p) { + throw_exception(std::bad_alloc()); + } + return p; + } + + //! Free previously allocated block of memory + void deallocate(T* p, std::size_t) { + scalable_free(p); + } + +#if TBB_ALLOCATOR_TRAITS_BROKEN + using pointer = value_type*; + using const_pointer = const value_type*; + using reference = value_type&; + using const_reference = const value_type&; + using difference_type = std::ptrdiff_t; + using size_type = std::size_t; + template struct rebind { + using other = scalable_allocator; + }; + //! Largest value for which method allocate might succeed. + size_type max_size() const noexcept { + size_type absolutemax = static_cast(-1) / sizeof (value_type); + return (absolutemax > 0 ? absolutemax : 1); + } + template + void construct(U *p, Args&&... args) + { ::new((void *)p) U(std::forward(args)...); } + void destroy(pointer p) { p->~value_type(); } + pointer address(reference x) const { return &x; } + const_pointer address(const_reference x) const { return &x; } +#endif // TBB_ALLOCATOR_TRAITS_BROKEN + +}; + +#if TBB_ALLOCATOR_TRAITS_BROKEN + template<> + class scalable_allocator { + public: + using pointer = void*; + using const_pointer = const void*; + using value_type = void; + template struct rebind { + using other = scalable_allocator; + }; + }; +#endif + +template +inline bool operator==(const scalable_allocator&, const scalable_allocator&) noexcept { return true; } + +#if !__TBB_CPP20_COMPARISONS_PRESENT +template +inline bool operator!=(const scalable_allocator&, const scalable_allocator&) noexcept { return false; } +#endif + +#if __TBB_CPP17_MEMORY_RESOURCE_PRESENT + +//! C++17 memory resource implementation for scalable allocator +//! ISO C++ Section 23.12.2 +class scalable_resource_impl : public std::pmr::memory_resource { +private: + void* do_allocate(std::size_t bytes, std::size_t alignment) override { + void* p = scalable_aligned_malloc(bytes, alignment); + if (!p) { + throw_exception(std::bad_alloc()); + } + return p; + } + + void do_deallocate(void* ptr, std::size_t /*bytes*/, std::size_t /*alignment*/) override { + scalable_free(ptr); + } + + //! Memory allocated by one instance of scalable_resource_impl could be deallocated by any + //! other instance of this class + bool do_is_equal(const std::pmr::memory_resource& other) const noexcept override { + return this == &other || +#if __TBB_USE_OPTIONAL_RTTI + dynamic_cast(&other) != nullptr; +#else + false; +#endif + } +}; + +//! Global scalable allocator memory resource provider +inline std::pmr::memory_resource* scalable_memory_resource() noexcept { + static tbb::detail::d1::scalable_resource_impl scalable_res; + return &scalable_res; +} + +#endif // __TBB_CPP17_MEMORY_RESOURCE_PRESENT + +} // namespace d1 +} // namespace detail + +inline namespace v1 { +using detail::d1::scalable_allocator; +#if __TBB_CPP17_MEMORY_RESOURCE_PRESENT +using detail::d1::scalable_memory_resource; +#endif +} // namespace v1 + +} // namespace tbb + +#endif /* __cplusplus */ + +#endif /* __TBB_scalable_allocator_H */ diff --git a/src/tbb/include/oneapi/tbb/spin_mutex.h b/src/tbb/include/oneapi/tbb/spin_mutex.h new file mode 100644 index 000000000..e38c47c9d --- /dev/null +++ b/src/tbb/include/oneapi/tbb/spin_mutex.h @@ -0,0 +1,134 @@ +/* + Copyright (c) 2005-2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_spin_mutex_H +#define __TBB_spin_mutex_H + +#include "detail/_namespace_injection.h" +#include "detail/_mutex_common.h" + +#include "profiling.h" + +#include "detail/_assert.h" +#include "detail/_utils.h" +#include "detail/_scoped_lock.h" + +#include + +namespace tbb { +namespace detail { +namespace d1 { + +#if __TBB_TSX_INTRINSICS_PRESENT +class rtm_mutex; +#endif + +/** A spin_mutex is a low-level synchronization primitive. + While locked, it causes the waiting threads to spin in a loop until the lock is released. + It should be used only for locking short critical sections + (typically less than 20 instructions) when fairness is not an issue. + If zero-initialized, the mutex is considered unheld. + @ingroup synchronization */ +class spin_mutex { +public: + //! Constructors + spin_mutex() noexcept : m_flag(false) { + create_itt_sync(this, "tbb::spin_mutex", ""); + }; + + //! Destructor + ~spin_mutex() = default; + + //! No Copy + spin_mutex(const spin_mutex&) = delete; + spin_mutex& operator=(const spin_mutex&) = delete; + + using scoped_lock = unique_scoped_lock; + + //! Mutex traits + static constexpr bool is_rw_mutex = false; + static constexpr bool is_recursive_mutex = false; + static constexpr bool is_fair_mutex = false; + + //! Acquire lock + /** Spin if the lock is taken */ + void lock() { + atomic_backoff backoff; + call_itt_notify(prepare, this); + while (m_flag.exchange(true)) backoff.pause(); + call_itt_notify(acquired, this); + } + + //! Try acquiring lock (non-blocking) + /** Return true if lock acquired; false otherwise. */ + bool try_lock() { + bool result = !m_flag.exchange(true); + if (result) { + call_itt_notify(acquired, this); + } + return result; + } + + //! Release lock + void unlock() { + call_itt_notify(releasing, this); + m_flag.store(false, std::memory_order_release); + } + +protected: + std::atomic m_flag; +}; // class spin_mutex + +#if TBB_USE_PROFILING_TOOLS +inline void set_name(spin_mutex& obj, const char* name) { + itt_set_sync_name(&obj, name); +} +#if (_WIN32||_WIN64) +inline void set_name(spin_mutex& obj, const wchar_t* name) { + itt_set_sync_name(&obj, name); +} +#endif //WIN +#else +inline void set_name(spin_mutex&, const char*) {} +#if (_WIN32||_WIN64) +inline void set_name(spin_mutex&, const wchar_t*) {} +#endif // WIN +#endif +} // namespace d1 +} // namespace detail + +inline namespace v1 { +using detail::d1::spin_mutex; +} // namespace v1 +namespace profiling { + using detail::d1::set_name; +} +} // namespace tbb + +#include "detail/_rtm_mutex.h" + +namespace tbb { +inline namespace v1 { +#if __TBB_TSX_INTRINSICS_PRESENT + using speculative_spin_mutex = detail::d1::rtm_mutex; +#else + using speculative_spin_mutex = detail::d1::spin_mutex; +#endif +} +} + +#endif /* __TBB_spin_mutex_H */ + diff --git a/src/tbb/include/oneapi/tbb/spin_rw_mutex.h b/src/tbb/include/oneapi/tbb/spin_rw_mutex.h new file mode 100644 index 000000000..3fdae3500 --- /dev/null +++ b/src/tbb/include/oneapi/tbb/spin_rw_mutex.h @@ -0,0 +1,229 @@ +/* + Copyright (c) 2005-2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_spin_rw_mutex_H +#define __TBB_spin_rw_mutex_H + +#include "detail/_namespace_injection.h" +#include "detail/_mutex_common.h" + +#include "profiling.h" + +#include "detail/_assert.h" +#include "detail/_utils.h" +#include "detail/_scoped_lock.h" + +#include + +namespace tbb { +namespace detail { +namespace d1 { + +#if __TBB_TSX_INTRINSICS_PRESENT +class rtm_rw_mutex; +#endif + +//! Fast, unfair, spinning reader-writer lock with backoff and writer-preference +/** @ingroup synchronization */ +class spin_rw_mutex { +public: + //! Constructors + spin_rw_mutex() noexcept : m_state(0) { + create_itt_sync(this, "tbb::spin_rw_mutex", ""); + } + + //! Destructor + ~spin_rw_mutex() { + __TBB_ASSERT(!m_state, "destruction of an acquired mutex"); + } + + //! No Copy + spin_rw_mutex(const spin_rw_mutex&) = delete; + spin_rw_mutex& operator=(const spin_rw_mutex&) = delete; + + using scoped_lock = rw_scoped_lock; + + //! Mutex traits + static constexpr bool is_rw_mutex = true; + static constexpr bool is_recursive_mutex = false; + static constexpr bool is_fair_mutex = false; + + //! Acquire lock + void lock() { + call_itt_notify(prepare, this); + for (atomic_backoff backoff; ; backoff.pause()) { + state_type s = m_state.load(std::memory_order_relaxed); + if (!(s & BUSY)) { // no readers, no writers + if (m_state.compare_exchange_strong(s, WRITER)) + break; // successfully stored writer flag + backoff.reset(); // we could be very close to complete op. + } else if (!(s & WRITER_PENDING)) { // no pending writers + m_state |= WRITER_PENDING; + } + } + call_itt_notify(acquired, this); + } + + //! Try acquiring lock (non-blocking) + /** Return true if lock acquired; false otherwise. */ + bool try_lock() { + // for a writer: only possible to acquire if no active readers or writers + state_type s = m_state.load(std::memory_order_relaxed); + if (!(s & BUSY)) { // no readers, no writers; mask is 1..1101 + if (m_state.compare_exchange_strong(s, WRITER)) { + call_itt_notify(acquired, this); + return true; // successfully stored writer flag + } + } + return false; + } + + //! Release lock + void unlock() { + call_itt_notify(releasing, this); + m_state &= READERS; + } + + //! Lock shared ownership mutex + void lock_shared() { + call_itt_notify(prepare, this); + for (atomic_backoff b; ; b.pause()) { + state_type s = m_state.load(std::memory_order_relaxed); + if (!(s & (WRITER | WRITER_PENDING))) { // no writer or write requests + state_type prev_state = m_state.fetch_add(ONE_READER); + if (!(prev_state & WRITER)) { + break; // successfully stored increased number of readers + } + // writer got there first, undo the increment + m_state -= ONE_READER; + } + } + call_itt_notify(acquired, this); + __TBB_ASSERT(m_state & READERS, "invalid state of a read lock: no readers"); + } + + //! Try lock shared ownership mutex + bool try_lock_shared() { + // for a reader: acquire if no active or waiting writers + state_type s = m_state.load(std::memory_order_relaxed); + if (!(s & (WRITER | WRITER_PENDING))) { // no writers + state_type prev_state = m_state.fetch_add(ONE_READER); + if (!(prev_state & WRITER)) { // got the lock + call_itt_notify(acquired, this); + return true; // successfully stored increased number of readers + } + // writer got there first, undo the increment + m_state -= ONE_READER; + } + return false; + } + + //! Unlock shared ownership mutex + void unlock_shared() { + __TBB_ASSERT(m_state & READERS, "invalid state of a read lock: no readers"); + call_itt_notify(releasing, this); + m_state -= ONE_READER; + } + +protected: + /** Internal non ISO C++ standard API **/ + //! This API is used through the scoped_lock class + + //! Upgrade reader to become a writer. + /** Returns whether the upgrade happened without releasing and re-acquiring the lock */ + bool upgrade() { + state_type s = m_state.load(std::memory_order_relaxed); + __TBB_ASSERT(s & READERS, "invalid state before upgrade: no readers "); + // Check and set writer-pending flag. + // Required conditions: either no pending writers, or we are the only reader + // (with multiple readers and pending writer, another upgrade could have been requested) + while ((s & READERS) == ONE_READER || !(s & WRITER_PENDING)) { + if (m_state.compare_exchange_strong(s, s | WRITER | WRITER_PENDING)) { + atomic_backoff backoff; + while ((m_state.load(std::memory_order_relaxed) & READERS) != ONE_READER) backoff.pause(); + __TBB_ASSERT((m_state & (WRITER_PENDING|WRITER)) == (WRITER_PENDING | WRITER), "invalid state when upgrading to writer"); + // Both new readers and writers are blocked at this time + m_state -= (ONE_READER + WRITER_PENDING); + return true; // successfully upgraded + } + } + // Slow reacquire + unlock_shared(); + lock(); + return false; + } + + //! Downgrade writer to a reader + void downgrade() { + call_itt_notify(releasing, this); + m_state += (ONE_READER - WRITER); + __TBB_ASSERT(m_state & READERS, "invalid state after downgrade: no readers"); + } + + using state_type = std::intptr_t; + static constexpr state_type WRITER = 1; + static constexpr state_type WRITER_PENDING = 2; + static constexpr state_type READERS = ~(WRITER | WRITER_PENDING); + static constexpr state_type ONE_READER = 4; + static constexpr state_type BUSY = WRITER | READERS; + friend scoped_lock; + //! State of lock + /** Bit 0 = writer is holding lock + Bit 1 = request by a writer to acquire lock (hint to readers to wait) + Bit 2..N = number of readers holding lock */ + std::atomic m_state; +}; // class spin_rw_mutex + +#if TBB_USE_PROFILING_TOOLS +inline void set_name(spin_rw_mutex& obj, const char* name) { + itt_set_sync_name(&obj, name); +} +#if (_WIN32||_WIN64) +inline void set_name(spin_rw_mutex& obj, const wchar_t* name) { + itt_set_sync_name(&obj, name); +} +#endif // WIN +#else +inline void set_name(spin_rw_mutex&, const char*) {} +#if (_WIN32||_WIN64) +inline void set_name(spin_rw_mutex&, const wchar_t*) {} +#endif // WIN +#endif +} // namespace d1 +} // namespace detail + +inline namespace v1 { +using detail::d1::spin_rw_mutex; +} // namespace v1 +namespace profiling { + using detail::d1::set_name; +} +} // namespace tbb + +#include "detail/_rtm_rw_mutex.h" + +namespace tbb { +inline namespace v1 { +#if __TBB_TSX_INTRINSICS_PRESENT + using speculative_spin_rw_mutex = detail::d1::rtm_rw_mutex; +#else + using speculative_spin_rw_mutex = detail::d1::spin_rw_mutex; +#endif +} +} + +#endif /* __TBB_spin_rw_mutex_H */ + diff --git a/inst/include/serial/tbb/tbb_annotate.h b/src/tbb/include/oneapi/tbb/task.h similarity index 52% rename from inst/include/serial/tbb/tbb_annotate.h rename to src/tbb/include/oneapi/tbb/task.h index c16defead..82ce1df6c 100644 --- a/inst/include/serial/tbb/tbb_annotate.h +++ b/src/tbb/include/oneapi/tbb/task.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2018 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,25 +12,26 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ -#ifndef __TBB_annotate_H -#define __TBB_annotate_H - -// Macros used by the Intel(R) Parallel Advisor. -#ifdef __TBB_NORMAL_EXECUTION - #define ANNOTATE_SITE_BEGIN( site ) - #define ANNOTATE_SITE_END( site ) - #define ANNOTATE_TASK_BEGIN( task ) - #define ANNOTATE_TASK_END( task ) - #define ANNOTATE_LOCK_ACQUIRE( lock ) - #define ANNOTATE_LOCK_RELEASE( lock ) -#else - #include -#endif - -#endif /* __TBB_annotate_H */ +#ifndef __TBB_task_H +#define __TBB_task_H + +#include "detail/_config.h" +#include "detail/_namespace_injection.h" +#include "detail/_task.h" + +namespace tbb { +inline namespace v1 { +namespace task { +#if __TBB_RESUMABLE_TASKS + using detail::d1::suspend_point; + using detail::d1::resume; + using detail::d1::suspend; +#endif /* __TBB_RESUMABLE_TASKS */ + using detail::d1::current_context; +} // namespace task +} // namespace v1 +} // namespace tbb + +#endif /* __TBB_task_H */ diff --git a/src/tbb/include/oneapi/tbb/task_arena.h b/src/tbb/include/oneapi/tbb/task_arena.h new file mode 100644 index 000000000..5ce41d99c --- /dev/null +++ b/src/tbb/include/oneapi/tbb/task_arena.h @@ -0,0 +1,499 @@ +/* + Copyright (c) 2005-2023 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_task_arena_H +#define __TBB_task_arena_H + +#include "detail/_config.h" + +#include "detail/_aligned_space.h" +#include "detail/_attach.h" +#include "detail/_exception.h" +#include "detail/_namespace_injection.h" +#include "detail/_small_object_pool.h" +#include "detail/_task.h" + +#include "detail/_task_handle.h" + +#if __TBB_ARENA_BINDING +#include "info.h" +#endif /*__TBB_ARENA_BINDING*/ + +namespace tbb { +namespace detail { + +namespace d1 { + +template +class task_arena_function : public delegate_base { + F &my_func; + aligned_space my_return_storage; + bool my_constructed{false}; + // The function should be called only once. + bool operator()() const override { + new (my_return_storage.begin()) R(my_func()); + return true; + } +public: + task_arena_function(F& f) : my_func(f) {} + // The function can be called only after operator() and only once. + R consume_result() { + my_constructed = true; + return std::move(*(my_return_storage.begin())); + } + ~task_arena_function() override { + if (my_constructed) { + my_return_storage.begin()->~R(); + } + } +}; + +template +class task_arena_function : public delegate_base { + F &my_func; + bool operator()() const override { + my_func(); + return true; + } +public: + task_arena_function(F& f) : my_func(f) {} + void consume_result() const {} + + friend class task_arena_base; +}; + +class task_arena_base; +class task_scheduler_observer; +} // namespace d1 + +namespace r1 { +class arena; +struct task_arena_impl; + +TBB_EXPORT void __TBB_EXPORTED_FUNC observe(d1::task_scheduler_observer&, bool); +TBB_EXPORT void __TBB_EXPORTED_FUNC initialize(d1::task_arena_base&); +TBB_EXPORT void __TBB_EXPORTED_FUNC terminate(d1::task_arena_base&); +TBB_EXPORT bool __TBB_EXPORTED_FUNC attach(d1::task_arena_base&); +TBB_EXPORT void __TBB_EXPORTED_FUNC execute(d1::task_arena_base&, d1::delegate_base&); +TBB_EXPORT void __TBB_EXPORTED_FUNC wait(d1::task_arena_base&); +TBB_EXPORT int __TBB_EXPORTED_FUNC max_concurrency(const d1::task_arena_base*); +TBB_EXPORT void __TBB_EXPORTED_FUNC isolate_within_arena(d1::delegate_base& d, std::intptr_t); + +TBB_EXPORT void __TBB_EXPORTED_FUNC enqueue(d1::task&, d1::task_arena_base*); +TBB_EXPORT void __TBB_EXPORTED_FUNC enqueue(d1::task&, d1::task_group_context&, d1::task_arena_base*); +TBB_EXPORT void __TBB_EXPORTED_FUNC submit(d1::task&, d1::task_group_context&, arena*, std::uintptr_t); +} // namespace r1 + +namespace d2 { +inline void enqueue_impl(task_handle&& th, d1::task_arena_base* ta) { + __TBB_ASSERT(th != nullptr, "Attempt to schedule empty task_handle"); + + auto& ctx = task_handle_accessor::ctx_of(th); + + // Do not access th after release + r1::enqueue(*task_handle_accessor::release(th), ctx, ta); +} +} //namespace d2 + +namespace d1 { + +static constexpr unsigned num_priority_levels = 3; +static constexpr int priority_stride = INT_MAX / (num_priority_levels + 1); + +class task_arena_base { + friend struct r1::task_arena_impl; + friend void r1::observe(d1::task_scheduler_observer&, bool); +public: + enum class priority : int { + low = 1 * priority_stride, + normal = 2 * priority_stride, + high = 3 * priority_stride + }; +#if __TBB_ARENA_BINDING + using constraints = tbb::detail::d1::constraints; +#endif /*__TBB_ARENA_BINDING*/ +protected: + //! Special settings + intptr_t my_version_and_traits; + + std::atomic my_initialization_state; + + //! nullptr if not currently initialized. + std::atomic my_arena; + static_assert(sizeof(std::atomic) == sizeof(r1::arena*), + "To preserve backward compatibility we need the equal size of an atomic pointer and a pointer"); + + //! Concurrency level for deferred initialization + int my_max_concurrency; + + //! Reserved slots for external threads + unsigned my_num_reserved_slots; + + //! Arena priority + priority my_priority; + + //! The NUMA node index to which the arena will be attached + numa_node_id my_numa_id; + + //! The core type index to which arena will be attached + core_type_id my_core_type; + + //! Number of threads per core + int my_max_threads_per_core; + + // Backward compatibility checks. + core_type_id core_type() const { + return (my_version_and_traits & core_type_support_flag) == core_type_support_flag ? my_core_type : automatic; + } + int max_threads_per_core() const { + return (my_version_and_traits & core_type_support_flag) == core_type_support_flag ? my_max_threads_per_core : automatic; + } + + enum { + default_flags = 0 + , core_type_support_flag = 1 + }; + + task_arena_base(int max_concurrency, unsigned reserved_for_masters, priority a_priority) + : my_version_and_traits(default_flags | core_type_support_flag) + , my_initialization_state(do_once_state::uninitialized) + , my_arena(nullptr) + , my_max_concurrency(max_concurrency) + , my_num_reserved_slots(reserved_for_masters) + , my_priority(a_priority) + , my_numa_id(automatic) + , my_core_type(automatic) + , my_max_threads_per_core(automatic) + {} + +#if __TBB_ARENA_BINDING + task_arena_base(const constraints& constraints_, unsigned reserved_for_masters, priority a_priority) + : my_version_and_traits(default_flags | core_type_support_flag) + , my_initialization_state(do_once_state::uninitialized) + , my_arena(nullptr) + , my_max_concurrency(constraints_.max_concurrency) + , my_num_reserved_slots(reserved_for_masters) + , my_priority(a_priority) + , my_numa_id(constraints_.numa_id) + , my_core_type(constraints_.core_type) + , my_max_threads_per_core(constraints_.max_threads_per_core) + {} +#endif /*__TBB_ARENA_BINDING*/ +public: + //! Typedef for number of threads that is automatic. + static const int automatic = -1; + static const int not_initialized = -2; +}; + +template +R isolate_impl(F& f) { + task_arena_function func(f); + r1::isolate_within_arena(func, /*isolation*/ 0); + return func.consume_result(); +} + +template +class enqueue_task : public task { + small_object_allocator m_allocator; + const F m_func; + + void finalize(const execution_data& ed) { + m_allocator.delete_object(this, ed); + } + task* execute(execution_data& ed) override { + m_func(); + finalize(ed); + return nullptr; + } + task* cancel(execution_data&) override { + __TBB_ASSERT_RELEASE(false, "Unhandled exception from enqueue task is caught"); + return nullptr; + } +public: + enqueue_task(const F& f, small_object_allocator& alloc) : m_allocator(alloc), m_func(f) {} + enqueue_task(F&& f, small_object_allocator& alloc) : m_allocator(alloc), m_func(std::move(f)) {} +}; + +template +void enqueue_impl(F&& f, task_arena_base* ta) { + small_object_allocator alloc{}; + r1::enqueue(*alloc.new_object::type>>(std::forward(f), alloc), ta); +} +/** 1-to-1 proxy representation class of scheduler's arena + * Constructors set up settings only, real construction is deferred till the first method invocation + * Destructor only removes one of the references to the inner arena representation. + * Final destruction happens when all the references (and the work) are gone. + */ +class task_arena : public task_arena_base { + + void mark_initialized() { + __TBB_ASSERT( my_arena.load(std::memory_order_relaxed), "task_arena initialization is incomplete" ); + my_initialization_state.store(do_once_state::initialized, std::memory_order_release); + } + + template + R execute_impl(F& f) { + initialize(); + task_arena_function func(f); + r1::execute(*this, func); + return func.consume_result(); + } +public: + //! Creates task_arena with certain concurrency limits + /** Sets up settings only, real construction is deferred till the first method invocation + * @arg max_concurrency specifies total number of slots in arena where threads work + * @arg reserved_for_masters specifies number of slots to be used by external threads only. + * Value of 1 is default and reflects behavior of implicit arenas. + **/ + task_arena(int max_concurrency_ = automatic, unsigned reserved_for_masters = 1, + priority a_priority = priority::normal) + : task_arena_base(max_concurrency_, reserved_for_masters, a_priority) + {} + +#if __TBB_ARENA_BINDING + //! Creates task arena pinned to certain NUMA node + task_arena(const constraints& constraints_, unsigned reserved_for_masters = 1, + priority a_priority = priority::normal) + : task_arena_base(constraints_, reserved_for_masters, a_priority) + {} + + //! Copies settings from another task_arena + task_arena(const task_arena &s) // copy settings but not the reference or instance + : task_arena_base( + constraints{} + .set_numa_id(s.my_numa_id) + .set_max_concurrency(s.my_max_concurrency) + .set_core_type(s.my_core_type) + .set_max_threads_per_core(s.my_max_threads_per_core) + , s.my_num_reserved_slots, s.my_priority) + {} +#else + //! Copies settings from another task_arena + task_arena(const task_arena& a) // copy settings but not the reference or instance + : task_arena_base(a.my_max_concurrency, a.my_num_reserved_slots, a.my_priority) + {} +#endif /*__TBB_ARENA_BINDING*/ + + //! Tag class used to indicate the "attaching" constructor + struct attach {}; + + //! Creates an instance of task_arena attached to the current arena of the thread + explicit task_arena( attach ) + : task_arena_base(automatic, 1, priority::normal) // use default settings if attach fails + { + if (r1::attach(*this)) { + mark_initialized(); + } + } + + //! Creates an instance of task_arena attached to the current arena of the thread + explicit task_arena(d1::attach) + : task_arena(attach{}) + {} + + //! Forces allocation of the resources for the task_arena as specified in constructor arguments + void initialize() { + atomic_do_once([this]{ r1::initialize(*this); }, my_initialization_state); + } + + //! Overrides concurrency level and forces initialization of internal representation + void initialize(int max_concurrency_, unsigned reserved_for_masters = 1, + priority a_priority = priority::normal) + { + __TBB_ASSERT(!my_arena.load(std::memory_order_relaxed), "Impossible to modify settings of an already initialized task_arena"); + if( !is_active() ) { + my_max_concurrency = max_concurrency_; + my_num_reserved_slots = reserved_for_masters; + my_priority = a_priority; + r1::initialize(*this); + mark_initialized(); + } + } + +#if __TBB_ARENA_BINDING + void initialize(constraints constraints_, unsigned reserved_for_masters = 1, + priority a_priority = priority::normal) + { + __TBB_ASSERT(!my_arena.load(std::memory_order_relaxed), "Impossible to modify settings of an already initialized task_arena"); + if( !is_active() ) { + my_numa_id = constraints_.numa_id; + my_max_concurrency = constraints_.max_concurrency; + my_core_type = constraints_.core_type; + my_max_threads_per_core = constraints_.max_threads_per_core; + my_num_reserved_slots = reserved_for_masters; + my_priority = a_priority; + r1::initialize(*this); + mark_initialized(); + } + } +#endif /*__TBB_ARENA_BINDING*/ + + //! Attaches this instance to the current arena of the thread + void initialize(attach) { + // TODO: decide if this call must be thread-safe + __TBB_ASSERT(!my_arena.load(std::memory_order_relaxed), "Impossible to modify settings of an already initialized task_arena"); + if( !is_active() ) { + if ( !r1::attach(*this) ) { + r1::initialize(*this); + } + mark_initialized(); + } + } + + //! Attaches this instance to the current arena of the thread + void initialize(d1::attach) { + initialize(attach{}); + } + + //! Removes the reference to the internal arena representation. + //! Not thread safe wrt concurrent invocations of other methods. + void terminate() { + if( is_active() ) { + r1::terminate(*this); + my_initialization_state.store(do_once_state::uninitialized, std::memory_order_relaxed); + } + } + + //! Removes the reference to the internal arena representation, and destroys the external object. + //! Not thread safe wrt concurrent invocations of other methods. + ~task_arena() { + terminate(); + } + + //! Returns true if the arena is active (initialized); false otherwise. + //! The name was chosen to match a task_scheduler_init method with the same semantics. + bool is_active() const { + return my_initialization_state.load(std::memory_order_acquire) == do_once_state::initialized; + } + + //! Enqueues a task into the arena to process a functor, and immediately returns. + //! Does not require the calling thread to join the arena + + template + void enqueue(F&& f) { + initialize(); + enqueue_impl(std::forward(f), this); + } + + //! Enqueues a task into the arena to process a functor wrapped in task_handle, and immediately returns. + //! Does not require the calling thread to join the arena + void enqueue(d2::task_handle&& th) { + initialize(); + d2::enqueue_impl(std::move(th), this); + } + + //! Joins the arena and executes a mutable functor, then returns + //! If not possible to join, wraps the functor into a task, enqueues it and waits for task completion + //! Can decrement the arena demand for workers, causing a worker to leave and free a slot to the calling thread + //! Since C++11, the method returns the value returned by functor (prior to C++11 it returns void). + template + auto execute(F&& f) -> decltype(f()) { + return execute_impl(f); + } + +#if __TBB_EXTRA_DEBUG + //! Returns my_num_reserved_slots + int debug_reserved_slots() const { + // Handle special cases inside the library + return my_num_reserved_slots; + } + + //! Returns my_max_concurrency + int debug_max_concurrency() const { + // Handle special cases inside the library + return my_max_concurrency; + } + + //! Wait for all work in the arena to be completed + //! Even submitted by other application threads + //! Joins arena if/when possible (in the same way as execute()) + void debug_wait_until_empty() { + initialize(); + r1::wait(*this); + } +#endif //__TBB_EXTRA_DEBUG + + //! Returns the maximal number of threads that can work inside the arena + int max_concurrency() const { + // Handle special cases inside the library + return (my_max_concurrency > 1) ? my_max_concurrency : r1::max_concurrency(this); + } + + friend void submit(task& t, task_arena& ta, task_group_context& ctx, bool as_critical) { + __TBB_ASSERT(ta.is_active(), nullptr); + call_itt_task_notify(releasing, &t); + r1::submit(t, ctx, ta.my_arena.load(std::memory_order_relaxed), as_critical ? 1 : 0); + } +}; + +//! Executes a mutable functor in isolation within the current task arena. +//! Since C++11, the method returns the value returned by functor (prior to C++11 it returns void). +template +inline auto isolate(F&& f) -> decltype(f()) { + return isolate_impl(f); +} + +//! Returns the index, aka slot number, of the calling thread in its current arena +inline int current_thread_index() { + slot_id idx = r1::execution_slot(nullptr); + return idx == slot_id(-1) ? task_arena_base::not_initialized : int(idx); +} + +#if __TBB_PREVIEW_TASK_GROUP_EXTENSIONS +inline bool is_inside_task() { + return nullptr != current_context(); +} +#endif //__TBB_PREVIEW_TASK_GROUP_EXTENSIONS + +//! Returns the maximal number of threads that can work inside the arena +inline int max_concurrency() { + return r1::max_concurrency(nullptr); +} + +inline void enqueue(d2::task_handle&& th) { + d2::enqueue_impl(std::move(th), nullptr); +} + +template +inline void enqueue(F&& f) { + enqueue_impl(std::forward(f), nullptr); +} + +using r1::submit; + +} // namespace d1 +} // namespace detail + +inline namespace v1 { +using detail::d1::task_arena; +using detail::d1::attach; + +#if __TBB_PREVIEW_TASK_GROUP_EXTENSIONS +using detail::d1::is_inside_task; +#endif + +namespace this_task_arena { +using detail::d1::current_thread_index; +using detail::d1::max_concurrency; +using detail::d1::isolate; + +using detail::d1::enqueue; +} // namespace this_task_arena + +} // inline namespace v1 + +} // namespace tbb +#endif /* __TBB_task_arena_H */ diff --git a/src/tbb/include/oneapi/tbb/task_group.h b/src/tbb/include/oneapi/tbb/task_group.h new file mode 100644 index 000000000..09c0adaa1 --- /dev/null +++ b/src/tbb/include/oneapi/tbb/task_group.h @@ -0,0 +1,712 @@ +/* + Copyright (c) 2005-2024 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_task_group_H +#define __TBB_task_group_H + +#include "detail/_config.h" +#include "detail/_namespace_injection.h" +#include "detail/_assert.h" +#include "detail/_utils.h" +#include "detail/_template_helpers.h" +#include "detail/_exception.h" +#include "detail/_task.h" +#include "detail/_small_object_pool.h" +#include "detail/_intrusive_list_node.h" +#include "detail/_task_handle.h" + +#include "profiling.h" + +#include + +#if _MSC_VER && !defined(__INTEL_COMPILER) + // Suppress warning: structure was padded due to alignment specifier + // #pragma warning(push) + // #pragma warning(disable:4324) +#endif + +namespace tbb { +namespace detail { + +namespace d1 { +class delegate_base; +class task_arena_base; +class task_group_context; +} + +namespace r1 { +// Forward declarations +class tbb_exception_ptr; +class cancellation_disseminator; +class thread_data; +class task_dispatcher; +template +class context_guard_helper; +struct task_arena_impl; +class context_list; + +TBB_EXPORT void __TBB_EXPORTED_FUNC execute(d1::task_arena_base&, d1::delegate_base&); +TBB_EXPORT void __TBB_EXPORTED_FUNC isolate_within_arena(d1::delegate_base&, std::intptr_t); + +TBB_EXPORT void __TBB_EXPORTED_FUNC initialize(d1::task_group_context&); +TBB_EXPORT void __TBB_EXPORTED_FUNC destroy(d1::task_group_context&); +TBB_EXPORT void __TBB_EXPORTED_FUNC reset(d1::task_group_context&); +TBB_EXPORT bool __TBB_EXPORTED_FUNC cancel_group_execution(d1::task_group_context&); +TBB_EXPORT bool __TBB_EXPORTED_FUNC is_group_execution_cancelled(d1::task_group_context&); +TBB_EXPORT void __TBB_EXPORTED_FUNC capture_fp_settings(d1::task_group_context&); + +struct task_group_context_impl; +} + +namespace d2 { + +namespace { +template +d1::task* task_ptr_or_nullptr(F&& f); +} + +template +class function_task : public task_handle_task { + //TODO: apply empty base optimization here + const F m_func; + +private: + d1::task* execute(d1::execution_data& ed) override { + __TBB_ASSERT(ed.context == &this->ctx(), "The task group context should be used for all tasks"); + task* res = task_ptr_or_nullptr(m_func); + finalize(&ed); + return res; + } + d1::task* cancel(d1::execution_data& ed) override { + finalize(&ed); + return nullptr; + } +public: + template + function_task(FF&& f, d1::wait_tree_vertex_interface* vertex, d1::task_group_context& ctx, d1::small_object_allocator& alloc) + : task_handle_task{vertex, ctx, alloc}, + m_func(std::forward(f)) {} +}; + +#if __TBB_PREVIEW_TASK_GROUP_EXTENSIONS +namespace { + template + d1::task* task_ptr_or_nullptr_impl(std::false_type, F&& f){ + task_handle th = std::forward(f)(); + return task_handle_accessor::release(th); + } + + template + d1::task* task_ptr_or_nullptr_impl(std::true_type, F&& f){ + std::forward(f)(); + return nullptr; + } + + template + d1::task* task_ptr_or_nullptr(F&& f){ + using is_void_t = std::is_void< + decltype(std::forward(f)()) + >; + + return task_ptr_or_nullptr_impl(is_void_t{}, std::forward(f)); + } +} +#else +namespace { + template + d1::task* task_ptr_or_nullptr(F&& f){ + std::forward(f)(); + return nullptr; + } +} // namespace +#endif // __TBB_PREVIEW_TASK_GROUP_EXTENSIONS +} // namespace d2 + +namespace d1 { + +// This structure is left here for backward compatibility check +struct context_list_node { + std::atomic prev{}; + std::atomic next{}; +}; + +//! Used to form groups of tasks +/** @ingroup task_scheduling + The context services explicit cancellation requests from user code, and unhandled + exceptions intercepted during tasks execution. Intercepting an exception results + in generating internal cancellation requests (which is processed in exactly the + same way as external ones). + + The context is associated with one or more root tasks and defines the cancellation + group that includes all the descendants of the corresponding root task(s). Association + is established when a context object is passed as an argument to the task::allocate_root() + method. See task_group_context::task_group_context for more details. + + The context can be bound to another one, and other contexts can be bound to it, + forming a tree-like structure: parent -> this -> children. Arrows here designate + cancellation propagation direction. If a task in a cancellation group is cancelled + all the other tasks in this group and groups bound to it (as children) get cancelled too. +**/ +class task_group_context : no_copy { +public: + enum traits_type { + fp_settings = 1 << 1, + concurrent_wait = 1 << 2, + default_traits = 0 + }; + enum kind_type { + isolated, + bound + }; +private: + //! Space for platform-specific FPU settings. + /** Must only be accessed inside TBB binaries, and never directly in user + code or inline methods. */ + std::uint64_t my_cpu_ctl_env; + + //! Specifies whether cancellation was requested for this task group. + std::atomic my_cancellation_requested; + + //! Versioning for run-time checks and behavioral traits of the context. + enum class task_group_context_version : std::uint8_t { + unused = 1 // ensure that new versions, if any, will not clash with previously used ones + }; + task_group_context_version my_version; + + //! The context traits. + struct context_traits { + bool fp_settings : 1; + bool concurrent_wait : 1; + bool bound : 1; + bool reserved1 : 1; + bool reserved2 : 1; + bool reserved3 : 1; + bool reserved4 : 1; + bool reserved5 : 1; + } my_traits; + + static_assert(sizeof(context_traits) == 1, "Traits shall fit into one byte."); + + static constexpr std::uint8_t may_have_children = 1; + //! The context internal state (currently only may_have_children). + std::atomic my_may_have_children; + + enum class state : std::uint8_t { + created, + locked, + isolated, + bound, + dead, + proxy = std::uint8_t(-1) //the context is not the real one, but proxy to other one + }; + + //! The synchronization machine state to manage lifetime. + std::atomic my_state; + + union { + //! Pointer to the context of the parent cancellation group. nullptr for isolated contexts. + task_group_context* my_parent; + + //! Pointer to the actual context 'this' context represents a proxy of. + task_group_context* my_actual_context; + }; + + //! Thread data instance that registered this context in its list. + r1::context_list* my_context_list; + static_assert(sizeof(std::atomic) == sizeof(r1::context_list*), "To preserve backward compatibility these types should have the same size"); + + //! Used to form the thread specific list of contexts without additional memory allocation. + /** A context is included into the list of the current thread when its binding to + its parent happens. Any context can be present in the list of one thread only. **/ + intrusive_list_node my_node; + static_assert(sizeof(intrusive_list_node) == sizeof(context_list_node), "To preserve backward compatibility these types should have the same size"); + + //! Pointer to the container storing exception being propagated across this task group. + std::atomic my_exception; + static_assert(sizeof(std::atomic) == sizeof(r1::tbb_exception_ptr*), + "backward compatibility check"); + + //! Used to set and maintain stack stitching point for Intel Performance Tools. + void* my_itt_caller; + + //! Description of algorithm for scheduler based instrumentation. + string_resource_index my_name; + + char padding[max_nfs_size + - sizeof(std::uint64_t) // my_cpu_ctl_env + - sizeof(std::atomic) // my_cancellation_requested + - sizeof(std::uint8_t) // my_version + - sizeof(context_traits) // my_traits + - sizeof(std::atomic) // my_state + - sizeof(std::atomic) // my_state + - sizeof(task_group_context*) // my_parent + - sizeof(r1::context_list*) // my_context_list + - sizeof(intrusive_list_node) // my_node + - sizeof(std::atomic) // my_exception + - sizeof(void*) // my_itt_caller + - sizeof(string_resource_index) // my_name + ]; + + task_group_context(context_traits t, string_resource_index name) + : my_version{task_group_context_version::unused}, my_name{name} + { + my_traits = t; // GCC4.8 issues warning list initialization for bitset (missing-field-initializers) + r1::initialize(*this); + } + + task_group_context(task_group_context* actual_context) + : my_version{task_group_context_version::unused} + , my_state{state::proxy} + , my_actual_context{actual_context} + { + __TBB_ASSERT(my_actual_context, "Passed pointer value points to nothing."); + my_name = actual_context->my_name; + + // no need to initialize 'this' context as it acts as a proxy for my_actual_context, which + // initialization is a user-side responsibility. + } + + static context_traits make_traits(kind_type relation_with_parent, std::uintptr_t user_traits) { + context_traits ct; + ct.fp_settings = (user_traits & fp_settings) == fp_settings; + ct.concurrent_wait = (user_traits & concurrent_wait) == concurrent_wait; + ct.bound = relation_with_parent == bound; + ct.reserved1 = ct.reserved2 = ct.reserved3 = ct.reserved4 = ct.reserved5 = false; + return ct; + } + + bool is_proxy() const { + return my_state.load(std::memory_order_relaxed) == state::proxy; + } + + task_group_context& actual_context() noexcept { + if (is_proxy()) { + __TBB_ASSERT(my_actual_context, "Actual task_group_context is not set."); + return *my_actual_context; + } + return *this; + } + + const task_group_context& actual_context() const noexcept { + if (is_proxy()) { + __TBB_ASSERT(my_actual_context, "Actual task_group_context is not set."); + return *my_actual_context; + } + return *this; + } + +public: + //! Default & binding constructor. + /** By default a bound context is created. That is this context will be bound + (as child) to the context of the currently executing task . Cancellation + requests passed to the parent context are propagated to all the contexts + bound to it. Similarly priority change is propagated from the parent context + to its children. + + If task_group_context::isolated is used as the argument, then the tasks associated + with this context will never be affected by events in any other context. + + Creating isolated contexts involve much less overhead, but they have limited + utility. Normally when an exception occurs in an algorithm that has nested + ones running, it is desirably to have all the nested algorithms cancelled + as well. Such a behavior requires nested algorithms to use bound contexts. + + There is one good place where using isolated algorithms is beneficial. It is + an external thread. That is if a particular algorithm is invoked directly from + the external thread (not from a TBB task), supplying it with explicitly + created isolated context will result in a faster algorithm startup. + + VERSIONING NOTE: + Implementation(s) of task_group_context constructor(s) cannot be made + entirely out-of-line because the run-time version must be set by the user + code. This will become critically important for binary compatibility, if + we ever have to change the size of the context object. **/ + + task_group_context(kind_type relation_with_parent = bound, + std::uintptr_t t = default_traits) + : task_group_context(make_traits(relation_with_parent, t), CUSTOM_CTX) {} + + // Custom constructor for instrumentation of oneTBB algorithm + task_group_context(string_resource_index name ) + : task_group_context(make_traits(bound, default_traits), name) {} + + // Do not introduce any logic on user side since it might break state propagation assumptions + ~task_group_context() { + // When 'this' serves as a proxy, the initialization does not happen - nor should the + // destruction. + if (!is_proxy()) + { + r1::destroy(*this); + } + } + + //! Forcefully reinitializes the context after the task tree it was associated with is completed. + /** Because the method assumes that all the tasks that used to be associated with + this context have already finished, calling it while the context is still + in use somewhere in the task hierarchy leads to undefined behavior. + + IMPORTANT: This method is not thread safe! + + The method does not change the context's parent if it is set. **/ + void reset() { + r1::reset(actual_context()); + } + + //! Initiates cancellation of all tasks in this cancellation group and its subordinate groups. + /** \return false if cancellation has already been requested, true otherwise. + + Note that canceling never fails. When false is returned, it just means that + another thread (or this one) has already sent cancellation request to this + context or to one of its ancestors (if this context is bound). It is guaranteed + that when this method is concurrently called on the same not yet cancelled + context, true will be returned by one and only one invocation. **/ + bool cancel_group_execution() { + return r1::cancel_group_execution(actual_context()); + } + + //! Returns true if the context received cancellation request. + bool is_group_execution_cancelled() { + return r1::is_group_execution_cancelled(actual_context()); + } + +#if __TBB_FP_CONTEXT + //! Captures the current FPU control settings to the context. + /** Because the method assumes that all the tasks that used to be associated with + this context have already finished, calling it while the context is still + in use somewhere in the task hierarchy leads to undefined behavior. + + IMPORTANT: This method is not thread safe! + + The method does not change the FPU control settings of the context's parent. **/ + void capture_fp_settings() { + r1::capture_fp_settings(actual_context()); + } +#endif + + //! Returns the user visible context trait + std::uintptr_t traits() const { + std::uintptr_t t{}; + const task_group_context& ctx = actual_context(); + t |= ctx.my_traits.fp_settings ? fp_settings : 0; + t |= ctx.my_traits.concurrent_wait ? concurrent_wait : 0; + return t; + } +private: + //// TODO: cleanup friends + friend class r1::cancellation_disseminator; + friend class r1::thread_data; + friend class r1::task_dispatcher; + template + friend class r1::context_guard_helper; + friend struct r1::task_arena_impl; + friend struct r1::task_group_context_impl; + friend class d2::task_group_base; +}; // class task_group_context + +static_assert(sizeof(task_group_context) == 128, "Wrong size of task_group_context"); + +inline bool is_current_task_group_canceling() { + task_group_context* ctx = current_context(); + return ctx ? ctx->is_group_execution_cancelled() : false; +} + +} // namespace d1 + +namespace d2 { + +enum task_group_status { + not_complete, + complete, + canceled +}; + +class task_group; +class structured_task_group; +#if TBB_PREVIEW_ISOLATED_TASK_GROUP +class isolated_task_group; +#endif + +template +class function_stack_task : public d1::task { + const F& m_func; + d1::wait_tree_vertex_interface* m_wait_tree_vertex; + + void finalize() { + m_wait_tree_vertex->release(); + } + task* execute(d1::execution_data&) override { + task* res = d2::task_ptr_or_nullptr(m_func); + finalize(); + return res; + } + task* cancel(d1::execution_data&) override { + finalize(); + return nullptr; + } +public: + function_stack_task(const F& f, d1::wait_tree_vertex_interface* vertex) : m_func(f), m_wait_tree_vertex(vertex) { + m_wait_tree_vertex->reserve(); + } +}; + +class task_group_base : no_copy { +protected: + d1::wait_context_vertex m_wait_vertex; + d1::task_group_context m_context; + + template + task_group_status internal_run_and_wait(const F& f) { + function_stack_task t{ f, r1::get_thread_reference_vertex(&m_wait_vertex) }; + + bool cancellation_status = false; + try_call([&] { + execute_and_wait(t, context(), m_wait_vertex.get_context(), context()); + }).on_completion([&] { + // TODO: the reset method is not thread-safe. Ensure the correct behavior. + cancellation_status = context().is_group_execution_cancelled(); + context().reset(); + }); + return cancellation_status ? canceled : complete; + } + + task_group_status internal_run_and_wait(d2::task_handle&& h) { + __TBB_ASSERT(h != nullptr, "Attempt to schedule empty task_handle"); + + using acs = d2::task_handle_accessor; + __TBB_ASSERT(&acs::ctx_of(h) == &context(), "Attempt to schedule task_handle into different task_group"); + + bool cancellation_status = false; + try_call([&] { + execute_and_wait(*acs::release(h), context(), m_wait_vertex.get_context(), context()); + }).on_completion([&] { + // TODO: the reset method is not thread-safe. Ensure the correct behavior. + cancellation_status = context().is_group_execution_cancelled(); + context().reset(); + }); + return cancellation_status ? canceled : complete; + } + + template + d1::task* prepare_task(F&& f) { + d1::small_object_allocator alloc{}; + return alloc.new_object::type>>(std::forward(f), + r1::get_thread_reference_vertex(&m_wait_vertex), context(), alloc); + } + + d1::task_group_context& context() noexcept { + return m_context.actual_context(); + } + + template + d2::task_handle prepare_task_handle(F&& f) { + d1::small_object_allocator alloc{}; + using function_task_t = d2::function_task::type>; + d2::task_handle_task* function_task_p = alloc.new_object(std::forward(f), + r1::get_thread_reference_vertex(&m_wait_vertex), context(), alloc); + + return d2::task_handle_accessor::construct(function_task_p); + } + +public: + task_group_base(uintptr_t traits = 0) + : m_wait_vertex(0) + , m_context(d1::task_group_context::bound, d1::task_group_context::default_traits | traits) + {} + + task_group_base(d1::task_group_context& ctx) + : m_wait_vertex(0) + , m_context(&ctx) + {} + + ~task_group_base() noexcept(false) { + if (m_wait_vertex.continue_execution()) { +#if __TBB_CPP17_UNCAUGHT_EXCEPTIONS_PRESENT + bool stack_unwinding_in_progress = std::uncaught_exceptions() > 0; +#else + bool stack_unwinding_in_progress = std::uncaught_exception(); +#endif + // Always attempt to do proper cleanup to avoid inevitable memory corruption + // in case of missing wait (for the sake of better testability & debuggability) + if (!context().is_group_execution_cancelled()) + cancel(); + d1::wait(m_wait_vertex.get_context(), context()); + if (!stack_unwinding_in_progress) + throw_exception(exception_id::missing_wait); + } + } + + task_group_status wait() { + bool cancellation_status = false; + try_call([&] { + d1::wait(m_wait_vertex.get_context(), context()); + }).on_completion([&] { + // TODO: the reset method is not thread-safe. Ensure the correct behavior. + cancellation_status = m_context.is_group_execution_cancelled(); + context().reset(); + }); + return cancellation_status ? canceled : complete; + } + + void cancel() { + context().cancel_group_execution(); + } +}; // class task_group_base + +class task_group : public task_group_base { +public: + task_group() : task_group_base(d1::task_group_context::concurrent_wait) {} + task_group(d1::task_group_context& ctx) : task_group_base(ctx) {} + + template + void run(F&& f) { + d1::spawn(*prepare_task(std::forward(f)), context()); + } + + void run(d2::task_handle&& h) { + __TBB_ASSERT(h != nullptr, "Attempt to schedule empty task_handle"); + + using acs = d2::task_handle_accessor; + __TBB_ASSERT(&acs::ctx_of(h) == &context(), "Attempt to schedule task_handle into different task_group"); + + d1::spawn(*acs::release(h), context()); + } + + template + d2::task_handle defer(F&& f) { + return prepare_task_handle(std::forward(f)); + + } + + template + task_group_status run_and_wait(const F& f) { + return internal_run_and_wait(f); + } + + task_group_status run_and_wait(d2::task_handle&& h) { + return internal_run_and_wait(std::move(h)); + } +}; // class task_group + +#if TBB_PREVIEW_ISOLATED_TASK_GROUP +class spawn_delegate : public d1::delegate_base { + d1::task* task_to_spawn; + d1::task_group_context& context; + bool operator()() const override { + spawn(*task_to_spawn, context); + return true; + } +public: + spawn_delegate(d1::task* a_task, d1::task_group_context& ctx) + : task_to_spawn(a_task), context(ctx) + {} +}; + +class wait_delegate : public d1::delegate_base { + bool operator()() const override { + status = tg.wait(); + return true; + } +protected: + task_group& tg; + task_group_status& status; +public: + wait_delegate(task_group& a_group, task_group_status& tgs) + : tg(a_group), status(tgs) {} +}; + +template +class run_wait_delegate : public wait_delegate { + F& func; + bool operator()() const override { + status = tg.run_and_wait(func); + return true; + } +public: + run_wait_delegate(task_group& a_group, F& a_func, task_group_status& tgs) + : wait_delegate(a_group, tgs), func(a_func) {} +}; + +class isolated_task_group : public task_group { + intptr_t this_isolation() { + return reinterpret_cast(this); + } +public: + isolated_task_group() : task_group() {} + + isolated_task_group(d1::task_group_context& ctx) : task_group(ctx) {} + + template + void run(F&& f) { + spawn_delegate sd(prepare_task(std::forward(f)), context()); + r1::isolate_within_arena(sd, this_isolation()); + } + + void run(d2::task_handle&& h) { + __TBB_ASSERT(h != nullptr, "Attempt to schedule empty task_handle"); + + using acs = d2::task_handle_accessor; + __TBB_ASSERT(&acs::ctx_of(h) == &context(), "Attempt to schedule task_handle into different task_group"); + + spawn_delegate sd(acs::release(h), context()); + r1::isolate_within_arena(sd, this_isolation()); + } + + template + task_group_status run_and_wait( const F& f ) { + task_group_status result = not_complete; + run_wait_delegate rwd(*this, f, result); + r1::isolate_within_arena(rwd, this_isolation()); + __TBB_ASSERT(result != not_complete, "premature exit from wait?"); + return result; + } + + task_group_status wait() { + task_group_status result = not_complete; + wait_delegate wd(*this, result); + r1::isolate_within_arena(wd, this_isolation()); + __TBB_ASSERT(result != not_complete, "premature exit from wait?"); + return result; + } +}; // class isolated_task_group +#endif // TBB_PREVIEW_ISOLATED_TASK_GROUP +} // namespace d2 +} // namespace detail + +inline namespace v1 { +using detail::d1::task_group_context; +using detail::d2::task_group; +#if TBB_PREVIEW_ISOLATED_TASK_GROUP +using detail::d2::isolated_task_group; +#endif + +using detail::d2::task_group_status; +using detail::d2::not_complete; +using detail::d2::complete; +using detail::d2::canceled; + +using detail::d1::is_current_task_group_canceling; +using detail::r1::missing_wait; + +using detail::d2::task_handle; +} + +} // namespace tbb + +#if _MSC_VER && !defined(__INTEL_COMPILER) + // #pragma warning(pop) // 4324 warning +#endif + +#endif // __TBB_task_group_H diff --git a/inst/include/tbb/task_scheduler_observer.h b/src/tbb/include/oneapi/tbb/task_scheduler_observer.h similarity index 56% rename from inst/include/tbb/task_scheduler_observer.h rename to src/tbb/include/oneapi/tbb/task_scheduler_observer.h index 5586ad4fb..f4f91e96d 100644 --- a/inst/include/tbb/task_scheduler_observer.h +++ b/src/tbb/include/oneapi/tbb/task_scheduler_observer.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,43 +12,129 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_task_scheduler_observer_H #define __TBB_task_scheduler_observer_H -#include "atomic.h" -#if __TBB_ARENA_OBSERVER || __TBB_SLEEP_PERMISSION +#include + +#include "detail/_namespace_injection.h" + #include "task_arena.h" -#endif -#if __TBB_SCHEDULER_OBSERVER namespace tbb { -namespace interface6 { +namespace detail { + +namespace d1 { class task_scheduler_observer; } -namespace internal { +namespace r1 { class observer_proxy; class observer_list; +//! Enable or disable observation +/** For local observers the method can be used only when the current thread +has the task scheduler initialized or is attached to an arena. +Repeated calls with the same state are no-ops. **/ +TBB_EXPORT void __TBB_EXPORTED_FUNC observe(d1::task_scheduler_observer&, bool state = true); +} + +namespace d1 { +class task_scheduler_observer { + friend class r1::observer_proxy; + friend class r1::observer_list; + friend void r1::observe(d1::task_scheduler_observer&, bool); + + //! Pointer to the proxy holding this observer. + /** Observers are proxied by the scheduler to maintain persistent lists of them. **/ + std::atomic my_proxy{ nullptr }; + + //! Counter preventing the observer from being destroyed while in use by the scheduler. + /** Valid only when observation is on. **/ + std::atomic my_busy_count{ 0 }; + + //! Contains task_arena pointer + task_arena* my_task_arena{ nullptr }; +public: + //! Returns true if observation is enabled, false otherwise. + bool is_observing() const { return my_proxy.load(std::memory_order_relaxed) != nullptr; } + + //! Entry notification + /** Invoked from inside observe(true) call and whenever a worker enters the arena + this observer is associated with. If a thread is already in the arena when + the observer is activated, the entry notification is called before it + executes the first stolen task. **/ + virtual void on_scheduler_entry( bool /*is_worker*/ ) {} + + //! Exit notification + /** Invoked from inside observe(false) call and whenever a worker leaves the + arena this observer is associated with. **/ + virtual void on_scheduler_exit( bool /*is_worker*/ ) {} + + //! Construct local or global observer in inactive state (observation disabled). + /** For a local observer entry/exit notifications are invoked whenever a worker + thread joins/leaves the arena of the observer's owner thread. If a thread is + already in the arena when the observer is activated, the entry notification is + called before it executes the first stolen task. **/ + explicit task_scheduler_observer() = default; + + //! Construct local observer for a given arena in inactive state (observation disabled). + /** entry/exit notifications are invoked whenever a thread joins/leaves arena. + If a thread is already in the arena when the observer is activated, the entry notification + is called before it executes the first stolen task. **/ + explicit task_scheduler_observer(task_arena& a) : my_task_arena(&a) {} + + /** Destructor protects instance of the observer from concurrent notification. + It is recommended to disable observation before destructor of a derived class starts, + otherwise it can lead to concurrent notification callback on partly destroyed object **/ + virtual ~task_scheduler_observer() { + if (my_proxy.load(std::memory_order_acquire)) { + observe(false); + } + } + + //! Enable or disable observation + /** Warning: concurrent invocations of this method are not safe. + Repeated calls with the same state are no-ops. **/ + void observe(bool state = true) { + if( state && !my_proxy.load(std::memory_order_relaxed) ) { + __TBB_ASSERT( my_busy_count.load(std::memory_order_relaxed) == 0, "Inconsistent state of task_scheduler_observer instance"); + } + r1::observe(*this, state); + } +}; + +} // namespace d1 +} // namespace detail + +inline namespace v1 { + using detail::d1::task_scheduler_observer; +} +} // namespace tbb + + +// Provided for backwards compatibility. +namespace tbb { +namespace interface6 { +class task_scheduler_observer; +} +namespace internal { + class task_scheduler_observer_v3 { - friend class observer_proxy; - friend class observer_list; + friend class tbb::detail::r1::observer_proxy; + friend class tbb::detail::r1::observer_list; friend class interface6::task_scheduler_observer; //! Pointer to the proxy holding this observer. /** Observers are proxied by the scheduler to maintain persistent lists of them. **/ - observer_proxy* my_proxy; + tbb::detail::r1::observer_proxy* my_proxy; //! Counter preventing the observer from being destroyed while in use by the scheduler. /** Valid only when observation is on. **/ - atomic my_busy_count; + std::atomic my_busy_count; public: //! Enable or disable observation @@ -62,7 +148,7 @@ class task_scheduler_observer_v3 { bool is_observing() const {return my_proxy!=NULL;} //! Construct observer with observation disabled. - task_scheduler_observer_v3() : my_proxy(NULL) { my_busy_count.store(0); } + task_scheduler_observer_v3() : my_proxy(NULL) { my_busy_count.store(0); } //! Entry notification /** Invoked from inside observe(true) call and whenever a worker enters the arena @@ -88,12 +174,11 @@ class task_scheduler_observer_v3 { } // namespace internal -#if __TBB_ARENA_OBSERVER || __TBB_SLEEP_PERMISSION namespace interface6 { class task_scheduler_observer : public internal::task_scheduler_observer_v3 { friend class internal::task_scheduler_observer_v3; - friend class internal::observer_proxy; - friend class internal::observer_list; + friend class tbb::detail::r1::observer_proxy; + friend class tbb::detail::r1::observer_list; /** Negative numbers with the largest absolute value to minimize probability of coincidence in case of a bug in busy count usage. **/ @@ -116,15 +201,9 @@ class task_scheduler_observer : public internal::task_scheduler_observer_v3 { constructor is obsolete too and will be changed in one of the future versions of the library. **/ explicit task_scheduler_observer( bool local = false ) { -#if __TBB_ARENA_OBSERVER my_context_tag = local? implicit_tag : global_tag; -#else - __TBB_ASSERT_EX( !local, NULL ); - my_context_tag = global_tag; -#endif } -#if __TBB_ARENA_OBSERVER //! Construct local observer for a given arena in inactive state (observation disabled). /** entry/exit notifications are invoked whenever a thread joins/leaves arena. If a thread is already in the arena when the observer is activated, the entry notification @@ -132,7 +211,6 @@ class task_scheduler_observer : public internal::task_scheduler_observer_v3 { explicit task_scheduler_observer( task_arena & a) { my_context_tag = (intptr_t)&a; } -#endif /* __TBB_ARENA_OBSERVER */ /** Destructor protects instance of the observer from concurrent notification. It is recommended to disable observation before destructor of a derived class starts, @@ -145,30 +223,14 @@ class task_scheduler_observer : public internal::task_scheduler_observer_v3 { void observe( bool state=true ) { if( state && !my_proxy ) { __TBB_ASSERT( !my_busy_count, "Inconsistent state of task_scheduler_observer instance"); - my_busy_count.store(v6_trait); + my_busy_count.store(v6_trait); } internal::task_scheduler_observer_v3::observe(state); } - -#if __TBB_SLEEP_PERMISSION - //! Return commands for may_sleep() - enum { keep_awake = false, allow_sleep = true }; - - //! The callback can be invoked by a worker thread before it goes to sleep. - /** If it returns false ('keep_awake'), the thread will keep spinning and looking for work. - It will not be called for master threads. **/ - virtual bool may_sleep() { return allow_sleep; } -#endif /*__TBB_SLEEP_PERMISSION*/ }; } //namespace interface6 -using interface6::task_scheduler_observer; -#else /*__TBB_ARENA_OBSERVER || __TBB_SLEEP_PERMISSION*/ -typedef tbb::internal::task_scheduler_observer_v3 task_scheduler_observer; -#endif /*__TBB_ARENA_OBSERVER || __TBB_SLEEP_PERMISSION*/ } // namespace tbb -#endif /* __TBB_SCHEDULER_OBSERVER */ - #endif /* __TBB_task_scheduler_observer_H */ diff --git a/src/tbb/include/oneapi/tbb/tbb_allocator.h b/src/tbb/include/oneapi/tbb/tbb_allocator.h new file mode 100644 index 000000000..5394b8188 --- /dev/null +++ b/src/tbb/include/oneapi/tbb/tbb_allocator.h @@ -0,0 +1,126 @@ +/* + Copyright (c) 2005-2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_tbb_allocator_H +#define __TBB_tbb_allocator_H + +#include "oneapi/tbb/detail/_utils.h" +#include "detail/_namespace_injection.h" +#include +#include + +#if __TBB_CPP17_MEMORY_RESOURCE_PRESENT +#include +#endif + +namespace tbb { +namespace detail { + +namespace r1 { +TBB_EXPORT void* __TBB_EXPORTED_FUNC allocate_memory(std::size_t size); +TBB_EXPORT void __TBB_EXPORTED_FUNC deallocate_memory(void* p); +TBB_EXPORT bool __TBB_EXPORTED_FUNC is_tbbmalloc_used(); +} + +namespace d1 { + +template +class tbb_allocator { +public: + using value_type = T; + using propagate_on_container_move_assignment = std::true_type; + + //! Always defined for TBB containers (supported since C++17 for std containers) + using is_always_equal = std::true_type; + + //! Specifies current allocator + enum malloc_type { + scalable, + standard + }; + + tbb_allocator() = default; + template tbb_allocator(const tbb_allocator&) noexcept {} + + //! Allocate space for n objects. + __TBB_nodiscard T* allocate(std::size_t n) { + return static_cast(r1::allocate_memory(n * sizeof(value_type))); + } + + //! Free previously allocated block of memory. + void deallocate(T* p, std::size_t) { + r1::deallocate_memory(p); + } + + //! Returns current allocator + static malloc_type allocator_type() { + return r1::is_tbbmalloc_used() ? standard : scalable; + } + +#if TBB_ALLOCATOR_TRAITS_BROKEN + using pointer = value_type*; + using const_pointer = const value_type*; + using reference = value_type&; + using const_reference = const value_type&; + using difference_type = std::ptrdiff_t; + using size_type = std::size_t; + template struct rebind { + using other = tbb_allocator; + }; + //! Largest value for which method allocate might succeed. + size_type max_size() const noexcept { + size_type max = ~(std::size_t(0)) / sizeof(value_type); + return (max > 0 ? max : 1); + } + template + void construct(U *p, Args&&... args) + { ::new (p) U(std::forward(args)...); } + void destroy( pointer p ) { p->~value_type(); } + pointer address(reference x) const { return &x; } + const_pointer address(const_reference x) const { return &x; } +#endif // TBB_ALLOCATOR_TRAITS_BROKEN +}; + +#if TBB_ALLOCATOR_TRAITS_BROKEN + template<> + class tbb_allocator { + public: + using pointer = void*; + using const_pointer = const void*; + using value_type = void; + template struct rebind { + using other = tbb_allocator; + }; + }; +#endif + +template +inline bool operator==(const tbb_allocator&, const tbb_allocator&) noexcept { return true; } + +#if !__TBB_CPP20_COMPARISONS_PRESENT +template +inline bool operator!=(const tbb_allocator&, const tbb_allocator&) noexcept { return false; } +#endif + +} // namespace d1 +} // namespace detail + +inline namespace v1 { +using detail::d1::tbb_allocator; +} // namespace v1 +} // namespace tbb + +#endif /* __TBB_tbb_allocator_H */ diff --git a/inst/include/tbb/tbbmalloc_proxy.h b/src/tbb/include/oneapi/tbb/tbbmalloc_proxy.h similarity index 89% rename from inst/include/tbb/tbbmalloc_proxy.h rename to src/tbb/include/oneapi/tbb/tbbmalloc_proxy.h index 76cbd6d72..0ba38f215 100644 --- a/inst/include/tbb/tbbmalloc_proxy.h +++ b/src/tbb/include/oneapi/tbb/tbbmalloc_proxy.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ /* @@ -25,7 +21,7 @@ Replacing the standard memory allocation routines in Microsoft* C/C++ RTL Include the following header to a source of any binary which is loaded during application startup -#include "tbb/tbbmalloc_proxy.h" +#include "oneapi/tbb/tbbmalloc_proxy.h" or add following parameters to the linker options for the binary which is loaded during application startup. It can be either exe-file or dll. @@ -63,4 +59,7 @@ struct __TBB_malloc_proxy_caller { #endif // _MSC_VER +/* Public Windows API */ +extern "C" int TBB_malloc_replacement_log(char *** function_replacement_log_ptr); + #endif //__TBB_tbbmalloc_proxy_H diff --git a/src/tbb/include/oneapi/tbb/tick_count.h b/src/tbb/include/oneapi/tbb/tick_count.h new file mode 100644 index 000000000..2caa56ba1 --- /dev/null +++ b/src/tbb/include/oneapi/tbb/tick_count.h @@ -0,0 +1,99 @@ +/* + Copyright (c) 2005-2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_tick_count_H +#define __TBB_tick_count_H + +#include + +#include "detail/_namespace_injection.h" + +namespace tbb { +namespace detail { +namespace d1 { + + +//! Absolute timestamp +/** @ingroup timing */ +class tick_count { +public: + using clock_type = typename std::conditional::type; + + //! Relative time interval. + class interval_t : public clock_type::duration { + public: + //! Construct a time interval representing zero time duration + interval_t() : clock_type::duration(clock_type::duration::zero()) {} + + //! Construct a time interval representing sec seconds time duration + explicit interval_t( double sec ) + : clock_type::duration(std::chrono::duration_cast(std::chrono::duration(sec))) {} + + //! Return the length of a time interval in seconds + double seconds() const { + return std::chrono::duration_cast>(*this).count(); + } + + //! Extract the intervals from the tick_counts and subtract them. + friend interval_t operator-( const tick_count& t1, const tick_count& t0 ); + + //! Add two intervals. + friend interval_t operator+( const interval_t& i, const interval_t& j ) { + return interval_t(std::chrono::operator+(i, j)); + } + + //! Subtract two intervals. + friend interval_t operator-( const interval_t& i, const interval_t& j ) { + return interval_t(std::chrono::operator-(i, j)); + } + + private: + explicit interval_t( clock_type::duration value_ ) : clock_type::duration(value_) {} + }; + + tick_count() = default; + + //! Return current time. + static tick_count now() { + return clock_type::now(); + } + + //! Subtract two timestamps to get the time interval between + friend interval_t operator-( const tick_count& t1, const tick_count& t0 ) { + return tick_count::interval_t(t1.my_time_point - t0.my_time_point); + } + + //! Return the resolution of the clock in seconds per tick. + static double resolution() { + return static_cast(interval_t::period::num) / interval_t::period::den; + } + +private: + clock_type::time_point my_time_point; + tick_count( clock_type::time_point tp ) : my_time_point(tp) {} +}; + +} // namespace d1 +} // namespace detail + +inline namespace v1 { + using detail::d1::tick_count; +} // namespace v1 + +} // namespace tbb + +#endif /* __TBB_tick_count_H */ diff --git a/src/tbb/include/oneapi/tbb/version.h b/src/tbb/include/oneapi/tbb/version.h new file mode 100644 index 000000000..c8f3ad50e --- /dev/null +++ b/src/tbb/include/oneapi/tbb/version.h @@ -0,0 +1,118 @@ +/* + Copyright (c) 2005-2024 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_version_H +#define __TBB_version_H + +// Exclude all includes during .rc files compilation +#ifndef RC_INVOKED + #include "detail/_config.h" + #include "detail/_namespace_injection.h" +#else + #define __TBB_STRING_AUX(x) #x + #define __TBB_STRING(x) __TBB_STRING_AUX(x) +#endif + +// Product version +#define TBB_VERSION_MAJOR 2022 +// Update version +#define TBB_VERSION_MINOR 0 +// "Patch" version for custom releases +#define TBB_VERSION_PATCH 0 +// Suffix string +#define __TBB_VERSION_SUFFIX "" +// Full official version string +#define TBB_VERSION_STRING \ + __TBB_STRING(TBB_VERSION_MAJOR) "." \ + __TBB_STRING(TBB_VERSION_MINOR) "." \ + __TBB_STRING(TBB_VERSION_PATCH) \ + __TBB_VERSION_SUFFIX + +// OneAPI oneTBB specification version +#define ONETBB_SPEC_VERSION "1.0" +// Full interface version +#define TBB_INTERFACE_VERSION 12140 +// Major interface version +#define TBB_INTERFACE_VERSION_MAJOR (TBB_INTERFACE_VERSION/1000) +// Minor interface version +#define TBB_INTERFACE_VERSION_MINOR (TBB_INTERFACE_VERSION%1000/10) + +// The binary compatibility version +// To be used in SONAME, manifests, etc. +#define __TBB_BINARY_VERSION 12 + +//! TBB_VERSION support +#ifndef TBB_ENDL +#define TBB_ENDL "\n" +#endif + +//TBB_REVAMP_TODO: consider enabling version_string.ver generation +//TBB_REVAMP_TODO: #include "version_string.ver" + +#define __TBB_ONETBB_SPEC_VERSION(N) #N ": SPECIFICATION VERSION\t" ONETBB_SPEC_VERSION TBB_ENDL +#define __TBB_VERSION_NUMBER(N) #N ": VERSION\t\t" TBB_VERSION_STRING TBB_ENDL +#define __TBB_INTERFACE_VERSION_NUMBER(N) #N ": INTERFACE VERSION\t" __TBB_STRING(TBB_INTERFACE_VERSION) TBB_ENDL + +#ifndef TBB_USE_DEBUG + #define __TBB_VERSION_USE_DEBUG(N) #N ": TBB_USE_DEBUG\tundefined" TBB_ENDL +#elif TBB_USE_DEBUG==0 + #define __TBB_VERSION_USE_DEBUG(N) #N ": TBB_USE_DEBUG\t0" TBB_ENDL +#elif TBB_USE_DEBUG==1 + #define __TBB_VERSION_USE_DEBUG(N) #N ": TBB_USE_DEBUG\t1" TBB_ENDL +#elif TBB_USE_DEBUG==2 + #define __TBB_VERSION_USE_DEBUG(N) #N ": TBB_USE_DEBUG\t2" TBB_ENDL +#else + #error Unexpected value for TBB_USE_DEBUG +#endif + +#ifndef TBB_USE_ASSERT + #define __TBB_VERSION_USE_ASSERT(N) #N ": TBB_USE_ASSERT\tundefined" TBB_ENDL +#elif TBB_USE_ASSERT==0 + #define __TBB_VERSION_USE_ASSERT(N) #N ": TBB_USE_ASSERT\t0" TBB_ENDL +#elif TBB_USE_ASSERT==1 + #define __TBB_VERSION_USE_ASSERT(N) #N ": TBB_USE_ASSERT\t1" TBB_ENDL +#elif TBB_USE_ASSERT==2 + #define __TBB_VERSION_USE_ASSERT(N) #N ": TBB_USE_ASSERT\t2" TBB_ENDL +#else + #error Unexpected value for TBB_USE_ASSERT +#endif + +#define TBB_VERSION_STRINGS_P(N) \ + __TBB_ONETBB_SPEC_VERSION(N) \ + __TBB_VERSION_NUMBER(N) \ + __TBB_INTERFACE_VERSION_NUMBER(N) \ + __TBB_VERSION_USE_DEBUG(N) \ + __TBB_VERSION_USE_ASSERT(N) + +#define TBB_VERSION_STRINGS TBB_VERSION_STRINGS_P(oneTBB) +#define TBBMALLOC_VERSION_STRINGS TBB_VERSION_STRINGS_P(TBBmalloc) + +//! The function returns the version string for the Intel(R) oneAPI Threading Building Blocks (oneTBB) +//! shared library being used. +/** + * The returned pointer is an address of a string in the shared library. + * It can be different than the TBB_VERSION_STRING obtained at compile time. + */ +extern "C" TBB_EXPORT const char* __TBB_EXPORTED_FUNC TBB_runtime_version(); + +//! The function returns the interface version of the oneTBB shared library being used. +/** + * The returned version is determined at runtime, not at compile/link time. + * It can be different than the value of TBB_INTERFACE_VERSION obtained at compile time. + */ +extern "C" TBB_EXPORT int __TBB_EXPORTED_FUNC TBB_runtime_interface_version(); + +#endif // __TBB_version_H diff --git a/src/tbb/include/serial/tbb/parallel_for.h b/src/tbb/include/serial/tbb/parallel_for.h deleted file mode 100644 index 83ca9d578..000000000 --- a/src/tbb/include/serial/tbb/parallel_for.h +++ /dev/null @@ -1,219 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_SERIAL_parallel_for_H -#define __TBB_SERIAL_parallel_for_H - -#include "tbb_annotate.h" - -#ifndef __TBB_NORMAL_EXECUTION -#include "tbb/blocked_range.h" -#include "tbb/partitioner.h" -#endif - -#if TBB_USE_EXCEPTIONS -#include -#include // required to construct std exception classes -#else -#include -#include -#endif - -namespace tbb { -namespace serial { -namespace interface9 { - -// parallel_for serial annotated implementation - -template< typename Range, typename Body, typename Partitioner > -class start_for : tbb::internal::no_copy { - Range my_range; - const Body my_body; - typename Partitioner::task_partition_type my_partition; - void execute(); - - //! Constructor for root task. - start_for( const Range& range, const Body& body, Partitioner& partitioner ) : - my_range( range ), - my_body( body ), - my_partition( partitioner ) - { - } - - //! Splitting constructor used to generate children. - /** this becomes left child. Newly constructed object is right child. */ - start_for( start_for& parent_, typename Partitioner::split_type& split_obj ) : - my_range( parent_.my_range, split_obj ), - my_body( parent_.my_body ), - my_partition( parent_.my_partition, split_obj ) - { - } - -public: - static void run( const Range& range, const Body& body, Partitioner& partitioner ) { - if( !range.empty() ) { - ANNOTATE_SITE_BEGIN( tbb_parallel_for ); - { - start_for a( range, body, partitioner ); - a.execute(); - } - ANNOTATE_SITE_END( tbb_parallel_for ); - } - } -}; - -template< typename Range, typename Body, typename Partitioner > -void start_for< Range, Body, Partitioner >::execute() { - if( !my_range.is_divisible() || !my_partition.is_divisible() ) { - ANNOTATE_TASK_BEGIN( tbb_parallel_for_range ); - { - my_body( my_range ); - } - ANNOTATE_TASK_END( tbb_parallel_for_range ); - } else { - typename Partitioner::split_type split_obj; - start_for b( *this, split_obj ); - this->execute(); // Execute the left interval first to keep the serial order. - b.execute(); // Execute the right interval then. - } -} - -//! Parallel iteration over range with default partitioner. -/** @ingroup algorithms **/ -template -void parallel_for( const Range& range, const Body& body ) { - serial::interface9::start_for::run(range,body,__TBB_DEFAULT_PARTITIONER()); -} - -//! Parallel iteration over range with simple partitioner. -/** @ingroup algorithms **/ -template -void parallel_for( const Range& range, const Body& body, const simple_partitioner& partitioner ) { - serial::interface9::start_for::run(range,body,partitioner); -} - -//! Parallel iteration over range with auto_partitioner. -/** @ingroup algorithms **/ -template -void parallel_for( const Range& range, const Body& body, const auto_partitioner& partitioner ) { - serial::interface9::start_for::run(range,body,partitioner); -} - -//! Parallel iteration over range with static_partitioner. -/** @ingroup algorithms **/ -template -void parallel_for( const Range& range, const Body& body, const static_partitioner& partitioner ) { - serial::interface9::start_for::run(range,body,partitioner); -} - -//! Parallel iteration over range with affinity_partitioner. -/** @ingroup algorithms **/ -template -void parallel_for( const Range& range, const Body& body, affinity_partitioner& partitioner ) { - serial::interface9::start_for::run(range,body,partitioner); -} - -//! Implementation of parallel iteration over stepped range of integers with explicit step and partitioner (ignored) -template -void parallel_for_impl(Index first, Index last, Index step, const Function& f, Partitioner& ) { - if (step <= 0 ) { -#if TBB_USE_EXCEPTIONS - throw std::invalid_argument( "nonpositive_step" ); -#else - std::cerr << "nonpositive step in a call to parallel_for" << std::endl; - std::abort(); -#endif - } else if (last > first) { - // Above "else" avoids "potential divide by zero" warning on some platforms - ANNOTATE_SITE_BEGIN( tbb_parallel_for ); - for( Index i = first; i < last; i = i + step ) { - ANNOTATE_TASK_BEGIN( tbb_parallel_for_iteration ); - { f( i ); } - ANNOTATE_TASK_END( tbb_parallel_for_iteration ); - } - ANNOTATE_SITE_END( tbb_parallel_for ); - } -} - -//! Parallel iteration over a range of integers with explicit step and default partitioner -template -void parallel_for(Index first, Index last, Index step, const Function& f) { - parallel_for_impl(first, last, step, f, auto_partitioner()); -} -//! Parallel iteration over a range of integers with explicit step and simple partitioner -template -void parallel_for(Index first, Index last, Index step, const Function& f, const simple_partitioner& p) { - parallel_for_impl(first, last, step, f, p); -} -//! Parallel iteration over a range of integers with explicit step and auto partitioner -template -void parallel_for(Index first, Index last, Index step, const Function& f, const auto_partitioner& p) { - parallel_for_impl(first, last, step, f, p); -} -//! Parallel iteration over a range of integers with explicit step and static partitioner -template -void parallel_for(Index first, Index last, Index step, const Function& f, const static_partitioner& p) { - parallel_for_impl(first, last, step, f, p); -} -//! Parallel iteration over a range of integers with explicit step and affinity partitioner -template -void parallel_for(Index first, Index last, Index step, const Function& f, affinity_partitioner& p) { - parallel_for_impl(first, last, step, f, p); -} - -//! Parallel iteration over a range of integers with default step and default partitioner -template -void parallel_for(Index first, Index last, const Function& f) { - parallel_for_impl(first, last, static_cast(1), f, auto_partitioner()); -} -//! Parallel iteration over a range of integers with default step and simple partitioner -template -void parallel_for(Index first, Index last, const Function& f, const simple_partitioner& p) { - parallel_for_impl(first, last, static_cast(1), f, p); -} -//! Parallel iteration over a range of integers with default step and auto partitioner -template - void parallel_for(Index first, Index last, const Function& f, const auto_partitioner& p) { - parallel_for_impl(first, last, static_cast(1), f, p); -} -//! Parallel iteration over a range of integers with default step and static partitioner -template -void parallel_for(Index first, Index last, const Function& f, const static_partitioner& p) { - parallel_for_impl(first, last, static_cast(1), f, p); -} -//! Parallel iteration over a range of integers with default step and affinity_partitioner -template -void parallel_for(Index first, Index last, const Function& f, affinity_partitioner& p) { - parallel_for_impl(first, last, static_cast(1), f, p); -} - -} // namespace interfaceX - -using interface9::parallel_for; - -} // namespace serial - -#ifndef __TBB_NORMAL_EXECUTION -using serial::interface9::parallel_for; -#endif - -} // namespace tbb - -#endif /* __TBB_SERIAL_parallel_for_H */ diff --git a/src/tbb/include/serial/tbb/tbb_annotate.h b/src/tbb/include/serial/tbb/tbb_annotate.h deleted file mode 100644 index 81c8d3a4d..000000000 --- a/src/tbb/include/serial/tbb/tbb_annotate.h +++ /dev/null @@ -1,36 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_annotate_H -#define __TBB_annotate_H - -// Macros used by the Intel(R) Parallel Advisor. -#ifdef __TBB_NORMAL_EXECUTION - #define ANNOTATE_SITE_BEGIN( site ) - #define ANNOTATE_SITE_END( site ) - #define ANNOTATE_TASK_BEGIN( task ) - #define ANNOTATE_TASK_END( task ) - #define ANNOTATE_LOCK_ACQUIRE( lock ) - #define ANNOTATE_LOCK_RELEASE( lock ) -#else - #include -#endif - -#endif /* __TBB_annotate_H */ diff --git a/src/tbb/include/tbb/aggregator.h b/src/tbb/include/tbb/aggregator.h deleted file mode 100644 index 6aecbb748..000000000 --- a/src/tbb/include/tbb/aggregator.h +++ /dev/null @@ -1,202 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB__aggregator_H -#define __TBB__aggregator_H - -#if !TBB_PREVIEW_AGGREGATOR -#error Set TBB_PREVIEW_AGGREGATOR before including aggregator.h -#endif - -#include "atomic.h" -#include "tbb_profiling.h" - -namespace tbb { -namespace interface6 { - -using namespace tbb::internal; - -class aggregator_operation { - template friend class aggregator_ext; - uintptr_t status; - aggregator_operation* my_next; -public: - enum aggregator_operation_status { agg_waiting=0, agg_finished }; - aggregator_operation() : status(agg_waiting), my_next(NULL) {} - /// Call start before handling this operation - void start() { call_itt_notify(acquired, &status); } - /// Call finish when done handling this operation - /** The operation will be released to its originating thread, and possibly deleted. */ - void finish() { itt_store_word_with_release(status, uintptr_t(agg_finished)); } - aggregator_operation* next() { return itt_hide_load_word(my_next);} - void set_next(aggregator_operation* n) { itt_hide_store_word(my_next, n); } -}; - -namespace internal { - -class basic_operation_base : public aggregator_operation { - friend class basic_handler; - virtual void apply_body() = 0; -public: - basic_operation_base() : aggregator_operation() {} - virtual ~basic_operation_base() {} -}; - -template -class basic_operation : public basic_operation_base, no_assign { - const Body& my_body; - void apply_body() __TBB_override { my_body(); } -public: - basic_operation(const Body& b) : basic_operation_base(), my_body(b) {} -}; - -class basic_handler { -public: - basic_handler() {} - void operator()(aggregator_operation* op_list) const { - while (op_list) { - // ITT note: &(op_list->status) tag is used to cover accesses to the operation data. - // The executing thread "acquires" the tag (see start()) and then performs - // the associated operation w/o triggering a race condition diagnostics. - // A thread that created the operation is waiting for its status (see execute_impl()), - // so when this thread is done with the operation, it will "release" the tag - // and update the status (see finish()) to give control back to the waiting thread. - basic_operation_base& request = static_cast(*op_list); - // IMPORTANT: need to advance op_list to op_list->next() before calling request.finish() - op_list = op_list->next(); - request.start(); - request.apply_body(); - request.finish(); - } - } -}; - -} // namespace internal - -//! Aggregator base class and expert interface -/** An aggregator for collecting operations coming from multiple sources and executing - them serially on a single thread. */ -template -class aggregator_ext : tbb::internal::no_copy { -public: - aggregator_ext(const handler_type& h) : handler_busy(0), handle_operations(h) { mailbox = NULL; } - - //! EXPERT INTERFACE: Enter a user-made operation into the aggregator's mailbox. - /** Details of user-made operations must be handled by user-provided handler */ - void process(aggregator_operation *op) { execute_impl(*op); } - -protected: - /** Place operation in mailbox, then either handle mailbox or wait for the operation - to be completed by a different thread. */ - void execute_impl(aggregator_operation& op) { - aggregator_operation* res; - - // ITT note: &(op.status) tag is used to cover accesses to this operation. This - // thread has created the operation, and now releases it so that the handler - // thread may handle the associated operation w/o triggering a race condition; - // thus this tag will be acquired just before the operation is handled in the - // handle_operations functor. - call_itt_notify(releasing, &(op.status)); - // insert the operation into the list - do { - // ITT may flag the following line as a race; it is a false positive: - // This is an atomic read; we don't provide itt_hide_load_word for atomics - op.my_next = res = mailbox; // NOT A RACE - } while (mailbox.compare_and_swap(&op, res) != res); - if (!res) { // first in the list; handle the operations - // ITT note: &mailbox tag covers access to the handler_busy flag, which this - // waiting handler thread will try to set before entering handle_operations. - call_itt_notify(acquired, &mailbox); - start_handle_operations(); - __TBB_ASSERT(op.status, NULL); - } - else { // not first; wait for op to be ready - call_itt_notify(prepare, &(op.status)); - spin_wait_while_eq(op.status, uintptr_t(aggregator_operation::agg_waiting)); - itt_load_word_with_acquire(op.status); - } - } - - -private: - //! An atomically updated list (aka mailbox) of aggregator_operations - atomic mailbox; - - //! Controls thread access to handle_operations - /** Behaves as boolean flag where 0=false, 1=true */ - uintptr_t handler_busy; - - handler_type handle_operations; - - //! Trigger the handling of operations when the handler is free - void start_handle_operations() { - aggregator_operation *pending_operations; - - // ITT note: &handler_busy tag covers access to mailbox as it is passed - // between active and waiting handlers. Below, the waiting handler waits until - // the active handler releases, and the waiting handler acquires &handler_busy as - // it becomes the active_handler. The release point is at the end of this - // function, when all operations in mailbox have been handled by the - // owner of this aggregator. - call_itt_notify(prepare, &handler_busy); - // get handler_busy: only one thread can possibly spin here at a time - spin_wait_until_eq(handler_busy, uintptr_t(0)); - call_itt_notify(acquired, &handler_busy); - // acquire fence not necessary here due to causality rule and surrounding atomics - __TBB_store_with_release(handler_busy, uintptr_t(1)); - - // ITT note: &mailbox tag covers access to the handler_busy flag itself. - // Capturing the state of the mailbox signifies that handler_busy has been - // set and a new active handler will now process that list's operations. - call_itt_notify(releasing, &mailbox); - // grab pending_operations - pending_operations = mailbox.fetch_and_store(NULL); - - // handle all the operations - handle_operations(pending_operations); - - // release the handler - itt_store_word_with_release(handler_busy, uintptr_t(0)); - } -}; - -//! Basic aggregator interface -class aggregator : private aggregator_ext { -public: - aggregator() : aggregator_ext(internal::basic_handler()) {} - //! BASIC INTERFACE: Enter a function for exclusive execution by the aggregator. - /** The calling thread stores the function object in a basic_operation and - places the operation in the aggregator's mailbox */ - template - void execute(const Body& b) { - internal::basic_operation op(b); - this->execute_impl(op); - } -}; - -} // namespace interface6 - -using interface6::aggregator; -using interface6::aggregator_ext; -using interface6::aggregator_operation; - -} // namespace tbb - -#endif // __TBB__aggregator_H diff --git a/src/tbb/include/tbb/atomic.h b/src/tbb/include/tbb/atomic.h deleted file mode 100644 index 72ec534e3..000000000 --- a/src/tbb/include/tbb/atomic.h +++ /dev/null @@ -1,558 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_atomic_H -#define __TBB_atomic_H - -#include - -#if _MSC_VER -#define __TBB_LONG_LONG __int64 -#else -#define __TBB_LONG_LONG long long -#endif /* _MSC_VER */ - -#include "tbb_machine.h" - -#if _MSC_VER && !__INTEL_COMPILER - // Suppress overzealous compiler warnings till the end of the file - #pragma warning (push) - #pragma warning (disable: 4244 4267 4512) -#endif - -namespace tbb { - -//! Specifies memory semantics. -enum memory_semantics { - //! Sequential consistency - full_fence, - //! Acquire - acquire, - //! Release - release, - //! No ordering - relaxed -}; - -//! @cond INTERNAL -namespace internal { - -#if __TBB_ALIGNAS_PRESENT - #define __TBB_DECL_ATOMIC_FIELD(t,f,a) alignas(a) t f; -#elif __TBB_ATTRIBUTE_ALIGNED_PRESENT - #define __TBB_DECL_ATOMIC_FIELD(t,f,a) t f __attribute__ ((aligned(a))); -#elif __TBB_DECLSPEC_ALIGN_PRESENT - #define __TBB_DECL_ATOMIC_FIELD(t,f,a) __declspec(align(a)) t f; -#else - #error Do not know syntax for forcing alignment. -#endif - -template -struct atomic_rep; // Primary template declared, but never defined. - -template<> -struct atomic_rep<1> { // Specialization - typedef int8_t word; -}; -template<> -struct atomic_rep<2> { // Specialization - typedef int16_t word; -}; -template<> -struct atomic_rep<4> { // Specialization -#if _MSC_VER && !_WIN64 - // Work-around that avoids spurious /Wp64 warnings - typedef intptr_t word; -#else - typedef int32_t word; -#endif -}; -#if __TBB_64BIT_ATOMICS -template<> -struct atomic_rep<8> { // Specialization - typedef int64_t word; -}; -#endif - -template -struct aligned_storage; - -//the specializations are needed to please MSVC syntax of __declspec(align()) which accept _literal_ constants only -#if __TBB_ATOMIC_CTORS - #define ATOMIC_STORAGE_PARTIAL_SPECIALIZATION(S) \ - template \ - struct aligned_storage { \ - __TBB_DECL_ATOMIC_FIELD(value_type,my_value,S) \ - aligned_storage() = default ; \ - constexpr aligned_storage(value_type value):my_value(value){} \ - }; \ - -#else - #define ATOMIC_STORAGE_PARTIAL_SPECIALIZATION(S) \ - template \ - struct aligned_storage { \ - __TBB_DECL_ATOMIC_FIELD(value_type,my_value,S) \ - }; \ - -#endif - -template -struct aligned_storage { - value_type my_value; -#if __TBB_ATOMIC_CTORS - aligned_storage() = default ; - constexpr aligned_storage(value_type value):my_value(value){} -#endif -}; - -ATOMIC_STORAGE_PARTIAL_SPECIALIZATION(2) -ATOMIC_STORAGE_PARTIAL_SPECIALIZATION(4) -#if __TBB_64BIT_ATOMICS -ATOMIC_STORAGE_PARTIAL_SPECIALIZATION(8) -#endif - -template -struct atomic_traits; // Primary template declared, but not defined. - -#define __TBB_DECL_FENCED_ATOMIC_PRIMITIVES(S,M) \ - template<> struct atomic_traits { \ - typedef atomic_rep::word word; \ - inline static word compare_and_swap( volatile void* location, word new_value, word comparand ) { \ - return __TBB_machine_cmpswp##S##M(location,new_value,comparand); \ - } \ - inline static word fetch_and_add( volatile void* location, word addend ) { \ - return __TBB_machine_fetchadd##S##M(location,addend); \ - } \ - inline static word fetch_and_store( volatile void* location, word value ) { \ - return __TBB_machine_fetchstore##S##M(location,value); \ - } \ - }; - -#define __TBB_DECL_ATOMIC_PRIMITIVES(S) \ - template \ - struct atomic_traits { \ - typedef atomic_rep::word word; \ - inline static word compare_and_swap( volatile void* location, word new_value, word comparand ) { \ - return __TBB_machine_cmpswp##S(location,new_value,comparand); \ - } \ - inline static word fetch_and_add( volatile void* location, word addend ) { \ - return __TBB_machine_fetchadd##S(location,addend); \ - } \ - inline static word fetch_and_store( volatile void* location, word value ) { \ - return __TBB_machine_fetchstore##S(location,value); \ - } \ - }; - -template -struct atomic_load_store_traits; // Primary template declaration - -#define __TBB_DECL_ATOMIC_LOAD_STORE_PRIMITIVES(M) \ - template<> struct atomic_load_store_traits { \ - template \ - inline static T load( const volatile T& location ) { \ - return __TBB_load_##M( location ); \ - } \ - template \ - inline static void store( volatile T& location, T value ) { \ - __TBB_store_##M( location, value ); \ - } \ - } - -#if __TBB_USE_FENCED_ATOMICS -__TBB_DECL_FENCED_ATOMIC_PRIMITIVES(1,full_fence) -__TBB_DECL_FENCED_ATOMIC_PRIMITIVES(2,full_fence) -__TBB_DECL_FENCED_ATOMIC_PRIMITIVES(4,full_fence) -__TBB_DECL_FENCED_ATOMIC_PRIMITIVES(1,acquire) -__TBB_DECL_FENCED_ATOMIC_PRIMITIVES(2,acquire) -__TBB_DECL_FENCED_ATOMIC_PRIMITIVES(4,acquire) -__TBB_DECL_FENCED_ATOMIC_PRIMITIVES(1,release) -__TBB_DECL_FENCED_ATOMIC_PRIMITIVES(2,release) -__TBB_DECL_FENCED_ATOMIC_PRIMITIVES(4,release) -__TBB_DECL_FENCED_ATOMIC_PRIMITIVES(1,relaxed) -__TBB_DECL_FENCED_ATOMIC_PRIMITIVES(2,relaxed) -__TBB_DECL_FENCED_ATOMIC_PRIMITIVES(4,relaxed) -#if __TBB_64BIT_ATOMICS -__TBB_DECL_FENCED_ATOMIC_PRIMITIVES(8,full_fence) -__TBB_DECL_FENCED_ATOMIC_PRIMITIVES(8,acquire) -__TBB_DECL_FENCED_ATOMIC_PRIMITIVES(8,release) -__TBB_DECL_FENCED_ATOMIC_PRIMITIVES(8,relaxed) -#endif -#else /* !__TBB_USE_FENCED_ATOMICS */ -__TBB_DECL_ATOMIC_PRIMITIVES(1) -__TBB_DECL_ATOMIC_PRIMITIVES(2) -__TBB_DECL_ATOMIC_PRIMITIVES(4) -#if __TBB_64BIT_ATOMICS -__TBB_DECL_ATOMIC_PRIMITIVES(8) -#endif -#endif /* !__TBB_USE_FENCED_ATOMICS */ - -__TBB_DECL_ATOMIC_LOAD_STORE_PRIMITIVES(full_fence); -__TBB_DECL_ATOMIC_LOAD_STORE_PRIMITIVES(acquire); -__TBB_DECL_ATOMIC_LOAD_STORE_PRIMITIVES(release); -__TBB_DECL_ATOMIC_LOAD_STORE_PRIMITIVES(relaxed); - -//! Additive inverse of 1 for type T. -/** Various compilers issue various warnings if -1 is used with various integer types. - The baroque expression below avoids all the warnings (we hope). */ -#define __TBB_MINUS_ONE(T) (T(T(0)-T(1))) - -//! Base class that provides basic functionality for atomic without fetch_and_add. -/** Works for any type T that has the same size as an integral type, has a trivial constructor/destructor, - and can be copied/compared by memcpy/memcmp. */ -template -struct atomic_impl { -protected: - aligned_storage my_storage; -private: - //TODO: rechecks on recent versions of gcc if union is still the _only_ way to do a conversion without warnings - //! Union type used to convert type T to underlying integral type. - template - union converter { - typedef typename atomic_rep::word bits_type; - converter(){} - converter(value_type a_value) : value(a_value) {} - value_type value; - bits_type bits; - }; - - template - static typename converter::bits_type to_bits(value_t value){ - return converter(value).bits; - } - template - static value_t to_value(typename converter::bits_type bits){ - converter u; - u.bits = bits; - return u.value; - } - - template - union ptr_converter; //Primary template declared, but never defined. - - template - union ptr_converter { - ptr_converter(){} - ptr_converter(value_t* a_value) : value(a_value) {} - value_t* value; - uintptr_t bits; - }; - //TODO: check if making to_bits accepting reference (thus unifying it with to_bits_ref) - //does not hurt performance - template - static typename converter::bits_type & to_bits_ref(value_t& value){ - //TODO: this #ifdef is temporary workaround, as union conversion seems to fail - //on suncc for 64 bit types for 32 bit target - #if !__SUNPRO_CC - return *(typename converter::bits_type*)ptr_converter(&value).bits; - #else - return *(typename converter::bits_type*)(&value); - #endif - } - - -public: - typedef T value_type; - -#if __TBB_ATOMIC_CTORS - atomic_impl() = default ; - constexpr atomic_impl(value_type value):my_storage(value){} -#endif - template - value_type fetch_and_store( value_type value ) { - return to_value( - internal::atomic_traits::fetch_and_store( &my_storage.my_value, to_bits(value) ) - ); - } - - value_type fetch_and_store( value_type value ) { - return fetch_and_store(value); - } - - template - value_type compare_and_swap( value_type value, value_type comparand ) { - return to_value( - internal::atomic_traits::compare_and_swap( &my_storage.my_value, to_bits(value), to_bits(comparand) ) - ); - } - - value_type compare_and_swap( value_type value, value_type comparand ) { - return compare_and_swap(value,comparand); - } - - operator value_type() const volatile { // volatile qualifier here for backwards compatibility - return to_value( - __TBB_load_with_acquire( to_bits_ref(my_storage.my_value) ) - ); - } - - template - value_type load () const { - return to_value( - internal::atomic_load_store_traits::load( to_bits_ref(my_storage.my_value) ) - ); - } - - value_type load () const { - return load(); - } - - template - void store ( value_type value ) { - internal::atomic_load_store_traits::store( to_bits_ref(my_storage.my_value), to_bits(value)); - } - - void store ( value_type value ) { - store( value ); - } - -protected: - value_type store_with_release( value_type rhs ) { - //TODO: unify with store - __TBB_store_with_release( to_bits_ref(my_storage.my_value), to_bits(rhs) ); - return rhs; - } -}; - -//! Base class that provides basic functionality for atomic with fetch_and_add. -/** I is the underlying type. - D is the difference type. - StepType should be char if I is an integral type, and T if I is a T*. */ -template -struct atomic_impl_with_arithmetic: atomic_impl { -public: - typedef I value_type; -#if __TBB_ATOMIC_CTORS - atomic_impl_with_arithmetic() = default ; - constexpr atomic_impl_with_arithmetic(value_type value): atomic_impl(value){} -#endif - template - value_type fetch_and_add( D addend ) { - return value_type(internal::atomic_traits::fetch_and_add( &this->my_storage.my_value, addend*sizeof(StepType) )); - } - - value_type fetch_and_add( D addend ) { - return fetch_and_add(addend); - } - - template - value_type fetch_and_increment() { - return fetch_and_add(1); - } - - value_type fetch_and_increment() { - return fetch_and_add(1); - } - - template - value_type fetch_and_decrement() { - return fetch_and_add(__TBB_MINUS_ONE(D)); - } - - value_type fetch_and_decrement() { - return fetch_and_add(__TBB_MINUS_ONE(D)); - } - -public: - value_type operator+=( D value ) { - return fetch_and_add(value)+value; - } - - value_type operator-=( D value ) { - // Additive inverse of value computed using binary minus, - // instead of unary minus, for sake of avoiding compiler warnings. - return operator+=(D(0)-value); - } - - value_type operator++() { - return fetch_and_add(1)+1; - } - - value_type operator--() { - return fetch_and_add(__TBB_MINUS_ONE(D))-1; - } - - value_type operator++(int) { - return fetch_and_add(1); - } - - value_type operator--(int) { - return fetch_and_add(__TBB_MINUS_ONE(D)); - } -}; - -} /* Internal */ -//! @endcond - -//! Primary template for atomic. -/** See the Reference for details. - @ingroup synchronization */ -template -struct atomic: internal::atomic_impl { -#if __TBB_ATOMIC_CTORS - atomic() = default; - constexpr atomic(T arg): internal::atomic_impl(arg) {} -#endif - T operator=( T rhs ) { - // "this" required here in strict ISO C++ because store_with_release is a dependent name - return this->store_with_release(rhs); - } - atomic& operator=( const atomic& rhs ) {this->store_with_release(rhs); return *this;} -}; - -#if __TBB_ATOMIC_CTORS - #define __TBB_DECL_ATOMIC(T) \ - template<> struct atomic: internal::atomic_impl_with_arithmetic { \ - atomic() = default; \ - constexpr atomic(T arg): internal::atomic_impl_with_arithmetic(arg) {} \ - \ - T operator=( T rhs ) {return store_with_release(rhs);} \ - atomic& operator=( const atomic& rhs ) {store_with_release(rhs); return *this;} \ - }; -#else - #define __TBB_DECL_ATOMIC(T) \ - template<> struct atomic: internal::atomic_impl_with_arithmetic { \ - T operator=( T rhs ) {return store_with_release(rhs);} \ - atomic& operator=( const atomic& rhs ) {store_with_release(rhs); return *this;} \ - }; -#endif - -#if __TBB_64BIT_ATOMICS -//TODO: consider adding non-default (and atomic) copy constructor for 32bit platform -__TBB_DECL_ATOMIC(__TBB_LONG_LONG) -__TBB_DECL_ATOMIC(unsigned __TBB_LONG_LONG) -#else -// test_atomic will verify that sizeof(long long)==8 -#endif -__TBB_DECL_ATOMIC(long) -__TBB_DECL_ATOMIC(unsigned long) - -#if _MSC_VER && !_WIN64 -#if __TBB_ATOMIC_CTORS -/* Special version of __TBB_DECL_ATOMIC that avoids gratuitous warnings from cl /Wp64 option. - It is identical to __TBB_DECL_ATOMIC(unsigned) except that it replaces operator=(T) - with an operator=(U) that explicitly converts the U to a T. Types T and U should be - type synonyms on the platform. Type U should be the wider variant of T from the - perspective of /Wp64. */ -#define __TBB_DECL_ATOMIC_ALT(T,U) \ - template<> struct atomic: internal::atomic_impl_with_arithmetic { \ - atomic() = default ; \ - constexpr atomic(T arg): internal::atomic_impl_with_arithmetic(arg) {} \ - T operator=( U rhs ) {return store_with_release(T(rhs));} \ - atomic& operator=( const atomic& rhs ) {store_with_release(rhs); return *this;} \ - }; -#else -#define __TBB_DECL_ATOMIC_ALT(T,U) \ - template<> struct atomic: internal::atomic_impl_with_arithmetic { \ - T operator=( U rhs ) {return store_with_release(T(rhs));} \ - atomic& operator=( const atomic& rhs ) {store_with_release(rhs); return *this;} \ - }; -#endif -__TBB_DECL_ATOMIC_ALT(unsigned,size_t) -__TBB_DECL_ATOMIC_ALT(int,ptrdiff_t) -#else -__TBB_DECL_ATOMIC(unsigned) -__TBB_DECL_ATOMIC(int) -#endif /* _MSC_VER && !_WIN64 */ - -__TBB_DECL_ATOMIC(unsigned short) -__TBB_DECL_ATOMIC(short) -__TBB_DECL_ATOMIC(char) -__TBB_DECL_ATOMIC(signed char) -__TBB_DECL_ATOMIC(unsigned char) - -#if !_MSC_VER || defined(_NATIVE_WCHAR_T_DEFINED) -__TBB_DECL_ATOMIC(wchar_t) -#endif /* _MSC_VER||!defined(_NATIVE_WCHAR_T_DEFINED) */ - -//! Specialization for atomic with arithmetic and operator->. -template struct atomic: internal::atomic_impl_with_arithmetic { -#if __TBB_ATOMIC_CTORS - atomic() = default ; - constexpr atomic(T* arg): internal::atomic_impl_with_arithmetic(arg) {} -#endif - T* operator=( T* rhs ) { - // "this" required here in strict ISO C++ because store_with_release is a dependent name - return this->store_with_release(rhs); - } - atomic& operator=( const atomic& rhs ) { - this->store_with_release(rhs); return *this; - } - T* operator->() const { - return (*this); - } -}; - -//! Specialization for atomic, for sake of not allowing arithmetic or operator->. -template<> struct atomic: internal::atomic_impl { -#if __TBB_ATOMIC_CTORS - atomic() = default ; - constexpr atomic(void* arg): internal::atomic_impl(arg) {} -#endif - void* operator=( void* rhs ) { - // "this" required here in strict ISO C++ because store_with_release is a dependent name - return this->store_with_release(rhs); - } - atomic& operator=( const atomic& rhs ) { - this->store_with_release(rhs); return *this; - } -}; - -// Helpers to workaround ugly syntax of calling template member function of a -// template class with template argument dependent on template parameters. - -template -T load ( const atomic& a ) { return a.template load(); } - -template -void store ( atomic& a, T value ) { a.template store(value); } - -namespace interface6{ -//! Make an atomic for use in an initialization (list), as an alternative to zero-initialization or normal assignment. -template -atomic make_atomic(T t) { - atomic a; - store(a,t); - return a; -} -} -using interface6::make_atomic; - -namespace internal { -template -void swap(atomic & lhs, atomic & rhs){ - T tmp = load(lhs); - store(lhs,load(rhs)); - store(rhs,tmp); -} - -// only to aid in the gradual conversion of ordinary variables to proper atomics -template -inline atomic& as_atomic( T& t ) { - return (atomic&)t; -} -} // namespace tbb::internal - -} // namespace tbb - -#if _MSC_VER && !__INTEL_COMPILER - #pragma warning (pop) -#endif // warnings are restored - -#endif /* __TBB_atomic_H */ diff --git a/src/tbb/include/tbb/blocked_range.h b/src/tbb/include/tbb/blocked_range.h index 1f3901ebb..316ec01ba 100644 --- a/src/tbb/include/tbb/blocked_range.h +++ b/src/tbb/include/tbb/blocked_range.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,147 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ -#ifndef __TBB_blocked_range_H -#define __TBB_blocked_range_H - -#include "tbb_stddef.h" - -namespace tbb { - -/** \page range_req Requirements on range concept - Class \c R implementing the concept of range must define: - - \code R::R( const R& ); \endcode Copy constructor - - \code R::~R(); \endcode Destructor - - \code bool R::is_divisible() const; \endcode True if range can be partitioned into two subranges - - \code bool R::empty() const; \endcode True if range is empty - - \code R::R( R& r, split ); \endcode Split range \c r into two subranges. -**/ - -//! A range over which to iterate. -/** @ingroup algorithms */ -template -class blocked_range { -public: - //! Type of a value - /** Called a const_iterator for sake of algorithms that need to treat a blocked_range - as an STL container. */ - typedef Value const_iterator; - - //! Type for size of a range - typedef std::size_t size_type; - - //! Construct range with default-constructed values for begin, end, and grainsize. - /** Requires that Value have a default constructor. */ - blocked_range() : my_end(), my_begin(), my_grainsize() {} - - //! Construct range over half-open interval [begin,end), with the given grainsize. - blocked_range( Value begin_, Value end_, size_type grainsize_=1 ) : - my_end(end_), my_begin(begin_), my_grainsize(grainsize_) - { - __TBB_ASSERT( my_grainsize>0, "grainsize must be positive" ); - } - - //! Beginning of range. - const_iterator begin() const {return my_begin;} - - //! One past last value in range. - const_iterator end() const {return my_end;} - - //! Size of the range - /** Unspecified if end() - friend class blocked_range2d; - - template - friend class blocked_range3d; -}; - -} // namespace tbb - -#endif /* __TBB_blocked_range_H */ +#include "../oneapi/tbb/blocked_range.h" diff --git a/src/tbb/include/tbb/blocked_range2d.h b/src/tbb/include/tbb/blocked_range2d.h index 1e3dfd171..1e1324078 100644 --- a/src/tbb/include/tbb/blocked_range2d.h +++ b/src/tbb/include/tbb/blocked_range2d.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,97 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ -#ifndef __TBB_blocked_range2d_H -#define __TBB_blocked_range2d_H - -#include "tbb_stddef.h" -#include "blocked_range.h" - -namespace tbb { - -//! A 2-dimensional range that models the Range concept. -/** @ingroup algorithms */ -template -class blocked_range2d { -public: - //! Type for size of an iteration range - typedef blocked_range row_range_type; - typedef blocked_range col_range_type; - -private: - row_range_type my_rows; - col_range_type my_cols; - -public: - - blocked_range2d( RowValue row_begin, RowValue row_end, typename row_range_type::size_type row_grainsize, - ColValue col_begin, ColValue col_end, typename col_range_type::size_type col_grainsize ) : - my_rows(row_begin,row_end,row_grainsize), - my_cols(col_begin,col_end,col_grainsize) - {} - - blocked_range2d( RowValue row_begin, RowValue row_end, - ColValue col_begin, ColValue col_end ) : - my_rows(row_begin,row_end), - my_cols(col_begin,col_end) - {} - - //! True if range is empty - bool empty() const { - // Yes, it is a logical OR here, not AND. - return my_rows.empty() || my_cols.empty(); - } - - //! True if range is divisible into two pieces. - bool is_divisible() const { - return my_rows.is_divisible() || my_cols.is_divisible(); - } - - blocked_range2d( blocked_range2d& r, split ) : - my_rows(r.my_rows), - my_cols(r.my_cols) - { - split split_obj; - do_split(r, split_obj); - } - -#if __TBB_USE_PROPORTIONAL_SPLIT_IN_BLOCKED_RANGES - //! Static field to support proportional split - static const bool is_splittable_in_proportion = true; - - blocked_range2d( blocked_range2d& r, proportional_split& proportion ) : - my_rows(r.my_rows), - my_cols(r.my_cols) - { - do_split(r, proportion); - } -#endif /* __TBB_USE_PROPORTIONAL_SPLIT_IN_BLOCKED_RANGES */ - - //! The rows of the iteration space - const row_range_type& rows() const {return my_rows;} - - //! The columns of the iteration space - const col_range_type& cols() const {return my_cols;} - -private: - - template - void do_split( blocked_range2d& r, Split& split_obj ) - { - if( my_rows.size()*double(my_cols.grainsize()) < my_cols.size()*double(my_rows.grainsize()) ) { - my_cols.my_begin = col_range_type::do_split(r.my_cols, split_obj); - } else { - my_rows.my_begin = row_range_type::do_split(r.my_rows, split_obj); - } - } -}; - -} // namespace tbb - -#endif /* __TBB_blocked_range2d_H */ +#include "../oneapi/tbb/blocked_range2d.h" diff --git a/src/tbb/include/tbb/blocked_range3d.h b/src/tbb/include/tbb/blocked_range3d.h index 5f3b3c860..332197966 100644 --- a/src/tbb/include/tbb/blocked_range3d.h +++ b/src/tbb/include/tbb/blocked_range3d.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,116 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ -#ifndef __TBB_blocked_range3d_H -#define __TBB_blocked_range3d_H - -#include "tbb_stddef.h" -#include "blocked_range.h" - -namespace tbb { - -//! A 3-dimensional range that models the Range concept. -/** @ingroup algorithms */ -template -class blocked_range3d { -public: - //! Type for size of an iteration range - typedef blocked_range page_range_type; - typedef blocked_range row_range_type; - typedef blocked_range col_range_type; - -private: - page_range_type my_pages; - row_range_type my_rows; - col_range_type my_cols; - -public: - - blocked_range3d( PageValue page_begin, PageValue page_end, - RowValue row_begin, RowValue row_end, - ColValue col_begin, ColValue col_end ) : - my_pages(page_begin,page_end), - my_rows(row_begin,row_end), - my_cols(col_begin,col_end) - {} - - blocked_range3d( PageValue page_begin, PageValue page_end, typename page_range_type::size_type page_grainsize, - RowValue row_begin, RowValue row_end, typename row_range_type::size_type row_grainsize, - ColValue col_begin, ColValue col_end, typename col_range_type::size_type col_grainsize ) : - my_pages(page_begin,page_end,page_grainsize), - my_rows(row_begin,row_end,row_grainsize), - my_cols(col_begin,col_end,col_grainsize) - {} - - //! True if range is empty - bool empty() const { - // Yes, it is a logical OR here, not AND. - return my_pages.empty() || my_rows.empty() || my_cols.empty(); - } - - //! True if range is divisible into two pieces. - bool is_divisible() const { - return my_pages.is_divisible() || my_rows.is_divisible() || my_cols.is_divisible(); - } - - blocked_range3d( blocked_range3d& r, split ) : - my_pages(r.my_pages), - my_rows(r.my_rows), - my_cols(r.my_cols) - { - split split_obj; - do_split(r, split_obj); - } - -#if __TBB_USE_PROPORTIONAL_SPLIT_IN_BLOCKED_RANGES - //! Static field to support proportional split - static const bool is_splittable_in_proportion = true; - - blocked_range3d( blocked_range3d& r, proportional_split& proportion ) : - my_pages(r.my_pages), - my_rows(r.my_rows), - my_cols(r.my_cols) - { - do_split(r, proportion); - } -#endif /* __TBB_USE_PROPORTIONAL_SPLIT_IN_BLOCKED_RANGES */ - - //! The pages of the iteration space - const page_range_type& pages() const {return my_pages;} - - //! The rows of the iteration space - const row_range_type& rows() const {return my_rows;} - - //! The columns of the iteration space - const col_range_type& cols() const {return my_cols;} - -private: - - template - void do_split( blocked_range3d& r, Split& split_obj) - { - if ( my_pages.size()*double(my_rows.grainsize()) < my_rows.size()*double(my_pages.grainsize()) ) { - if ( my_rows.size()*double(my_cols.grainsize()) < my_cols.size()*double(my_rows.grainsize()) ) { - my_cols.my_begin = col_range_type::do_split(r.my_cols, split_obj); - } else { - my_rows.my_begin = row_range_type::do_split(r.my_rows, split_obj); - } - } else { - if ( my_pages.size()*double(my_cols.grainsize()) < my_cols.size()*double(my_pages.grainsize()) ) { - my_cols.my_begin = col_range_type::do_split(r.my_cols, split_obj); - } else { - my_pages.my_begin = page_range_type::do_split(r.my_pages, split_obj); - } - } - } -}; - -} // namespace tbb - -#endif /* __TBB_blocked_range3d_H */ +#include "../oneapi/tbb/blocked_range3d.h" diff --git a/src/tbb/src/tbb/mac32-tbb-export.def b/src/tbb/include/tbb/blocked_rangeNd.h similarity index 82% rename from src/tbb/src/tbb/mac32-tbb-export.def rename to src/tbb/include/tbb/blocked_rangeNd.h index 5ed01b1a0..0c0fb7303 100644 --- a/src/tbb/src/tbb/mac32-tbb-export.def +++ b/src/tbb/include/tbb/blocked_rangeNd.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2017-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,12 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ -#define __TBB_SYMBOL( sym ) _##sym -#include "mac32-tbb-export.lst" - +#include "../oneapi/tbb/blocked_rangeNd.h" diff --git a/src/tbb/include/tbb/cache_aligned_allocator.h b/src/tbb/include/tbb/cache_aligned_allocator.h index d435e785c..2d3c66a74 100644 --- a/src/tbb/include/tbb/cache_aligned_allocator.h +++ b/src/tbb/include/tbb/cache_aligned_allocator.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,126 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ -#ifndef __TBB_cache_aligned_allocator_H -#define __TBB_cache_aligned_allocator_H - -#include -#include "tbb_stddef.h" -#if __TBB_ALLOCATOR_CONSTRUCT_VARIADIC - #include // std::forward -#endif - -namespace tbb { - -//! @cond INTERNAL -namespace internal { - //! Cache/sector line size. - /** @ingroup memory_allocation */ - size_t __TBB_EXPORTED_FUNC NFS_GetLineSize(); - - //! Allocate memory on cache/sector line boundary. - /** @ingroup memory_allocation */ - void* __TBB_EXPORTED_FUNC NFS_Allocate( size_t n_element, size_t element_size, void* hint ); - - //! Free memory allocated by NFS_Allocate. - /** Freeing a NULL pointer is allowed, but has no effect. - @ingroup memory_allocation */ - void __TBB_EXPORTED_FUNC NFS_Free( void* ); -} -//! @endcond - -#if _MSC_VER && !defined(__INTEL_COMPILER) - // Workaround for erroneous "unreferenced parameter" warning in method destroy. - #pragma warning (push) - #pragma warning (disable: 4100) -#endif - -//! Meets "allocator" requirements of ISO C++ Standard, Section 20.1.5 -/** The members are ordered the same way they are in section 20.4.1 - of the ISO C++ standard. - @ingroup memory_allocation */ -template -class cache_aligned_allocator { -public: - typedef typename internal::allocator_type::value_type value_type; - typedef value_type* pointer; - typedef const value_type* const_pointer; - typedef value_type& reference; - typedef const value_type& const_reference; - typedef size_t size_type; - typedef ptrdiff_t difference_type; - template struct rebind { - typedef cache_aligned_allocator other; - }; - - cache_aligned_allocator() throw() {} - cache_aligned_allocator( const cache_aligned_allocator& ) throw() {} - template cache_aligned_allocator(const cache_aligned_allocator&) throw() {} - - pointer address(reference x) const {return &x;} - const_pointer address(const_reference x) const {return &x;} - - //! Allocate space for n objects, starting on a cache/sector line. - pointer allocate( size_type n, const void* hint=0 ) { - // The "hint" argument is always ignored in NFS_Allocate thus const_cast shouldn't hurt - return pointer(internal::NFS_Allocate( n, sizeof(value_type), const_cast(hint) )); - } - - //! Free block of memory that starts on a cache line - void deallocate( pointer p, size_type ) { - internal::NFS_Free(p); - } - - //! Largest value for which method allocate might succeed. - size_type max_size() const throw() { - return (~size_t(0)-internal::NFS_MaxLineSize)/sizeof(value_type); - } - - //! Copy-construct value at location pointed to by p. -#if __TBB_ALLOCATOR_CONSTRUCT_VARIADIC - template - void construct(U *p, Args&&... args) - { ::new((void *)p) U(std::forward(args)...); } -#else // __TBB_ALLOCATOR_CONSTRUCT_VARIADIC -#if __TBB_CPP11_RVALUE_REF_PRESENT - void construct( pointer p, value_type&& value ) {::new((void*)(p)) value_type(std::move(value));} -#endif - void construct( pointer p, const value_type& value ) {::new((void*)(p)) value_type(value);} -#endif // __TBB_ALLOCATOR_CONSTRUCT_VARIADIC - - //! Destroy value at location pointed to by p. - void destroy( pointer p ) {p->~value_type();} -}; - -#if _MSC_VER && !defined(__INTEL_COMPILER) - #pragma warning (pop) -#endif // warning 4100 is back - -//! Analogous to std::allocator, as defined in ISO C++ Standard, Section 20.4.1 -/** @ingroup memory_allocation */ -template<> -class cache_aligned_allocator { -public: - typedef void* pointer; - typedef const void* const_pointer; - typedef void value_type; - template struct rebind { - typedef cache_aligned_allocator other; - }; -}; - -template -inline bool operator==( const cache_aligned_allocator&, const cache_aligned_allocator& ) {return true;} - -template -inline bool operator!=( const cache_aligned_allocator&, const cache_aligned_allocator& ) {return false;} - -} // namespace tbb - -#endif /* __TBB_cache_aligned_allocator_H */ +#include "../oneapi/tbb/cache_aligned_allocator.h" diff --git a/src/tbb/src/tbb/mac64-tbb-export.def b/src/tbb/include/tbb/collaborative_call_once.h similarity index 82% rename from src/tbb/src/tbb/mac64-tbb-export.def rename to src/tbb/include/tbb/collaborative_call_once.h index e2c037053..68cbbcddb 100644 --- a/src/tbb/src/tbb/mac64-tbb-export.def +++ b/src/tbb/include/tbb/collaborative_call_once.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,12 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ -#define __TBB_SYMBOL( sym ) _##sym -#include "mac64-tbb-export.lst" - +#include "../oneapi/tbb/collaborative_call_once.h" diff --git a/src/tbb/include/tbb/combinable.h b/src/tbb/include/tbb/combinable.h index 0063dbb46..50295ec72 100644 --- a/src/tbb/include/tbb/combinable.h +++ b/src/tbb/include/tbb/combinable.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,74 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ -#ifndef __TBB_combinable_H -#define __TBB_combinable_H - -#include "enumerable_thread_specific.h" -#include "cache_aligned_allocator.h" - -namespace tbb { -/** \name combinable - **/ -//@{ -//! Thread-local storage with optional reduction -/** @ingroup containers */ - template - class combinable { - - private: - typedef typename tbb::cache_aligned_allocator my_alloc; - typedef typename tbb::enumerable_thread_specific my_ets_type; - my_ets_type my_ets; - - public: - - combinable() { } - - template - explicit combinable( finit _finit) : my_ets(_finit) { } - - //! destructor - ~combinable() { } - - combinable( const combinable& other) : my_ets(other.my_ets) { } - -#if __TBB_ETS_USE_CPP11 - combinable( combinable&& other) : my_ets( std::move(other.my_ets)) { } -#endif - - combinable & operator=( const combinable & other) { - my_ets = other.my_ets; - return *this; - } - -#if __TBB_ETS_USE_CPP11 - combinable & operator=( combinable && other) { - my_ets=std::move(other.my_ets); - return *this; - } -#endif - - void clear() { my_ets.clear(); } - - T& local() { return my_ets.local(); } - - T& local(bool & exists) { return my_ets.local(exists); } - - // combine_func_t has signature T(T,T) or T(const T&, const T&) - template - T combine(combine_func_t f_combine) { return my_ets.combine(f_combine); } - - // combine_func_t has signature void(T) or void(const T&) - template - void combine_each(combine_func_t f_combine) { my_ets.combine_each(f_combine); } - - }; -} // namespace tbb -#endif /* __TBB_combinable_H */ +#include "../oneapi/tbb/combinable.h" diff --git a/src/tbb/include/tbb/compat/condition_variable b/src/tbb/include/tbb/compat/condition_variable deleted file mode 100644 index 43edfc032..000000000 --- a/src/tbb/include/tbb/compat/condition_variable +++ /dev/null @@ -1,476 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_condition_variable_H -#define __TBB_condition_variable_H - -#if _WIN32||_WIN64 -#include "../machine/windows_api.h" - -namespace tbb { -namespace interface5 { -namespace internal { -struct condition_variable_using_event -{ - //! Event for blocking waiting threads. - HANDLE event; - //! Protects invariants involving n_waiters, release_count, and epoch. - CRITICAL_SECTION mutex; - //! Number of threads waiting on this condition variable - int n_waiters; - //! Number of threads remaining that should no longer wait on this condition variable. - int release_count; - //! To keep threads from waking up prematurely with earlier signals. - unsigned epoch; -}; -}}} // namespace tbb::interface5::internal - -#ifndef CONDITION_VARIABLE_INIT -typedef void* CONDITION_VARIABLE; -typedef CONDITION_VARIABLE* PCONDITION_VARIABLE; -#endif - -#else /* if not _WIN32||_WIN64 */ -#include // some systems need it for ETIMEDOUT -#include -#if __linux__ -#include -#else /* generic Unix */ -#include -#endif -#endif /* _WIN32||_WIN64 */ - -#include "../tbb_stddef.h" -#include "../mutex.h" -#include "../tbb_thread.h" -#include "../tbb_exception.h" -#include "../tbb_profiling.h" - -namespace tbb { - -namespace interface5 { - -// C++0x standard working draft 30.4.3 -// Lock tag types -struct defer_lock_t { }; //! do not acquire ownership of the mutex -struct try_to_lock_t { }; //! try to acquire ownership of the mutex without blocking -struct adopt_lock_t { }; //! assume the calling thread has already -const defer_lock_t defer_lock = {}; -const try_to_lock_t try_to_lock = {}; -const adopt_lock_t adopt_lock = {}; - -// C++0x standard working draft 30.4.3.1 -//! lock_guard -template -class lock_guard : tbb::internal::no_copy { -public: - //! mutex type - typedef M mutex_type; - - //! Constructor - /** precondition: If mutex_type is not a recursive mutex, the calling thread - does not own the mutex m. */ - explicit lock_guard(mutex_type& m) : pm(m) {m.lock();} - - //! Adopt_lock constructor - /** precondition: the calling thread owns the mutex m. */ - lock_guard(mutex_type& m, adopt_lock_t) : pm(m) {} - - //! Destructor - ~lock_guard() { pm.unlock(); } -private: - mutex_type& pm; -}; - -// C++0x standard working draft 30.4.3.2 -//! unique_lock -template -class unique_lock : tbb::internal::no_copy { - friend class condition_variable; -public: - typedef M mutex_type; - - // 30.4.3.2.1 construct/copy/destroy - // NB: Without constructors that take an r-value reference to a unique_lock, the following constructor is of little use. - //! Constructor - /** postcondition: pm==0 && owns==false */ - unique_lock() : pm(NULL), owns(false) {} - - //! Constructor - /** precondition: if mutex_type is not a recursive mutex, the calling thread - does not own the mutex m. If the precondition is not met, a deadlock occurs. - postcondition: pm==&m and owns==true */ - explicit unique_lock(mutex_type& m) : pm(&m) {m.lock(); owns=true;} - - //! Defer_lock constructor - /** postcondition: pm==&m and owns==false */ - unique_lock(mutex_type& m, defer_lock_t) : pm(&m), owns(false) {} - - //! Try_to_lock constructor - /** precondition: if mutex_type is not a recursive mutex, the calling thread - does not own the mutex m. If the precondition is not met, a deadlock occurs. - postcondition: pm==&m and owns==res where res is the value returned by - the call to m.try_lock(). */ - unique_lock(mutex_type& m, try_to_lock_t) : pm(&m) {owns = m.try_lock();} - - //! Adopt_lock constructor - /** precondition: the calling thread owns the mutex. If it does not, mutex->unlock() would fail. - postcondition: pm==&m and owns==true */ - unique_lock(mutex_type& m, adopt_lock_t) : pm(&m), owns(true) {} - - //! Timed unique_lock acquisition. - /** To avoid requiring support for namespace chrono, this method deviates from the working draft in that - it uses tbb::tick_count::interval_t to specify the time duration. */ - unique_lock(mutex_type& m, const tick_count::interval_t &i) : pm(&m) {owns = try_lock_for( i );} - -#if __TBB_CPP11_RVALUE_REF_PRESENT - //! Move constructor - /** postconditions: pm == src_p.pm and owns == src_p.owns (where src_p is the state of src just prior to this - construction), src.pm == 0 and src.owns == false. */ - unique_lock(unique_lock && src): pm(NULL), owns(false) {this->swap(src);} - - //! Move assignment - /** effects: If owns calls pm->unlock(). - Postconditions: pm == src_p.pm and owns == src_p.owns (where src_p is the state of src just prior to this - assignment), src.pm == 0 and src.owns == false. */ - unique_lock& operator=(unique_lock && src) { - if (owns) - this->unlock(); - pm = NULL; - this->swap(src); - return *this; - } -#endif // __TBB_CPP11_RVALUE_REF_PRESENT - - //! Destructor - ~unique_lock() { if( owns ) pm->unlock(); } - - // 30.4.3.2.2 locking - //! Lock the mutex and own it. - void lock() { - if( pm ) { - if( !owns ) { - pm->lock(); - owns = true; - } else - throw_exception_v4( tbb::internal::eid_possible_deadlock ); - } else - throw_exception_v4( tbb::internal::eid_operation_not_permitted ); - __TBB_ASSERT( owns, NULL ); - } - - //! Try to lock the mutex. - /** If successful, note that this lock owns it. Otherwise, set it false. */ - bool try_lock() { - if( pm ) { - if( !owns ) - owns = pm->try_lock(); - else - throw_exception_v4( tbb::internal::eid_possible_deadlock ); - } else - throw_exception_v4( tbb::internal::eid_operation_not_permitted ); - return owns; - } - - //! Try to lock the mutex. - bool try_lock_for( const tick_count::interval_t &i ); - - //! Unlock the mutex - /** And note that this lock no longer owns it. */ - void unlock() { - if( owns ) { - pm->unlock(); - owns = false; - } else - throw_exception_v4( tbb::internal::eid_operation_not_permitted ); - __TBB_ASSERT( !owns, NULL ); - } - - // 30.4.3.2.3 modifiers - //! Swap the two unique locks - void swap(unique_lock& u) { - mutex_type* t_pm = u.pm; u.pm = pm; pm = t_pm; - bool t_owns = u.owns; u.owns = owns; owns = t_owns; - } - - //! Release control over the mutex. - mutex_type* release() { - mutex_type* o_pm = pm; - pm = NULL; - owns = false; - return o_pm; - } - - // 30.4.3.2.4 observers - //! Does this lock own the mutex? - bool owns_lock() const { return owns; } - - // TODO: Un-comment 'explicit' when the last non-C++0x compiler support is dropped - //! Does this lock own the mutex? - /*explicit*/ operator bool() const { return owns; } - - //! Return the mutex that this lock currently has. - mutex_type* mutex() const { return pm; } - -private: - mutex_type* pm; - bool owns; -}; - -template -bool unique_lock::try_lock_for( const tick_count::interval_t &i) -{ - const int unique_lock_tick = 100; /* microseconds; 0.1 milliseconds */ - // the smallest wait-time is 0.1 milliseconds. - bool res = pm->try_lock(); - int duration_in_micro; - if( !res && (duration_in_micro=int(i.seconds()*1e6))>unique_lock_tick ) { - tick_count::interval_t i_100( double(unique_lock_tick)/1e6 /* seconds */); // 100 microseconds = 0.1*10E-3 - do { - this_tbb_thread::sleep(i_100); // sleep for 100 micro seconds - duration_in_micro -= unique_lock_tick; - res = pm->try_lock(); - } while( !res && duration_in_micro>unique_lock_tick ); - } - return (owns=res); -} - -//! Swap the two unique locks that have the mutexes of same type -template -void swap(unique_lock& x, unique_lock& y) { x.swap( y ); } - -namespace internal { - -#if _WIN32||_WIN64 -union condvar_impl_t { - condition_variable_using_event cv_event; - CONDITION_VARIABLE cv_native; -}; -void __TBB_EXPORTED_FUNC internal_initialize_condition_variable( condvar_impl_t& cv ); -void __TBB_EXPORTED_FUNC internal_destroy_condition_variable( condvar_impl_t& cv ); -void __TBB_EXPORTED_FUNC internal_condition_variable_notify_one( condvar_impl_t& cv ); -void __TBB_EXPORTED_FUNC internal_condition_variable_notify_all( condvar_impl_t& cv ); -bool __TBB_EXPORTED_FUNC internal_condition_variable_wait( condvar_impl_t& cv, mutex* mtx, const tick_count::interval_t* i = NULL ); - -#else /* if !(_WIN32||_WIN64), i.e., POSIX threads */ -typedef pthread_cond_t condvar_impl_t; -#endif - -} // namespace internal - -//! cv_status -/** C++0x standard working draft 30.5 */ -enum cv_status { no_timeout, timeout }; - -//! condition variable -/** C++0x standard working draft 30.5.1 - @ingroup synchronization */ -class condition_variable : tbb::internal::no_copy { -public: - //! Constructor - condition_variable() { -#if _WIN32||_WIN64 - internal_initialize_condition_variable( my_cv ); -#else - pthread_cond_init( &my_cv, NULL ); -#endif - } - - //! Destructor - ~condition_variable() { - //precondition: There shall be no thread blocked on *this. -#if _WIN32||_WIN64 - internal_destroy_condition_variable( my_cv ); -#else - pthread_cond_destroy( &my_cv ); -#endif - } - - //! Notify one thread and wake it up - void notify_one() { -#if _WIN32||_WIN64 - internal_condition_variable_notify_one( my_cv ); -#else - pthread_cond_signal( &my_cv ); -#endif - } - - //! Notify all threads - void notify_all() { -#if _WIN32||_WIN64 - internal_condition_variable_notify_all( my_cv ); -#else - pthread_cond_broadcast( &my_cv ); -#endif - } - - //! Release the mutex associated with the lock and wait on this condition variable - void wait(unique_lock& lock); - - //! Wait on this condition variable while pred is false - template - void wait(unique_lock& lock, Predicate pred) { - while( !pred() ) - wait( lock ); - } - - //! Timed version of wait() - cv_status wait_for(unique_lock& lock, const tick_count::interval_t &i ); - - //! Timed version of the predicated wait - /** The loop terminates when pred() returns true or when the time duration specified by rel_time (i) has elapsed. */ - template - bool wait_for(unique_lock& lock, const tick_count::interval_t &i, Predicate pred) - { - while( !pred() ) { - cv_status st = wait_for( lock, i ); - if( st==timeout ) - return pred(); - } - return true; - } - - // C++0x standard working draft. 30.2.3 - typedef internal::condvar_impl_t* native_handle_type; - - native_handle_type native_handle() { return (native_handle_type) &my_cv; } - -private: - internal::condvar_impl_t my_cv; -}; - - -#if _WIN32||_WIN64 -inline void condition_variable::wait( unique_lock& lock ) -{ - __TBB_ASSERT( lock.owns, NULL ); - lock.owns = false; - if( !internal_condition_variable_wait( my_cv, lock.mutex() ) ) { - int ec = GetLastError(); - // on Windows 7, SleepConditionVariableCS() may return ERROR_TIMEOUT while the doc says it returns WAIT_TIMEOUT - __TBB_ASSERT_EX( ec!=WAIT_TIMEOUT&&ec!=ERROR_TIMEOUT, NULL ); - lock.owns = true; - throw_exception_v4( tbb::internal::eid_condvar_wait_failed ); - } - lock.owns = true; -} - -inline cv_status condition_variable::wait_for( unique_lock& lock, const tick_count::interval_t& i ) -{ - cv_status rc = no_timeout; - __TBB_ASSERT( lock.owns, NULL ); - lock.owns = false; - // condvar_wait could be SleepConditionVariableCS (or SleepConditionVariableSRW) or our own pre-vista cond_var_wait() - if( !internal_condition_variable_wait( my_cv, lock.mutex(), &i ) ) { - int ec = GetLastError(); - if( ec==WAIT_TIMEOUT || ec==ERROR_TIMEOUT ) - rc = timeout; - else { - lock.owns = true; - throw_exception_v4( tbb::internal::eid_condvar_wait_failed ); - } - } - lock.owns = true; - return rc; -} - -#else /* !(_WIN32||_WIN64) */ -inline void condition_variable::wait( unique_lock& lock ) -{ - __TBB_ASSERT( lock.owns, NULL ); - lock.owns = false; - if( pthread_cond_wait( &my_cv, lock.mutex()->native_handle() ) ) { - lock.owns = true; - throw_exception_v4( tbb::internal::eid_condvar_wait_failed ); - } - // upon successful return, the mutex has been locked and is owned by the calling thread. - lock.owns = true; -} - -inline cv_status condition_variable::wait_for( unique_lock& lock, const tick_count::interval_t& i ) -{ -#if __linux__ - struct timespec req; - double sec = i.seconds(); - clock_gettime( CLOCK_REALTIME, &req ); - req.tv_sec += static_cast(sec); - req.tv_nsec += static_cast( (sec - static_cast(sec))*1e9 ); -#else /* generic Unix */ - struct timeval tv; - struct timespec req; - double sec = i.seconds(); - int status = gettimeofday(&tv, NULL); - __TBB_ASSERT_EX( status==0, "gettimeofday failed" ); - req.tv_sec = tv.tv_sec + static_cast(sec); - req.tv_nsec = tv.tv_usec*1000 + static_cast( (sec - static_cast(sec))*1e9 ); -#endif /*(choice of OS) */ - if( req.tv_nsec>=1e9 ) { - req.tv_sec += 1; - req.tv_nsec -= static_cast(1e9); - } - __TBB_ASSERT( 0<=req.tv_nsec && req.tv_nsec<1e9, NULL ); - - int ec; - cv_status rc = no_timeout; - __TBB_ASSERT( lock.owns, NULL ); - lock.owns = false; - if( ( ec=pthread_cond_timedwait( &my_cv, lock.mutex()->native_handle(), &req ) ) ) { - if( ec==ETIMEDOUT ) - rc = timeout; - else { - __TBB_ASSERT( lock.try_lock()==false, NULL ); - lock.owns = true; - throw_exception_v4( tbb::internal::eid_condvar_wait_failed ); - } - } - lock.owns = true; - return rc; -} -#endif /* !(_WIN32||_WIN64) */ - -} // namespace interface5 - -__TBB_DEFINE_PROFILING_SET_NAME(interface5::condition_variable) - -} // namespace tbb - -#if TBB_IMPLEMENT_CPP0X - -namespace std { - -using tbb::interface5::defer_lock_t; -using tbb::interface5::try_to_lock_t; -using tbb::interface5::adopt_lock_t; -using tbb::interface5::defer_lock; -using tbb::interface5::try_to_lock; -using tbb::interface5::adopt_lock; -using tbb::interface5::lock_guard; -using tbb::interface5::unique_lock; -using tbb::interface5::swap; /* this is for void std::swap(unique_lock&,unique_lock&) */ -using tbb::interface5::condition_variable; -using tbb::interface5::cv_status; -using tbb::interface5::timeout; -using tbb::interface5::no_timeout; - -} // namespace std - -#endif /* TBB_IMPLEMENT_CPP0X */ - -#endif /* __TBB_condition_variable_H */ diff --git a/src/tbb/include/tbb/compat/ppl.h b/src/tbb/include/tbb/compat/ppl.h deleted file mode 100644 index 840dfb22f..000000000 --- a/src/tbb/include/tbb/compat/ppl.h +++ /dev/null @@ -1,62 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_compat_ppl_H -#define __TBB_compat_ppl_H - -#include "../task_group.h" -#include "../parallel_invoke.h" -#include "../parallel_for_each.h" -#include "../parallel_for.h" -#include "../tbb_exception.h" -#include "../critical_section.h" -#include "../reader_writer_lock.h" -#include "../combinable.h" - -namespace Concurrency { - -#if __TBB_TASK_GROUP_CONTEXT - using tbb::task_handle; - using tbb::task_group_status; - using tbb::task_group; - using tbb::structured_task_group; - using tbb::invalid_multiple_scheduling; - using tbb::missing_wait; - using tbb::make_task; - - using tbb::not_complete; - using tbb::complete; - using tbb::canceled; - - using tbb::is_current_task_group_canceling; -#endif /* __TBB_TASK_GROUP_CONTEXT */ - - using tbb::parallel_invoke; - using tbb::strict_ppl::parallel_for; - using tbb::parallel_for_each; - using tbb::critical_section; - using tbb::reader_writer_lock; - using tbb::combinable; - - using tbb::improper_lock; - -} // namespace Concurrency - -#endif /* __TBB_compat_ppl_H */ diff --git a/src/tbb/include/tbb/compat/thread b/src/tbb/include/tbb/compat/thread deleted file mode 100644 index 0edd92899..000000000 --- a/src/tbb/include/tbb/compat/thread +++ /dev/null @@ -1,60 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_thread_H -#define __TBB_thread_H - -#include "../tbb_config.h" - -#if TBB_IMPLEMENT_CPP0X - -#include "../tbb_thread.h" - -namespace std { - -typedef tbb::tbb_thread thread; - -namespace this_thread { - using tbb::this_tbb_thread::get_id; - using tbb::this_tbb_thread::yield; - - inline void sleep_for(const tbb::tick_count::interval_t& rel_time) { - tbb::internal::thread_sleep_v3( rel_time ); - } -} - -} // namespace std - -#else /* TBB_IMPLEMENT_CPP0X */ - -#define __TBB_COMPAT_THREAD_RECURSION_PROTECTOR 1 -#include -#undef __TBB_COMPAT_THREAD_RECURSION_PROTECTOR - -#endif /* TBB_IMPLEMENT_CPP0X */ - -#else /* __TBB_thread_H */ - -#if __TBB_COMPAT_THREAD_RECURSION_PROTECTOR -#error The tbb/compat/thread header attempts to include itself. \ - Please make sure that {TBBROOT}/include/tbb/compat is NOT in include paths. -#endif - -#endif /* __TBB_thread_H */ diff --git a/src/tbb/include/tbb/compat/tuple b/src/tbb/include/tbb/compat/tuple deleted file mode 100644 index 5767c49e7..000000000 --- a/src/tbb/include/tbb/compat/tuple +++ /dev/null @@ -1,488 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_tuple_H -#define __TBB_tuple_H - -#include -#include "../tbb_stddef.h" - -// build preprocessor variables for varying number of arguments -// Need the leading comma so the empty __TBB_T_PACK will not cause a syntax error. -#if __TBB_VARIADIC_MAX <= 5 -#define __TBB_T_PACK -#define __TBB_U_PACK -#define __TBB_TYPENAME_T_PACK -#define __TBB_TYPENAME_U_PACK -#define __TBB_NULL_TYPE_PACK -#define __TBB_REF_T_PARAM_PACK -#define __TBB_CONST_REF_T_PARAM_PACK -#define __TBB_T_PARAM_LIST_PACK -#define __TBB_CONST_NULL_REF_PACK -// -#elif __TBB_VARIADIC_MAX == 6 -#define __TBB_T_PACK ,__T5 -#define __TBB_U_PACK ,__U5 -#define __TBB_TYPENAME_T_PACK , typename __T5 -#define __TBB_TYPENAME_U_PACK , typename __U5 -#define __TBB_NULL_TYPE_PACK , null_type -#define __TBB_REF_T_PARAM_PACK ,__T5& t5 -#define __TBB_CONST_REF_T_PARAM_PACK ,const __T5& t5 -#define __TBB_T_PARAM_LIST_PACK ,t5 -#define __TBB_CONST_NULL_REF_PACK , const null_type& -// -#elif __TBB_VARIADIC_MAX == 7 -#define __TBB_T_PACK ,__T5, __T6 -#define __TBB_U_PACK ,__U5, __U6 -#define __TBB_TYPENAME_T_PACK , typename __T5 , typename __T6 -#define __TBB_TYPENAME_U_PACK , typename __U5 , typename __U6 -#define __TBB_NULL_TYPE_PACK , null_type, null_type -#define __TBB_REF_T_PARAM_PACK ,__T5& t5, __T6& t6 -#define __TBB_CONST_REF_T_PARAM_PACK ,const __T5& t5, const __T6& t6 -#define __TBB_T_PARAM_LIST_PACK ,t5 ,t6 -#define __TBB_CONST_NULL_REF_PACK , const null_type&, const null_type& -// -#elif __TBB_VARIADIC_MAX == 8 -#define __TBB_T_PACK ,__T5, __T6, __T7 -#define __TBB_U_PACK ,__U5, __U6, __U7 -#define __TBB_TYPENAME_T_PACK , typename __T5 , typename __T6, typename __T7 -#define __TBB_TYPENAME_U_PACK , typename __U5 , typename __U6, typename __U7 -#define __TBB_NULL_TYPE_PACK , null_type, null_type, null_type -#define __TBB_REF_T_PARAM_PACK ,__T5& t5, __T6& t6, __T7& t7 -#define __TBB_CONST_REF_T_PARAM_PACK , const __T5& t5, const __T6& t6, const __T7& t7 -#define __TBB_T_PARAM_LIST_PACK ,t5 ,t6 ,t7 -#define __TBB_CONST_NULL_REF_PACK , const null_type&, const null_type&, const null_type& -// -#elif __TBB_VARIADIC_MAX == 9 -#define __TBB_T_PACK ,__T5, __T6, __T7, __T8 -#define __TBB_U_PACK ,__U5, __U6, __U7, __U8 -#define __TBB_TYPENAME_T_PACK , typename __T5, typename __T6, typename __T7, typename __T8 -#define __TBB_TYPENAME_U_PACK , typename __U5, typename __U6, typename __U7, typename __U8 -#define __TBB_NULL_TYPE_PACK , null_type, null_type, null_type, null_type -#define __TBB_REF_T_PARAM_PACK ,__T5& t5, __T6& t6, __T7& t7, __T8& t8 -#define __TBB_CONST_REF_T_PARAM_PACK , const __T5& t5, const __T6& t6, const __T7& t7, const __T8& t8 -#define __TBB_T_PARAM_LIST_PACK ,t5 ,t6 ,t7 ,t8 -#define __TBB_CONST_NULL_REF_PACK , const null_type&, const null_type&, const null_type&, const null_type& -// -#elif __TBB_VARIADIC_MAX >= 10 -#define __TBB_T_PACK ,__T5, __T6, __T7, __T8, __T9 -#define __TBB_U_PACK ,__U5, __U6, __U7, __U8, __U9 -#define __TBB_TYPENAME_T_PACK , typename __T5, typename __T6, typename __T7, typename __T8, typename __T9 -#define __TBB_TYPENAME_U_PACK , typename __U5, typename __U6, typename __U7, typename __U8, typename __U9 -#define __TBB_NULL_TYPE_PACK , null_type, null_type, null_type, null_type, null_type -#define __TBB_REF_T_PARAM_PACK ,__T5& t5, __T6& t6, __T7& t7, __T8& t8, __T9& t9 -#define __TBB_CONST_REF_T_PARAM_PACK , const __T5& t5, const __T6& t6, const __T7& t7, const __T8& t8, const __T9& t9 -#define __TBB_T_PARAM_LIST_PACK ,t5 ,t6 ,t7 ,t8 ,t9 -#define __TBB_CONST_NULL_REF_PACK , const null_type&, const null_type&, const null_type&, const null_type&, const null_type& -#endif - - - -namespace tbb { -namespace interface5 { - -namespace internal { -struct null_type { }; -} -using internal::null_type; - -// tuple forward declaration -template = 6 -, typename __T5=null_type -#if __TBB_VARIADIC_MAX >= 7 -, typename __T6=null_type -#if __TBB_VARIADIC_MAX >= 8 -, typename __T7=null_type -#if __TBB_VARIADIC_MAX >= 9 -, typename __T8=null_type -#if __TBB_VARIADIC_MAX >= 10 -, typename __T9=null_type -#endif -#endif -#endif -#endif -#endif -> -class tuple; - -namespace internal { - -// const null_type temp -inline const null_type cnull() { return null_type(); } - -// cons forward declaration -template struct cons; - -// type of a component of the cons -template -struct component { - typedef typename __T::tail_type next; - typedef typename component<__N-1,next>::type type; -}; - -template -struct component<0,__T> { - typedef typename __T::head_type type; -}; - -template<> -struct component<0,null_type> { - typedef null_type type; -}; - -// const version of component - -template -struct component<__N, const __T> -{ - typedef typename __T::tail_type next; - typedef const typename component<__N-1,next>::type type; -}; - -template -struct component<0, const __T> -{ - typedef const typename __T::head_type type; -}; - - -// helper class for getting components of cons -template< int __N> -struct get_helper { -template -inline static typename component<__N, cons<__HT,__TT> >::type& get(cons<__HT,__TT>& ti) { - return get_helper<__N-1>::get(ti.tail); -} -template -inline static typename component<__N, cons<__HT,__TT> >::type const& get(const cons<__HT,__TT>& ti) { - return get_helper<__N-1>::get(ti.tail); -} -}; - -template<> -struct get_helper<0> { -template -inline static typename component<0, cons<__HT,__TT> >::type& get(cons<__HT,__TT>& ti) { - return ti.head; -} -template -inline static typename component<0, cons<__HT,__TT> >::type const& get(const cons<__HT,__TT>& ti) { - return ti.head; -} -}; - -// traits adaptor -template -struct tuple_traits { - typedef cons <__T0, typename tuple_traits<__T1, __T2, __T3, __T4 __TBB_T_PACK , null_type>::U > U; -}; - -template -struct tuple_traits<__T0, null_type, null_type, null_type, null_type __TBB_NULL_TYPE_PACK > { - typedef cons<__T0, null_type> U; -}; - -template<> -struct tuple_traits { - typedef null_type U; -}; - - -// core cons defs -template -struct cons{ - - typedef __HT head_type; - typedef __TT tail_type; - - head_type head; - tail_type tail; - - static const int length = 1 + tail_type::length; - - // default constructors - explicit cons() : head(), tail() { } - - // non-default constructors - cons(head_type& h, const tail_type& t) : head(h), tail(t) { } - - template - cons(const __T0& t0, const __T1& t1, const __T2& t2, const __T3& t3, const __T4& t4 __TBB_CONST_REF_T_PARAM_PACK) : - head(t0), tail(t1, t2, t3, t4 __TBB_T_PARAM_LIST_PACK, cnull()) { } - - template - cons(__T0& t0, __T1& t1, __T2& t2, __T3& t3, __T4& t4 __TBB_REF_T_PARAM_PACK) : - head(t0), tail(t1, t2, t3, t4 __TBB_T_PARAM_LIST_PACK , cnull()) { } - - template - cons(const cons<__HT1,__TT1>& other) : head(other.head), tail(other.tail) { } - - cons& operator=(const cons& other) { head = other.head; tail = other.tail; return *this; } - - friend bool operator==(const cons& me, const cons& other) { - return me.head == other.head && me.tail == other.tail; - } - friend bool operator<(const cons& me, const cons& other) { - return me.head < other.head || (!(other.head < me.head) && me.tail < other.tail); - } - friend bool operator>(const cons& me, const cons& other) { return other=(const cons& me, const cons& other) { return !(meother); } - - template - friend bool operator==(const cons<__HT,__TT>& me, const cons<__HT1,__TT1>& other) { - return me.head == other.head && me.tail == other.tail; - } - - template - friend bool operator<(const cons<__HT,__TT>& me, const cons<__HT1,__TT1>& other) { - return me.head < other.head || (!(other.head < me.head) && me.tail < other.tail); - } - - template - friend bool operator>(const cons<__HT,__TT>& me, const cons<__HT1,__TT1>& other) { return other - friend bool operator!=(const cons<__HT,__TT>& me, const cons<__HT1,__TT1>& other) { return !(me==other); } - - template - friend bool operator>=(const cons<__HT,__TT>& me, const cons<__HT1,__TT1>& other) { return !(me - friend bool operator<=(const cons<__HT,__TT>& me, const cons<__HT1,__TT1>& other) { return !(me>other); } - - -}; // cons - - -template -struct cons<__HT,null_type> { - - typedef __HT head_type; - typedef null_type tail_type; - - head_type head; - - static const int length = 1; - - // default constructor - cons() : head() { /*std::cout << "default constructor 1\n";*/ } - - cons(const null_type&, const null_type&, const null_type&, const null_type&, const null_type& __TBB_CONST_NULL_REF_PACK) : head() { /*std::cout << "default constructor 2\n";*/ } - - // non-default constructor - template - cons(__T1& t1, const null_type&, const null_type&, const null_type&, const null_type& __TBB_CONST_NULL_REF_PACK) : head(t1) { /*std::cout << "non-default a1, t1== " << t1 << "\n";*/} - - cons(head_type& h, const null_type& = null_type() ) : head(h) { } - cons(const head_type& t0, const null_type&, const null_type&, const null_type&, const null_type& __TBB_CONST_NULL_REF_PACK) : head(t0) { } - - // converting constructor - template - cons(__HT1 h1, const null_type&, const null_type&, const null_type&, const null_type& __TBB_CONST_NULL_REF_PACK) : head(h1) { } - - // copy constructor - template - cons( const cons<__HT1, null_type>& other) : head(other.head) { } - - // assignment operator - cons& operator=(const cons& other) { head = other.head; return *this; } - - friend bool operator==(const cons& me, const cons& other) { return me.head == other.head; } - friend bool operator<(const cons& me, const cons& other) { return me.head < other.head; } - friend bool operator>(const cons& me, const cons& other) { return otherother); } - friend bool operator>=(const cons& me, const cons& other) {return !(me - friend bool operator==(const cons<__HT,null_type>& me, const cons<__HT1,null_type>& other) { - return me.head == other.head; - } - - template - friend bool operator<(const cons<__HT,null_type>& me, const cons<__HT1,null_type>& other) { - return me.head < other.head; - } - - template - friend bool operator>(const cons<__HT,null_type>& me, const cons<__HT1,null_type>& other) { return other - friend bool operator!=(const cons<__HT,null_type>& me, const cons<__HT1,null_type>& other) { return !(me==other); } - - template - friend bool operator<=(const cons<__HT,null_type>& me, const cons<__HT1,null_type>& other) { return !(me>other); } - - template - friend bool operator>=(const cons<__HT,null_type>& me, const cons<__HT1,null_type>& other) { return !(me -struct cons { typedef null_type tail_type; static const int length = 0; }; - -// wrapper for default constructor -template -inline const __T wrap_dcons(__T*) { return __T(); } - -} // namespace internal - -// tuple definition -template -class tuple : public internal::tuple_traits<__T0, __T1, __T2, __T3, __T4 __TBB_T_PACK >::U { - // friends - template friend class tuple_size; - template friend struct tuple_element; - - // stl components - typedef tuple<__T0,__T1,__T2,__T3,__T4 __TBB_T_PACK > value_type; - typedef value_type *pointer; - typedef const value_type *const_pointer; - typedef value_type &reference; - typedef const value_type &const_reference; - typedef size_t size_type; - - typedef typename internal::tuple_traits<__T0,__T1,__T2,__T3, __T4 __TBB_T_PACK >::U my_cons; - -public: - tuple(const __T0& t0=internal::wrap_dcons((__T0*)NULL) - ,const __T1& t1=internal::wrap_dcons((__T1*)NULL) - ,const __T2& t2=internal::wrap_dcons((__T2*)NULL) - ,const __T3& t3=internal::wrap_dcons((__T3*)NULL) - ,const __T4& t4=internal::wrap_dcons((__T4*)NULL) -#if __TBB_VARIADIC_MAX >= 6 - ,const __T5& t5=internal::wrap_dcons((__T5*)NULL) -#if __TBB_VARIADIC_MAX >= 7 - ,const __T6& t6=internal::wrap_dcons((__T6*)NULL) -#if __TBB_VARIADIC_MAX >= 8 - ,const __T7& t7=internal::wrap_dcons((__T7*)NULL) -#if __TBB_VARIADIC_MAX >= 9 - ,const __T8& t8=internal::wrap_dcons((__T8*)NULL) -#if __TBB_VARIADIC_MAX >= 10 - ,const __T9& t9=internal::wrap_dcons((__T9*)NULL) -#endif -#endif -#endif -#endif -#endif - ) : - my_cons(t0,t1,t2,t3,t4 __TBB_T_PARAM_LIST_PACK) { } - - template - struct internal_tuple_element { - typedef typename internal::component<__N,my_cons>::type type; - }; - - template - typename internal_tuple_element<__N>::type& get() { return internal::get_helper<__N>::get(*this); } - - template - typename internal_tuple_element<__N>::type const& get() const { return internal::get_helper<__N>::get(*this); } - - template - tuple& operator=(const internal::cons<__U1,__U2>& other) { - my_cons::operator=(other); - return *this; - } - - template - tuple& operator=(const std::pair<__U1,__U2>& other) { - // __TBB_ASSERT(tuple_size::value == 2, "Invalid size for pair to tuple assignment"); - this->head = other.first; - this->tail.head = other.second; - return *this; - } - - friend bool operator==(const tuple& me, const tuple& other) {return static_cast(me)==(other);} - friend bool operator<(const tuple& me, const tuple& other) {return static_cast(me)<(other);} - friend bool operator>(const tuple& me, const tuple& other) {return static_cast(me)>(other);} - friend bool operator!=(const tuple& me, const tuple& other) {return static_cast(me)!=(other);} - friend bool operator>=(const tuple& me, const tuple& other) {return static_cast(me)>=(other);} - friend bool operator<=(const tuple& me, const tuple& other) {return static_cast(me)<=(other);} - -}; // tuple - -// empty tuple -template<> -class tuple : public null_type { -}; - -// helper classes - -template < typename __T> -class tuple_size { -public: - static const size_t value = 1 + tuple_size::value; -}; - -template <> -class tuple_size > { -public: - static const size_t value = 0; -}; - -template <> -class tuple_size { -public: - static const size_t value = 0; -}; - -template -struct tuple_element { - typedef typename internal::component<__N, typename __T::my_cons>::type type; -}; - -template -inline static typename tuple_element<__N,tuple<__T0,__T1,__T2,__T3,__T4 __TBB_T_PACK > >::type& - get(tuple<__T0,__T1,__T2,__T3,__T4 __TBB_T_PACK >& t) { return internal::get_helper<__N>::get(t); } - -template -inline static typename tuple_element<__N,tuple<__T0,__T1,__T2,__T3,__T4 __TBB_T_PACK > >::type const& - get(const tuple<__T0,__T1,__T2,__T3,__T4 __TBB_T_PACK >& t) { return internal::get_helper<__N>::get(t); } - -} // interface5 -} // tbb - -#if !__TBB_CPP11_TUPLE_PRESENT -namespace tbb { - namespace flow { - using tbb::interface5::tuple; - using tbb::interface5::tuple_size; - using tbb::interface5::tuple_element; - using tbb::interface5::get; - } -} -#endif - -#undef __TBB_T_PACK -#undef __TBB_U_PACK -#undef __TBB_TYPENAME_T_PACK -#undef __TBB_TYPENAME_U_PACK -#undef __TBB_NULL_TYPE_PACK -#undef __TBB_REF_T_PARAM_PACK -#undef __TBB_CONST_REF_T_PARAM_PACK -#undef __TBB_T_PARAM_LIST_PACK -#undef __TBB_CONST_NULL_REF_PACK - -#endif /* __TBB_tuple_H */ diff --git a/src/tbb/include/tbb/concurrent_hash_map.h b/src/tbb/include/tbb/concurrent_hash_map.h index b25550a40..68652c596 100644 --- a/src/tbb/include/tbb/concurrent_hash_map.h +++ b/src/tbb/include/tbb/concurrent_hash_map.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,1488 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ -#ifndef __TBB_concurrent_hash_map_H -#define __TBB_concurrent_hash_map_H - -#include "tbb_stddef.h" -#include -#include // Need std::pair -#include // Need std::memset -#include __TBB_STD_SWAP_HEADER - -#include "cache_aligned_allocator.h" -#include "tbb_allocator.h" -#include "spin_rw_mutex.h" -#include "atomic.h" -#include "tbb_exception.h" -#include "tbb_profiling.h" -#include "internal/_tbb_hash_compare_impl.h" -#if __TBB_INITIALIZER_LISTS_PRESENT -#include -#endif -#if TBB_USE_PERFORMANCE_WARNINGS || __TBB_STATISTICS -#include -#endif -#if __TBB_STATISTICS -#include -#endif - -namespace tbb { - -namespace interface5 { - - template, typename A = tbb_allocator > > - class concurrent_hash_map; - - //! @cond INTERNAL - namespace internal { - using namespace tbb::internal; - - - //! Type of a hash code. - typedef size_t hashcode_t; - //! Node base type - struct hash_map_node_base : tbb::internal::no_copy { - //! Mutex type - typedef spin_rw_mutex mutex_t; - //! Scoped lock type for mutex - typedef mutex_t::scoped_lock scoped_t; - //! Next node in chain - hash_map_node_base *next; - mutex_t mutex; - }; - //! Incompleteness flag value - static hash_map_node_base *const rehash_req = reinterpret_cast(size_t(3)); - //! Rehashed empty bucket flag - static hash_map_node_base *const empty_rehashed = reinterpret_cast(size_t(0)); - //! base class of concurrent_hash_map - class hash_map_base { - public: - //! Size type - typedef size_t size_type; - //! Type of a hash code. - typedef size_t hashcode_t; - //! Segment index type - typedef size_t segment_index_t; - //! Node base type - typedef hash_map_node_base node_base; - //! Bucket type - struct bucket : tbb::internal::no_copy { - //! Mutex type for buckets - typedef spin_rw_mutex mutex_t; - //! Scoped lock type for mutex - typedef mutex_t::scoped_lock scoped_t; - mutex_t mutex; - node_base *node_list; - }; - //! Count of segments in the first block - static size_type const embedded_block = 1; - //! Count of segments in the first block - static size_type const embedded_buckets = 1< my_mask; - //! Segment pointers table. Also prevents false sharing between my_mask and my_size - segments_table_t my_table; - //! Size of container in stored items - atomic my_size; // It must be in separate cache line from my_mask due to performance effects - //! Zero segment - bucket my_embedded_segment[embedded_buckets]; -#if __TBB_STATISTICS - atomic my_info_resizes; // concurrent ones - mutable atomic my_info_restarts; // race collisions - atomic my_info_rehashes; // invocations of rehash_bucket -#endif - //! Constructor - hash_map_base() { - std::memset( (void*) this, 0, pointers_per_table*sizeof(segment_ptr_t) // 32*4=128 or 64*8=512 - + sizeof(my_size) + sizeof(my_mask) // 4+4 or 8+8 - + embedded_buckets*sizeof(bucket) ); // n*8 or n*16 - for( size_type i = 0; i < embedded_block; i++ ) // fill the table - my_table[i] = my_embedded_segment + segment_base(i); - my_mask = embedded_buckets - 1; - __TBB_ASSERT( embedded_block <= first_block, "The first block number must include embedded blocks"); -#if __TBB_STATISTICS - my_info_resizes = 0; // concurrent ones - my_info_restarts = 0; // race collisions - my_info_rehashes = 0; // invocations of rehash_bucket -#endif - } - - //! @return segment index of given index in the array - static segment_index_t segment_index_of( size_type index ) { - return segment_index_t( __TBB_Log2( index|1 ) ); - } - - //! @return the first array index of given segment - static segment_index_t segment_base( segment_index_t k ) { - return (segment_index_t(1)<(ptr) > uintptr_t(63); - } - - //! Initialize buckets - static void init_buckets( segment_ptr_t ptr, size_type sz, bool is_initial ) { - if( is_initial ) std::memset((void*) ptr, 0, sz*sizeof(bucket) ); - else for(size_type i = 0; i < sz; i++, ptr++) { - *reinterpret_cast(&ptr->mutex) = 0; - ptr->node_list = rehash_req; - } - } - - //! Add node @arg n to bucket @arg b - static void add_to_bucket( bucket *b, node_base *n ) { - __TBB_ASSERT(b->node_list != rehash_req, NULL); - n->next = b->node_list; - b->node_list = n; // its under lock and flag is set - } - - //! Exception safety helper - struct enable_segment_failsafe : tbb::internal::no_copy { - segment_ptr_t *my_segment_ptr; - enable_segment_failsafe(segments_table_t &table, segment_index_t k) : my_segment_ptr(&table[k]) {} - ~enable_segment_failsafe() { - if( my_segment_ptr ) *my_segment_ptr = 0; // indicate no allocation in progress - } - }; - - //! Enable segment - void enable_segment( segment_index_t k, bool is_initial = false ) { - __TBB_ASSERT( k, "Zero segment must be embedded" ); - enable_segment_failsafe watchdog( my_table, k ); - cache_aligned_allocator alloc; - size_type sz; - __TBB_ASSERT( !is_valid(my_table[k]), "Wrong concurrent assignment"); - if( k >= first_block ) { - sz = segment_size( k ); - segment_ptr_t ptr = alloc.allocate( sz ); - init_buckets( ptr, sz, is_initial ); - itt_hide_store_word( my_table[k], ptr ); - sz <<= 1;// double it to get entire capacity of the container - } else { // the first block - __TBB_ASSERT( k == embedded_block, "Wrong segment index" ); - sz = segment_size( first_block ); - segment_ptr_t ptr = alloc.allocate( sz - embedded_buckets ); - init_buckets( ptr, sz - embedded_buckets, is_initial ); - ptr -= segment_base(embedded_block); - for(segment_index_t i = embedded_block; i < first_block; i++) // calc the offsets - itt_hide_store_word( my_table[i], ptr + segment_base(i) ); - } - itt_store_word_with_release( my_mask, sz-1 ); - watchdog.my_segment_ptr = 0; - } - - //! Get bucket by (masked) hashcode - bucket *get_bucket( hashcode_t h ) const throw() { // TODO: add throw() everywhere? - segment_index_t s = segment_index_of( h ); - h -= segment_base(s); - segment_ptr_t seg = my_table[s]; - __TBB_ASSERT( is_valid(seg), "hashcode must be cut by valid mask for allocated segments" ); - return &seg[h]; - } - - // internal serial rehashing helper - void mark_rehashed_levels( hashcode_t h ) throw () { - segment_index_t s = segment_index_of( h ); - while( segment_ptr_t seg = my_table[++s] ) - if( seg[h].node_list == rehash_req ) { - seg[h].node_list = empty_rehashed; - mark_rehashed_levels( h + ((hashcode_t)1<node_list) != rehash_req ) - { -#if __TBB_STATISTICS - my_info_restarts++; // race collisions -#endif - return true; - } - } - return false; - } - - //! Insert a node and check for load factor. @return segment index to enable. - segment_index_t insert_new_node( bucket *b, node_base *n, hashcode_t mask ) { - size_type sz = ++my_size; // prefix form is to enforce allocation after the first item inserted - add_to_bucket( b, n ); - // check load factor - if( sz >= mask ) { // TODO: add custom load_factor - segment_index_t new_seg = __TBB_Log2( mask+1 ); //optimized segment_index_of - __TBB_ASSERT( is_valid(my_table[new_seg-1]), "new allocations must not publish new mask until segment has allocated"); - static const segment_ptr_t is_allocating = (segment_ptr_t)2; - if( !itt_hide_load_word(my_table[new_seg]) - && as_atomic(my_table[new_seg]).compare_and_swap(is_allocating, NULL) == NULL ) - return new_seg; // The value must be processed - } - return 0; - } - - //! Prepare enough segments for number of buckets - void reserve(size_type buckets) { - if( !buckets-- ) return; - bool is_initial = !my_size; - for( size_type m = my_mask; buckets > m; m = my_mask ) - enable_segment( segment_index_of( m+1 ), is_initial ); - } - //! Swap hash_map_bases - void internal_swap(hash_map_base &table) { - using std::swap; - swap(this->my_mask, table.my_mask); - swap(this->my_size, table.my_size); - for(size_type i = 0; i < embedded_buckets; i++) - swap(this->my_embedded_segment[i].node_list, table.my_embedded_segment[i].node_list); - for(size_type i = embedded_block; i < pointers_per_table; i++) - swap(this->my_table[i], table.my_table[i]); - } - }; - - template - class hash_map_range; - - //! Meets requirements of a forward iterator for STL */ - /** Value is either the T or const T type of the container. - @ingroup containers */ - template - class hash_map_iterator - : public std::iterator - { - typedef Container map_type; - typedef typename Container::node node; - typedef hash_map_base::node_base node_base; - typedef hash_map_base::bucket bucket; - - template - friend bool operator==( const hash_map_iterator& i, const hash_map_iterator& j ); - - template - friend bool operator!=( const hash_map_iterator& i, const hash_map_iterator& j ); - - template - friend ptrdiff_t operator-( const hash_map_iterator& i, const hash_map_iterator& j ); - - template - friend class hash_map_iterator; - - template - friend class hash_map_range; - - void advance_to_next_bucket() { // TODO?: refactor to iterator_base class - size_t k = my_index+1; - __TBB_ASSERT( my_bucket, "advancing an invalid iterator?"); - while( k <= my_map->my_mask ) { - // Following test uses 2's-complement wizardry - if( k&(k-2) ) // not the beginning of a segment - ++my_bucket; - else my_bucket = my_map->get_bucket( k ); - my_node = static_cast( my_bucket->node_list ); - if( hash_map_base::is_valid(my_node) ) { - my_index = k; return; - } - ++k; - } - my_bucket = 0; my_node = 0; my_index = k; // the end - } -#if !defined(_MSC_VER) || defined(__INTEL_COMPILER) - template - friend class interface5::concurrent_hash_map; -#else - public: // workaround -#endif - //! concurrent_hash_map over which we are iterating. - const Container *my_map; - - //! Index in hash table for current item - size_t my_index; - - //! Pointer to bucket - const bucket *my_bucket; - - //! Pointer to node that has current item - node *my_node; - - hash_map_iterator( const Container &map, size_t index, const bucket *b, node_base *n ); - - public: - //! Construct undefined iterator - hash_map_iterator(): my_map(), my_index(), my_bucket(), my_node() {} - hash_map_iterator( const hash_map_iterator &other ) : - my_map(other.my_map), - my_index(other.my_index), - my_bucket(other.my_bucket), - my_node(other.my_node) - {} - Value& operator*() const { - __TBB_ASSERT( hash_map_base::is_valid(my_node), "iterator uninitialized or at end of container?" ); - return my_node->item; - } - Value* operator->() const {return &operator*();} - hash_map_iterator& operator++(); - - //! Post increment - hash_map_iterator operator++(int) { - hash_map_iterator old(*this); - operator++(); - return old; - } - }; - - template - hash_map_iterator::hash_map_iterator( const Container &map, size_t index, const bucket *b, node_base *n ) : - my_map(&map), - my_index(index), - my_bucket(b), - my_node( static_cast(n) ) - { - if( b && !hash_map_base::is_valid(n) ) - advance_to_next_bucket(); - } - - template - hash_map_iterator& hash_map_iterator::operator++() { - my_node = static_cast( my_node->next ); - if( !my_node ) advance_to_next_bucket(); - return *this; - } - - template - bool operator==( const hash_map_iterator& i, const hash_map_iterator& j ) { - return i.my_node == j.my_node && i.my_map == j.my_map; - } - - template - bool operator!=( const hash_map_iterator& i, const hash_map_iterator& j ) { - return i.my_node != j.my_node || i.my_map != j.my_map; - } - - //! Range class used with concurrent_hash_map - /** @ingroup containers */ - template - class hash_map_range { - typedef typename Iterator::map_type map_type; - Iterator my_begin; - Iterator my_end; - mutable Iterator my_midpoint; - size_t my_grainsize; - //! Set my_midpoint to point approximately half way between my_begin and my_end. - void set_midpoint() const; - template friend class hash_map_range; - public: - //! Type for size of a range - typedef std::size_t size_type; - typedef typename Iterator::value_type value_type; - typedef typename Iterator::reference reference; - typedef typename Iterator::difference_type difference_type; - typedef Iterator iterator; - - //! True if range is empty. - bool empty() const {return my_begin==my_end;} - - //! True if range can be partitioned into two subranges. - bool is_divisible() const { - return my_midpoint!=my_end; - } - //! Split range. - hash_map_range( hash_map_range& r, split ) : - my_end(r.my_end), - my_grainsize(r.my_grainsize) - { - r.my_end = my_begin = r.my_midpoint; - __TBB_ASSERT( !empty(), "Splitting despite the range is not divisible" ); - __TBB_ASSERT( !r.empty(), "Splitting despite the range is not divisible" ); - set_midpoint(); - r.set_midpoint(); - } - //! type conversion - template - hash_map_range( hash_map_range& r) : - my_begin(r.my_begin), - my_end(r.my_end), - my_midpoint(r.my_midpoint), - my_grainsize(r.my_grainsize) - {} - //! Init range with container and grainsize specified - hash_map_range( const map_type &map, size_type grainsize_ = 1 ) : - my_begin( Iterator( map, 0, map.my_embedded_segment, map.my_embedded_segment->node_list ) ), - my_end( Iterator( map, map.my_mask + 1, 0, 0 ) ), - my_grainsize( grainsize_ ) - { - __TBB_ASSERT( grainsize_>0, "grainsize must be positive" ); - set_midpoint(); - } - const Iterator& begin() const {return my_begin;} - const Iterator& end() const {return my_end;} - //! The grain size for this range. - size_type grainsize() const {return my_grainsize;} - }; - - template - void hash_map_range::set_midpoint() const { - // Split by groups of nodes - size_t m = my_end.my_index-my_begin.my_index; - if( m > my_grainsize ) { - m = my_begin.my_index + m/2u; - hash_map_base::bucket *b = my_begin.my_map->get_bucket(m); - my_midpoint = Iterator(*my_begin.my_map,m,b,b->node_list); - } else { - my_midpoint = my_end; - } - __TBB_ASSERT( my_begin.my_index <= my_midpoint.my_index, - "my_begin is after my_midpoint" ); - __TBB_ASSERT( my_midpoint.my_index <= my_end.my_index, - "my_midpoint is after my_end" ); - __TBB_ASSERT( my_begin != my_midpoint || my_begin == my_end, - "[my_begin, my_midpoint) range should not be empty" ); - } - - } // internal -//! @endcond - -#if _MSC_VER && !defined(__INTEL_COMPILER) - // Suppress "conditional expression is constant" warning. - #pragma warning( push ) - #pragma warning( disable: 4127 ) -#endif - -//! Unordered map from Key to T. -/** concurrent_hash_map is associative container with concurrent access. - -@par Compatibility - The class meets all Container Requirements from C++ Standard (See ISO/IEC 14882:2003(E), clause 23.1). - -@par Exception Safety - - Hash function is not permitted to throw an exception. User-defined types Key and T are forbidden from throwing an exception in destructors. - - If exception happens during insert() operations, it has no effect (unless exception raised by HashCompare::hash() function during grow_segment). - - If exception happens during operator=() operation, the container can have a part of source items, and methods size() and empty() can return wrong results. - -@par Changes since TBB 2.1 - - Replaced internal algorithm and data structure. Patent is pending. - - Added buckets number argument for constructor - -@par Changes since TBB 2.0 - - Fixed exception-safety - - Added template argument for allocator - - Added allocator argument in constructors - - Added constructor from a range of iterators - - Added several new overloaded insert() methods - - Added get_allocator() - - Added swap() - - Added count() - - Added overloaded erase(accessor &) and erase(const_accessor&) - - Added equal_range() [const] - - Added [const_]pointer, [const_]reference, and allocator_type types - - Added global functions: operator==(), operator!=(), and swap() - - @ingroup containers */ -template -class concurrent_hash_map : protected internal::hash_map_base { - template - friend class internal::hash_map_iterator; - - template - friend class internal::hash_map_range; - -public: - typedef Key key_type; - typedef T mapped_type; - typedef std::pair value_type; - typedef hash_map_base::size_type size_type; - typedef ptrdiff_t difference_type; - typedef value_type *pointer; - typedef const value_type *const_pointer; - typedef value_type &reference; - typedef const value_type &const_reference; - typedef internal::hash_map_iterator iterator; - typedef internal::hash_map_iterator const_iterator; - typedef internal::hash_map_range range_type; - typedef internal::hash_map_range const_range_type; - typedef Allocator allocator_type; - -protected: - friend class const_accessor; - struct node; - typedef typename Allocator::template rebind::other node_allocator_type; - node_allocator_type my_allocator; - HashCompare my_hash_compare; - - struct node : public node_base { - value_type item; - node( const Key &key ) : item(key, T()) {} - node( const Key &key, const T &t ) : item(key, t) {} -#if __TBB_CPP11_RVALUE_REF_PRESENT - node( const Key &key, T &&t ) : item(key, std::move(t)) {} - node( value_type&& i ) : item(std::move(i)){} -#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT - template - node( Args&&... args ) : item(std::forward(args)...) {} -#if __TBB_COPY_FROM_NON_CONST_REF_BROKEN - node( value_type& i ) : item(const_cast(i)) {} -#endif //__TBB_COPY_FROM_NON_CONST_REF_BROKEN -#endif //__TBB_CPP11_VARIADIC_TEMPLATES_PRESENT -#endif //__TBB_CPP11_RVALUE_REF_PRESENT - node( const value_type& i ) : item(i) {} - - // exception-safe allocation, see C++ Standard 2003, clause 5.3.4p17 - void *operator new( size_t /*size*/, node_allocator_type &a ) { - void *ptr = a.allocate(1); - if(!ptr) - tbb::internal::throw_exception(tbb::internal::eid_bad_alloc); - return ptr; - } - // match placement-new form above to be called if exception thrown in constructor - void operator delete( void *ptr, node_allocator_type &a ) { a.deallocate(static_cast(ptr),1); } - }; - - void delete_node( node_base *n ) { - my_allocator.destroy( static_cast(n) ); - my_allocator.deallocate( static_cast(n), 1); - } - - static node* allocate_node_copy_construct(node_allocator_type& allocator, const Key &key, const T * t){ - return new( allocator ) node(key, *t); - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - static node* allocate_node_move_construct(node_allocator_type& allocator, const Key &key, const T * t){ - return new( allocator ) node(key, std::move(*const_cast(t))); - } -#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT - template - static node* allocate_node_emplace_construct(node_allocator_type& allocator, Args&&... args){ - return new( allocator ) node(std::forward(args)...); - } -#endif //#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT -#endif - - static node* allocate_node_default_construct(node_allocator_type& allocator, const Key &key, const T * ){ - return new( allocator ) node(key); - } - - static node* do_not_allocate_node(node_allocator_type& , const Key &, const T * ){ - __TBB_ASSERT(false,"this dummy function should not be called"); - return NULL; - } - - node *search_bucket( const key_type &key, bucket *b ) const { - node *n = static_cast( b->node_list ); - while( is_valid(n) && !my_hash_compare.equal(key, n->item.first) ) - n = static_cast( n->next ); - __TBB_ASSERT(n != internal::rehash_req, "Search can be executed only for rehashed bucket"); - return n; - } - - //! bucket accessor is to find, rehash, acquire a lock, and access a bucket - class bucket_accessor : public bucket::scoped_t { - bucket *my_b; - public: - bucket_accessor( concurrent_hash_map *base, const hashcode_t h, bool writer = false ) { acquire( base, h, writer ); } - //! find a bucket by masked hashcode, optionally rehash, and acquire the lock - inline void acquire( concurrent_hash_map *base, const hashcode_t h, bool writer = false ) { - my_b = base->get_bucket( h ); - // TODO: actually, notification is unnecessary here, just hiding double-check - if( itt_load_word_with_acquire(my_b->node_list) == internal::rehash_req - && try_acquire( my_b->mutex, /*write=*/true ) ) - { - if( my_b->node_list == internal::rehash_req ) base->rehash_bucket( my_b, h ); //recursive rehashing - } - else bucket::scoped_t::acquire( my_b->mutex, writer ); - __TBB_ASSERT( my_b->node_list != internal::rehash_req, NULL); - } - //! check whether bucket is locked for write - bool is_writer() { return bucket::scoped_t::is_writer; } - //! get bucket pointer - bucket *operator() () { return my_b; } - }; - - // TODO refactor to hash_base - void rehash_bucket( bucket *b_new, const hashcode_t h ) { - __TBB_ASSERT( *(intptr_t*)(&b_new->mutex), "b_new must be locked (for write)"); - __TBB_ASSERT( h > 1, "The lowermost buckets can't be rehashed" ); - __TBB_store_with_release(b_new->node_list, internal::empty_rehashed); // mark rehashed - hashcode_t mask = ( 1u<<__TBB_Log2( h ) ) - 1; // get parent mask from the topmost bit -#if __TBB_STATISTICS - my_info_rehashes++; // invocations of rehash_bucket -#endif - - bucket_accessor b_old( this, h & mask ); - - mask = (mask<<1) | 1; // get full mask for new bucket - __TBB_ASSERT( (mask&(mask+1))==0 && (h & mask) == h, NULL ); - restart: - for( node_base **p = &b_old()->node_list, *n = __TBB_load_with_acquire(*p); is_valid(n); n = *p ) { - hashcode_t c = my_hash_compare.hash( static_cast(n)->item.first ); -#if TBB_USE_ASSERT - hashcode_t bmask = h & (mask>>1); - bmask = bmask==0? 1 : ( 1u<<(__TBB_Log2( bmask )+1 ) ) - 1; // minimal mask of parent bucket - __TBB_ASSERT( (c & bmask) == (h & bmask), "hash() function changed for key in table" ); -#endif - if( (c & mask) == h ) { - if( !b_old.is_writer() ) - if( !b_old.upgrade_to_writer() ) { - goto restart; // node ptr can be invalid due to concurrent erase - } - *p = n->next; // exclude from b_old - add_to_bucket( b_new, n ); - } else p = &n->next; // iterate to next item - } - } - - struct call_clear_on_leave { - concurrent_hash_map* my_ch_map; - call_clear_on_leave( concurrent_hash_map* a_ch_map ) : my_ch_map(a_ch_map) {} - void dismiss() {my_ch_map = 0;} - ~call_clear_on_leave(){ - if (my_ch_map){ - my_ch_map->clear(); - } - } - }; -public: - - class accessor; - //! Combines data access, locking, and garbage collection. - class const_accessor : private node::scoped_t /*which derived from no_copy*/ { - friend class concurrent_hash_map; - friend class accessor; - public: - //! Type of value - typedef const typename concurrent_hash_map::value_type value_type; - - //! True if result is empty. - bool empty() const { return !my_node; } - - //! Set to null - void release() { - if( my_node ) { - node::scoped_t::release(); - my_node = 0; - } - } - - //! Return reference to associated value in hash table. - const_reference operator*() const { - __TBB_ASSERT( my_node, "attempt to dereference empty accessor" ); - return my_node->item; - } - - //! Return pointer to associated value in hash table. - const_pointer operator->() const { - return &operator*(); - } - - //! Create empty result - const_accessor() : my_node(NULL) {} - - //! Destroy result after releasing the underlying reference. - ~const_accessor() { - my_node = NULL; // scoped lock's release() is called in its destructor - } - protected: - bool is_writer() { return node::scoped_t::is_writer; } - node *my_node; - hashcode_t my_hash; - }; - - //! Allows write access to elements and combines data access, locking, and garbage collection. - class accessor: public const_accessor { - public: - //! Type of value - typedef typename concurrent_hash_map::value_type value_type; - - //! Return reference to associated value in hash table. - reference operator*() const { - __TBB_ASSERT( this->my_node, "attempt to dereference empty accessor" ); - return this->my_node->item; - } - - //! Return pointer to associated value in hash table. - pointer operator->() const { - return &operator*(); - } - }; - - //! Construct empty table. - explicit concurrent_hash_map( const allocator_type &a = allocator_type() ) - : internal::hash_map_base(), my_allocator(a) - {} - - //! Construct empty table with n preallocated buckets. This number serves also as initial concurrency level. - concurrent_hash_map( size_type n, const allocator_type &a = allocator_type() ) - : my_allocator(a) - { - reserve( n ); - } - - //! Copy constructor - concurrent_hash_map( const concurrent_hash_map &table, const allocator_type &a = allocator_type() ) - : internal::hash_map_base(), my_allocator(a) - { - internal_copy(table); - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - //! Move constructor - concurrent_hash_map( concurrent_hash_map &&table ) - : internal::hash_map_base(), my_allocator(std::move(table.get_allocator())) - { - swap(table); - } - - //! Move constructor - concurrent_hash_map( concurrent_hash_map &&table, const allocator_type &a ) - : internal::hash_map_base(), my_allocator(a) - { - if (a == table.get_allocator()){ - this->swap(table); - }else{ - call_clear_on_leave scope_guard(this); - internal_copy(std::make_move_iterator(table.begin()), std::make_move_iterator(table.end())); - scope_guard.dismiss(); - } - } -#endif //__TBB_CPP11_RVALUE_REF_PRESENT - - //! Construction with copying iteration range and given allocator instance - template - concurrent_hash_map( I first, I last, const allocator_type &a = allocator_type() ) - : my_allocator(a) - { - reserve( std::distance(first, last) ); // TODO: load_factor? - internal_copy(first, last); - } - -#if __TBB_INITIALIZER_LISTS_PRESENT - //! Construct empty table with n preallocated buckets. This number serves also as initial concurrency level. - concurrent_hash_map( std::initializer_list il, const allocator_type &a = allocator_type() ) - : my_allocator(a) - { - reserve(il.size()); - internal_copy(il.begin(), il.end()); - } - -#endif //__TBB_INITIALIZER_LISTS_PRESENT - - //! Assignment - concurrent_hash_map& operator=( const concurrent_hash_map &table ) { - if( this!=&table ) { - clear(); - internal_copy(table); - } - return *this; - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - //! Move Assignment - concurrent_hash_map& operator=( concurrent_hash_map &&table ) { - if(this != &table){ - typedef typename tbb::internal::allocator_traits::propagate_on_container_move_assignment pocma_t; - if(pocma_t::value || this->my_allocator == table.my_allocator) { - concurrent_hash_map trash (std::move(*this)); - //TODO: swapping allocators here may be a problem, replace with single direction moving iff pocma is set - this->swap(table); - } else { - //do per element move - concurrent_hash_map moved_copy(std::move(table), this->my_allocator); - this->swap(moved_copy); - } - } - return *this; - } -#endif //__TBB_CPP11_RVALUE_REF_PRESENT - -#if __TBB_INITIALIZER_LISTS_PRESENT - //! Assignment - concurrent_hash_map& operator=( std::initializer_list il ) { - clear(); - reserve(il.size()); - internal_copy(il.begin(), il.end()); - return *this; - } -#endif //__TBB_INITIALIZER_LISTS_PRESENT - - - //! Rehashes and optionally resizes the whole table. - /** Useful to optimize performance before or after concurrent operations. - Also enables using of find() and count() concurrent methods in serial context. */ - void rehash(size_type n = 0); - - //! Clear table - void clear(); - - //! Clear table and destroy it. - ~concurrent_hash_map() { clear(); } - - //------------------------------------------------------------------------ - // Parallel algorithm support - //------------------------------------------------------------------------ - range_type range( size_type grainsize=1 ) { - return range_type( *this, grainsize ); - } - const_range_type range( size_type grainsize=1 ) const { - return const_range_type( *this, grainsize ); - } - - //------------------------------------------------------------------------ - // STL support - not thread-safe methods - //------------------------------------------------------------------------ - iterator begin() { return iterator( *this, 0, my_embedded_segment, my_embedded_segment->node_list ); } - iterator end() { return iterator( *this, 0, 0, 0 ); } - const_iterator begin() const { return const_iterator( *this, 0, my_embedded_segment, my_embedded_segment->node_list ); } - const_iterator end() const { return const_iterator( *this, 0, 0, 0 ); } - std::pair equal_range( const Key& key ) { return internal_equal_range( key, end() ); } - std::pair equal_range( const Key& key ) const { return internal_equal_range( key, end() ); } - - //! Number of items in table. - size_type size() const { return my_size; } - - //! True if size()==0. - bool empty() const { return my_size == 0; } - - //! Upper bound on size. - size_type max_size() const {return (~size_type(0))/sizeof(node);} - - //! Returns the current number of buckets - size_type bucket_count() const { return my_mask+1; } - - //! return allocator object - allocator_type get_allocator() const { return this->my_allocator; } - - //! swap two instances. Iterators are invalidated - void swap( concurrent_hash_map &table ); - - //------------------------------------------------------------------------ - // concurrent map operations - //------------------------------------------------------------------------ - - //! Return count of items (0 or 1) - size_type count( const Key &key ) const { - return const_cast(this)->lookup(/*insert*/false, key, NULL, NULL, /*write=*/false, &do_not_allocate_node ); - } - - //! Find item and acquire a read lock on the item. - /** Return true if item is found, false otherwise. */ - bool find( const_accessor &result, const Key &key ) const { - result.release(); - return const_cast(this)->lookup(/*insert*/false, key, NULL, &result, /*write=*/false, &do_not_allocate_node ); - } - - //! Find item and acquire a write lock on the item. - /** Return true if item is found, false otherwise. */ - bool find( accessor &result, const Key &key ) { - result.release(); - return lookup(/*insert*/false, key, NULL, &result, /*write=*/true, &do_not_allocate_node ); - } - - //! Insert item (if not already present) and acquire a read lock on the item. - /** Returns true if item is new. */ - bool insert( const_accessor &result, const Key &key ) { - result.release(); - return lookup(/*insert*/true, key, NULL, &result, /*write=*/false, &allocate_node_default_construct ); - } - - //! Insert item (if not already present) and acquire a write lock on the item. - /** Returns true if item is new. */ - bool insert( accessor &result, const Key &key ) { - result.release(); - return lookup(/*insert*/true, key, NULL, &result, /*write=*/true, &allocate_node_default_construct ); - } - - //! Insert item by copying if there is no such key present already and acquire a read lock on the item. - /** Returns true if item is new. */ - bool insert( const_accessor &result, const value_type &value ) { - result.release(); - return lookup(/*insert*/true, value.first, &value.second, &result, /*write=*/false, &allocate_node_copy_construct ); - } - - //! Insert item by copying if there is no such key present already and acquire a write lock on the item. - /** Returns true if item is new. */ - bool insert( accessor &result, const value_type &value ) { - result.release(); - return lookup(/*insert*/true, value.first, &value.second, &result, /*write=*/true, &allocate_node_copy_construct ); - } - - //! Insert item by copying if there is no such key present already - /** Returns true if item is inserted. */ - bool insert( const value_type &value ) { - return lookup(/*insert*/true, value.first, &value.second, NULL, /*write=*/false, &allocate_node_copy_construct ); - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - //! Insert item by copying if there is no such key present already and acquire a read lock on the item. - /** Returns true if item is new. */ - bool insert( const_accessor &result, value_type && value ) { - return generic_move_insert(result, std::move(value)); - } - - //! Insert item by copying if there is no such key present already and acquire a write lock on the item. - /** Returns true if item is new. */ - bool insert( accessor &result, value_type && value ) { - return generic_move_insert(result, std::move(value)); - } - - //! Insert item by copying if there is no such key present already - /** Returns true if item is inserted. */ - bool insert( value_type && value ) { - return generic_move_insert(accessor_not_used(), std::move(value)); - } - -#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT - //! Insert item by copying if there is no such key present already and acquire a read lock on the item. - /** Returns true if item is new. */ - template - bool emplace( const_accessor &result, Args&&... args ) { - return generic_emplace(result, std::forward(args)...); - } - - //! Insert item by copying if there is no such key present already and acquire a write lock on the item. - /** Returns true if item is new. */ - template - bool emplace( accessor &result, Args&&... args ) { - return generic_emplace(result, std::forward(args)...); - } - - //! Insert item by copying if there is no such key present already - /** Returns true if item is inserted. */ - template - bool emplace( Args&&... args ) { - return generic_emplace(accessor_not_used(), std::forward(args)...); - } -#endif //__TBB_CPP11_VARIADIC_TEMPLATES_PRESENT -#endif //__TBB_CPP11_RVALUE_REF_PRESENT - - //! Insert range [first, last) - template - void insert( I first, I last ) { - for ( ; first != last; ++first ) - insert( *first ); - } - -#if __TBB_INITIALIZER_LISTS_PRESENT - //! Insert initializer list - void insert( std::initializer_list il ) { - insert( il.begin(), il.end() ); - } -#endif //__TBB_INITIALIZER_LISTS_PRESENT - - //! Erase item. - /** Return true if item was erased by particularly this call. */ - bool erase( const Key& key ); - - //! Erase item by const_accessor. - /** Return true if item was erased by particularly this call. */ - bool erase( const_accessor& item_accessor ) { - return exclude( item_accessor ); - } - - //! Erase item by accessor. - /** Return true if item was erased by particularly this call. */ - bool erase( accessor& item_accessor ) { - return exclude( item_accessor ); - } - -protected: - //! Insert or find item and optionally acquire a lock on the item. - bool lookup(bool op_insert, const Key &key, const T *t, const_accessor *result, bool write, node* (*allocate_node)(node_allocator_type& , const Key &, const T * ), node *tmp_n = 0 ) ; - - struct accessor_not_used { void release(){}}; - friend const_accessor* accessor_location( accessor_not_used const& ){ return NULL;} - friend const_accessor* accessor_location( const_accessor & a ) { return &a;} - - friend bool is_write_access_needed( accessor const& ) { return true;} - friend bool is_write_access_needed( const_accessor const& ) { return false;} - friend bool is_write_access_needed( accessor_not_used const& ) { return false;} - -#if __TBB_CPP11_RVALUE_REF_PRESENT - template - bool generic_move_insert( Accessor && result, value_type && value ) { - result.release(); - return lookup(/*insert*/true, value.first, &value.second, accessor_location(result), is_write_access_needed(result), &allocate_node_move_construct ); - } - -#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT - template - bool generic_emplace( Accessor && result, Args &&... args ) { - result.release(); - node * node_ptr = allocate_node_emplace_construct(my_allocator, std::forward(args)...); - return lookup(/*insert*/true, node_ptr->item.first, NULL, accessor_location(result), is_write_access_needed(result), &do_not_allocate_node, node_ptr ); - } -#endif //__TBB_CPP11_VARIADIC_TEMPLATES_PRESENT -#endif //__TBB_CPP11_RVALUE_REF_PRESENT - - //! delete item by accessor - bool exclude( const_accessor &item_accessor ); - - //! Returns an iterator for an item defined by the key, or for the next item after it (if upper==true) - template - std::pair internal_equal_range( const Key& key, I end ) const; - - //! Copy "source" to *this, where *this must start out empty. - void internal_copy( const concurrent_hash_map& source ); - - template - void internal_copy( I first, I last ); - - //! Fast find when no concurrent erasure is used. For internal use inside TBB only! - /** Return pointer to item with given key, or NULL if no such item exists. - Must not be called concurrently with erasure operations. */ - const_pointer internal_fast_find( const Key& key ) const { - hashcode_t h = my_hash_compare.hash( key ); - hashcode_t m = (hashcode_t) itt_load_word_with_acquire( my_mask ); - node *n; - restart: - __TBB_ASSERT((m&(m+1))==0, "data structure is invalid"); - bucket *b = get_bucket( h & m ); - // TODO: actually, notification is unnecessary here, just hiding double-check - if( itt_load_word_with_acquire(b->node_list) == internal::rehash_req ) - { - bucket::scoped_t lock; - if( lock.try_acquire( b->mutex, /*write=*/true ) ) { - if( b->node_list == internal::rehash_req) - const_cast(this)->rehash_bucket( b, h & m ); //recursive rehashing - } - else lock.acquire( b->mutex, /*write=*/false ); - __TBB_ASSERT(b->node_list!=internal::rehash_req,NULL); - } - n = search_bucket( key, b ); - if( n ) - return &n->item; - else if( check_mask_race( h, m ) ) - goto restart; - return 0; - } -}; - -template -bool concurrent_hash_map::lookup( bool op_insert, const Key &key, const T *t, const_accessor *result, bool write, node* (*allocate_node)(node_allocator_type& , const Key&, const T*), node *tmp_n ) { - __TBB_ASSERT( !result || !result->my_node, NULL ); - bool return_value; - hashcode_t const h = my_hash_compare.hash( key ); - hashcode_t m = (hashcode_t) itt_load_word_with_acquire( my_mask ); - segment_index_t grow_segment = 0; - node *n; - restart: - {//lock scope - __TBB_ASSERT((m&(m+1))==0, "data structure is invalid"); - return_value = false; - // get bucket - bucket_accessor b( this, h & m ); - - // find a node - n = search_bucket( key, b() ); - if( op_insert ) { - // [opt] insert a key - if( !n ) { - if( !tmp_n ) { - tmp_n = allocate_node(my_allocator, key, t); - } - if( !b.is_writer() && !b.upgrade_to_writer() ) { // TODO: improved insertion - // Rerun search_list, in case another thread inserted the item during the upgrade. - n = search_bucket( key, b() ); - if( is_valid(n) ) { // unfortunately, it did - b.downgrade_to_reader(); - goto exists; - } - } - if( check_mask_race(h, m) ) - goto restart; // b.release() is done in ~b(). - // insert and set flag to grow the container - grow_segment = insert_new_node( b(), n = tmp_n, m ); - tmp_n = 0; - return_value = true; - } - } else { // find or count - if( !n ) { - if( check_mask_race( h, m ) ) - goto restart; // b.release() is done in ~b(). TODO: replace by continue - return false; - } - return_value = true; - } - exists: - if( !result ) goto check_growth; - // TODO: the following seems as generic/regular operation - // acquire the item - if( !result->try_acquire( n->mutex, write ) ) { - for( tbb::internal::atomic_backoff backoff(true);; ) { - if( result->try_acquire( n->mutex, write ) ) break; - if( !backoff.bounded_pause() ) { - // the wait takes really long, restart the operation - b.release(); - __TBB_ASSERT( !op_insert || !return_value, "Can't acquire new item in locked bucket?" ); - __TBB_Yield(); - m = (hashcode_t) itt_load_word_with_acquire( my_mask ); - goto restart; - } - } - } - }//lock scope - result->my_node = n; - result->my_hash = h; -check_growth: - // [opt] grow the container - if( grow_segment ) { -#if __TBB_STATISTICS - my_info_resizes++; // concurrent ones -#endif - enable_segment( grow_segment ); - } - if( tmp_n ) // if op_insert only - delete_node( tmp_n ); - return return_value; -} - -template -template -std::pair concurrent_hash_map::internal_equal_range( const Key& key, I end_ ) const { - hashcode_t h = my_hash_compare.hash( key ); - hashcode_t m = my_mask; - __TBB_ASSERT((m&(m+1))==0, "data structure is invalid"); - h &= m; - bucket *b = get_bucket( h ); - while( b->node_list == internal::rehash_req ) { - m = ( 1u<<__TBB_Log2( h ) ) - 1; // get parent mask from the topmost bit - b = get_bucket( h &= m ); - } - node *n = search_bucket( key, b ); - if( !n ) - return std::make_pair(end_, end_); - iterator lower(*this, h, b, n), upper(lower); - return std::make_pair(lower, ++upper); -} - -template -bool concurrent_hash_map::exclude( const_accessor &item_accessor ) { - __TBB_ASSERT( item_accessor.my_node, NULL ); - node_base *const n = item_accessor.my_node; - hashcode_t const h = item_accessor.my_hash; - hashcode_t m = (hashcode_t) itt_load_word_with_acquire( my_mask ); - do { - // get bucket - bucket_accessor b( this, h & m, /*writer=*/true ); - node_base **p = &b()->node_list; - while( *p && *p != n ) - p = &(*p)->next; - if( !*p ) { // someone else was first - if( check_mask_race( h, m ) ) - continue; - item_accessor.release(); - return false; - } - __TBB_ASSERT( *p == n, NULL ); - *p = n->next; // remove from container - my_size--; - break; - } while(true); - if( !item_accessor.is_writer() ) // need to get exclusive lock - item_accessor.upgrade_to_writer(); // return value means nothing here - item_accessor.release(); - delete_node( n ); // Only one thread can delete it - return true; -} - -template -bool concurrent_hash_map::erase( const Key &key ) { - node_base *n; - hashcode_t const h = my_hash_compare.hash( key ); - hashcode_t m = (hashcode_t) itt_load_word_with_acquire( my_mask ); -restart: - {//lock scope - // get bucket - bucket_accessor b( this, h & m ); - search: - node_base **p = &b()->node_list; - n = *p; - while( is_valid(n) && !my_hash_compare.equal(key, static_cast(n)->item.first ) ) { - p = &n->next; - n = *p; - } - if( !n ) { // not found, but mask could be changed - if( check_mask_race( h, m ) ) - goto restart; - return false; - } - else if( !b.is_writer() && !b.upgrade_to_writer() ) { - if( check_mask_race( h, m ) ) // contended upgrade, check mask - goto restart; - goto search; - } - *p = n->next; - my_size--; - } - { - typename node::scoped_t item_locker( n->mutex, /*write=*/true ); - } - // note: there should be no threads pretending to acquire this mutex again, do not try to upgrade const_accessor! - delete_node( n ); // Only one thread can delete it due to write lock on the bucket - return true; -} - -template -void concurrent_hash_map::swap(concurrent_hash_map &table) { - //TODO: respect C++11 allocator_traits::propogate_on_constainer_swap - using std::swap; - swap(this->my_allocator, table.my_allocator); - swap(this->my_hash_compare, table.my_hash_compare); - internal_swap(table); -} - -template -void concurrent_hash_map::rehash(size_type sz) { - reserve( sz ); // TODO: add reduction of number of buckets as well - hashcode_t mask = my_mask; - hashcode_t b = (mask+1)>>1; // size or first index of the last segment - __TBB_ASSERT((b&(b-1))==0, NULL); // zero or power of 2 - bucket *bp = get_bucket( b ); // only the last segment should be scanned for rehashing - for(; b <= mask; b++, bp++ ) { - node_base *n = bp->node_list; - __TBB_ASSERT( is_valid(n) || n == internal::empty_rehashed || n == internal::rehash_req, "Broken internal structure" ); - __TBB_ASSERT( *reinterpret_cast(&bp->mutex) == 0, "concurrent or unexpectedly terminated operation during rehash() execution" ); - if( n == internal::rehash_req ) { // rehash bucket, conditional because rehashing of a previous bucket may affect this one - hashcode_t h = b; bucket *b_old = bp; - do { - __TBB_ASSERT( h > 1, "The lowermost buckets can't be rehashed" ); - hashcode_t m = ( 1u<<__TBB_Log2( h ) ) - 1; // get parent mask from the topmost bit - b_old = get_bucket( h &= m ); - } while( b_old->node_list == internal::rehash_req ); - // now h - is index of the root rehashed bucket b_old - mark_rehashed_levels( h ); // mark all non-rehashed children recursively across all segments - for( node_base **p = &b_old->node_list, *q = *p; is_valid(q); q = *p ) { - hashcode_t c = my_hash_compare.hash( static_cast(q)->item.first ); - if( (c & mask) != h ) { // should be rehashed - *p = q->next; // exclude from b_old - bucket *b_new = get_bucket( c & mask ); - __TBB_ASSERT( b_new->node_list != internal::rehash_req, "hash() function changed for key in table or internal error" ); - add_to_bucket( b_new, q ); - } else p = &q->next; // iterate to next item - } - } - } -#if TBB_USE_PERFORMANCE_WARNINGS - int current_size = int(my_size), buckets = int(mask)+1, empty_buckets = 0, overpopulated_buckets = 0; // usage statistics - static bool reported = false; -#endif -#if TBB_USE_ASSERT || TBB_USE_PERFORMANCE_WARNINGS - for( b = 0; b <= mask; b++ ) {// only last segment should be scanned for rehashing - if( b & (b-2) ) ++bp; // not the beginning of a segment - else bp = get_bucket( b ); - node_base *n = bp->node_list; - __TBB_ASSERT( *reinterpret_cast(&bp->mutex) == 0, "concurrent or unexpectedly terminated operation during rehash() execution" ); - __TBB_ASSERT( is_valid(n) || n == internal::empty_rehashed, "Broken internal structure" ); -#if TBB_USE_PERFORMANCE_WARNINGS - if( n == internal::empty_rehashed ) empty_buckets++; - else if( n->next ) overpopulated_buckets++; -#endif -#if TBB_USE_ASSERT - for( ; is_valid(n); n = n->next ) { - hashcode_t h = my_hash_compare.hash( static_cast(n)->item.first ) & mask; - __TBB_ASSERT( h == b, "hash() function changed for key in table or internal error" ); - } -#endif - } -#endif // TBB_USE_ASSERT || TBB_USE_PERFORMANCE_WARNINGS -#if TBB_USE_PERFORMANCE_WARNINGS - if( buckets > current_size) empty_buckets -= buckets - current_size; - else overpopulated_buckets -= current_size - buckets; // TODO: load_factor? - if( !reported && buckets >= 512 && ( 2*empty_buckets > current_size || 2*overpopulated_buckets > current_size ) ) { - tbb::internal::runtime_warning( - "Performance is not optimal because the hash function produces bad randomness in lower bits in %s.\nSize: %d Empties: %d Overlaps: %d", -#if __TBB_USE_OPTIONAL_RTTI - typeid(*this).name(), -#else - "concurrent_hash_map", -#endif - current_size, empty_buckets, overpopulated_buckets ); - reported = true; - } -#endif -} - -template -void concurrent_hash_map::clear() { - hashcode_t m = my_mask; - __TBB_ASSERT((m&(m+1))==0, "data structure is invalid"); -#if TBB_USE_ASSERT || TBB_USE_PERFORMANCE_WARNINGS || __TBB_STATISTICS -#if TBB_USE_PERFORMANCE_WARNINGS || __TBB_STATISTICS - int current_size = int(my_size), buckets = int(m)+1, empty_buckets = 0, overpopulated_buckets = 0; // usage statistics - static bool reported = false; -#endif - bucket *bp = 0; - // check consistency - for( segment_index_t b = 0; b <= m; b++ ) { - if( b & (b-2) ) ++bp; // not the beginning of a segment - else bp = get_bucket( b ); - node_base *n = bp->node_list; - __TBB_ASSERT( is_valid(n) || n == internal::empty_rehashed || n == internal::rehash_req, "Broken internal structure" ); - __TBB_ASSERT( *reinterpret_cast(&bp->mutex) == 0, "concurrent or unexpectedly terminated operation during clear() execution" ); -#if TBB_USE_PERFORMANCE_WARNINGS || __TBB_STATISTICS - if( n == internal::empty_rehashed ) empty_buckets++; - else if( n == internal::rehash_req ) buckets--; - else if( n->next ) overpopulated_buckets++; -#endif -#if __TBB_EXTRA_DEBUG - for(; is_valid(n); n = n->next ) { - hashcode_t h = my_hash_compare.hash( static_cast(n)->item.first ); - h &= m; - __TBB_ASSERT( h == b || get_bucket(h)->node_list == internal::rehash_req, "hash() function changed for key in table or internal error" ); - } -#endif - } -#if TBB_USE_PERFORMANCE_WARNINGS || __TBB_STATISTICS -#if __TBB_STATISTICS - printf( "items=%d buckets: capacity=%d rehashed=%d empty=%d overpopulated=%d" - " concurrent: resizes=%u rehashes=%u restarts=%u\n", - current_size, int(m+1), buckets, empty_buckets, overpopulated_buckets, - unsigned(my_info_resizes), unsigned(my_info_rehashes), unsigned(my_info_restarts) ); - my_info_resizes = 0; // concurrent ones - my_info_restarts = 0; // race collisions - my_info_rehashes = 0; // invocations of rehash_bucket -#endif - if( buckets > current_size) empty_buckets -= buckets - current_size; - else overpopulated_buckets -= current_size - buckets; // TODO: load_factor? - if( !reported && buckets >= 512 && ( 2*empty_buckets > current_size || 2*overpopulated_buckets > current_size ) ) { - tbb::internal::runtime_warning( - "Performance is not optimal because the hash function produces bad randomness in lower bits in %s.\nSize: %d Empties: %d Overlaps: %d", -#if __TBB_USE_OPTIONAL_RTTI - typeid(*this).name(), -#else - "concurrent_hash_map", -#endif - current_size, empty_buckets, overpopulated_buckets ); - reported = true; - } -#endif -#endif//TBB_USE_ASSERT || TBB_USE_PERFORMANCE_WARNINGS || __TBB_STATISTICS - my_size = 0; - segment_index_t s = segment_index_of( m ); - __TBB_ASSERT( s+1 == pointers_per_table || !my_table[s+1], "wrong mask or concurrent grow" ); - cache_aligned_allocator alloc; - do { - __TBB_ASSERT( is_valid( my_table[s] ), "wrong mask or concurrent grow" ); - segment_ptr_t buckets_ptr = my_table[s]; - size_type sz = segment_size( s ? s : 1 ); - for( segment_index_t i = 0; i < sz; i++ ) - for( node_base *n = buckets_ptr[i].node_list; is_valid(n); n = buckets_ptr[i].node_list ) { - buckets_ptr[i].node_list = n->next; - delete_node( n ); - } - if( s >= first_block) // the first segment or the next - alloc.deallocate( buckets_ptr, sz ); - else if( s == embedded_block && embedded_block != first_block ) - alloc.deallocate( buckets_ptr, segment_size(first_block)-embedded_buckets ); - if( s >= embedded_block ) my_table[s] = 0; - } while(s-- > 0); - my_mask = embedded_buckets - 1; -} - -template -void concurrent_hash_map::internal_copy( const concurrent_hash_map& source ) { - reserve( source.my_size ); // TODO: load_factor? - hashcode_t mask = source.my_mask; - if( my_mask == mask ) { // optimized version - bucket *dst = 0, *src = 0; - bool rehash_required = false; - for( hashcode_t k = 0; k <= mask; k++ ) { - if( k & (k-2) ) ++dst,src++; // not the beginning of a segment - else { dst = get_bucket( k ); src = source.get_bucket( k ); } - __TBB_ASSERT( dst->node_list != internal::rehash_req, "Invalid bucket in destination table"); - node *n = static_cast( src->node_list ); - if( n == internal::rehash_req ) { // source is not rehashed, items are in previous buckets - rehash_required = true; - dst->node_list = internal::rehash_req; - } else for(; n; n = static_cast( n->next ) ) { - add_to_bucket( dst, new( my_allocator ) node(n->item.first, n->item.second) ); - ++my_size; // TODO: replace by non-atomic op - } - } - if( rehash_required ) rehash(); - } else internal_copy( source.begin(), source.end() ); -} - -template -template -void concurrent_hash_map::internal_copy(I first, I last) { - hashcode_t m = my_mask; - for(; first != last; ++first) { - hashcode_t h = my_hash_compare.hash( (*first).first ); - bucket *b = get_bucket( h & m ); - __TBB_ASSERT( b->node_list != internal::rehash_req, "Invalid bucket in destination table"); - node *n = new( my_allocator ) node(*first); - add_to_bucket( b, n ); - ++my_size; // TODO: replace by non-atomic op - } -} - -} // namespace interface5 - -using interface5::concurrent_hash_map; - - -template -inline bool operator==(const concurrent_hash_map &a, const concurrent_hash_map &b) { - if(a.size() != b.size()) return false; - typename concurrent_hash_map::const_iterator i(a.begin()), i_end(a.end()); - typename concurrent_hash_map::const_iterator j, j_end(b.end()); - for(; i != i_end; ++i) { - j = b.equal_range(i->first).first; - if( j == j_end || !(i->second == j->second) ) return false; - } - return true; -} - -template -inline bool operator!=(const concurrent_hash_map &a, const concurrent_hash_map &b) -{ return !(a == b); } - -template -inline void swap(concurrent_hash_map &a, concurrent_hash_map &b) -{ a.swap( b ); } - -#if _MSC_VER && !defined(__INTEL_COMPILER) - #pragma warning( pop ) -#endif // warning 4127 is back - -} // namespace tbb - -#endif /* __TBB_concurrent_hash_map_H */ +#include "../oneapi/tbb/concurrent_hash_map.h" diff --git a/src/tbb/include/tbb/concurrent_lru_cache.h b/src/tbb/include/tbb/concurrent_lru_cache.h index dc1532cfb..2757a234b 100644 --- a/src/tbb/include/tbb/concurrent_lru_cache.h +++ b/src/tbb/include/tbb/concurrent_lru_cache.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,276 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ -#ifndef __TBB_concurrent_lru_cache_H -#define __TBB_concurrent_lru_cache_H - -#if ! TBB_PREVIEW_CONCURRENT_LRU_CACHE - #error Set TBB_PREVIEW_CONCURRENT_LRU_CACHE to include concurrent_lru_cache.h -#endif - -#include "tbb_stddef.h" - -#include -#include -#include // std::find -#if __TBB_CPP11_RVALUE_REF_PRESENT -#include // std::move -#endif - -#include "atomic.h" -#include "internal/_aggregator_impl.h" - -namespace tbb{ -namespace interface6 { - - -template -class concurrent_lru_cache : internal::no_assign{ -private: - typedef concurrent_lru_cache self_type; - typedef value_functor_type value_function_type; - typedef std::size_t ref_counter_type; - struct map_value_type; - typedef std::map map_storage_type; - typedef std::list lru_list_type; - struct map_value_type { - value_type my_value; - ref_counter_type my_ref_counter; - typename lru_list_type::iterator my_lru_list_iterator; - bool my_is_ready; - - map_value_type (value_type const& a_value, ref_counter_type a_ref_counter, typename lru_list_type::iterator a_lru_list_iterator, bool a_is_ready) - : my_value(a_value), my_ref_counter(a_ref_counter), my_lru_list_iterator (a_lru_list_iterator), my_is_ready(a_is_ready) - {} - }; - - class handle_object; - - struct aggregator_operation; - typedef aggregator_operation aggregated_operation_type; - typedef tbb::internal::aggregating_functor aggregator_function_type; - friend class tbb::internal::aggregating_functor; - typedef tbb::internal::aggregator aggregator_type; - -private: - value_function_type my_value_function; - std::size_t const my_number_of_lru_history_items; - map_storage_type my_map_storage; - lru_list_type my_lru_list; - aggregator_type my_aggregator; - -public: - typedef handle_object handle; - -public: - concurrent_lru_cache(value_function_type f, std::size_t number_of_lru_history_items) - : my_value_function(f),my_number_of_lru_history_items(number_of_lru_history_items) - { - my_aggregator.initialize_handler(aggregator_function_type(this)); - } - - handle_object operator[](key_type k){ - retrieve_aggregator_operation op(k); - my_aggregator.execute(&op); - if (op.is_new_value_needed()){ - op.result().second.my_value = my_value_function(k); - __TBB_store_with_release(op.result().second.my_is_ready, true); - }else{ - tbb::internal::spin_wait_while_eq(op.result().second.my_is_ready,false); - } - return handle_object(*this,op.result()); - } -private: - void signal_end_of_usage(typename map_storage_type::reference value_ref){ - signal_end_of_usage_aggregator_operation op(value_ref); - my_aggregator.execute(&op); - } - -private: -#if !__TBB_CPP11_RVALUE_REF_PRESENT - struct handle_move_t:no_assign{ - concurrent_lru_cache & my_cache_ref; - typename map_storage_type::reference my_map_record_ref; - handle_move_t(concurrent_lru_cache & cache_ref, typename map_storage_type::reference value_ref):my_cache_ref(cache_ref),my_map_record_ref(value_ref) {}; - }; -#endif - class handle_object { - concurrent_lru_cache * my_cache_pointer; - typename map_storage_type::pointer my_map_record_ptr; - public: - handle_object() : my_cache_pointer(), my_map_record_ptr() {} - handle_object(concurrent_lru_cache& cache_ref, typename map_storage_type::reference value_ref) : my_cache_pointer(&cache_ref), my_map_record_ptr(&value_ref) {} - operator bool() const { - return (my_cache_pointer && my_map_record_ptr); - } -#if __TBB_CPP11_RVALUE_REF_PRESENT - // TODO: add check for double moved objects by special dedicated field - handle_object(handle_object&& src) : my_cache_pointer(src.my_cache_pointer), my_map_record_ptr(src.my_map_record_ptr) { - __TBB_ASSERT((src.my_cache_pointer && src.my_map_record_ptr) || (!src.my_cache_pointer && !src.my_map_record_ptr), "invalid state of moving object?"); - src.my_cache_pointer = NULL; - src.my_map_record_ptr = NULL; - } - handle_object& operator=(handle_object&& src) { - __TBB_ASSERT((src.my_cache_pointer && src.my_map_record_ptr) || (!src.my_cache_pointer && !src.my_map_record_ptr), "invalid state of moving object?"); - if (my_cache_pointer) { - my_cache_pointer->signal_end_of_usage(*my_map_record_ptr); - } - my_cache_pointer = src.my_cache_pointer; - my_map_record_ptr = src.my_map_record_ptr; - src.my_cache_pointer = NULL; - src.my_map_record_ptr = NULL; - return *this; - } -#else - handle_object(handle_move_t m) : my_cache_pointer(&m.my_cache_ref), my_map_record_ptr(&m.my_map_record_ref) {} - handle_object& operator=(handle_move_t m) { - if (my_cache_pointer) { - my_cache_pointer->signal_end_of_usage(*my_map_record_ptr); - } - my_cache_pointer = &m.my_cache_ref; - my_map_record_ptr = &m.my_map_record_ref; - return *this; - } - operator handle_move_t(){ - return move(*this); - } -#endif // __TBB_CPP11_RVALUE_REF_PRESENT - value_type& value(){ - __TBB_ASSERT(my_cache_pointer,"get value from already moved object?"); - __TBB_ASSERT(my_map_record_ptr,"get value from an invalid or already moved object?"); - return my_map_record_ptr->second.my_value; - } - ~handle_object(){ - if (my_cache_pointer){ - my_cache_pointer->signal_end_of_usage(*my_map_record_ptr); - } - } - private: -#if __TBB_CPP11_RVALUE_REF_PRESENT - // For source compatibility with C++03 - friend handle_object&& move(handle_object& h){ - return std::move(h); - } -#else - friend handle_move_t move(handle_object& h){ - return handle_object::move(h); - } - // TODO: add check for double moved objects by special dedicated field - static handle_move_t move(handle_object& h){ - __TBB_ASSERT((h.my_cache_pointer && h.my_map_record_ptr) || (!h.my_cache_pointer && !h.my_map_record_ptr), "invalid state of moving object?"); - concurrent_lru_cache * cache_pointer = h.my_cache_pointer; - typename map_storage_type::pointer map_record_ptr = h.my_map_record_ptr; - h.my_cache_pointer = NULL; - h.my_map_record_ptr = NULL; - return handle_move_t(*cache_pointer, *map_record_ptr); - } -#endif // __TBB_CPP11_RVALUE_REF_PRESENT - private: - void operator=(handle_object&); -#if __SUNPRO_CC - // Presumably due to a compiler error, private copy constructor - // breaks expressions like handle h = cache[key]; - public: -#endif - handle_object(handle_object &); - }; -private: - //TODO: looks like aggregator_operation is a perfect match for statically typed variant type - struct aggregator_operation : tbb::internal::aggregated_operation{ - enum e_op_type {op_retive, op_signal_end_of_usage}; - //TODO: try to use pointer to function apply_visitor here - //TODO: try virtual functions and measure the difference - e_op_type my_operation_type; - aggregator_operation(e_op_type operation_type): my_operation_type(operation_type) {} - void cast_and_handle(self_type& container ){ - if (my_operation_type==op_retive){ - static_cast(this)->handle(container); - }else{ - static_cast(this)->handle(container); - } - } - }; - struct retrieve_aggregator_operation : aggregator_operation, private internal::no_assign { - key_type my_key; - typename map_storage_type::pointer my_result_map_record_pointer; - bool my_is_new_value_needed; - retrieve_aggregator_operation(key_type key):aggregator_operation(aggregator_operation::op_retive),my_key(key),my_is_new_value_needed(false){} - void handle(self_type& container ){ - my_result_map_record_pointer = & container.retrieve_serial(my_key,my_is_new_value_needed); - } - typename map_storage_type::reference result(){ return * my_result_map_record_pointer; } - bool is_new_value_needed(){return my_is_new_value_needed;} - }; - struct signal_end_of_usage_aggregator_operation : aggregator_operation, private internal::no_assign { - typename map_storage_type::reference my_map_record_ref; - signal_end_of_usage_aggregator_operation(typename map_storage_type::reference map_record_ref):aggregator_operation(aggregator_operation::op_signal_end_of_usage),my_map_record_ref(map_record_ref){} - void handle(self_type& container ){ - container.signal_end_of_usage_serial(my_map_record_ref); - } - }; - -private: - void handle_operations(aggregator_operation* op_list){ - while(op_list){ - op_list->cast_and_handle(*this); - aggregator_operation* tmp = op_list; - op_list=op_list->next; - tbb::internal::itt_store_word_with_release(tmp->status, uintptr_t(1)); - } - } - -private: - typename map_storage_type::reference retrieve_serial(key_type k, bool& is_new_value_needed){ - typename map_storage_type::iterator it = my_map_storage.find(k); - if (it == my_map_storage.end()){ - it = my_map_storage.insert(it,std::make_pair(k,map_value_type(value_type(),0,my_lru_list.end(),false))); - is_new_value_needed = true; - }else { - typename lru_list_type::iterator list_it = it->second.my_lru_list_iterator; - if (list_it!=my_lru_list.end()) { - __TBB_ASSERT(!it->second.my_ref_counter,"item to be evicted should not have a live references"); - //item is going to be used. Therefore it is not a subject for eviction - //so - remove it from LRU history. - my_lru_list.erase(list_it); - it->second.my_lru_list_iterator= my_lru_list.end(); - } - } - ++(it->second.my_ref_counter); - return *it; - } - - void signal_end_of_usage_serial(typename map_storage_type::reference map_record_ref){ - typename map_storage_type::iterator it = my_map_storage.find(map_record_ref.first); - __TBB_ASSERT(it!=my_map_storage.end(),"cache should not return past-end iterators to outer world"); - __TBB_ASSERT(&(*it) == &map_record_ref,"dangling reference has been returned to outside world? data race ?"); - __TBB_ASSERT( my_lru_list.end()== std::find(my_lru_list.begin(),my_lru_list.end(),it), - "object in use should not be in list of unused objects "); - if (! --(it->second.my_ref_counter)){ - //it was the last reference so put it to the LRU history - if (my_lru_list.size()>=my_number_of_lru_history_items){ - //evict items in order to get a space - size_t number_of_elements_to_evict = 1 + my_lru_list.size() - my_number_of_lru_history_items; - for (size_t i=0; isecond.my_ref_counter,"item to be evicted should not have a live references"); - my_lru_list.pop_back(); - my_map_storage.erase(it_to_evict); - } - } - my_lru_list.push_front(it); - it->second.my_lru_list_iterator = my_lru_list.begin(); - } - } -}; -} // namespace interface6 - -using interface6::concurrent_lru_cache; - -} // namespace tbb -#endif //__TBB_concurrent_lru_cache_H +#include "../oneapi/tbb/concurrent_lru_cache.h" diff --git a/src/tbb/src/rml/test/test_rml_omp_c_linkage.c b/src/tbb/include/tbb/concurrent_map.h similarity index 82% rename from src/tbb/src/rml/test/test_rml_omp_c_linkage.c rename to src/tbb/include/tbb/concurrent_map.h index a59a58645..84f59d7e6 100644 --- a/src/tbb/src/rml/test/test_rml_omp_c_linkage.c +++ b/src/tbb/include/tbb/concurrent_map.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2019-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,15 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ -void Cplusplus(); - -int main() { - Cplusplus(); - return 0; -} +#include "../oneapi/tbb/concurrent_map.h" diff --git a/src/tbb/include/tbb/concurrent_priority_queue.h b/src/tbb/include/tbb/concurrent_priority_queue.h index 89297d851..3b27130b1 100644 --- a/src/tbb/include/tbb/concurrent_priority_queue.h +++ b/src/tbb/include/tbb/concurrent_priority_queue.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,479 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ -#ifndef __TBB_concurrent_priority_queue_H -#define __TBB_concurrent_priority_queue_H - -#include "atomic.h" -#include "cache_aligned_allocator.h" -#include "tbb_exception.h" -#include "tbb_stddef.h" -#include "tbb_profiling.h" -#include "internal/_aggregator_impl.h" -#include -#include -#include -#include __TBB_STD_SWAP_HEADER - -#if __TBB_INITIALIZER_LISTS_PRESENT - #include -#endif - -#if __TBB_CPP11_IS_COPY_CONSTRUCTIBLE_PRESENT - #include -#endif - -namespace tbb { -namespace interface5 { -namespace internal { -#if __TBB_CPP11_IS_COPY_CONSTRUCTIBLE_PRESENT - template::value> - struct use_element_copy_constructor { - typedef tbb::internal::true_type type; - }; - template - struct use_element_copy_constructor { - typedef tbb::internal::false_type type; - }; -#else - template - struct use_element_copy_constructor { - typedef tbb::internal::true_type type; - }; -#endif -} // namespace internal - -using namespace tbb::internal; - -//! Concurrent priority queue -template , typename A=cache_aligned_allocator > -class concurrent_priority_queue { - public: - //! Element type in the queue. - typedef T value_type; - - //! Reference type - typedef T& reference; - - //! Const reference type - typedef const T& const_reference; - - //! Integral type for representing size of the queue. - typedef size_t size_type; - - //! Difference type for iterator - typedef ptrdiff_t difference_type; - - //! Allocator type - typedef A allocator_type; - - //! Constructs a new concurrent_priority_queue with default capacity - explicit concurrent_priority_queue(const allocator_type& a = allocator_type()) : mark(0), my_size(0), data(a) - { - my_aggregator.initialize_handler(my_functor_t(this)); - } - - //! Constructs a new concurrent_priority_queue with init_sz capacity - explicit concurrent_priority_queue(size_type init_capacity, const allocator_type& a = allocator_type()) : - mark(0), my_size(0), data(a) - { - data.reserve(init_capacity); - my_aggregator.initialize_handler(my_functor_t(this)); - } - - //! [begin,end) constructor - template - concurrent_priority_queue(InputIterator begin, InputIterator end, const allocator_type& a = allocator_type()) : - mark(0), data(begin, end, a) - { - my_aggregator.initialize_handler(my_functor_t(this)); - heapify(); - my_size = data.size(); - } - -#if __TBB_INITIALIZER_LISTS_PRESENT - //! Constructor from std::initializer_list - concurrent_priority_queue(std::initializer_list init_list, const allocator_type &a = allocator_type()) : - mark(0),data(init_list.begin(), init_list.end(), a) - { - my_aggregator.initialize_handler(my_functor_t(this)); - heapify(); - my_size = data.size(); - } -#endif //# __TBB_INITIALIZER_LISTS_PRESENT - - //! Copy constructor - /** This operation is unsafe if there are pending concurrent operations on the src queue. */ - explicit concurrent_priority_queue(const concurrent_priority_queue& src) : mark(src.mark), - my_size(src.my_size), data(src.data.begin(), src.data.end(), src.data.get_allocator()) - { - my_aggregator.initialize_handler(my_functor_t(this)); - heapify(); - } - - //! Copy constructor with specific allocator - /** This operation is unsafe if there are pending concurrent operations on the src queue. */ - concurrent_priority_queue(const concurrent_priority_queue& src, const allocator_type& a) : mark(src.mark), - my_size(src.my_size), data(src.data.begin(), src.data.end(), a) - { - my_aggregator.initialize_handler(my_functor_t(this)); - heapify(); - } - - //! Assignment operator - /** This operation is unsafe if there are pending concurrent operations on the src queue. */ - concurrent_priority_queue& operator=(const concurrent_priority_queue& src) { - if (this != &src) { - vector_t(src.data.begin(), src.data.end(), src.data.get_allocator()).swap(data); - mark = src.mark; - my_size = src.my_size; - } - return *this; - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - //! Move constructor - /** This operation is unsafe if there are pending concurrent operations on the src queue. */ - concurrent_priority_queue(concurrent_priority_queue&& src) : mark(src.mark), - my_size(src.my_size), data(std::move(src.data)) - { - my_aggregator.initialize_handler(my_functor_t(this)); - } - - //! Move constructor with specific allocator - /** This operation is unsafe if there are pending concurrent operations on the src queue. */ - concurrent_priority_queue(concurrent_priority_queue&& src, const allocator_type& a) : mark(src.mark), - my_size(src.my_size), -#if __TBB_ALLOCATOR_TRAITS_PRESENT - data(std::move(src.data), a) -#else - // Some early version of C++11 STL vector does not have a constructor of vector(vector&& , allocator). - // It seems that the reason is absence of support of allocator_traits (stateful allocators). - data(a) -#endif //__TBB_ALLOCATOR_TRAITS_PRESENT - { - my_aggregator.initialize_handler(my_functor_t(this)); -#if !__TBB_ALLOCATOR_TRAITS_PRESENT - if (a != src.data.get_allocator()){ - data.reserve(src.data.size()); - data.assign(std::make_move_iterator(src.data.begin()), std::make_move_iterator(src.data.end())); - }else{ - data = std::move(src.data); - } -#endif //!__TBB_ALLOCATOR_TRAITS_PRESENT - } - - //! Move assignment operator - /** This operation is unsafe if there are pending concurrent operations on the src queue. */ - concurrent_priority_queue& operator=( concurrent_priority_queue&& src) { - if (this != &src) { - mark = src.mark; - my_size = src.my_size; -#if !__TBB_ALLOCATOR_TRAITS_PRESENT - if (data.get_allocator() != src.data.get_allocator()){ - vector_t(std::make_move_iterator(src.data.begin()), std::make_move_iterator(src.data.end()), data.get_allocator()).swap(data); - }else -#endif //!__TBB_ALLOCATOR_TRAITS_PRESENT - { - data = std::move(src.data); - } - } - return *this; - } -#endif //__TBB_CPP11_RVALUE_REF_PRESENT - - //! Assign the queue from [begin,end) range, not thread-safe - template - void assign(InputIterator begin, InputIterator end) { - vector_t(begin, end, data.get_allocator()).swap(data); - mark = 0; - my_size = data.size(); - heapify(); - } - -#if __TBB_INITIALIZER_LISTS_PRESENT - //! Assign the queue from std::initializer_list, not thread-safe - void assign(std::initializer_list il) { this->assign(il.begin(), il.end()); } - - //! Assign from std::initializer_list, not thread-safe - concurrent_priority_queue& operator=(std::initializer_list il) { - this->assign(il.begin(), il.end()); - return *this; - } -#endif //# __TBB_INITIALIZER_LISTS_PRESENT - - //! Returns true if empty, false otherwise - /** Returned value may not reflect results of pending operations. - This operation reads shared data and will trigger a race condition. */ - bool empty() const { return size()==0; } - - //! Returns the current number of elements contained in the queue - /** Returned value may not reflect results of pending operations. - This operation reads shared data and will trigger a race condition. */ - size_type size() const { return __TBB_load_with_acquire(my_size); } - - //! Pushes elem onto the queue, increasing capacity of queue if necessary - /** This operation can be safely used concurrently with other push, try_pop or emplace operations. */ - void push(const_reference elem) { -#if __TBB_CPP11_IS_COPY_CONSTRUCTIBLE_PRESENT - __TBB_STATIC_ASSERT( std::is_copy_constructible::value, "The type is not copy constructible. Copying push operation is impossible." ); -#endif - cpq_operation op_data(elem, PUSH_OP); - my_aggregator.execute(&op_data); - if (op_data.status == FAILED) // exception thrown - throw_exception(eid_bad_alloc); - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - //! Pushes elem onto the queue, increasing capacity of queue if necessary - /** This operation can be safely used concurrently with other push, try_pop or emplace operations. */ - void push(value_type &&elem) { - cpq_operation op_data(elem, PUSH_RVALUE_OP); - my_aggregator.execute(&op_data); - if (op_data.status == FAILED) // exception thrown - throw_exception(eid_bad_alloc); - } - -#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT - //! Constructs a new element using args as the arguments for its construction and pushes it onto the queue */ - /** This operation can be safely used concurrently with other push, try_pop or emplace operations. */ - template - void emplace(Args&&... args) { - push(value_type(std::forward(args)...)); - } -#endif /* __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT */ -#endif /* __TBB_CPP11_RVALUE_REF_PRESENT */ - - //! Gets a reference to and removes highest priority element - /** If a highest priority element was found, sets elem and returns true, - otherwise returns false. - This operation can be safely used concurrently with other push, try_pop or emplace operations. */ - bool try_pop(reference elem) { - cpq_operation op_data(POP_OP); - op_data.elem = &elem; - my_aggregator.execute(&op_data); - return op_data.status==SUCCEEDED; - } - - //! Clear the queue; not thread-safe - /** This operation is unsafe if there are pending concurrent operations on the queue. - Resets size, effectively emptying queue; does not free space. - May not clear elements added in pending operations. */ - void clear() { - data.clear(); - mark = 0; - my_size = 0; - } - - //! Swap this queue with another; not thread-safe - /** This operation is unsafe if there are pending concurrent operations on the queue. */ - void swap(concurrent_priority_queue& q) { - using std::swap; - data.swap(q.data); - swap(mark, q.mark); - swap(my_size, q.my_size); - } - - //! Return allocator object - allocator_type get_allocator() const { return data.get_allocator(); } - - private: - enum operation_type {INVALID_OP, PUSH_OP, POP_OP, PUSH_RVALUE_OP}; - enum operation_status { WAIT=0, SUCCEEDED, FAILED }; - - class cpq_operation : public aggregated_operation { - public: - operation_type type; - union { - value_type *elem; - size_type sz; - }; - cpq_operation(const_reference e, operation_type t) : - type(t), elem(const_cast(&e)) {} - cpq_operation(operation_type t) : type(t) {} - }; - - class my_functor_t { - concurrent_priority_queue *cpq; - public: - my_functor_t() {} - my_functor_t(concurrent_priority_queue *cpq_) : cpq(cpq_) {} - void operator()(cpq_operation* op_list) { - cpq->handle_operations(op_list); - } - }; - - typedef tbb::internal::aggregator< my_functor_t, cpq_operation > aggregator_t; - aggregator_t my_aggregator; - //! Padding added to avoid false sharing - char padding1[NFS_MaxLineSize - sizeof(aggregator_t)]; - //! The point at which unsorted elements begin - size_type mark; - __TBB_atomic size_type my_size; - Compare compare; - //! Padding added to avoid false sharing - char padding2[NFS_MaxLineSize - (2*sizeof(size_type)) - sizeof(Compare)]; - //! Storage for the heap of elements in queue, plus unheapified elements - /** data has the following structure: - - binary unheapified - heap elements - ____|_______|____ - | | | - v v v - [_|...|_|_|...|_| |...| ] - 0 ^ ^ ^ - | | |__capacity - | |__my_size - |__mark - - Thus, data stores the binary heap starting at position 0 through - mark-1 (it may be empty). Then there are 0 or more elements - that have not yet been inserted into the heap, in positions - mark through my_size-1. */ - typedef std::vector vector_t; - vector_t data; - - void handle_operations(cpq_operation *op_list) { - cpq_operation *tmp, *pop_list=NULL; - - __TBB_ASSERT(mark == data.size(), NULL); - - // First pass processes all constant (amortized; reallocation may happen) time pushes and pops. - while (op_list) { - // ITT note: &(op_list->status) tag is used to cover accesses to op_list - // node. This thread is going to handle the operation, and so will acquire it - // and perform the associated operation w/o triggering a race condition; the - // thread that created the operation is waiting on the status field, so when - // this thread is done with the operation, it will perform a - // store_with_release to give control back to the waiting thread in - // aggregator::insert_operation. - call_itt_notify(acquired, &(op_list->status)); - __TBB_ASSERT(op_list->type != INVALID_OP, NULL); - tmp = op_list; - op_list = itt_hide_load_word(op_list->next); - if (tmp->type == POP_OP) { - if (mark < data.size() && - compare(data[0], data[data.size()-1])) { - // there are newly pushed elems and the last one - // is higher than top - *(tmp->elem) = tbb::internal::move(data[data.size()-1]); - __TBB_store_with_release(my_size, my_size-1); - itt_store_word_with_release(tmp->status, uintptr_t(SUCCEEDED)); - data.pop_back(); - __TBB_ASSERT(mark<=data.size(), NULL); - } - else { // no convenient item to pop; postpone - itt_hide_store_word(tmp->next, pop_list); - pop_list = tmp; - } - } else { // PUSH_OP or PUSH_RVALUE_OP - __TBB_ASSERT(tmp->type == PUSH_OP || tmp->type == PUSH_RVALUE_OP, "Unknown operation" ); - __TBB_TRY{ - if (tmp->type == PUSH_OP) { - push_back_helper(*(tmp->elem), typename internal::use_element_copy_constructor::type()); - } else { - data.push_back(tbb::internal::move(*(tmp->elem))); - } - __TBB_store_with_release(my_size, my_size + 1); - itt_store_word_with_release(tmp->status, uintptr_t(SUCCEEDED)); - } __TBB_CATCH(...) { - itt_store_word_with_release(tmp->status, uintptr_t(FAILED)); - } - } - } - - // second pass processes pop operations - while (pop_list) { - tmp = pop_list; - pop_list = itt_hide_load_word(pop_list->next); - __TBB_ASSERT(tmp->type == POP_OP, NULL); - if (data.empty()) { - itt_store_word_with_release(tmp->status, uintptr_t(FAILED)); - } - else { - __TBB_ASSERT(mark<=data.size(), NULL); - if (mark < data.size() && - compare(data[0], data[data.size()-1])) { - // there are newly pushed elems and the last one is - // higher than top - *(tmp->elem) = tbb::internal::move(data[data.size()-1]); - __TBB_store_with_release(my_size, my_size-1); - itt_store_word_with_release(tmp->status, uintptr_t(SUCCEEDED)); - data.pop_back(); - } - else { // extract top and push last element down heap - *(tmp->elem) = tbb::internal::move(data[0]); - __TBB_store_with_release(my_size, my_size-1); - itt_store_word_with_release(tmp->status, uintptr_t(SUCCEEDED)); - reheap(); - } - } - } - - // heapify any leftover pushed elements before doing the next - // batch of operations - if (mark0) mark = 1; - for (; mark>1; - if (!compare(data[parent], to_place)) break; - data[cur_pos] = tbb::internal::move(data[parent]); - cur_pos = parent; - } while( cur_pos ); - data[cur_pos] = tbb::internal::move(to_place); - } - } - - //! Re-heapify after an extraction - /** Re-heapify by pushing last element down the heap from the root. */ - void reheap() { - size_type cur_pos=0, child=1; - - while (child < mark) { - size_type target = child; - if (child+1 < mark && compare(data[child], data[child+1])) - ++target; - // target now has the higher priority child - if (compare(data[target], data[data.size()-1])) break; - data[cur_pos] = tbb::internal::move(data[target]); - cur_pos = target; - child = (cur_pos<<1)+1; - } - if (cur_pos != data.size()-1) - data[cur_pos] = tbb::internal::move(data[data.size()-1]); - data.pop_back(); - if (mark > data.size()) mark = data.size(); - } - - void push_back_helper(const T& t, tbb::internal::true_type) { - data.push_back(t); - } - - void push_back_helper(const T&, tbb::internal::false_type) { - __TBB_ASSERT( false, "The type is not copy constructible. Copying push operation is impossible." ); - } -}; - -} // namespace interface5 - -using interface5::concurrent_priority_queue; - -} // namespace tbb - -#endif /* __TBB_concurrent_priority_queue_H */ +#include "../oneapi/tbb/concurrent_priority_queue.h" diff --git a/src/tbb/include/tbb/concurrent_queue.h b/src/tbb/include/tbb/concurrent_queue.h index 81db58a33..d81a58b88 100644 --- a/src/tbb/include/tbb/concurrent_queue.h +++ b/src/tbb/include/tbb/concurrent_queue.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,447 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ -#ifndef __TBB_concurrent_queue_H -#define __TBB_concurrent_queue_H - -#include "internal/_concurrent_queue_impl.h" - -namespace tbb { - -namespace strict_ppl { - -//! A high-performance thread-safe non-blocking concurrent queue. -/** Multiple threads may each push and pop concurrently. - Assignment construction is not allowed. - @ingroup containers */ -template > -class concurrent_queue: public internal::concurrent_queue_base_v3 { - template friend class internal::concurrent_queue_iterator; - - //! Allocator type - typedef typename A::template rebind::other page_allocator_type; - page_allocator_type my_allocator; - - //! Allocates a block of size n (bytes) - virtual void *allocate_block( size_t n ) __TBB_override { - void *b = reinterpret_cast(my_allocator.allocate( n )); - if( !b ) - internal::throw_exception(internal::eid_bad_alloc); - return b; - } - - //! Deallocates block created by allocate_block. - virtual void deallocate_block( void *b, size_t n ) __TBB_override { - my_allocator.deallocate( reinterpret_cast(b), n ); - } - - static void copy_construct_item(T* location, const void* src){ - new (location) T(*static_cast(src)); - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - static void move_construct_item(T* location, const void* src) { - new (location) T( std::move(*static_cast(const_cast(src))) ); - } -#endif /* __TBB_CPP11_RVALUE_REF_PRESENT */ -public: - //! Element type in the queue. - typedef T value_type; - - //! Reference type - typedef T& reference; - - //! Const reference type - typedef const T& const_reference; - - //! Integral type for representing size of the queue. - typedef size_t size_type; - - //! Difference type for iterator - typedef ptrdiff_t difference_type; - - //! Allocator type - typedef A allocator_type; - - //! Construct empty queue - explicit concurrent_queue(const allocator_type& a = allocator_type()) : - my_allocator( a ) - { - } - - //! [begin,end) constructor - template - concurrent_queue( InputIterator begin, InputIterator end, const allocator_type& a = allocator_type()) : - my_allocator( a ) - { - for( ; begin != end; ++begin ) - this->push(*begin); - } - - //! Copy constructor - concurrent_queue( const concurrent_queue& src, const allocator_type& a = allocator_type()) : - internal::concurrent_queue_base_v3(), my_allocator( a ) - { - this->assign( src, copy_construct_item ); - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - //! Move constructors - concurrent_queue( concurrent_queue&& src ) : - internal::concurrent_queue_base_v3(), my_allocator( std::move(src.my_allocator) ) - { - this->internal_swap( src ); - } - - concurrent_queue( concurrent_queue&& src, const allocator_type& a ) : - internal::concurrent_queue_base_v3(), my_allocator( a ) - { - // checking that memory allocated by one instance of allocator can be deallocated - // with another - if( my_allocator == src.my_allocator) { - this->internal_swap( src ); - } else { - // allocators are different => performing per-element move - this->assign( src, move_construct_item ); - src.clear(); - } - } -#endif /* __TBB_CPP11_RVALUE_REF_PRESENT */ - - //! Destroy queue - ~concurrent_queue(); - - //! Enqueue an item at tail of queue. - void push( const T& source ) { - this->internal_push( &source, copy_construct_item ); - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - void push( T&& source ) { - this->internal_push( &source, move_construct_item ); - } - -#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT - template - void emplace( Arguments&&... args ) { - push( T(std::forward( args )...) ); - } -#endif //__TBB_CPP11_VARIADIC_TEMPLATES_PRESENT -#endif /* __TBB_CPP11_RVALUE_REF_PRESENT */ - - //! Attempt to dequeue an item from head of queue. - /** Does not wait for item to become available. - Returns true if successful; false otherwise. */ - bool try_pop( T& result ) { - return this->internal_try_pop( &result ); - } - - //! Return the number of items in the queue; thread unsafe - size_type unsafe_size() const {return this->internal_size();} - - //! Equivalent to size()==0. - bool empty() const {return this->internal_empty();} - - //! Clear the queue. not thread-safe. - void clear() ; - - //! Return allocator object - allocator_type get_allocator() const { return this->my_allocator; } - - typedef internal::concurrent_queue_iterator iterator; - typedef internal::concurrent_queue_iterator const_iterator; - - //------------------------------------------------------------------------ - // The iterators are intended only for debugging. They are slow and not thread safe. - //------------------------------------------------------------------------ - iterator unsafe_begin() {return iterator(*this);} - iterator unsafe_end() {return iterator();} - const_iterator unsafe_begin() const {return const_iterator(*this);} - const_iterator unsafe_end() const {return const_iterator();} -} ; - -template -concurrent_queue::~concurrent_queue() { - clear(); - this->internal_finish_clear(); -} - -template -void concurrent_queue::clear() { - T value; - while( !empty() ) try_pop(value); -} - -} // namespace strict_ppl - -//! A high-performance thread-safe blocking concurrent bounded queue. -/** This is the pre-PPL TBB concurrent queue which supports boundedness and blocking semantics. - Note that method names agree with the PPL-style concurrent queue. - Multiple threads may each push and pop concurrently. - Assignment construction is not allowed. - @ingroup containers */ -template > -class concurrent_bounded_queue: public internal::concurrent_queue_base_v8 { - template friend class internal::concurrent_queue_iterator; - - //! Allocator type - typedef typename A::template rebind::other page_allocator_type; - page_allocator_type my_allocator; - - typedef typename concurrent_queue_base_v3::padded_page padded_page; - typedef typename concurrent_queue_base_v3::copy_specifics copy_specifics; - - //! Class used to ensure exception-safety of method "pop" - class destroyer: internal::no_copy { - T& my_value; - public: - destroyer( T& value ) : my_value(value) {} - ~destroyer() {my_value.~T();} - }; - - T& get_ref( page& p, size_t index ) { - __TBB_ASSERT( index(static_cast(&p))->last)[index]; - } - - virtual void copy_item( page& dst, size_t index, const void* src ) __TBB_override { - new( &get_ref(dst,index) ) T(*static_cast(src)); - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - virtual void move_item( page& dst, size_t index, const void* src ) __TBB_override { - new( &get_ref(dst,index) ) T( std::move(*static_cast(const_cast(src))) ); - } -#else - virtual void move_item( page&, size_t, const void* ) __TBB_override { - __TBB_ASSERT( false, "Unreachable code" ); - } -#endif - - virtual void copy_page_item( page& dst, size_t dindex, const page& src, size_t sindex ) __TBB_override { - new( &get_ref(dst,dindex) ) T( get_ref( const_cast(src), sindex ) ); - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - virtual void move_page_item( page& dst, size_t dindex, const page& src, size_t sindex ) __TBB_override { - new( &get_ref(dst,dindex) ) T( std::move(get_ref( const_cast(src), sindex )) ); - } -#else - virtual void move_page_item( page&, size_t, const page&, size_t ) __TBB_override { - __TBB_ASSERT( false, "Unreachable code" ); - } -#endif - - virtual void assign_and_destroy_item( void* dst, page& src, size_t index ) __TBB_override { - T& from = get_ref(src,index); - destroyer d(from); - *static_cast(dst) = tbb::internal::move( from ); - } - - virtual page *allocate_page() __TBB_override { - size_t n = sizeof(padded_page) + (items_per_page-1)*sizeof(T); - page *p = reinterpret_cast(my_allocator.allocate( n )); - if( !p ) - internal::throw_exception(internal::eid_bad_alloc); - return p; - } - - virtual void deallocate_page( page *p ) __TBB_override { - size_t n = sizeof(padded_page) + (items_per_page-1)*sizeof(T); - my_allocator.deallocate( reinterpret_cast(p), n ); - } - -public: - //! Element type in the queue. - typedef T value_type; - - //! Allocator type - typedef A allocator_type; - - //! Reference type - typedef T& reference; - - //! Const reference type - typedef const T& const_reference; - - //! Integral type for representing size of the queue. - /** Note that the size_type is a signed integral type. - This is because the size can be negative if there are pending pops without corresponding pushes. */ - typedef std::ptrdiff_t size_type; - - //! Difference type for iterator - typedef std::ptrdiff_t difference_type; - - //! Construct empty queue - explicit concurrent_bounded_queue(const allocator_type& a = allocator_type()) : - concurrent_queue_base_v8( sizeof(T) ), my_allocator( a ) - { - } - - //! Copy constructor - concurrent_bounded_queue( const concurrent_bounded_queue& src, const allocator_type& a = allocator_type()) - : concurrent_queue_base_v8( sizeof(T) ), my_allocator( a ) - { - assign( src ); - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - //! Move constructors - concurrent_bounded_queue( concurrent_bounded_queue&& src ) - : concurrent_queue_base_v8( sizeof(T) ), my_allocator( std::move(src.my_allocator) ) - { - internal_swap( src ); - } - - concurrent_bounded_queue( concurrent_bounded_queue&& src, const allocator_type& a ) - : concurrent_queue_base_v8( sizeof(T) ), my_allocator( a ) - { - // checking that memory allocated by one instance of allocator can be deallocated - // with another - if( my_allocator == src.my_allocator) { - this->internal_swap( src ); - } else { - // allocators are different => performing per-element move - this->move_content( src ); - src.clear(); - } - } -#endif /* __TBB_CPP11_RVALUE_REF_PRESENT */ - - //! [begin,end) constructor - template - concurrent_bounded_queue( InputIterator begin, InputIterator end, - const allocator_type& a = allocator_type()) - : concurrent_queue_base_v8( sizeof(T) ), my_allocator( a ) - { - for( ; begin != end; ++begin ) - internal_push_if_not_full(&*begin); - } - - //! Destroy queue - ~concurrent_bounded_queue(); - - //! Enqueue an item at tail of queue. - void push( const T& source ) { - internal_push( &source ); - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - //! Move an item at tail of queue. - void push( T&& source ) { - internal_push_move( &source ); - } - -#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT - template - void emplace( Arguments&&... args ) { - push( T(std::forward( args )...) ); - } -#endif /* __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT */ -#endif /* __TBB_CPP11_RVALUE_REF_PRESENT */ - - //! Dequeue item from head of queue. - /** Block until an item becomes available, and then dequeue it. */ - void pop( T& destination ) { - internal_pop( &destination ); - } - -#if TBB_USE_EXCEPTIONS - //! Abort all pending queue operations - void abort() { - internal_abort(); - } -#endif - - //! Enqueue an item at tail of queue if queue is not already full. - /** Does not wait for queue to become not full. - Returns true if item is pushed; false if queue was already full. */ - bool try_push( const T& source ) { - return internal_push_if_not_full( &source ); - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - //! Move an item at tail of queue if queue is not already full. - /** Does not wait for queue to become not full. - Returns true if item is pushed; false if queue was already full. */ - bool try_push( T&& source ) { - return internal_push_move_if_not_full( &source ); - } -#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT - template - bool try_emplace( Arguments&&... args ) { - return try_push( T(std::forward( args )...) ); - } -#endif /* __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT */ -#endif /* __TBB_CPP11_RVALUE_REF_PRESENT */ - - //! Attempt to dequeue an item from head of queue. - /** Does not wait for item to become available. - Returns true if successful; false otherwise. */ - bool try_pop( T& destination ) { - return internal_pop_if_present( &destination ); - } - - //! Return number of pushes minus number of pops. - /** Note that the result can be negative if there are pops waiting for the - corresponding pushes. The result can also exceed capacity() if there - are push operations in flight. */ - size_type size() const {return internal_size();} - - //! Equivalent to size()<=0. - bool empty() const {return internal_empty();} - - //! Maximum number of allowed elements - size_type capacity() const { - return my_capacity; - } - - //! Set the capacity - /** Setting the capacity to 0 causes subsequent try_push operations to always fail, - and subsequent push operations to block forever. */ - void set_capacity( size_type new_capacity ) { - internal_set_capacity( new_capacity, sizeof(T) ); - } - - //! return allocator object - allocator_type get_allocator() const { return this->my_allocator; } - - //! clear the queue. not thread-safe. - void clear() ; - - typedef internal::concurrent_queue_iterator iterator; - typedef internal::concurrent_queue_iterator const_iterator; - - //------------------------------------------------------------------------ - // The iterators are intended only for debugging. They are slow and not thread safe. - //------------------------------------------------------------------------ - iterator unsafe_begin() {return iterator(*this);} - iterator unsafe_end() {return iterator();} - const_iterator unsafe_begin() const {return const_iterator(*this);} - const_iterator unsafe_end() const {return const_iterator();} - -}; - -template -concurrent_bounded_queue::~concurrent_bounded_queue() { - clear(); - internal_finish_clear(); -} - -template -void concurrent_bounded_queue::clear() { - T value; - while( try_pop(value) ) /*noop*/; -} - -using strict_ppl::concurrent_queue; - -} // namespace tbb - -#endif /* __TBB_concurrent_queue_H */ +#include "../oneapi/tbb/concurrent_queue.h" diff --git a/src/tbb/src/rml/client/omp_dynamic_link.cpp b/src/tbb/include/tbb/concurrent_set.h similarity index 76% rename from src/tbb/src/rml/client/omp_dynamic_link.cpp rename to src/tbb/include/tbb/concurrent_set.h index e9a8d2e42..cf4652f59 100644 --- a/src/tbb/src/rml/client/omp_dynamic_link.cpp +++ b/src/tbb/include/tbb/concurrent_set.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2019-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,13 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ -#include "omp_dynamic_link.h" -#include "library_assert.h" -#include "tbb/dynamic_link.cpp" // Refers to src/tbb, not include/tbb - +#include "../oneapi/tbb/concurrent_set.h" diff --git a/src/tbb/include/tbb/concurrent_unordered_map.h b/src/tbb/include/tbb/concurrent_unordered_map.h index c959a7ec7..9475c06cf 100644 --- a/src/tbb/include/tbb/concurrent_unordered_map.h +++ b/src/tbb/include/tbb/concurrent_unordered_map.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,288 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ -/* Container implementations in this header are based on PPL implementations - provided by Microsoft. */ - -#ifndef __TBB_concurrent_unordered_map_H -#define __TBB_concurrent_unordered_map_H - -#include "internal/_concurrent_unordered_impl.h" - -namespace tbb -{ - -namespace interface5 { - -// Template class for hash map traits -template -class concurrent_unordered_map_traits -{ -protected: - typedef std::pair value_type; - typedef Key key_type; - typedef Hash_compare hash_compare; - typedef typename Allocator::template rebind::other allocator_type; - enum { allow_multimapping = Allow_multimapping }; - - concurrent_unordered_map_traits() : my_hash_compare() {} - concurrent_unordered_map_traits(const hash_compare& hc) : my_hash_compare(hc) {} - - template - static const Key& get_key(const std::pair& value) { - return (value.first); - } - - hash_compare my_hash_compare; // the comparator predicate for keys -}; - -template , typename Key_equality = std::equal_to, - typename Allocator = tbb::tbb_allocator > > -class concurrent_unordered_map : - public internal::concurrent_unordered_base< concurrent_unordered_map_traits, Allocator, false> > -{ - // Base type definitions - typedef internal::hash_compare hash_compare; - typedef concurrent_unordered_map_traits traits_type; - typedef internal::concurrent_unordered_base< traits_type > base_type; -#if __TBB_EXTRA_DEBUG -public: -#endif - using traits_type::allow_multimapping; -public: - using base_type::end; - using base_type::find; - using base_type::insert; - - // Type definitions - typedef Key key_type; - typedef typename base_type::value_type value_type; - typedef T mapped_type; - typedef Hasher hasher; - typedef Key_equality key_equal; - typedef hash_compare key_compare; - - typedef typename base_type::allocator_type allocator_type; - typedef typename base_type::pointer pointer; - typedef typename base_type::const_pointer const_pointer; - typedef typename base_type::reference reference; - typedef typename base_type::const_reference const_reference; - - typedef typename base_type::size_type size_type; - typedef typename base_type::difference_type difference_type; - - typedef typename base_type::iterator iterator; - typedef typename base_type::const_iterator const_iterator; - typedef typename base_type::iterator local_iterator; - typedef typename base_type::const_iterator const_local_iterator; - - // Construction/destruction/copying - explicit concurrent_unordered_map(size_type n_of_buckets = base_type::initial_bucket_number, - const hasher& _Hasher = hasher(), const key_equal& _Key_equality = key_equal(), - const allocator_type& a = allocator_type()) - : base_type(n_of_buckets, key_compare(_Hasher, _Key_equality), a) - {} - - explicit concurrent_unordered_map(const Allocator& a) : base_type(base_type::initial_bucket_number, key_compare(), a) - {} - - template - concurrent_unordered_map(Iterator first, Iterator last, size_type n_of_buckets = base_type::initial_bucket_number, - const hasher& _Hasher = hasher(), const key_equal& _Key_equality = key_equal(), - const allocator_type& a = allocator_type()) - : base_type(n_of_buckets, key_compare(_Hasher, _Key_equality), a) - { - insert(first, last); - } - -#if __TBB_INITIALIZER_LISTS_PRESENT - //! Constructor from initializer_list - concurrent_unordered_map(std::initializer_list il, size_type n_of_buckets = base_type::initial_bucket_number, - const hasher& _Hasher = hasher(), const key_equal& _Key_equality = key_equal(), - const allocator_type& a = allocator_type()) - : base_type(n_of_buckets, key_compare(_Hasher, _Key_equality), a) - { - this->insert(il.begin(),il.end()); - } -#endif //# __TBB_INITIALIZER_LISTS_PRESENT - -#if __TBB_CPP11_RVALUE_REF_PRESENT -#if !__TBB_IMPLICIT_MOVE_PRESENT - concurrent_unordered_map(const concurrent_unordered_map& table) - : base_type(table) - {} - - concurrent_unordered_map& operator=(const concurrent_unordered_map& table) - { - return static_cast(base_type::operator=(table)); - } - - concurrent_unordered_map(concurrent_unordered_map&& table) - : base_type(std::move(table)) - {} - - concurrent_unordered_map& operator=(concurrent_unordered_map&& table) - { - return static_cast(base_type::operator=(std::move(table))); - } -#endif //!__TBB_IMPLICIT_MOVE_PRESENT - - concurrent_unordered_map(concurrent_unordered_map&& table, const Allocator& a) : base_type(std::move(table), a) - {} -#endif //__TBB_CPP11_RVALUE_REF_PRESENT - - concurrent_unordered_map(const concurrent_unordered_map& table, const Allocator& a) - : base_type(table, a) - {} - - // Observers - mapped_type& operator[](const key_type& key) - { - iterator where = find(key); - - if (where == end()) - { - where = insert(std::pair(key, mapped_type())).first; - } - - return ((*where).second); - } - - mapped_type& at(const key_type& key) - { - iterator where = find(key); - - if (where == end()) - { - tbb::internal::throw_exception(tbb::internal::eid_invalid_key); - } - - return ((*where).second); - } - - const mapped_type& at(const key_type& key) const - { - const_iterator where = find(key); - - if (where == end()) - { - tbb::internal::throw_exception(tbb::internal::eid_invalid_key); - } - - return ((*where).second); - } -}; - -template < typename Key, typename T, typename Hasher = tbb::tbb_hash, typename Key_equality = std::equal_to, - typename Allocator = tbb::tbb_allocator > > -class concurrent_unordered_multimap : - public internal::concurrent_unordered_base< concurrent_unordered_map_traits< Key, T, - internal::hash_compare, Allocator, true> > -{ - // Base type definitions - typedef internal::hash_compare hash_compare; - typedef concurrent_unordered_map_traits traits_type; - typedef internal::concurrent_unordered_base base_type; -#if __TBB_EXTRA_DEBUG -public: -#endif - using traits_type::allow_multimapping; -public: - using base_type::insert; - - // Type definitions - typedef Key key_type; - typedef typename base_type::value_type value_type; - typedef T mapped_type; - typedef Hasher hasher; - typedef Key_equality key_equal; - typedef hash_compare key_compare; - - typedef typename base_type::allocator_type allocator_type; - typedef typename base_type::pointer pointer; - typedef typename base_type::const_pointer const_pointer; - typedef typename base_type::reference reference; - typedef typename base_type::const_reference const_reference; - - typedef typename base_type::size_type size_type; - typedef typename base_type::difference_type difference_type; - - typedef typename base_type::iterator iterator; - typedef typename base_type::const_iterator const_iterator; - typedef typename base_type::iterator local_iterator; - typedef typename base_type::const_iterator const_local_iterator; - - // Construction/destruction/copying - explicit concurrent_unordered_multimap(size_type n_of_buckets = base_type::initial_bucket_number, - const hasher& _Hasher = hasher(), const key_equal& _Key_equality = key_equal(), - const allocator_type& a = allocator_type()) - : base_type(n_of_buckets, key_compare(_Hasher, _Key_equality), a) - {} - - explicit concurrent_unordered_multimap(const Allocator& a) : base_type(base_type::initial_bucket_number, key_compare(), a) - {} - - template - concurrent_unordered_multimap(Iterator first, Iterator last, size_type n_of_buckets = base_type::initial_bucket_number, - const hasher& _Hasher = hasher(), const key_equal& _Key_equality = key_equal(), - const allocator_type& a = allocator_type()) - : base_type(n_of_buckets,key_compare(_Hasher,_Key_equality), a) - { - insert(first, last); - } - -#if __TBB_INITIALIZER_LISTS_PRESENT - //! Constructor from initializer_list - concurrent_unordered_multimap(std::initializer_list il, size_type n_of_buckets = base_type::initial_bucket_number, - const hasher& _Hasher = hasher(), const key_equal& _Key_equality = key_equal(), - const allocator_type& a = allocator_type()) - : base_type(n_of_buckets, key_compare(_Hasher, _Key_equality), a) - { - this->insert(il.begin(),il.end()); - } -#endif //# __TBB_INITIALIZER_LISTS_PRESENT - -#if __TBB_CPP11_RVALUE_REF_PRESENT -#if !__TBB_IMPLICIT_MOVE_PRESENT - concurrent_unordered_multimap(const concurrent_unordered_multimap& table) - : base_type(table) - {} - - concurrent_unordered_multimap& operator=(const concurrent_unordered_multimap& table) - { - return static_cast(base_type::operator=(table)); - } - - concurrent_unordered_multimap(concurrent_unordered_multimap&& table) - : base_type(std::move(table)) - {} - - concurrent_unordered_multimap& operator=(concurrent_unordered_multimap&& table) - { - return static_cast(base_type::operator=(std::move(table))); - } -#endif //!__TBB_IMPLICIT_MOVE_PRESENT - - concurrent_unordered_multimap(concurrent_unordered_multimap&& table, const Allocator& a) : base_type(std::move(table), a) - {} -#endif //__TBB_CPP11_RVALUE_REF_PRESENT - - concurrent_unordered_multimap(const concurrent_unordered_multimap& table, const Allocator& a) - : base_type(table, a) - {} -}; -} // namespace interface5 - -using interface5::concurrent_unordered_map; -using interface5::concurrent_unordered_multimap; - -} // namespace tbb - -#endif// __TBB_concurrent_unordered_map_H +#include "../oneapi/tbb/concurrent_unordered_map.h" diff --git a/src/tbb/include/tbb/concurrent_unordered_set.h b/src/tbb/include/tbb/concurrent_unordered_set.h index a26fee1a7..81a8f9c37 100644 --- a/src/tbb/include/tbb/concurrent_unordered_set.h +++ b/src/tbb/include/tbb/concurrent_unordered_set.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,245 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ -/* Container implementations in this header are based on PPL implementations - provided by Microsoft. */ - -#ifndef __TBB_concurrent_unordered_set_H -#define __TBB_concurrent_unordered_set_H - -#include "internal/_concurrent_unordered_impl.h" - -namespace tbb -{ - -namespace interface5 { - -// Template class for hash set traits -template -class concurrent_unordered_set_traits -{ -protected: - typedef Key value_type; - typedef Key key_type; - typedef Hash_compare hash_compare; - typedef typename Allocator::template rebind::other allocator_type; - enum { allow_multimapping = Allow_multimapping }; - - concurrent_unordered_set_traits() : my_hash_compare() {} - concurrent_unordered_set_traits(const hash_compare& hc) : my_hash_compare(hc) {} - - static const Key& get_key(const value_type& value) { - return value; - } - - hash_compare my_hash_compare; // the comparator predicate for keys -}; - -template , typename Key_equality = std::equal_to, typename Allocator = tbb::tbb_allocator > -class concurrent_unordered_set : public internal::concurrent_unordered_base< concurrent_unordered_set_traits, Allocator, false> > -{ - // Base type definitions - typedef internal::hash_compare hash_compare; - typedef concurrent_unordered_set_traits traits_type; - typedef internal::concurrent_unordered_base< traits_type > base_type; -#if __TBB_EXTRA_DEBUG -public: -#endif - using traits_type::allow_multimapping; -public: - using base_type::insert; - - // Type definitions - typedef Key key_type; - typedef typename base_type::value_type value_type; - typedef Key mapped_type; - typedef Hasher hasher; - typedef Key_equality key_equal; - typedef hash_compare key_compare; - - typedef typename base_type::allocator_type allocator_type; - typedef typename base_type::pointer pointer; - typedef typename base_type::const_pointer const_pointer; - typedef typename base_type::reference reference; - typedef typename base_type::const_reference const_reference; - - typedef typename base_type::size_type size_type; - typedef typename base_type::difference_type difference_type; - - typedef typename base_type::iterator iterator; - typedef typename base_type::const_iterator const_iterator; - typedef typename base_type::iterator local_iterator; - typedef typename base_type::const_iterator const_local_iterator; - - // Construction/destruction/copying - explicit concurrent_unordered_set(size_type n_of_buckets = base_type::initial_bucket_number, const hasher& a_hasher = hasher(), - const key_equal& a_keyeq = key_equal(), const allocator_type& a = allocator_type()) - : base_type(n_of_buckets, key_compare(a_hasher, a_keyeq), a) - {} - - explicit concurrent_unordered_set(const Allocator& a) : base_type(base_type::initial_bucket_number, key_compare(), a) - {} - - template - concurrent_unordered_set(Iterator first, Iterator last, size_type n_of_buckets = base_type::initial_bucket_number, const hasher& a_hasher = hasher(), - const key_equal& a_keyeq = key_equal(), const allocator_type& a = allocator_type()) - : base_type(n_of_buckets, key_compare(a_hasher, a_keyeq), a) - { - insert(first, last); - } - -#if __TBB_INITIALIZER_LISTS_PRESENT - //! Constructor from initializer_list - concurrent_unordered_set(std::initializer_list il, size_type n_of_buckets = base_type::initial_bucket_number, const hasher& a_hasher = hasher(), - const key_equal& a_keyeq = key_equal(), const allocator_type& a = allocator_type()) - : base_type(n_of_buckets, key_compare(a_hasher, a_keyeq), a) - { - this->insert(il.begin(),il.end()); - } -#endif //# __TBB_INITIALIZER_LISTS_PRESENT - -#if __TBB_CPP11_RVALUE_REF_PRESENT -#if !__TBB_IMPLICIT_MOVE_PRESENT - concurrent_unordered_set(const concurrent_unordered_set& table) - : base_type(table) - {} - - concurrent_unordered_set& operator=(const concurrent_unordered_set& table) - { - return static_cast(base_type::operator=(table)); - } - - concurrent_unordered_set(concurrent_unordered_set&& table) - : base_type(std::move(table)) - {} - - concurrent_unordered_set& operator=(concurrent_unordered_set&& table) - { - return static_cast(base_type::operator=(std::move(table))); - } -#endif //!__TBB_IMPLICIT_MOVE_PRESENT - - concurrent_unordered_set(concurrent_unordered_set&& table, const Allocator& a) - : base_type(std::move(table), a) - {} -#endif //__TBB_CPP11_RVALUE_REF_PRESENT - - concurrent_unordered_set(const concurrent_unordered_set& table, const Allocator& a) - : base_type(table, a) - {} - -}; - -template , typename Key_equality = std::equal_to, - typename Allocator = tbb::tbb_allocator > -class concurrent_unordered_multiset : - public internal::concurrent_unordered_base< concurrent_unordered_set_traits, Allocator, true> > -{ - // Base type definitions - typedef internal::hash_compare hash_compare; - typedef concurrent_unordered_set_traits traits_type; - typedef internal::concurrent_unordered_base< traits_type > base_type; -#if __TBB_EXTRA_DEBUG -public: -#endif - using traits_type::allow_multimapping; -public: - using base_type::insert; - - // Type definitions - typedef Key key_type; - typedef typename base_type::value_type value_type; - typedef Key mapped_type; - typedef Hasher hasher; - typedef Key_equality key_equal; - typedef hash_compare key_compare; - - typedef typename base_type::allocator_type allocator_type; - typedef typename base_type::pointer pointer; - typedef typename base_type::const_pointer const_pointer; - typedef typename base_type::reference reference; - typedef typename base_type::const_reference const_reference; - - typedef typename base_type::size_type size_type; - typedef typename base_type::difference_type difference_type; - - typedef typename base_type::iterator iterator; - typedef typename base_type::const_iterator const_iterator; - typedef typename base_type::iterator local_iterator; - typedef typename base_type::const_iterator const_local_iterator; - - // Construction/destruction/copying - explicit concurrent_unordered_multiset(size_type n_of_buckets = base_type::initial_bucket_number, - const hasher& _Hasher = hasher(), const key_equal& _Key_equality = key_equal(), - const allocator_type& a = allocator_type()) - : base_type(n_of_buckets, key_compare(_Hasher, _Key_equality), a) - {} - - explicit concurrent_unordered_multiset(const Allocator& a) : base_type(base_type::initial_bucket_number, key_compare(), a) - {} - - template - concurrent_unordered_multiset(Iterator first, Iterator last, size_type n_of_buckets = base_type::initial_bucket_number, - const hasher& _Hasher = hasher(), const key_equal& _Key_equality = key_equal(), - const allocator_type& a = allocator_type()) - : base_type(n_of_buckets, key_compare(_Hasher, _Key_equality), a) - { - insert(first, last); - } - -#if __TBB_INITIALIZER_LISTS_PRESENT - //! Constructor from initializer_list - concurrent_unordered_multiset(std::initializer_list il, size_type n_of_buckets = base_type::initial_bucket_number, const hasher& a_hasher = hasher(), - const key_equal& a_keyeq = key_equal(), const allocator_type& a = allocator_type()) - : base_type(n_of_buckets, key_compare(a_hasher, a_keyeq), a) - { - this->insert(il.begin(),il.end()); - } -#endif //# __TBB_INITIALIZER_LISTS_PRESENT - -#if __TBB_CPP11_RVALUE_REF_PRESENT -#if !__TBB_IMPLICIT_MOVE_PRESENT - concurrent_unordered_multiset(const concurrent_unordered_multiset& table) - : base_type(table) - {} - - concurrent_unordered_multiset& operator=(const concurrent_unordered_multiset& table) - { - return static_cast(base_type::operator=(table)); - } - - concurrent_unordered_multiset(concurrent_unordered_multiset&& table) - : base_type(std::move(table)) - {} - - concurrent_unordered_multiset& operator=(concurrent_unordered_multiset&& table) - { - return static_cast(base_type::operator=(std::move(table))); - } -#endif //!__TBB_IMPLICIT_MOVE_PRESENT - - concurrent_unordered_multiset(concurrent_unordered_multiset&& table, const Allocator& a) - : base_type(std::move(table), a) - { - } -#endif //__TBB_CPP11_RVALUE_REF_PRESENT - - concurrent_unordered_multiset(const concurrent_unordered_multiset& table, const Allocator& a) - : base_type(table, a) - {} -}; -} // namespace interface5 - -using interface5::concurrent_unordered_set; -using interface5::concurrent_unordered_multiset; - -} // namespace tbb - -#endif// __TBB_concurrent_unordered_set_H +#include "../oneapi/tbb/concurrent_unordered_set.h" diff --git a/src/tbb/include/tbb/concurrent_vector.h b/src/tbb/include/tbb/concurrent_vector.h index b9bd55591..c1fc97c62 100644 --- a/src/tbb/include/tbb/concurrent_vector.h +++ b/src/tbb/include/tbb/concurrent_vector.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,1352 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ -#ifndef __TBB_concurrent_vector_H -#define __TBB_concurrent_vector_H - -#include "tbb_stddef.h" -#include "tbb_exception.h" -#include "atomic.h" -#include "cache_aligned_allocator.h" -#include "blocked_range.h" -#include "tbb_machine.h" -#include "tbb_profiling.h" -#include -#include // for memset() -#include __TBB_STD_SWAP_HEADER -#include -#include - -#if _MSC_VER==1500 && !__INTEL_COMPILER - // VS2008/VC9 seems to have an issue; limits pull in math.h - #pragma warning( push ) - #pragma warning( disable: 4985 ) -#endif -#include /* std::numeric_limits */ -#if _MSC_VER==1500 && !__INTEL_COMPILER - #pragma warning( pop ) -#endif - -#if __TBB_INITIALIZER_LISTS_PRESENT - #include -#endif - -#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) - // Workaround for overzealous compiler warnings in /Wp64 mode - #pragma warning (push) -#if defined(_Wp64) - #pragma warning (disable: 4267) -#endif - #pragma warning (disable: 4127) //warning C4127: conditional expression is constant -#endif - -namespace tbb { - -template > -class concurrent_vector; - -//! @cond INTERNAL -namespace internal { - - template - class vector_iterator; - - //! Bad allocation marker - static void *const vector_allocation_error_flag = reinterpret_cast(size_t(63)); - - //! Exception helper function - template - void handle_unconstructed_elements(T* array, size_t n_of_elements){ - std::memset((void*) array, 0, n_of_elements * sizeof( T )); - } - - //! Base class of concurrent vector implementation. - /** @ingroup containers */ - class concurrent_vector_base_v3 { - protected: - - // Basic types declarations - typedef size_t segment_index_t; - typedef size_t size_type; - - // Using enumerations due to Mac linking problems of static const variables - enum { - // Size constants - default_initial_segments = 1, // 2 initial items - //! Number of slots for segment pointers inside the class - pointers_per_short_table = 3, // to fit into 8 words of entire structure - pointers_per_long_table = sizeof(segment_index_t) * 8 // one segment per bit - }; - public: - struct segment_not_used {}; - struct segment_allocated {}; - struct segment_allocation_failed {}; - protected: - class segment_t; - class segment_value_t { - void* array; - private: - //TODO: More elegant way to grant access to selected functions _only_? - friend class segment_t; - explicit segment_value_t(void* an_array):array(an_array) {} - public: - friend bool operator==(segment_value_t const& lhs, segment_not_used ) { return lhs.array == 0;} - friend bool operator==(segment_value_t const& lhs, segment_allocated) { return lhs.array > internal::vector_allocation_error_flag;} - friend bool operator==(segment_value_t const& lhs, segment_allocation_failed) { return lhs.array == internal::vector_allocation_error_flag;} - template - friend bool operator!=(segment_value_t const& lhs, argument_type arg) { return ! (lhs == arg);} - - template - T* pointer() const { return static_cast(const_cast(array)); } - }; - - friend void enforce_segment_allocated(segment_value_t const& s, internal::exception_id exception = eid_bad_last_alloc){ - if(s != segment_allocated()){ - internal::throw_exception(exception); - } - } - - // Segment pointer. - class segment_t { - atomic array; - public: - segment_t(){ store(segment_not_used());} - //Copy ctor and assignment operator are defined to ease using of stl algorithms. - //These algorithms usually not a synchronization point, so, semantic is - //intentionally relaxed here. - segment_t(segment_t const& rhs ){ array.store(rhs.array.load());} - - void swap(segment_t & rhs ){ - tbb::internal::swap(array, rhs.array); - } - - segment_t& operator=(segment_t const& rhs ){ - array.store(rhs.array.load()); - return *this; - } - - template - segment_value_t load() const { return segment_value_t(array.load());} - - template - void store(segment_not_used) { - array.store(0); - } - - template - void store(segment_allocation_failed) { - __TBB_ASSERT(load() != segment_allocated(),"transition from \"allocated\" to \"allocation failed\" state looks non-logical"); - array.store(internal::vector_allocation_error_flag); - } - - template - void store(void* allocated_segment_pointer) __TBB_NOEXCEPT(true) { - __TBB_ASSERT(segment_value_t(allocated_segment_pointer) == segment_allocated(), - "other overloads of store should be used for marking segment as not_used or allocation_failed" ); - array.store(allocated_segment_pointer); - } - -#if TBB_USE_ASSERT - ~segment_t() { - __TBB_ASSERT(load() != segment_allocated(), "should have been freed by clear" ); - } -#endif /* TBB_USE_ASSERT */ - }; - friend void swap(segment_t & , segment_t & ) __TBB_NOEXCEPT(true); - - // Data fields - - //! allocator function pointer - void* (*vector_allocator_ptr)(concurrent_vector_base_v3 &, size_t); - - //! count of segments in the first block - atomic my_first_block; - - //! Requested size of vector - atomic my_early_size; - - //! Pointer to the segments table - atomic my_segment; - - //! embedded storage of segment pointers - segment_t my_storage[pointers_per_short_table]; - - // Methods - - concurrent_vector_base_v3() { - //Here the semantic is intentionally relaxed. - //The reason this is next: - //Object that is in middle of construction (i.e. its constructor is not yet finished) - //cannot be used concurrently until the construction is finished. - //Thus to flag other threads that construction is finished, some synchronization with - //acquire-release semantic should be done by the (external) code that uses the vector. - //So, no need to do the synchronization inside the vector. - - my_early_size.store(0); - my_first_block.store(0); // here is not default_initial_segments - my_segment.store(my_storage); - } - - __TBB_EXPORTED_METHOD ~concurrent_vector_base_v3(); - - //these helpers methods use the fact that segments are allocated so - //that every segment size is a (increasing) power of 2. - //with one exception 0 segment has size of 2 as well segment 1; - //e.g. size of segment with index of 3 is 2^3=8; - static segment_index_t segment_index_of( size_type index ) { - return segment_index_t( __TBB_Log2( index|1 ) ); - } - - static segment_index_t segment_base( segment_index_t k ) { - return (segment_index_t(1)< - friend class vector_iterator; - - }; - - inline void swap(concurrent_vector_base_v3::segment_t & lhs, concurrent_vector_base_v3::segment_t & rhs) __TBB_NOEXCEPT(true) { - lhs.swap(rhs); - } - - typedef concurrent_vector_base_v3 concurrent_vector_base; - - //! Meets requirements of a forward iterator for STL and a Value for a blocked_range.*/ - /** Value is either the T or const T type of the container. - @ingroup containers */ - template - class vector_iterator - { - //! concurrent_vector over which we are iterating. - Container* my_vector; - - //! Index into the vector - size_t my_index; - - //! Caches my_vector->internal_subscript(my_index) - /** NULL if cached value is not available */ - mutable Value* my_item; - - template - friend vector_iterator operator+( ptrdiff_t offset, const vector_iterator& v ); - - template - friend bool operator==( const vector_iterator& i, const vector_iterator& j ); - - template - friend bool operator<( const vector_iterator& i, const vector_iterator& j ); - - template - friend ptrdiff_t operator-( const vector_iterator& i, const vector_iterator& j ); - - template - friend class internal::vector_iterator; - -#if !__TBB_TEMPLATE_FRIENDS_BROKEN - template - friend class tbb::concurrent_vector; -#else -public: -#endif - - vector_iterator( const Container& vector, size_t index, void *ptr = 0 ) : - my_vector(const_cast(&vector)), - my_index(index), - my_item(static_cast(ptr)) - {} - - public: - //! Default constructor - vector_iterator() : my_vector(NULL), my_index(~size_t(0)), my_item(NULL) {} - - vector_iterator( const vector_iterator& other ) : - my_vector(other.my_vector), - my_index(other.my_index), - my_item(other.my_item) - {} - - vector_iterator operator+( ptrdiff_t offset ) const { - return vector_iterator( *my_vector, my_index+offset ); - } - vector_iterator &operator+=( ptrdiff_t offset ) { - my_index+=offset; - my_item = NULL; - return *this; - } - vector_iterator operator-( ptrdiff_t offset ) const { - return vector_iterator( *my_vector, my_index-offset ); - } - vector_iterator &operator-=( ptrdiff_t offset ) { - my_index-=offset; - my_item = NULL; - return *this; - } - Value& operator*() const { - Value* item = my_item; - if( !item ) { - item = my_item = &my_vector->internal_subscript(my_index); - } - __TBB_ASSERT( item==&my_vector->internal_subscript(my_index), "corrupt cache" ); - return *item; - } - Value& operator[]( ptrdiff_t k ) const { - return my_vector->internal_subscript(my_index+k); - } - Value* operator->() const {return &operator*();} - - //! Pre increment - vector_iterator& operator++() { - size_t element_index = ++my_index; - if( my_item ) { - //TODO: consider using of knowledge about "first_block optimization" here as well? - if( concurrent_vector_base::is_first_element_in_segment(element_index)) { - //if the iterator crosses a segment boundary, the pointer become invalid - //as possibly next segment is in another memory location - my_item= NULL; - } else { - ++my_item; - } - } - return *this; - } - - //! Pre decrement - vector_iterator& operator--() { - __TBB_ASSERT( my_index>0, "operator--() applied to iterator already at beginning of concurrent_vector" ); - size_t element_index = my_index--; - if( my_item ) { - if(concurrent_vector_base::is_first_element_in_segment(element_index)) { - //if the iterator crosses a segment boundary, the pointer become invalid - //as possibly next segment is in another memory location - my_item= NULL; - } else { - --my_item; - } - } - return *this; - } - - //! Post increment - vector_iterator operator++(int) { - vector_iterator result = *this; - operator++(); - return result; - } - - //! Post decrement - vector_iterator operator--(int) { - vector_iterator result = *this; - operator--(); - return result; - } - - // STL support - - typedef ptrdiff_t difference_type; - typedef Value value_type; - typedef Value* pointer; - typedef Value& reference; - typedef std::random_access_iterator_tag iterator_category; - }; - - template - vector_iterator operator+( ptrdiff_t offset, const vector_iterator& v ) { - return vector_iterator( *v.my_vector, v.my_index+offset ); - } - - template - bool operator==( const vector_iterator& i, const vector_iterator& j ) { - return i.my_index==j.my_index && i.my_vector == j.my_vector; - } - - template - bool operator!=( const vector_iterator& i, const vector_iterator& j ) { - return !(i==j); - } - - template - bool operator<( const vector_iterator& i, const vector_iterator& j ) { - return i.my_index - bool operator>( const vector_iterator& i, const vector_iterator& j ) { - return j - bool operator>=( const vector_iterator& i, const vector_iterator& j ) { - return !(i - bool operator<=( const vector_iterator& i, const vector_iterator& j ) { - return !(j - ptrdiff_t operator-( const vector_iterator& i, const vector_iterator& j ) { - return ptrdiff_t(i.my_index)-ptrdiff_t(j.my_index); - } - - template - class allocator_base { - public: - typedef typename A::template - rebind::other allocator_type; - allocator_type my_allocator; - - allocator_base(const allocator_type &a = allocator_type() ) : my_allocator(a) {} - - }; - -} // namespace internal -//! @endcond - -//! Concurrent vector container -/** concurrent_vector is a container having the following main properties: - - It provides random indexed access to its elements. The index of the first element is 0. - - It ensures safe concurrent growing its size (different threads can safely append new elements). - - Adding new elements does not invalidate existing iterators and does not change indices of existing items. - -@par Compatibility - The class meets all Container Requirements and Reversible Container Requirements from - C++ Standard (See ISO/IEC 14882:2003(E), clause 23.1). But it doesn't meet - Sequence Requirements due to absence of insert() and erase() methods. - -@par Exception Safety - Methods working with memory allocation and/or new elements construction can throw an - exception if allocator fails to allocate memory or element's default constructor throws one. - Concurrent vector's element of type T must conform to the following requirements: - - Throwing an exception is forbidden for destructor of T. - - Default constructor of T must not throw an exception OR its non-virtual destructor must safely work when its object memory is zero-initialized. - . - Otherwise, the program's behavior is undefined. -@par - If an exception happens inside growth or assignment operation, an instance of the vector becomes invalid unless it is stated otherwise in the method documentation. - Invalid state means: - - There are no guarantees that all items were initialized by a constructor. The rest of items is zero-filled, including item where exception happens. - - An invalid vector instance cannot be repaired; it is unable to grow anymore. - - Size and capacity reported by the vector are incorrect, and calculated as if the failed operation were successful. - - Attempt to access not allocated elements using operator[] or iterators results in access violation or segmentation fault exception, and in case of using at() method a C++ exception is thrown. - . - If a concurrent grow operation successfully completes, all the elements it has added to the vector will remain valid and accessible even if one of subsequent grow operations fails. - -@par Fragmentation - Unlike an STL vector, a concurrent_vector does not move existing elements if it needs - to allocate more memory. The container is divided into a series of contiguous arrays of - elements. The first reservation, growth, or assignment operation determines the size of - the first array. Using small number of elements as initial size incurs fragmentation that - may increase element access time. Internal layout can be optimized by method compact() that - merges several smaller arrays into one solid. - -@par Changes since TBB 2.1 - - Fixed guarantees of concurrent_vector::size() and grow_to_at_least() methods to assure elements are allocated. - - Methods end()/rbegin()/back() are partly thread-safe since they use size() to get the end of vector - - Added resize() methods (not thread-safe) - - Added cbegin/cend/crbegin/crend methods - - Changed return type of methods grow* and push_back to iterator - -@par Changes since TBB 2.0 - - Implemented exception-safety guarantees - - Added template argument for allocator - - Added allocator argument in constructors - - Faster index calculation - - First growth call specifies a number of segments to be merged in the first allocation. - - Fixed memory blow up for swarm of vector's instances of small size - - Added grow_by(size_type n, const_reference t) growth using copying constructor to init new items. - - Added STL-like constructors. - - Added operators ==, < and derivatives - - Added at() method, approved for using after an exception was thrown inside the vector - - Added get_allocator() method. - - Added assign() methods - - Added compact() method to defragment first segments - - Added swap() method - - range() defaults on grainsize = 1 supporting auto grainsize algorithms. - - @ingroup containers */ -template -class concurrent_vector: protected internal::allocator_base, - private internal::concurrent_vector_base { -private: - template - class generic_range_type: public blocked_range { - public: - typedef T value_type; - typedef T& reference; - typedef const T& const_reference; - typedef I iterator; - typedef ptrdiff_t difference_type; - generic_range_type( I begin_, I end_, size_t grainsize_ = 1) : blocked_range(begin_,end_,grainsize_) {} - template - generic_range_type( const generic_range_type& r) : blocked_range(r.begin(),r.end(),r.grainsize()) {} - generic_range_type( generic_range_type& r, split ) : blocked_range(r,split()) {} - }; - - template - friend class internal::vector_iterator; - -public: - //------------------------------------------------------------------------ - // STL compatible types - //------------------------------------------------------------------------ - typedef internal::concurrent_vector_base_v3::size_type size_type; - typedef typename internal::allocator_base::allocator_type allocator_type; - - typedef T value_type; - typedef ptrdiff_t difference_type; - typedef T& reference; - typedef const T& const_reference; - typedef T *pointer; - typedef const T *const_pointer; - - typedef internal::vector_iterator iterator; - typedef internal::vector_iterator const_iterator; - -#if !defined(_MSC_VER) || _CPPLIB_VER>=300 - // Assume ISO standard definition of std::reverse_iterator - typedef std::reverse_iterator reverse_iterator; - typedef std::reverse_iterator const_reverse_iterator; -#else - // Use non-standard std::reverse_iterator - typedef std::reverse_iterator reverse_iterator; - typedef std::reverse_iterator const_reverse_iterator; -#endif /* defined(_MSC_VER) && (_MSC_VER<1300) */ - - //------------------------------------------------------------------------ - // Parallel algorithm support - //------------------------------------------------------------------------ - typedef generic_range_type range_type; - typedef generic_range_type const_range_type; - - //------------------------------------------------------------------------ - // STL compatible constructors & destructors - //------------------------------------------------------------------------ - - //! Construct empty vector. - explicit concurrent_vector(const allocator_type &a = allocator_type()) - : internal::allocator_base(a), internal::concurrent_vector_base() - { - vector_allocator_ptr = &internal_allocator; - } - - //Constructors are not required to have synchronization - //(for more details see comment in the concurrent_vector_base constructor). -#if __TBB_INITIALIZER_LISTS_PRESENT - //! Constructor from initializer_list - concurrent_vector(std::initializer_list init_list, const allocator_type &a = allocator_type()) - : internal::allocator_base(a), internal::concurrent_vector_base() - { - vector_allocator_ptr = &internal_allocator; - __TBB_TRY { - internal_assign_iterators(init_list.begin(), init_list.end()); - } __TBB_CATCH(...) { - segment_t *table = my_segment.load();; - internal_free_segments( table, internal_clear(&destroy_array), my_first_block.load()); - __TBB_RETHROW(); - } - - } -#endif //# __TBB_INITIALIZER_LISTS_PRESENT - - //! Copying constructor - concurrent_vector( const concurrent_vector& vector, const allocator_type& a = allocator_type() ) - : internal::allocator_base(a), internal::concurrent_vector_base() - { - vector_allocator_ptr = &internal_allocator; - __TBB_TRY { - internal_copy(vector, sizeof(T), ©_array); - } __TBB_CATCH(...) { - segment_t *table = my_segment.load(); - internal_free_segments( table, internal_clear(&destroy_array), my_first_block.load()); - __TBB_RETHROW(); - } - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - //! Move constructor - //TODO add __TBB_NOEXCEPT(true) and static_assert(std::has_nothrow_move_constructor::value) - concurrent_vector( concurrent_vector&& source) - : internal::allocator_base(std::move(source)), internal::concurrent_vector_base() - { - vector_allocator_ptr = &internal_allocator; - concurrent_vector_base_v3::internal_swap(source); - } - - concurrent_vector( concurrent_vector&& source, const allocator_type& a) - : internal::allocator_base(a), internal::concurrent_vector_base() - { - vector_allocator_ptr = &internal_allocator; - //C++ standard requires instances of an allocator being compared for equality, - //which means that memory allocated by one instance is possible to deallocate with the other one. - if (a == source.my_allocator) { - concurrent_vector_base_v3::internal_swap(source); - } else { - __TBB_TRY { - internal_copy(source, sizeof(T), &move_array); - } __TBB_CATCH(...) { - segment_t *table = my_segment.load(); - internal_free_segments( table, internal_clear(&destroy_array), my_first_block.load()); - __TBB_RETHROW(); - } - } - } - -#endif - - //! Copying constructor for vector with different allocator type - template - concurrent_vector( const concurrent_vector& vector, const allocator_type& a = allocator_type() ) - : internal::allocator_base(a), internal::concurrent_vector_base() - { - vector_allocator_ptr = &internal_allocator; - __TBB_TRY { - internal_copy(vector.internal_vector_base(), sizeof(T), ©_array); - } __TBB_CATCH(...) { - segment_t *table = my_segment.load(); - internal_free_segments( table, internal_clear(&destroy_array), my_first_block.load() ); - __TBB_RETHROW(); - } - } - - //! Construction with initial size specified by argument n - explicit concurrent_vector(size_type n) - { - vector_allocator_ptr = &internal_allocator; - __TBB_TRY { - internal_resize( n, sizeof(T), max_size(), NULL, &destroy_array, &initialize_array ); - } __TBB_CATCH(...) { - segment_t *table = my_segment.load(); - internal_free_segments( table, internal_clear(&destroy_array), my_first_block.load() ); - __TBB_RETHROW(); - } - } - - //! Construction with initial size specified by argument n, initialization by copying of t, and given allocator instance - concurrent_vector(size_type n, const_reference t, const allocator_type& a = allocator_type()) - : internal::allocator_base(a) - { - vector_allocator_ptr = &internal_allocator; - __TBB_TRY { - internal_resize( n, sizeof(T), max_size(), static_cast(&t), &destroy_array, &initialize_array_by ); - } __TBB_CATCH(...) { - segment_t *table = my_segment.load(); - internal_free_segments( table, internal_clear(&destroy_array), my_first_block.load() ); - __TBB_RETHROW(); - } - } - - //! Construction with copying iteration range and given allocator instance - template - concurrent_vector(I first, I last, const allocator_type &a = allocator_type()) - : internal::allocator_base(a) - { - vector_allocator_ptr = &internal_allocator; - __TBB_TRY { - internal_assign_range(first, last, static_cast::is_integer> *>(0) ); - } __TBB_CATCH(...) { - segment_t *table = my_segment.load(); - internal_free_segments( table, internal_clear(&destroy_array), my_first_block.load() ); - __TBB_RETHROW(); - } - } - - //! Assignment - concurrent_vector& operator=( const concurrent_vector& vector ) { - if( this != &vector ) - internal_assign(vector, sizeof(T), &destroy_array, &assign_array, ©_array); - return *this; - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - //TODO: add __TBB_NOEXCEPT() - //! Move assignment - concurrent_vector& operator=( concurrent_vector&& other ) { - __TBB_ASSERT(this != &other, "Move assignment to itself is prohibited "); - typedef typename tbb::internal::allocator_traits::propagate_on_container_move_assignment pocma_t; - if(pocma_t::value || this->my_allocator == other.my_allocator) { - concurrent_vector trash (std::move(*this)); - internal_swap(other); - if (pocma_t::value) { - this->my_allocator = std::move(other.my_allocator); - } - } else { - internal_assign(other, sizeof(T), &destroy_array, &move_assign_array, &move_array); - } - return *this; - } -#endif - //TODO: add an template assignment operator? (i.e. with different element type) - - //! Assignment for vector with different allocator type - template - concurrent_vector& operator=( const concurrent_vector& vector ) { - if( static_cast( this ) != static_cast( &vector ) ) - internal_assign(vector.internal_vector_base(), - sizeof(T), &destroy_array, &assign_array, ©_array); - return *this; - } - -#if __TBB_INITIALIZER_LISTS_PRESENT - //! Assignment for initializer_list - concurrent_vector& operator=( std::initializer_list init_list ) { - internal_clear(&destroy_array); - internal_assign_iterators(init_list.begin(), init_list.end()); - return *this; - } -#endif //#if __TBB_INITIALIZER_LISTS_PRESENT - - //------------------------------------------------------------------------ - // Concurrent operations - //------------------------------------------------------------------------ - //! Grow by "delta" elements. - /** Returns iterator pointing to the first new element. */ - iterator grow_by( size_type delta ) { - return iterator(*this, delta ? internal_grow_by( delta, sizeof(T), &initialize_array, NULL ) : my_early_size.load()); - } - - //! Grow by "delta" elements using copying constructor. - /** Returns iterator pointing to the first new element. */ - iterator grow_by( size_type delta, const_reference t ) { - return iterator(*this, delta ? internal_grow_by( delta, sizeof(T), &initialize_array_by, static_cast(&t) ) : my_early_size.load()); - } - - /** Returns iterator pointing to the first new element. */ - template - iterator grow_by( I first, I last ) { - typename std::iterator_traits::difference_type delta = std::distance(first, last); - __TBB_ASSERT( delta >= 0, NULL); - - return iterator(*this, delta ? internal_grow_by(delta, sizeof(T), ©_range, static_cast(&first)) : my_early_size.load()); - } - -#if __TBB_INITIALIZER_LISTS_PRESENT - /** Returns iterator pointing to the first new element. */ - iterator grow_by( std::initializer_list init_list ) { - return grow_by( init_list.begin(), init_list.end() ); - } -#endif //#if __TBB_INITIALIZER_LISTS_PRESENT - - //! Append minimal sequence of elements such that size()>=n. - /** The new elements are default constructed. Blocks until all elements in range [0..n) are allocated. - May return while other elements are being constructed by other threads. - Returns iterator that points to beginning of appended sequence. - If no elements were appended, returns iterator pointing to nth element. */ - iterator grow_to_at_least( size_type n ) { - size_type m=0; - if( n ) { - m = internal_grow_to_at_least_with_result( n, sizeof(T), &initialize_array, NULL ); - if( m>n ) m=n; - } - return iterator(*this, m); - }; - - /** Analogous to grow_to_at_least( size_type n ) with exception that the new - elements are initialized by copying of t instead of default construction. */ - iterator grow_to_at_least( size_type n, const_reference t ) { - size_type m=0; - if( n ) { - m = internal_grow_to_at_least_with_result( n, sizeof(T), &initialize_array_by, &t); - if( m>n ) m=n; - } - return iterator(*this, m); - }; - - //! Push item - /** Returns iterator pointing to the new element. */ - iterator push_back( const_reference item ) - { - push_back_helper prolog(*this); - new(prolog.internal_push_back_result()) T(item); - return prolog.return_iterator_and_dismiss(); - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - //! Push item, move-aware - /** Returns iterator pointing to the new element. */ - iterator push_back( T&& item ) - { - push_back_helper prolog(*this); - new(prolog.internal_push_back_result()) T(std::move(item)); - return prolog.return_iterator_and_dismiss(); - } -#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT - //! Push item, create item "in place" with provided arguments - /** Returns iterator pointing to the new element. */ - template - iterator emplace_back( Args&&... args ) - { - push_back_helper prolog(*this); - new(prolog.internal_push_back_result()) T(std::forward(args)...); - return prolog.return_iterator_and_dismiss(); - } -#endif //__TBB_CPP11_VARIADIC_TEMPLATES_PRESENT -#endif //__TBB_CPP11_RVALUE_REF_PRESENT - //! Get reference to element at given index. - /** This method is thread-safe for concurrent reads, and also while growing the vector, - as long as the calling thread has checked that index < size(). */ - reference operator[]( size_type index ) { - return internal_subscript(index); - } - - //! Get const reference to element at given index. - const_reference operator[]( size_type index ) const { - return internal_subscript(index); - } - - //! Get reference to element at given index. Throws exceptions on errors. - reference at( size_type index ) { - return internal_subscript_with_exceptions(index); - } - - //! Get const reference to element at given index. Throws exceptions on errors. - const_reference at( size_type index ) const { - return internal_subscript_with_exceptions(index); - } - - //! Get range for iterating with parallel algorithms - range_type range( size_t grainsize = 1 ) { - return range_type( begin(), end(), grainsize ); - } - - //! Get const range for iterating with parallel algorithms - const_range_type range( size_t grainsize = 1 ) const { - return const_range_type( begin(), end(), grainsize ); - } - - //------------------------------------------------------------------------ - // Capacity - //------------------------------------------------------------------------ - //! Return size of vector. It may include elements under construction - size_type size() const { - size_type sz = my_early_size, cp = internal_capacity(); - return cp < sz ? cp : sz; - } - - //! Return false if vector is not empty or has elements under construction at least. - bool empty() const {return !my_early_size;} - - //! Maximum size to which array can grow without allocating more memory. Concurrent allocations are not included in the value. - size_type capacity() const {return internal_capacity();} - - //! Allocate enough space to grow to size n without having to allocate more memory later. - /** Like most of the methods provided for STL compatibility, this method is *not* thread safe. - The capacity afterwards may be bigger than the requested reservation. */ - void reserve( size_type n ) { - if( n ) - internal_reserve(n, sizeof(T), max_size()); - } - - //! Resize the vector. Not thread-safe. - void resize( size_type n ) { - internal_resize( n, sizeof(T), max_size(), NULL, &destroy_array, &initialize_array ); - } - - //! Resize the vector, copy t for new elements. Not thread-safe. - void resize( size_type n, const_reference t ) { - internal_resize( n, sizeof(T), max_size(), static_cast(&t), &destroy_array, &initialize_array_by ); - } - - //! Optimize memory usage and fragmentation. - void shrink_to_fit(); - - //! Upper bound on argument to reserve. - size_type max_size() const {return (~size_type(0))/sizeof(T);} - - //------------------------------------------------------------------------ - // STL support - //------------------------------------------------------------------------ - - //! start iterator - iterator begin() {return iterator(*this,0);} - //! end iterator - iterator end() {return iterator(*this,size());} - //! start const iterator - const_iterator begin() const {return const_iterator(*this,0);} - //! end const iterator - const_iterator end() const {return const_iterator(*this,size());} - //! start const iterator - const_iterator cbegin() const {return const_iterator(*this,0);} - //! end const iterator - const_iterator cend() const {return const_iterator(*this,size());} - //! reverse start iterator - reverse_iterator rbegin() {return reverse_iterator(end());} - //! reverse end iterator - reverse_iterator rend() {return reverse_iterator(begin());} - //! reverse start const iterator - const_reverse_iterator rbegin() const {return const_reverse_iterator(end());} - //! reverse end const iterator - const_reverse_iterator rend() const {return const_reverse_iterator(begin());} - //! reverse start const iterator - const_reverse_iterator crbegin() const {return const_reverse_iterator(end());} - //! reverse end const iterator - const_reverse_iterator crend() const {return const_reverse_iterator(begin());} - //! the first item - reference front() { - __TBB_ASSERT( size()>0, NULL); - const segment_value_t& segment_value = my_segment[0].template load(); - return (segment_value.template pointer())[0]; - } - //! the first item const - const_reference front() const { - __TBB_ASSERT( size()>0, NULL); - const segment_value_t& segment_value = my_segment[0].template load(); - return (segment_value.template pointer())[0]; - } - //! the last item - reference back() { - __TBB_ASSERT( size()>0, NULL); - return internal_subscript( size()-1 ); - } - //! the last item const - const_reference back() const { - __TBB_ASSERT( size()>0, NULL); - return internal_subscript( size()-1 ); - } - //! return allocator object - allocator_type get_allocator() const { return this->my_allocator; } - - //! assign n items by copying t item - void assign(size_type n, const_reference t) { - clear(); - internal_resize( n, sizeof(T), max_size(), static_cast(&t), &destroy_array, &initialize_array_by ); - } - - //! assign range [first, last) - template - void assign(I first, I last) { - clear(); internal_assign_range( first, last, static_cast::is_integer> *>(0) ); - } - -#if __TBB_INITIALIZER_LISTS_PRESENT - //! assigns an initializer list - void assign(std::initializer_list init_list) { - clear(); internal_assign_iterators( init_list.begin(), init_list.end()); - } -#endif //# __TBB_INITIALIZER_LISTS_PRESENT - - //! swap two instances - void swap(concurrent_vector &vector) { - using std::swap; - if( this != &vector ) { - concurrent_vector_base_v3::internal_swap(static_cast(vector)); - swap(this->my_allocator, vector.my_allocator); - } - } - - //! Clear container while keeping memory allocated. - /** To free up the memory, use in conjunction with method compact(). Not thread safe **/ - void clear() { - internal_clear(&destroy_array); - } - - //! Clear and destroy vector. - ~concurrent_vector() { - segment_t *table = my_segment.load(); - internal_free_segments( table, internal_clear(&destroy_array), my_first_block.load() ); - // base class destructor call should be then - } - - const internal::concurrent_vector_base_v3 &internal_vector_base() const { return *this; } -private: - //! Allocate k items - static void *internal_allocator(internal::concurrent_vector_base_v3 &vb, size_t k) { - return static_cast&>(vb).my_allocator.allocate(k); - } - //! Free k segments from table - void internal_free_segments(segment_t table[], segment_index_t k, segment_index_t first_block); - - //! Get reference to element at given index. - T& internal_subscript( size_type index ) const; - - //! Get reference to element at given index with errors checks - T& internal_subscript_with_exceptions( size_type index ) const; - - //! assign n items by copying t - void internal_assign_n(size_type n, const_pointer p) { - internal_resize( n, sizeof(T), max_size(), static_cast(p), &destroy_array, p? &initialize_array_by : &initialize_array ); - } - - //! helper class - template class is_integer_tag; - - //! assign integer items by copying when arguments are treated as iterators. See C++ Standard 2003 23.1.1p9 - template - void internal_assign_range(I first, I last, is_integer_tag *) { - internal_assign_n(static_cast(first), &static_cast(last)); - } - //! inline proxy assign by iterators - template - void internal_assign_range(I first, I last, is_integer_tag *) { - internal_assign_iterators(first, last); - } - //! assign by iterators - template - void internal_assign_iterators(I first, I last); - - //these functions are marked __TBB_EXPORTED_FUNC as they are called from within the library - - //! Construct n instances of T, starting at "begin". - static void __TBB_EXPORTED_FUNC initialize_array( void* begin, const void*, size_type n ); - - //! Copy-construct n instances of T, starting at "begin". - static void __TBB_EXPORTED_FUNC initialize_array_by( void* begin, const void* src, size_type n ); - - //! Copy-construct n instances of T by copying single element pointed to by src, starting at "dst". - static void __TBB_EXPORTED_FUNC copy_array( void* dst, const void* src, size_type n ); - -#if __TBB_MOVE_IF_NOEXCEPT_PRESENT - //! Either opy or move-construct n instances of T, starting at "dst" by copying according element of src array. - static void __TBB_EXPORTED_FUNC move_array_if_noexcept( void* dst, const void* src, size_type n ); -#endif //__TBB_MOVE_IF_NO_EXCEPT_PRESENT - -#if __TBB_CPP11_RVALUE_REF_PRESENT - //! Move-construct n instances of T, starting at "dst" by copying according element of src array. - static void __TBB_EXPORTED_FUNC move_array( void* dst, const void* src, size_type n ); - - //! Move-assign (using operator=) n instances of T, starting at "dst" by assigning according element of src array. - static void __TBB_EXPORTED_FUNC move_assign_array( void* dst, const void* src, size_type n ); -#endif - //! Copy-construct n instances of T, starting at "dst" by iterator range of [p_type_erased_iterator, p_type_erased_iterator+n). - template - static void __TBB_EXPORTED_FUNC copy_range( void* dst, const void* p_type_erased_iterator, size_type n ); - - //! Assign (using operator=) n instances of T, starting at "dst" by assigning according element of src array. - static void __TBB_EXPORTED_FUNC assign_array( void* dst, const void* src, size_type n ); - - //! Destroy n instances of T, starting at "begin". - static void __TBB_EXPORTED_FUNC destroy_array( void* begin, size_type n ); - - //! Exception-aware helper class for filling a segment by exception-danger operators of user class - class internal_loop_guide : internal::no_copy { - public: - const pointer array; - const size_type n; - size_type i; - - static const T* as_const_pointer(const void *ptr) { return static_cast(ptr); } - static T* as_pointer(const void *src) { return static_cast(const_cast(src)); } - - internal_loop_guide(size_type ntrials, void *ptr) - : array(as_pointer(ptr)), n(ntrials), i(0) {} - void init() { for(; i < n; ++i) new( &array[i] ) T(); } - void init(const void *src) { for(; i < n; ++i) new( &array[i] ) T(*as_const_pointer(src)); } - void copy(const void *src) { for(; i < n; ++i) new( &array[i] ) T(as_const_pointer(src)[i]); } - void assign(const void *src) { for(; i < n; ++i) array[i] = as_const_pointer(src)[i]; } -#if __TBB_CPP11_RVALUE_REF_PRESENT - void move_assign(const void *src) { for(; i < n; ++i) array[i] = std::move(as_pointer(src)[i]); } - void move_construct(const void *src) { for(; i < n; ++i) new( &array[i] ) T( std::move(as_pointer(src)[i]) ); } -#endif -#if __TBB_MOVE_IF_NOEXCEPT_PRESENT - void move_construct_if_noexcept(const void *src) { for(; i < n; ++i) new( &array[i] ) T( std::move_if_noexcept(as_pointer(src)[i]) ); } -#endif //__TBB_MOVE_IF_NOEXCEPT_PRESENT - - //TODO: rename to construct_range - template void iterate(I &src) { for(; i < n; ++i, ++src) new( &array[i] ) T( *src ); } - ~internal_loop_guide() { - if(i < n) {// if an exception was raised, fill the rest of items with zeros - internal::handle_unconstructed_elements(array+i, n-i); - } - } - }; - - struct push_back_helper : internal::no_copy{ - struct element_construction_guard : internal::no_copy{ - pointer element; - - element_construction_guard(pointer an_element) : element (an_element){} - void dismiss(){ element = NULL; } - ~element_construction_guard(){ - if (element){ - internal::handle_unconstructed_elements(element, 1); - } - } - }; - - concurrent_vector & v; - size_type k; - element_construction_guard g; - - push_back_helper(concurrent_vector & vector) : - v(vector), - g (static_cast(v.internal_push_back(sizeof(T),k))) - {} - - pointer internal_push_back_result(){ return g.element;} - iterator return_iterator_and_dismiss(){ - pointer ptr = g.element; - g.dismiss(); - return iterator(v, k, ptr); - } - }; -}; - -#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) -#pragma warning (push) -#pragma warning (disable: 4701) // potentially uninitialized local variable "old" -#endif -template -void concurrent_vector::shrink_to_fit() { - internal_segments_table old; - __TBB_TRY { - internal_array_op2 copy_or_move_array = -#if __TBB_MOVE_IF_NOEXCEPT_PRESENT - &move_array_if_noexcept -#else - ©_array -#endif - ; - if( internal_compact( sizeof(T), &old, &destroy_array, copy_or_move_array ) ) - internal_free_segments( old.table, pointers_per_long_table, old.first_block ); // free joined and unnecessary segments - } __TBB_CATCH(...) { - if( old.first_block ) // free segment allocated for compacting. Only for support of exceptions in ctor of user T[ype] - internal_free_segments( old.table, 1, old.first_block ); - __TBB_RETHROW(); - } -} -#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) -#pragma warning (pop) -#endif // warning 4701 is back - -template -void concurrent_vector::internal_free_segments(segment_t table[], segment_index_t k, segment_index_t first_block) { - // Free the arrays - while( k > first_block ) { - --k; - segment_value_t segment_value = table[k].load(); - table[k].store(segment_not_used()); - if( segment_value == segment_allocated() ) // check for correct segment pointer - this->my_allocator.deallocate( (segment_value.pointer()), segment_size(k) ); - } - segment_value_t segment_value = table[0].load(); - if( segment_value == segment_allocated() ) { - __TBB_ASSERT( first_block > 0, NULL ); - while(k > 0) table[--k].store(segment_not_used()); - this->my_allocator.deallocate( (segment_value.pointer()), segment_size(first_block) ); - } -} - -template -T& concurrent_vector::internal_subscript( size_type index ) const { - //TODO: unify both versions of internal_subscript - __TBB_ASSERT( index < my_early_size, "index out of bounds" ); - size_type j = index; - segment_index_t k = segment_base_index_of( j ); - __TBB_ASSERT( my_segment.load() != my_storage || k < pointers_per_short_table, "index is being allocated" ); - //no need in load with acquire (load) since thread works in own space or gets - //the information about added elements via some form of external synchronization - //TODO: why not make a load of my_segment relaxed as well ? - //TODO: add an assertion that my_segment[k] is properly aligned to please ITT - segment_value_t segment_value = my_segment[k].template load(); - __TBB_ASSERT( segment_value != segment_allocation_failed(), "the instance is broken by bad allocation. Use at() instead" ); - __TBB_ASSERT( segment_value != segment_not_used(), "index is being allocated" ); - return (( segment_value.pointer()))[j]; -} - -template -T& concurrent_vector::internal_subscript_with_exceptions( size_type index ) const { - if( index >= my_early_size ) - internal::throw_exception(internal::eid_out_of_range); // throw std::out_of_range - size_type j = index; - segment_index_t k = segment_base_index_of( j ); - //TODO: refactor this condition into separate helper function, e.g. fits_into_small_table - if( my_segment.load() == my_storage && k >= pointers_per_short_table ) - internal::throw_exception(internal::eid_segment_range_error); // throw std::range_error - // no need in load with acquire (load) since thread works in own space or gets - //the information about added elements via some form of external synchronization - //TODO: why not make a load of my_segment relaxed as well ? - //TODO: add an assertion that my_segment[k] is properly aligned to please ITT - segment_value_t segment_value = my_segment[k].template load(); - enforce_segment_allocated(segment_value, internal::eid_index_range_error); - return (segment_value.pointer())[j]; -} - -template template -void concurrent_vector::internal_assign_iterators(I first, I last) { - __TBB_ASSERT(my_early_size == 0, NULL); - size_type n = std::distance(first, last); - if( !n ) return; - internal_reserve(n, sizeof(T), max_size()); - my_early_size = n; - segment_index_t k = 0; - //TODO: unify segment iteration code with concurrent_base_v3::helper - size_type sz = segment_size( my_first_block ); - while( sz < n ) { - internal_loop_guide loop(sz, my_segment[k].template load().template pointer()); - loop.iterate(first); - n -= sz; - if( !k ) k = my_first_block; - else { ++k; sz <<= 1; } - } - internal_loop_guide loop(n, my_segment[k].template load().template pointer()); - loop.iterate(first); -} - -template -void concurrent_vector::initialize_array( void* begin, const void *, size_type n ) { - internal_loop_guide loop(n, begin); loop.init(); -} - -template -void concurrent_vector::initialize_array_by( void* begin, const void *src, size_type n ) { - internal_loop_guide loop(n, begin); loop.init(src); -} - -template -void concurrent_vector::copy_array( void* dst, const void* src, size_type n ) { - internal_loop_guide loop(n, dst); loop.copy(src); -} - -#if __TBB_CPP11_RVALUE_REF_PRESENT -template -void concurrent_vector::move_array( void* dst, const void* src, size_type n ) { - internal_loop_guide loop(n, dst); loop.move_construct(src); -} -template -void concurrent_vector::move_assign_array( void* dst, const void* src, size_type n ) { - internal_loop_guide loop(n, dst); loop.move_assign(src); -} -#endif - -#if __TBB_MOVE_IF_NOEXCEPT_PRESENT -template -void concurrent_vector::move_array_if_noexcept( void* dst, const void* src, size_type n ) { - internal_loop_guide loop(n, dst); loop.move_construct_if_noexcept(src); -} -#endif //__TBB_MOVE_IF_NOEXCEPT_PRESENT - -template -template -void concurrent_vector::copy_range( void* dst, const void* p_type_erased_iterator, size_type n ){ - I & iterator ((*const_cast(static_cast(p_type_erased_iterator)))); - internal_loop_guide loop(n, dst); loop.iterate(iterator); -} - -template -void concurrent_vector::assign_array( void* dst, const void* src, size_type n ) { - internal_loop_guide loop(n, dst); loop.assign(src); -} - -#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) - // Workaround for overzealous compiler warning - #pragma warning (push) - #pragma warning (disable: 4189) -#endif -template -void concurrent_vector::destroy_array( void* begin, size_type n ) { - T* array = static_cast(begin); - for( size_type j=n; j>0; --j ) - array[j-1].~T(); // destructors are supposed to not throw any exceptions -} -#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) - #pragma warning (pop) -#endif // warning 4189 is back - -// concurrent_vector's template functions -template -inline bool operator==(const concurrent_vector &a, const concurrent_vector &b) { - //TODO: call size() only once per vector (in operator==) - // Simply: return a.size() == b.size() && std::equal(a.begin(), a.end(), b.begin()); - if(a.size() != b.size()) return false; - typename concurrent_vector::const_iterator i(a.begin()); - typename concurrent_vector::const_iterator j(b.begin()); - for(; i != a.end(); ++i, ++j) - if( !(*i == *j) ) return false; - return true; -} - -template -inline bool operator!=(const concurrent_vector &a, const concurrent_vector &b) -{ return !(a == b); } - -template -inline bool operator<(const concurrent_vector &a, const concurrent_vector &b) -{ return (std::lexicographical_compare(a.begin(), a.end(), b.begin(), b.end())); } - -template -inline bool operator>(const concurrent_vector &a, const concurrent_vector &b) -{ return b < a; } - -template -inline bool operator<=(const concurrent_vector &a, const concurrent_vector &b) -{ return !(b < a); } - -template -inline bool operator>=(const concurrent_vector &a, const concurrent_vector &b) -{ return !(a < b); } - -template -inline void swap(concurrent_vector &a, concurrent_vector &b) -{ a.swap( b ); } - -} // namespace tbb - -#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) - #pragma warning (pop) -#endif // warning 4267,4127 are back - -#endif /* __TBB_concurrent_vector_H */ +#include "../oneapi/tbb/concurrent_vector.h" diff --git a/src/tbb/include/tbb/critical_section.h b/src/tbb/include/tbb/critical_section.h deleted file mode 100644 index 324b3e138..000000000 --- a/src/tbb/include/tbb/critical_section.h +++ /dev/null @@ -1,133 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef _TBB_CRITICAL_SECTION_H_ -#define _TBB_CRITICAL_SECTION_H_ - -#if _WIN32||_WIN64 -#include "machine/windows_api.h" -#else -#include -#include -#endif // _WIN32||WIN64 - -#include "tbb_stddef.h" -#include "tbb_thread.h" -#include "tbb_exception.h" - -#include "tbb_profiling.h" - -namespace tbb { - - namespace internal { -class critical_section_v4 : internal::no_copy { -#if _WIN32||_WIN64 - CRITICAL_SECTION my_impl; -#else - pthread_mutex_t my_impl; -#endif - tbb_thread::id my_tid; -public: - - void __TBB_EXPORTED_METHOD internal_construct(); - - critical_section_v4() { -#if _WIN32||_WIN64 - InitializeCriticalSectionEx( &my_impl, 4000, 0 ); -#else - pthread_mutex_init(&my_impl, NULL); -#endif - internal_construct(); - } - - ~critical_section_v4() { - __TBB_ASSERT(my_tid == tbb_thread::id(), "Destroying a still-held critical section"); -#if _WIN32||_WIN64 - DeleteCriticalSection(&my_impl); -#else - pthread_mutex_destroy(&my_impl); -#endif - } - - class scoped_lock : internal::no_copy { - private: - critical_section_v4 &my_crit; - public: - scoped_lock( critical_section_v4& lock_me) :my_crit(lock_me) { - my_crit.lock(); - } - - ~scoped_lock() { - my_crit.unlock(); - } - }; - - void lock() { - tbb_thread::id local_tid = this_tbb_thread::get_id(); - if(local_tid == my_tid) throw_exception( eid_improper_lock ); -#if _WIN32||_WIN64 - EnterCriticalSection( &my_impl ); -#else - int rval = pthread_mutex_lock(&my_impl); - __TBB_ASSERT_EX(!rval, "critical_section::lock: pthread_mutex_lock failed"); -#endif - __TBB_ASSERT(my_tid == tbb_thread::id(), NULL); - my_tid = local_tid; - } - - bool try_lock() { - bool gotlock; - tbb_thread::id local_tid = this_tbb_thread::get_id(); - if(local_tid == my_tid) return false; -#if _WIN32||_WIN64 - gotlock = TryEnterCriticalSection( &my_impl ) != 0; -#else - int rval = pthread_mutex_trylock(&my_impl); - // valid returns are 0 (locked) and [EBUSY] - __TBB_ASSERT(rval == 0 || rval == EBUSY, "critical_section::trylock: pthread_mutex_trylock failed"); - gotlock = rval == 0; -#endif - if(gotlock) { - my_tid = local_tid; - } - return gotlock; - } - - void unlock() { - __TBB_ASSERT(this_tbb_thread::get_id() == my_tid, "thread unlocking critical_section is not thread that locked it"); - my_tid = tbb_thread::id(); -#if _WIN32||_WIN64 - LeaveCriticalSection( &my_impl ); -#else - int rval = pthread_mutex_unlock(&my_impl); - __TBB_ASSERT_EX(!rval, "critical_section::unlock: pthread_mutex_unlock failed"); -#endif - } - - static const bool is_rw_mutex = false; - static const bool is_recursive_mutex = false; - static const bool is_fair_mutex = true; -}; // critical_section_v4 -} // namespace internal -typedef internal::critical_section_v4 critical_section; - -__TBB_DEFINE_PROFILING_SET_NAME(critical_section) -} // namespace tbb -#endif // _TBB_CRITICAL_SECTION_H_ diff --git a/src/tbb/include/tbb/enumerable_thread_specific.h b/src/tbb/include/tbb/enumerable_thread_specific.h index f31f533b9..9d6050d64 100644 --- a/src/tbb/include/tbb/enumerable_thread_specific.h +++ b/src/tbb/include/tbb/enumerable_thread_specific.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,1126 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ -#ifndef __TBB_enumerable_thread_specific_H -#define __TBB_enumerable_thread_specific_H - -#include "atomic.h" -#include "concurrent_vector.h" -#include "tbb_thread.h" -#include "tbb_allocator.h" -#include "cache_aligned_allocator.h" -#include "aligned_space.h" -#include "internal/_template_helpers.h" -#include "internal/_tbb_hash_compare_impl.h" -#include "tbb_profiling.h" -#include // for memcpy - -#if _WIN32||_WIN64 -#include "machine/windows_api.h" -#else -#include -#endif - -#define __TBB_ETS_USE_CPP11 \ - (__TBB_CPP11_RVALUE_REF_PRESENT && __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT \ - && __TBB_CPP11_DECLTYPE_PRESENT && __TBB_CPP11_LAMBDAS_PRESENT) - -namespace tbb { - -//! enum for selecting between single key and key-per-instance versions -enum ets_key_usage_type { ets_key_per_instance, ets_no_key }; - -namespace interface6 { - - // Forward declaration to use in internal classes - template - class enumerable_thread_specific; - - //! @cond - namespace internal { - - using namespace tbb::internal; - - template - class ets_base: tbb::internal::no_copy { - protected: - typedef tbb_thread::id key_type; -#if __TBB_PROTECTED_NESTED_CLASS_BROKEN - public: -#endif - struct slot; - - struct array { - array* next; - size_t lg_size; - slot& at( size_t k ) { - return ((slot*)(void*)(this+1))[k]; - } - size_t size() const {return size_t(1)<>(8*sizeof(size_t)-lg_size); - } - }; - struct slot { - key_type key; - void* ptr; - bool empty() const {return key == key_type();} - bool match( key_type k ) const {return key == k;} - bool claim( key_type k ) { - // TODO: maybe claim ptr, because key_type is not guaranteed to fit into word size - return atomic_compare_and_swap(key, k, key_type()) == key_type(); - } - }; -#if __TBB_PROTECTED_NESTED_CLASS_BROKEN - protected: -#endif - - //! Root of linked list of arrays of decreasing size. - /** NULL if and only if my_count==0. - Each array in the list is half the size of its predecessor. */ - atomic my_root; - atomic my_count; - virtual void* create_local() = 0; - virtual void* create_array(size_t _size) = 0; // _size in bytes - virtual void free_array(void* ptr, size_t _size) = 0; // _size in bytes - array* allocate( size_t lg_size ) { - size_t n = size_t(1)<(create_array( sizeof(array)+n*sizeof(slot) )); - a->lg_size = lg_size; - std::memset( a+1, 0, n*sizeof(slot) ); - return a; - } - void free(array* a) { - size_t n = size_t(1)<<(a->lg_size); - free_array( (void *)a, size_t(sizeof(array)+n*sizeof(slot)) ); - } - - ets_base() {my_root=NULL; my_count=0;} - virtual ~ets_base(); // g++ complains if this is not virtual - void* table_lookup( bool& exists ); - void table_clear(); - // The following functions are not used in concurrent context, - // so we don't need synchronization and ITT annotations there. - void table_elementwise_copy( const ets_base& other, - void*(*add_element)(ets_base&, void*) ) { - __TBB_ASSERT(!my_root,NULL); - __TBB_ASSERT(!my_count,NULL); - if( !other.my_root ) return; - array* root = my_root = allocate(other.my_root->lg_size); - root->next = NULL; - my_count = other.my_count; - size_t mask = root->mask(); - for( array* r=other.my_root; r; r=r->next ) { - for( size_t i=0; isize(); ++i ) { - slot& s1 = r->at(i); - if( !s1.empty() ) { - for( size_t j = root->start(tbb::tbb_hash()(s1.key)); ; j=(j+1)&mask ) { - slot& s2 = root->at(j); - if( s2.empty() ) { - s2.ptr = add_element(*this, s1.ptr); - s2.key = s1.key; - break; - } - else if( s2.match(s1.key) ) - break; - } - } - } - } - } - void table_swap( ets_base& other ) { - __TBB_ASSERT(this!=&other, "Don't swap an instance with itself"); - tbb::internal::swap(my_root, other.my_root); - tbb::internal::swap(my_count, other.my_count); - } - }; - - template - ets_base::~ets_base() { - __TBB_ASSERT(!my_root, NULL); - } - - template - void ets_base::table_clear() { - while( array* r = my_root ) { - my_root = r->next; - free(r); - } - my_count = 0; - } - - template - void* ets_base::table_lookup( bool& exists ) { - const key_type k = tbb::this_tbb_thread::get_id(); - - __TBB_ASSERT(k != key_type(),NULL); - void* found; - size_t h = tbb::tbb_hash()(k); - for( array* r=my_root; r; r=r->next ) { - call_itt_notify(acquired,r); - size_t mask=r->mask(); - for(size_t i = r->start(h); ;i=(i+1)&mask) { - slot& s = r->at(i); - if( s.empty() ) break; - if( s.match(k) ) { - if( r==my_root ) { - // Success at top level - exists = true; - return s.ptr; - } else { - // Success at some other level. Need to insert at top level. - exists = true; - found = s.ptr; - goto insert; - } - } - } - } - // Key does not yet exist. The density of slots in the table does not exceed 0.5, - // for if this will occur a new table is allocated with double the current table - // size, which is swapped in as the new root table. So an empty slot is guaranteed. - exists = false; - found = create_local(); - { - size_t c = ++my_count; - array* r = my_root; - call_itt_notify(acquired,r); - if( !r || c>r->size()/2 ) { - size_t s = r ? r->lg_size : 2; - while( c>size_t(1)<<(s-1) ) ++s; - array* a = allocate(s); - for(;;) { - a->next = r; - call_itt_notify(releasing,a); - array* new_r = my_root.compare_and_swap(a,r); - if( new_r==r ) break; - call_itt_notify(acquired, new_r); - if( new_r->lg_size>=s ) { - // Another thread inserted an equal or bigger array, so our array is superfluous. - free(a); - break; - } - r = new_r; - } - } - } - insert: - // Whether a slot has been found in an older table, or if it has been inserted at this level, - // it has already been accounted for in the total. Guaranteed to be room for it, and it is - // not present, so search for empty slot and use it. - array* ir = my_root; - call_itt_notify(acquired, ir); - size_t mask = ir->mask(); - for(size_t i = ir->start(h);;i=(i+1)&mask) { - slot& s = ir->at(i); - if( s.empty() ) { - if( s.claim(k) ) { - s.ptr = found; - return found; - } - } - } - } - - //! Specialization that exploits native TLS - template <> - class ets_base: protected ets_base { - typedef ets_base super; -#if _WIN32||_WIN64 -#if __TBB_WIN8UI_SUPPORT - typedef DWORD tls_key_t; - void create_key() { my_key = FlsAlloc(NULL); } - void destroy_key() { FlsFree(my_key); } - void set_tls(void * value) { FlsSetValue(my_key, (LPVOID)value); } - void* get_tls() { return (void *)FlsGetValue(my_key); } -#else - typedef DWORD tls_key_t; - void create_key() { my_key = TlsAlloc(); } - void destroy_key() { TlsFree(my_key); } - void set_tls(void * value) { TlsSetValue(my_key, (LPVOID)value); } - void* get_tls() { return (void *)TlsGetValue(my_key); } -#endif -#else - typedef pthread_key_t tls_key_t; - void create_key() { pthread_key_create(&my_key, NULL); } - void destroy_key() { pthread_key_delete(my_key); } - void set_tls( void * value ) const { pthread_setspecific(my_key, value); } - void* get_tls() const { return pthread_getspecific(my_key); } -#endif - tls_key_t my_key; - virtual void* create_local() __TBB_override = 0; - virtual void* create_array(size_t _size) __TBB_override = 0; // _size in bytes - virtual void free_array(void* ptr, size_t _size) __TBB_override = 0; // size in bytes - protected: - ets_base() {create_key();} - ~ets_base() {destroy_key();} - void* table_lookup( bool& exists ) { - void* found = get_tls(); - if( found ) { - exists=true; - } else { - found = super::table_lookup(exists); - set_tls(found); - } - return found; - } - void table_clear() { - destroy_key(); - create_key(); - super::table_clear(); - } - void table_swap( ets_base& other ) { - using std::swap; - __TBB_ASSERT(this!=&other, "Don't swap an instance with itself"); - swap(my_key, other.my_key); - super::table_swap(other); - } - }; - - //! Random access iterator for traversing the thread local copies. - template< typename Container, typename Value > - class enumerable_thread_specific_iterator -#if defined(_WIN64) && defined(_MSC_VER) - // Ensure that Microsoft's internal template function _Val_type works correctly. - : public std::iterator -#endif /* defined(_WIN64) && defined(_MSC_VER) */ - { - //! current position in the concurrent_vector - - Container *my_container; - typename Container::size_type my_index; - mutable Value *my_value; - - template - friend enumerable_thread_specific_iterator - operator+( ptrdiff_t offset, const enumerable_thread_specific_iterator& v ); - - template - friend bool operator==( const enumerable_thread_specific_iterator& i, - const enumerable_thread_specific_iterator& j ); - - template - friend bool operator<( const enumerable_thread_specific_iterator& i, - const enumerable_thread_specific_iterator& j ); - - template - friend ptrdiff_t operator-( const enumerable_thread_specific_iterator& i, - const enumerable_thread_specific_iterator& j ); - - template - friend class enumerable_thread_specific_iterator; - - public: - - enumerable_thread_specific_iterator( const Container &container, typename Container::size_type index ) : - my_container(&const_cast(container)), my_index(index), my_value(NULL) {} - - //! Default constructor - enumerable_thread_specific_iterator() : my_container(NULL), my_index(0), my_value(NULL) {} - - template - enumerable_thread_specific_iterator( const enumerable_thread_specific_iterator& other ) : - my_container( other.my_container ), my_index( other.my_index), my_value( const_cast(other.my_value) ) {} - - enumerable_thread_specific_iterator operator+( ptrdiff_t offset ) const { - return enumerable_thread_specific_iterator(*my_container, my_index + offset); - } - - enumerable_thread_specific_iterator &operator+=( ptrdiff_t offset ) { - my_index += offset; - my_value = NULL; - return *this; - } - - enumerable_thread_specific_iterator operator-( ptrdiff_t offset ) const { - return enumerable_thread_specific_iterator( *my_container, my_index-offset ); - } - - enumerable_thread_specific_iterator &operator-=( ptrdiff_t offset ) { - my_index -= offset; - my_value = NULL; - return *this; - } - - Value& operator*() const { - Value* value = my_value; - if( !value ) { - value = my_value = (*my_container)[my_index].value(); - } - __TBB_ASSERT( value==(*my_container)[my_index].value(), "corrupt cache" ); - return *value; - } - - Value& operator[]( ptrdiff_t k ) const { - return (*my_container)[my_index + k].value; - } - - Value* operator->() const {return &operator*();} - - enumerable_thread_specific_iterator& operator++() { - ++my_index; - my_value = NULL; - return *this; - } - - enumerable_thread_specific_iterator& operator--() { - --my_index; - my_value = NULL; - return *this; - } - - //! Post increment - enumerable_thread_specific_iterator operator++(int) { - enumerable_thread_specific_iterator result = *this; - ++my_index; - my_value = NULL; - return result; - } - - //! Post decrement - enumerable_thread_specific_iterator operator--(int) { - enumerable_thread_specific_iterator result = *this; - --my_index; - my_value = NULL; - return result; - } - - // STL support - typedef ptrdiff_t difference_type; - typedef Value value_type; - typedef Value* pointer; - typedef Value& reference; - typedef std::random_access_iterator_tag iterator_category; - }; - - template - enumerable_thread_specific_iterator - operator+( ptrdiff_t offset, const enumerable_thread_specific_iterator& v ) { - return enumerable_thread_specific_iterator( v.my_container, v.my_index + offset ); - } - - template - bool operator==( const enumerable_thread_specific_iterator& i, - const enumerable_thread_specific_iterator& j ) { - return i.my_index==j.my_index && i.my_container == j.my_container; - } - - template - bool operator!=( const enumerable_thread_specific_iterator& i, - const enumerable_thread_specific_iterator& j ) { - return !(i==j); - } - - template - bool operator<( const enumerable_thread_specific_iterator& i, - const enumerable_thread_specific_iterator& j ) { - return i.my_index - bool operator>( const enumerable_thread_specific_iterator& i, - const enumerable_thread_specific_iterator& j ) { - return j - bool operator>=( const enumerable_thread_specific_iterator& i, - const enumerable_thread_specific_iterator& j ) { - return !(i - bool operator<=( const enumerable_thread_specific_iterator& i, - const enumerable_thread_specific_iterator& j ) { - return !(j - ptrdiff_t operator-( const enumerable_thread_specific_iterator& i, - const enumerable_thread_specific_iterator& j ) { - return i.my_index-j.my_index; - } - - template - class segmented_iterator -#if defined(_WIN64) && defined(_MSC_VER) - : public std::iterator -#endif - { - template - friend bool operator==(const segmented_iterator& i, const segmented_iterator& j); - - template - friend bool operator!=(const segmented_iterator& i, const segmented_iterator& j); - - template - friend class segmented_iterator; - - public: - - segmented_iterator() {my_segcont = NULL;} - - segmented_iterator( const SegmentedContainer& _segmented_container ) : - my_segcont(const_cast(&_segmented_container)), - outer_iter(my_segcont->end()) { } - - ~segmented_iterator() {} - - typedef typename SegmentedContainer::iterator outer_iterator; - typedef typename SegmentedContainer::value_type InnerContainer; - typedef typename InnerContainer::iterator inner_iterator; - - // STL support - typedef ptrdiff_t difference_type; - typedef Value value_type; - typedef typename SegmentedContainer::size_type size_type; - typedef Value* pointer; - typedef Value& reference; - typedef std::input_iterator_tag iterator_category; - - // Copy Constructor - template - segmented_iterator(const segmented_iterator& other) : - my_segcont(other.my_segcont), - outer_iter(other.outer_iter), - // can we assign a default-constructed iterator to inner if we're at the end? - inner_iter(other.inner_iter) - {} - - // assignment - template - segmented_iterator& operator=( const segmented_iterator& other) { - if(this != &other) { - my_segcont = other.my_segcont; - outer_iter = other.outer_iter; - if(outer_iter != my_segcont->end()) inner_iter = other.inner_iter; - } - return *this; - } - - // allow assignment of outer iterator to segmented iterator. Once it is - // assigned, move forward until a non-empty inner container is found or - // the end of the outer container is reached. - segmented_iterator& operator=(const outer_iterator& new_outer_iter) { - __TBB_ASSERT(my_segcont != NULL, NULL); - // check that this iterator points to something inside the segmented container - for(outer_iter = new_outer_iter ;outer_iter!=my_segcont->end(); ++outer_iter) { - if( !outer_iter->empty() ) { - inner_iter = outer_iter->begin(); - break; - } - } - return *this; - } - - // pre-increment - segmented_iterator& operator++() { - advance_me(); - return *this; - } - - // post-increment - segmented_iterator operator++(int) { - segmented_iterator tmp = *this; - operator++(); - return tmp; - } - - bool operator==(const outer_iterator& other_outer) const { - __TBB_ASSERT(my_segcont != NULL, NULL); - return (outer_iter == other_outer && - (outer_iter == my_segcont->end() || inner_iter == outer_iter->begin())); - } - - bool operator!=(const outer_iterator& other_outer) const { - return !operator==(other_outer); - - } - - // (i)* RHS - reference operator*() const { - __TBB_ASSERT(my_segcont != NULL, NULL); - __TBB_ASSERT(outer_iter != my_segcont->end(), "Dereferencing a pointer at end of container"); - __TBB_ASSERT(inner_iter != outer_iter->end(), NULL); // should never happen - return *inner_iter; - } - - // i-> - pointer operator->() const { return &operator*();} - - private: - SegmentedContainer* my_segcont; - outer_iterator outer_iter; - inner_iterator inner_iter; - - void advance_me() { - __TBB_ASSERT(my_segcont != NULL, NULL); - __TBB_ASSERT(outer_iter != my_segcont->end(), NULL); // not true if there are no inner containers - __TBB_ASSERT(inner_iter != outer_iter->end(), NULL); // not true if the inner containers are all empty. - ++inner_iter; - while(inner_iter == outer_iter->end() && ++outer_iter != my_segcont->end()) { - inner_iter = outer_iter->begin(); - } - } - }; // segmented_iterator - - template - bool operator==( const segmented_iterator& i, - const segmented_iterator& j ) { - if(i.my_segcont != j.my_segcont) return false; - if(i.my_segcont == NULL) return true; - if(i.outer_iter != j.outer_iter) return false; - if(i.outer_iter == i.my_segcont->end()) return true; - return i.inner_iter == j.inner_iter; - } - - // != - template - bool operator!=( const segmented_iterator& i, - const segmented_iterator& j ) { - return !(i==j); - } - - template - struct construct_by_default: tbb::internal::no_assign { - void construct(void*where) {new(where) T();} // C++ note: the () in T() ensure zero initialization. - construct_by_default( int ) {} - }; - - template - struct construct_by_exemplar: tbb::internal::no_assign { - const T exemplar; - void construct(void*where) {new(where) T(exemplar);} - construct_by_exemplar( const T& t ) : exemplar(t) {} -#if __TBB_ETS_USE_CPP11 - construct_by_exemplar( T&& t ) : exemplar(std::move(t)) {} -#endif - }; - - template - struct construct_by_finit: tbb::internal::no_assign { - Finit f; - void construct(void* where) {new(where) T(f());} - construct_by_finit( const Finit& f_ ) : f(f_) {} -#if __TBB_ETS_USE_CPP11 - construct_by_finit( Finit&& f_ ) : f(std::move(f_)) {} -#endif - }; - -#if __TBB_ETS_USE_CPP11 - template - struct construct_by_args: tbb::internal::no_assign { - internal::stored_pack pack; - void construct(void* where) { - internal::call( [where](const typename strip

::type&... args ){ - new(where) T(args...); - }, pack ); - } - construct_by_args( P&& ... args ) : pack(std::forward

(args)...) {} - }; -#endif - - // storage for initialization function pointer - // TODO: consider removing the template parameter T here and in callback_leaf - template - class callback_base { - public: - // Clone *this - virtual callback_base* clone() const = 0; - // Destruct and free *this - virtual void destroy() = 0; - // Need virtual destructor to satisfy GCC compiler warning - virtual ~callback_base() { } - // Construct T at where - virtual void construct(void* where) = 0; - }; - - template - class callback_leaf: public callback_base, Constructor { -#if __TBB_ETS_USE_CPP11 - template callback_leaf( P&& ... params ) : Constructor(std::forward

(params)...) {} -#else - template callback_leaf( const X& x ) : Constructor(x) {} -#endif - // TODO: make the construction/destruction consistent (use allocator.construct/destroy) - typedef typename tbb::tbb_allocator my_allocator_type; - - callback_base* clone() const __TBB_override { - return make(*this); - } - - void destroy() __TBB_override { - my_allocator_type().destroy(this); - my_allocator_type().deallocate(this,1); - } - - void construct(void* where) __TBB_override { - Constructor::construct(where); - } - public: -#if __TBB_ETS_USE_CPP11 - template - static callback_base* make( P&& ... params ) { - void* where = my_allocator_type().allocate(1); - return new(where) callback_leaf( std::forward

(params)... ); - } -#else - template - static callback_base* make( const X& x ) { - void* where = my_allocator_type().allocate(1); - return new(where) callback_leaf(x); - } -#endif - }; - - //! Template for recording construction of objects in table - /** All maintenance of the space will be done explicitly on push_back, - and all thread local copies must be destroyed before the concurrent - vector is deleted. - - The flag is_built is initialized to false. When the local is - successfully-constructed, set the flag to true or call value_committed(). - If the constructor throws, the flag will be false. - */ - template - struct ets_element { - tbb::aligned_space my_space; - bool is_built; - ets_element() { is_built = false; } // not currently-built - U* value() { return my_space.begin(); } - U* value_committed() { is_built = true; return my_space.begin(); } - ~ets_element() { - if(is_built) { - my_space.begin()->~U(); - is_built = false; - } - } - }; - - // A predicate that can be used for a compile-time compatibility check of ETS instances - // Ideally, it should have been declared inside the ETS class, but unfortunately - // in that case VS2013 does not enable the variadic constructor. - template struct is_compatible_ets { static const bool value = false; }; - template - struct is_compatible_ets< T, enumerable_thread_specific > { static const bool value = internal::is_same_type::value; }; - -#if __TBB_ETS_USE_CPP11 - // A predicate that checks whether, for a variable 'foo' of type T, foo() is a valid expression - template - class is_callable_no_args { - private: - typedef char yes[1]; - typedef char no [2]; - - template static yes& decide( decltype(declval()())* ); - template static no& decide(...); - public: - static const bool value = (sizeof(decide(NULL)) == sizeof(yes)); - }; -#endif - - } // namespace internal - //! @endcond - - //! The enumerable_thread_specific container - /** enumerable_thread_specific has the following properties: - - thread-local copies are lazily created, with default, exemplar or function initialization. - - thread-local copies do not move (during lifetime, and excepting clear()) so the address of a copy is invariant. - - the contained objects need not have operator=() defined if combine is not used. - - enumerable_thread_specific containers may be copy-constructed or assigned. - - thread-local copies can be managed by hash-table, or can be accessed via TLS storage for speed. - - outside of parallel contexts, the contents of all thread-local copies are accessible by iterator or using combine or combine_each methods - - @par Segmented iterator - When the thread-local objects are containers with input_iterators defined, a segmented iterator may - be used to iterate over all the elements of all thread-local copies. - - @par combine and combine_each - - Both methods are defined for enumerable_thread_specific. - - combine() requires the type T have operator=() defined. - - neither method modifies the contents of the object (though there is no guarantee that the applied methods do not modify the object.) - - Both are evaluated in serial context (the methods are assumed to be non-benign.) - - @ingroup containers */ - template , - ets_key_usage_type ETS_key_type=ets_no_key > - class enumerable_thread_specific: internal::ets_base { - - template friend class enumerable_thread_specific; - - typedef internal::padded< internal::ets_element > padded_element; - - //! A generic range, used to create range objects from the iterators - template - class generic_range_type: public blocked_range { - public: - typedef T value_type; - typedef T& reference; - typedef const T& const_reference; - typedef I iterator; - typedef ptrdiff_t difference_type; - generic_range_type( I begin_, I end_, size_t grainsize_ = 1) : blocked_range(begin_,end_,grainsize_) {} - template - generic_range_type( const generic_range_type& r) : blocked_range(r.begin(),r.end(),r.grainsize()) {} - generic_range_type( generic_range_type& r, split ) : blocked_range(r,split()) {} - }; - - typedef typename Allocator::template rebind< padded_element >::other padded_allocator_type; - typedef tbb::concurrent_vector< padded_element, padded_allocator_type > internal_collection_type; - - internal::callback_base *my_construct_callback; - - internal_collection_type my_locals; - - // TODO: consider unifying the callback mechanism for all create_local* methods below - // (likely non-compatible and requires interface version increase) - void* create_local() __TBB_override { - padded_element& lref = *my_locals.grow_by(1); - my_construct_callback->construct(lref.value()); - return lref.value_committed(); - } - - static void* create_local_by_copy( internal::ets_base& base, void* p ) { - enumerable_thread_specific& ets = static_cast(base); - padded_element& lref = *ets.my_locals.grow_by(1); - new(lref.value()) T(*static_cast(p)); - return lref.value_committed(); - } - -#if __TBB_ETS_USE_CPP11 - static void* create_local_by_move( internal::ets_base& base, void* p ) { - enumerable_thread_specific& ets = static_cast(base); - padded_element& lref = *ets.my_locals.grow_by(1); - new(lref.value()) T(std::move(*static_cast(p))); - return lref.value_committed(); - } -#endif - - typedef typename Allocator::template rebind< uintptr_t >::other array_allocator_type; - - // _size is in bytes - void* create_array(size_t _size) __TBB_override { - size_t nelements = (_size + sizeof(uintptr_t) -1) / sizeof(uintptr_t); - return array_allocator_type().allocate(nelements); - } - - void free_array( void* _ptr, size_t _size) __TBB_override { - size_t nelements = (_size + sizeof(uintptr_t) -1) / sizeof(uintptr_t); - array_allocator_type().deallocate( reinterpret_cast(_ptr),nelements); - } - - public: - - //! Basic types - typedef Allocator allocator_type; - typedef T value_type; - typedef T& reference; - typedef const T& const_reference; - typedef T* pointer; - typedef const T* const_pointer; - typedef typename internal_collection_type::size_type size_type; - typedef typename internal_collection_type::difference_type difference_type; - - // Iterator types - typedef typename internal::enumerable_thread_specific_iterator< internal_collection_type, value_type > iterator; - typedef typename internal::enumerable_thread_specific_iterator< internal_collection_type, const value_type > const_iterator; - - // Parallel range types - typedef generic_range_type< iterator > range_type; - typedef generic_range_type< const_iterator > const_range_type; - - //! Default constructor. Each local instance of T is default constructed. - enumerable_thread_specific() : my_construct_callback( - internal::callback_leaf >::make(/*dummy argument*/0) - ){} - - //! Constructor with initializer functor. Each local instance of T is constructed by T(finit()). - template ::type>::value>::type -#endif - > - explicit enumerable_thread_specific( Finit finit ) : my_construct_callback( - internal::callback_leaf >::make( tbb::internal::move(finit) ) - ){} - - //! Constructor with exemplar. Each local instance of T is copy-constructed from the exemplar. - explicit enumerable_thread_specific( const T& exemplar ) : my_construct_callback( - internal::callback_leaf >::make( exemplar ) - ){} - -#if __TBB_ETS_USE_CPP11 - explicit enumerable_thread_specific( T&& exemplar ) : my_construct_callback( - internal::callback_leaf >::make( std::move(exemplar) ) - ){} - - //! Variadic constructor with initializer arguments. Each local instance of T is constructed by T(args...) - template ::type>::value - && !internal::is_compatible_ets::type>::value - && !internal::is_same_type::type>::value - >::type> - enumerable_thread_specific( P1&& arg1, P&& ... args ) : my_construct_callback( - internal::callback_leaf >::make( std::forward(arg1), std::forward

(args)... ) - ){} -#endif - - //! Destructor - ~enumerable_thread_specific() { - if(my_construct_callback) my_construct_callback->destroy(); - // Deallocate the hash table before overridden free_array() becomes inaccessible - this->internal::ets_base::table_clear(); - } - - //! returns reference to local, discarding exists - reference local() { - bool exists; - return local(exists); - } - - //! Returns reference to calling thread's local copy, creating one if necessary - reference local(bool& exists) { - void* ptr = this->table_lookup(exists); - return *(T*)ptr; - } - - //! Get the number of local copies - size_type size() const { return my_locals.size(); } - - //! true if there have been no local copies created - bool empty() const { return my_locals.empty(); } - - //! begin iterator - iterator begin() { return iterator( my_locals, 0 ); } - //! end iterator - iterator end() { return iterator(my_locals, my_locals.size() ); } - - //! begin const iterator - const_iterator begin() const { return const_iterator(my_locals, 0); } - - //! end const iterator - const_iterator end() const { return const_iterator(my_locals, my_locals.size()); } - - //! Get range for parallel algorithms - range_type range( size_t grainsize=1 ) { return range_type( begin(), end(), grainsize ); } - - //! Get const range for parallel algorithms - const_range_type range( size_t grainsize=1 ) const { return const_range_type( begin(), end(), grainsize ); } - - //! Destroys local copies - void clear() { - my_locals.clear(); - this->table_clear(); - // callback is not destroyed - } - - private: - - template - void internal_copy(const enumerable_thread_specific& other) { -#if __TBB_ETS_USE_CPP11 && TBB_USE_ASSERT - // this tests is_compatible_ets - __TBB_STATIC_ASSERT( (internal::is_compatible_ets::type>::value), "is_compatible_ets fails" ); -#endif - // Initialize my_construct_callback first, so that it is valid even if rest of this routine throws an exception. - my_construct_callback = other.my_construct_callback->clone(); - __TBB_ASSERT(my_locals.size()==0,NULL); - my_locals.reserve(other.size()); - this->table_elementwise_copy( other, create_local_by_copy ); - } - - void internal_swap(enumerable_thread_specific& other) { - using std::swap; - __TBB_ASSERT( this!=&other, NULL ); - swap(my_construct_callback, other.my_construct_callback); - // concurrent_vector::swap() preserves storage space, - // so addresses to the vector kept in ETS hash table remain valid. - swap(my_locals, other.my_locals); - this->internal::ets_base::table_swap(other); - } - -#if __TBB_ETS_USE_CPP11 - template - void internal_move(enumerable_thread_specific&& other) { -#if TBB_USE_ASSERT - // this tests is_compatible_ets - __TBB_STATIC_ASSERT( (internal::is_compatible_ets::type>::value), "is_compatible_ets fails" ); -#endif - my_construct_callback = other.my_construct_callback; - other.my_construct_callback = NULL; - __TBB_ASSERT(my_locals.size()==0,NULL); - my_locals.reserve(other.size()); - this->table_elementwise_copy( other, create_local_by_move ); - } -#endif - - public: - - enumerable_thread_specific( const enumerable_thread_specific& other ) - : internal::ets_base() /* prevents GCC warnings with -Wextra */ - { - internal_copy(other); - } - - template - enumerable_thread_specific( const enumerable_thread_specific& other ) - { - internal_copy(other); - } - -#if __TBB_ETS_USE_CPP11 - enumerable_thread_specific( enumerable_thread_specific&& other ) : my_construct_callback() - { - internal_swap(other); - } - - template - enumerable_thread_specific( enumerable_thread_specific&& other ) : my_construct_callback() - { - internal_move(std::move(other)); - } -#endif - - enumerable_thread_specific& operator=( const enumerable_thread_specific& other ) - { - if( this != &other ) { - this->clear(); - my_construct_callback->destroy(); - internal_copy( other ); - } - return *this; - } - - template - enumerable_thread_specific& operator=( const enumerable_thread_specific& other ) - { - __TBB_ASSERT( static_cast(this)!=static_cast(&other), NULL ); // Objects of different types - this->clear(); - my_construct_callback->destroy(); - internal_copy(other); - return *this; - } - -#if __TBB_ETS_USE_CPP11 - enumerable_thread_specific& operator=( enumerable_thread_specific&& other ) - { - if( this != &other ) - internal_swap(other); - return *this; - } - - template - enumerable_thread_specific& operator=( enumerable_thread_specific&& other ) - { - __TBB_ASSERT( static_cast(this)!=static_cast(&other), NULL ); // Objects of different types - this->clear(); - my_construct_callback->destroy(); - internal_move(std::move(other)); - return *this; - } -#endif - - // combine_func_t has signature T(T,T) or T(const T&, const T&) - template - T combine(combine_func_t f_combine) { - if(begin() == end()) { - internal::ets_element location; - my_construct_callback->construct(location.value()); - return *location.value_committed(); - } - const_iterator ci = begin(); - T my_result = *ci; - while(++ci != end()) - my_result = f_combine( my_result, *ci ); - return my_result; - } - - // combine_func_t takes T by value or by [const] reference, and returns nothing - template - void combine_each(combine_func_t f_combine) { - for(iterator ci = begin(); ci != end(); ++ci) { - f_combine( *ci ); - } - } - - }; // enumerable_thread_specific - - template< typename Container > - class flattened2d { - - // This intermediate typedef is to address issues with VC7.1 compilers - typedef typename Container::value_type conval_type; - - public: - - //! Basic types - typedef typename conval_type::size_type size_type; - typedef typename conval_type::difference_type difference_type; - typedef typename conval_type::allocator_type allocator_type; - typedef typename conval_type::value_type value_type; - typedef typename conval_type::reference reference; - typedef typename conval_type::const_reference const_reference; - typedef typename conval_type::pointer pointer; - typedef typename conval_type::const_pointer const_pointer; - - typedef typename internal::segmented_iterator iterator; - typedef typename internal::segmented_iterator const_iterator; - - flattened2d( const Container &c, typename Container::const_iterator b, typename Container::const_iterator e ) : - my_container(const_cast(&c)), my_begin(b), my_end(e) { } - - explicit flattened2d( const Container &c ) : - my_container(const_cast(&c)), my_begin(c.begin()), my_end(c.end()) { } - - iterator begin() { return iterator(*my_container) = my_begin; } - iterator end() { return iterator(*my_container) = my_end; } - const_iterator begin() const { return const_iterator(*my_container) = my_begin; } - const_iterator end() const { return const_iterator(*my_container) = my_end; } - - size_type size() const { - size_type tot_size = 0; - for(typename Container::const_iterator i = my_begin; i != my_end; ++i) { - tot_size += i->size(); - } - return tot_size; - } - - private: - - Container *my_container; - typename Container::const_iterator my_begin; - typename Container::const_iterator my_end; - - }; - - template - flattened2d flatten2d(const Container &c, const typename Container::const_iterator b, const typename Container::const_iterator e) { - return flattened2d(c, b, e); - } - - template - flattened2d flatten2d(const Container &c) { - return flattened2d(c); - } - -} // interface6 - -namespace internal { -using interface6::internal::segmented_iterator; -} - -using interface6::enumerable_thread_specific; -using interface6::flattened2d; -using interface6::flatten2d; - -} // namespace tbb - -#endif +#include "../oneapi/tbb/enumerable_thread_specific.h" diff --git a/src/tbb/include/tbb/flow_graph.h b/src/tbb/include/tbb/flow_graph.h index fb839ac18..40da468fe 100644 --- a/src/tbb/include/tbb/flow_graph.h +++ b/src/tbb/include/tbb/flow_graph.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,3930 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - -*/ - -#ifndef __TBB_flow_graph_H -#define __TBB_flow_graph_H - -#include "tbb_stddef.h" -#include "atomic.h" -#include "spin_mutex.h" -#include "null_mutex.h" -#include "spin_rw_mutex.h" -#include "null_rw_mutex.h" -#include "task.h" -#include "cache_aligned_allocator.h" -#include "tbb_exception.h" -#include "internal/_template_helpers.h" -#include "internal/_aggregator_impl.h" -#include "tbb_profiling.h" -#include "task_arena.h" -#include "flow_graph_abstractions.h" - -#if __TBB_PREVIEW_ASYNC_MSG -#include // std::vector in internal::async_storage -#include // std::shared_ptr in async_msg -#endif - -#if __TBB_PREVIEW_STREAMING_NODE -// For streaming_node -#include // std::array -#include // std::unordered_map -#include // std::decay, std::true_type, std::false_type -#endif // __TBB_PREVIEW_STREAMING_NODE - -#if TBB_DEPRECATED_FLOW_ENQUEUE -#define FLOW_SPAWN(a) tbb::task::enqueue((a)) -#else -#define FLOW_SPAWN(a) tbb::task::spawn((a)) -#endif - -// use the VC10 or gcc version of tuple if it is available. -#if __TBB_CPP11_TUPLE_PRESENT - #include -namespace tbb { - namespace flow { - using std::tuple; - using std::tuple_size; - using std::tuple_element; - using std::get; - } -} -#else - #include "compat/tuple" -#endif - -#include -#include - -/** @file - \brief The graph related classes and functions - - There are some applications that best express dependencies as messages - passed between nodes in a graph. These messages may contain data or - simply act as signals that a predecessors has completed. The graph - class and its associated node classes can be used to express such - applications. */ -namespace tbb { -namespace flow { - -//! An enumeration the provides the two most common concurrency levels: unlimited and serial -enum concurrency { unlimited = 0, serial = 1 }; - -namespace interface10 { - -//! A generic null type -struct null_type {}; - -//! An empty class used for messages that mean "I'm done" -class continue_msg {}; - -//! Forward declaration section -template< typename T > class sender; -template< typename T > class receiver; -class continue_receiver; -template< typename T > class limiter_node; // needed for resetting decrementer -template< typename R, typename B > class run_and_put_task; - -namespace internal { - -template class successor_cache; -template class broadcast_cache; -template class round_robin_cache; -template class predecessor_cache; -template class reservable_predecessor_cache; - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES -// Holder of edges both for caches and for those nodes which do not have predecessor caches. -// C == receiver< ... > or sender< ... >, depending. -template -class edge_container { - -public: - typedef std::list > edge_list_type; - - void add_edge(C &s) { - built_edges.push_back(&s); - } - - void delete_edge(C &s) { - for (typename edge_list_type::iterator i = built_edges.begin(); i != built_edges.end(); ++i) { - if (*i == &s) { - (void)built_edges.erase(i); - return; // only remove one predecessor per request - } - } - } - - void copy_edges(edge_list_type &v) { - v = built_edges; - } - - size_t edge_count() { - return (size_t)(built_edges.size()); - } - - void clear() { - built_edges.clear(); - } - - // methods remove the statement from all predecessors/successors liste in the edge - // container. - template< typename S > void sender_extract(S &s); - template< typename R > void receiver_extract(R &r); - -private: - edge_list_type built_edges; -}; // class edge_container -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ - -} // namespace internal - -} // namespace interface10 -} // namespace flow -} // namespace tbb - -//! The graph class -#include "internal/_flow_graph_impl.h" - -namespace tbb { -namespace flow { -namespace interface10 { - -// enqueue left task if necessary. Returns the non-enqueued task if there is one. -static inline tbb::task *combine_tasks(graph& g, tbb::task * left, tbb::task * right) { - // if no RHS task, don't change left. - if (right == NULL) return left; - // right != NULL - if (left == NULL) return right; - if (left == SUCCESSFULLY_ENQUEUED) return right; - // left contains a task - if (right != SUCCESSFULLY_ENQUEUED) { - // both are valid tasks - internal::spawn_in_graph_arena(g, *left); - return right; - } - return left; -} - -#if __TBB_PREVIEW_ASYNC_MSG - -template < typename T > class async_msg; - -namespace internal { - -template < typename T > class async_storage; - -template< typename T, typename = void > -struct async_helpers { - typedef async_msg async_type; - typedef T filtered_type; - - static const bool is_async_type = false; - - static const void* to_void_ptr(const T& t) { - return static_cast(&t); - } - - static void* to_void_ptr(T& t) { - return static_cast(&t); - } - - static const T& from_void_ptr(const void* p) { - return *static_cast(p); - } - - static T& from_void_ptr(void* p) { - return *static_cast(p); - } - - static task* try_put_task_wrapper_impl(receiver* const this_recv, const void *p, bool is_async) { - if (is_async) { - // This (T) is NOT async and incoming 'A t' IS async - // Get data from async_msg - const async_msg& msg = async_helpers< async_msg >::from_void_ptr(p); - task* const new_task = msg.my_storage->subscribe(*this_recv, this_recv->graph_reference()); - // finalize() must be called after subscribe() because set() can be called in finalize() - // and 'this_recv' client must be subscribed by this moment - msg.finalize(); - return new_task; - } - else { - // Incoming 't' is NOT async - return this_recv->try_put_task(from_void_ptr(p)); - } - } -}; - -template< typename T > -struct async_helpers< T, typename std::enable_if< std::is_base_of, T>::value >::type > { - typedef T async_type; - typedef typename T::async_msg_data_type filtered_type; - - static const bool is_async_type = true; - - // Receiver-classes use const interfaces - static const void* to_void_ptr(const T& t) { - return static_cast(&static_cast&>(t)); - } - - static void* to_void_ptr(T& t) { - return static_cast(&static_cast&>(t)); - } - - // Sender-classes use non-const interfaces - static const T& from_void_ptr(const void* p) { - return *static_cast(static_cast*>(p)); - } - - static T& from_void_ptr(void* p) { - return *static_cast(static_cast*>(p)); - } - - // Used in receiver class - static task* try_put_task_wrapper_impl(receiver* const this_recv, const void *p, bool is_async) { - if (is_async) { - // Both are async - return this_recv->try_put_task(from_void_ptr(p)); - } - else { - // This (T) is async and incoming 'X t' is NOT async - // Create async_msg for X - const filtered_type& t = async_helpers::from_void_ptr(p); - const T msg(t); - return this_recv->try_put_task(msg); - } - } -}; - -class untyped_receiver; - -class untyped_sender { - template< typename, typename > friend class internal::predecessor_cache; - template< typename, typename > friend class internal::reservable_predecessor_cache; -public: - //! The successor type for this node - typedef untyped_receiver successor_type; - - virtual ~untyped_sender() {} - - // NOTE: Following part of PUBLIC section is copy-paste from original sender class - - // TODO: Prevent untyped successor registration - - //! Add a new successor to this node - virtual bool register_successor( successor_type &r ) = 0; - - //! Removes a successor from this node - virtual bool remove_successor( successor_type &r ) = 0; - - //! Releases the reserved item - virtual bool try_release( ) { return false; } - - //! Consumes the reserved item - virtual bool try_consume( ) { return false; } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - //! interface to record edges for traversal & deletion - typedef internal::edge_container built_successors_type; - typedef built_successors_type::edge_list_type successor_list_type; - virtual built_successors_type &built_successors() = 0; - virtual void internal_add_built_successor( successor_type & ) = 0; - virtual void internal_delete_built_successor( successor_type & ) = 0; - virtual void copy_successors( successor_list_type &) = 0; - virtual size_t successor_count() = 0; -#endif -protected: - //! Request an item from the sender - template< typename X > - bool try_get( X &t ) { - return try_get_wrapper( internal::async_helpers::to_void_ptr(t), internal::async_helpers::is_async_type ); - } - - //! Reserves an item in the sender - template< typename X > - bool try_reserve( X &t ) { - return try_reserve_wrapper( internal::async_helpers::to_void_ptr(t), internal::async_helpers::is_async_type ); - } - - virtual bool try_get_wrapper( void* p, bool is_async ) = 0; - virtual bool try_reserve_wrapper( void* p, bool is_async ) = 0; -}; - -class untyped_receiver { - template< typename, typename > friend class run_and_put_task; - template< typename > friend class limiter_node; - - template< typename, typename > friend class internal::broadcast_cache; - template< typename, typename > friend class internal::round_robin_cache; - template< typename, typename > friend class internal::successor_cache; - -#if __TBB_PREVIEW_OPENCL_NODE - template< typename, typename > friend class proxy_dependency_receiver; -#endif /* __TBB_PREVIEW_OPENCL_NODE */ -public: - //! The predecessor type for this node - typedef untyped_sender predecessor_type; - - //! Destructor - virtual ~untyped_receiver() {} - - //! Put an item to the receiver - template - bool try_put(const X& t) { - task *res = try_put_task(t); - if (!res) return false; - if (res != SUCCESSFULLY_ENQUEUED) internal::spawn_in_graph_arena(graph_reference(), *res); - return true; - } - - // NOTE: Following part of PUBLIC section is copy-paste from original receiver class - - // TODO: Prevent untyped predecessor registration - - //! Add a predecessor to the node - virtual bool register_predecessor( predecessor_type & ) { return false; } - - //! Remove a predecessor from the node - virtual bool remove_predecessor( predecessor_type & ) { return false; } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - typedef internal::edge_container built_predecessors_type; - typedef built_predecessors_type::edge_list_type predecessor_list_type; - virtual built_predecessors_type &built_predecessors() = 0; - virtual void internal_add_built_predecessor( predecessor_type & ) = 0; - virtual void internal_delete_built_predecessor( predecessor_type & ) = 0; - virtual void copy_predecessors( predecessor_list_type & ) = 0; - virtual size_t predecessor_count() = 0; -#endif -protected: - template - task *try_put_task(const X& t) { - return try_put_task_wrapper( internal::async_helpers::to_void_ptr(t), internal::async_helpers::is_async_type ); - } - - virtual task* try_put_task_wrapper( const void* p, bool is_async ) = 0; - - virtual graph& graph_reference() = 0; - - // NOTE: Following part of PROTECTED and PRIVATE sections is copy-paste from original receiver class - - //! put receiver back in initial state - virtual void reset_receiver(reset_flags f = rf_reset_protocol) = 0; - - virtual bool is_continue_receiver() { return false; } -}; - -} // namespace internal - -//! Pure virtual template class that defines a sender of messages of type T -template< typename T > -class sender : public internal::untyped_sender { -public: - //! The output type of this sender - typedef T output_type; - - typedef typename internal::async_helpers::filtered_type filtered_type; - - //! Request an item from the sender - virtual bool try_get( T & ) { return false; } - - //! Reserves an item in the sender - virtual bool try_reserve( T & ) { return false; } - -protected: - virtual bool try_get_wrapper( void* p, bool is_async ) __TBB_override { - // Both async OR both are NOT async - if ( internal::async_helpers::is_async_type == is_async ) { - return try_get( internal::async_helpers::from_void_ptr(p) ); - } - // Else: this (T) is async OR incoming 't' is async - __TBB_ASSERT(false, "async_msg interface does not support 'pull' protocol in try_get()"); - return false; - } - - virtual bool try_reserve_wrapper( void* p, bool is_async ) __TBB_override { - // Both async OR both are NOT async - if ( internal::async_helpers::is_async_type == is_async ) { - return try_reserve( internal::async_helpers::from_void_ptr(p) ); - } - // Else: this (T) is async OR incoming 't' is async - __TBB_ASSERT(false, "async_msg interface does not support 'pull' protocol in try_reserve()"); - return false; - } -}; // class sender - -//! Pure virtual template class that defines a receiver of messages of type T -template< typename T > -class receiver : public internal::untyped_receiver { - template< typename > friend class internal::async_storage; - template< typename, typename > friend struct internal::async_helpers; -public: - //! The input type of this receiver - typedef T input_type; - - typedef typename internal::async_helpers::filtered_type filtered_type; - - //! Put an item to the receiver - bool try_put( const typename internal::async_helpers::filtered_type& t ) { - return internal::untyped_receiver::try_put(t); - } - - bool try_put( const typename internal::async_helpers::async_type& t ) { - return internal::untyped_receiver::try_put(t); - } - -protected: - virtual task* try_put_task_wrapper( const void *p, bool is_async ) __TBB_override { - return internal::async_helpers::try_put_task_wrapper_impl(this, p, is_async); - } - - //! Put item to successor; return task to run the successor if possible. - virtual task *try_put_task(const T& t) = 0; - -}; // class receiver - -#else // __TBB_PREVIEW_ASYNC_MSG - -//! Pure virtual template class that defines a sender of messages of type T -template< typename T > -class sender { -public: - //! The output type of this sender - typedef T output_type; - - //! The successor type for this node - typedef receiver successor_type; - - virtual ~sender() {} - - // NOTE: Following part of PUBLIC section is partly copy-pasted in sender under #if __TBB_PREVIEW_ASYNC_MSG - - //! Add a new successor to this node - virtual bool register_successor( successor_type &r ) = 0; - - //! Removes a successor from this node - virtual bool remove_successor( successor_type &r ) = 0; - - //! Request an item from the sender - virtual bool try_get( T & ) { return false; } - - //! Reserves an item in the sender - virtual bool try_reserve( T & ) { return false; } - - //! Releases the reserved item - virtual bool try_release( ) { return false; } - - //! Consumes the reserved item - virtual bool try_consume( ) { return false; } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - //! interface to record edges for traversal & deletion - typedef typename internal::edge_container built_successors_type; - typedef typename built_successors_type::edge_list_type successor_list_type; - virtual built_successors_type &built_successors() = 0; - virtual void internal_add_built_successor( successor_type & ) = 0; - virtual void internal_delete_built_successor( successor_type & ) = 0; - virtual void copy_successors( successor_list_type &) = 0; - virtual size_t successor_count() = 0; -#endif -}; // class sender - -//! Pure virtual template class that defines a receiver of messages of type T -template< typename T > -class receiver { -public: - //! The input type of this receiver - typedef T input_type; - - //! The predecessor type for this node - typedef sender predecessor_type; - - //! Destructor - virtual ~receiver() {} - - //! Put an item to the receiver - bool try_put( const T& t ) { - task *res = try_put_task(t); - if (!res) return false; - if (res != SUCCESSFULLY_ENQUEUED) internal::spawn_in_graph_arena(graph_reference(), *res); - return true; - } - - //! put item to successor; return task to run the successor if possible. -protected: - template< typename R, typename B > friend class run_and_put_task; - template< typename X, typename Y > friend class internal::broadcast_cache; - template< typename X, typename Y > friend class internal::round_robin_cache; - virtual task *try_put_task(const T& t) = 0; - virtual graph& graph_reference() = 0; -public: - // NOTE: Following part of PUBLIC and PROTECTED sections is copy-pasted in receiver under #if __TBB_PREVIEW_ASYNC_MSG - - //! Add a predecessor to the node - virtual bool register_predecessor( predecessor_type & ) { return false; } - - //! Remove a predecessor from the node - virtual bool remove_predecessor( predecessor_type & ) { return false; } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - typedef typename internal::edge_container built_predecessors_type; - typedef typename built_predecessors_type::edge_list_type predecessor_list_type; - virtual built_predecessors_type &built_predecessors() = 0; - virtual void internal_add_built_predecessor( predecessor_type & ) = 0; - virtual void internal_delete_built_predecessor( predecessor_type & ) = 0; - virtual void copy_predecessors( predecessor_list_type & ) = 0; - virtual size_t predecessor_count() = 0; -#endif - -protected: - //! put receiver back in initial state - template friend class limiter_node; - virtual void reset_receiver(reset_flags f = rf_reset_protocol) = 0; - - template friend class internal::successor_cache; - virtual bool is_continue_receiver() { return false; } - -#if __TBB_PREVIEW_OPENCL_NODE - template< typename, typename > friend class proxy_dependency_receiver; -#endif /* __TBB_PREVIEW_OPENCL_NODE */ -}; // class receiver - -#endif // __TBB_PREVIEW_ASYNC_MSG - -//! Base class for receivers of completion messages -/** These receivers automatically reset, but cannot be explicitly waited on */ -class continue_receiver : public receiver< continue_msg > { -public: - - //! The input type - typedef continue_msg input_type; - - //! The predecessor type for this node - typedef receiver::predecessor_type predecessor_type; - - //! Constructor - explicit continue_receiver( int number_of_predecessors = 0 ) { - my_predecessor_count = my_initial_predecessor_count = number_of_predecessors; - my_current_count = 0; - } - - //! Copy constructor - continue_receiver( const continue_receiver& src ) : receiver() { - my_predecessor_count = my_initial_predecessor_count = src.my_initial_predecessor_count; - my_current_count = 0; - } - - //! Increments the trigger threshold - bool register_predecessor( predecessor_type & ) __TBB_override { - spin_mutex::scoped_lock l(my_mutex); - ++my_predecessor_count; - return true; - } - - //! Decrements the trigger threshold - /** Does not check to see if the removal of the predecessor now makes the current count - exceed the new threshold. So removing a predecessor while the graph is active can cause - unexpected results. */ - bool remove_predecessor( predecessor_type & ) __TBB_override { - spin_mutex::scoped_lock l(my_mutex); - --my_predecessor_count; - return true; - } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - typedef internal::edge_container built_predecessors_type; - typedef built_predecessors_type::edge_list_type predecessor_list_type; - built_predecessors_type &built_predecessors() __TBB_override { return my_built_predecessors; } - - void internal_add_built_predecessor( predecessor_type &s) __TBB_override { - spin_mutex::scoped_lock l(my_mutex); - my_built_predecessors.add_edge( s ); - } - - void internal_delete_built_predecessor( predecessor_type &s) __TBB_override { - spin_mutex::scoped_lock l(my_mutex); - my_built_predecessors.delete_edge(s); - } - - void copy_predecessors( predecessor_list_type &v) __TBB_override { - spin_mutex::scoped_lock l(my_mutex); - my_built_predecessors.copy_edges(v); - } - - size_t predecessor_count() __TBB_override { - spin_mutex::scoped_lock l(my_mutex); - return my_built_predecessors.edge_count(); - } - -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ - -protected: - template< typename R, typename B > friend class run_and_put_task; - template friend class internal::broadcast_cache; - template friend class internal::round_robin_cache; - // execute body is supposed to be too small to create a task for. - task *try_put_task( const input_type & ) __TBB_override { - { - spin_mutex::scoped_lock l(my_mutex); - if ( ++my_current_count < my_predecessor_count ) - return SUCCESSFULLY_ENQUEUED; - else - my_current_count = 0; - } - task * res = execute(); - return res? res : SUCCESSFULLY_ENQUEUED; - } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - // continue_receiver must contain its own built_predecessors because it does - // not have a node_cache. - built_predecessors_type my_built_predecessors; -#endif - spin_mutex my_mutex; - int my_predecessor_count; - int my_current_count; - int my_initial_predecessor_count; - // the friend declaration in the base class did not eliminate the "protected class" - // error in gcc 4.1.2 - template friend class limiter_node; - - void reset_receiver( reset_flags f ) __TBB_override { - my_current_count = 0; - if (f & rf_clear_edges) { -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - my_built_predecessors.clear(); -#endif - my_predecessor_count = my_initial_predecessor_count; - } - } - - //! Does whatever should happen when the threshold is reached - /** This should be very fast or else spawn a task. This is - called while the sender is blocked in the try_put(). */ - virtual task * execute() = 0; - template friend class internal::successor_cache; - bool is_continue_receiver() __TBB_override { return true; } - -}; // class continue_receiver - -} // interfaceX - -#if __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING - template - K key_from_message( const T &t ) { - return t.key(); - } -#endif /* __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING */ - - using interface10::sender; - using interface10::receiver; - using interface10::continue_receiver; -} // flow -} // tbb - -#include "internal/_flow_graph_trace_impl.h" -#include "internal/_tbb_hash_compare_impl.h" - -namespace tbb { -namespace flow { -namespace interface10 { - -#include "internal/_flow_graph_body_impl.h" -#include "internal/_flow_graph_cache_impl.h" -#include "internal/_flow_graph_types_impl.h" -#if __TBB_PREVIEW_ASYNC_MSG -#include "internal/_flow_graph_async_msg_impl.h" -#endif -using namespace internal::graph_policy_namespace; - -template -graph_iterator::graph_iterator(C *g, bool begin) : my_graph(g), current_node(NULL) -{ - if (begin) current_node = my_graph->my_nodes; - //else it is an end iterator by default -} - -template -typename graph_iterator::reference graph_iterator::operator*() const { - __TBB_ASSERT(current_node, "graph_iterator at end"); - return *operator->(); -} - -template -typename graph_iterator::pointer graph_iterator::operator->() const { - return current_node; -} - -template -void graph_iterator::internal_forward() { - if (current_node) current_node = current_node->next; -} - -//! Constructs a graph with isolated task_group_context -inline graph::graph() : my_nodes(NULL), my_nodes_last(NULL), my_task_arena(NULL) { - prepare_task_arena(); - own_context = true; - cancelled = false; - caught_exception = false; - my_context = new task_group_context(); - my_root_task = (new (task::allocate_root(*my_context)) empty_task); - my_root_task->set_ref_count(1); - tbb::internal::fgt_graph(this); - my_is_active = true; -} - -inline graph::graph(task_group_context& use_this_context) : - my_context(&use_this_context), my_nodes(NULL), my_nodes_last(NULL), my_task_arena(NULL) { - prepare_task_arena(); - own_context = false; - my_root_task = (new (task::allocate_root(*my_context)) empty_task); - my_root_task->set_ref_count(1); - tbb::internal::fgt_graph(this); - my_is_active = true; -} - -inline void graph::reserve_wait() { - if (my_root_task) { - my_root_task->increment_ref_count(); - tbb::internal::fgt_reserve_wait(this); - } -} - -inline void graph::release_wait() { - if (my_root_task) { - tbb::internal::fgt_release_wait(this); - my_root_task->decrement_ref_count(); - } -} - -inline void graph::register_node(graph_node *n) { - n->next = NULL; - { - spin_mutex::scoped_lock lock(nodelist_mutex); - n->prev = my_nodes_last; - if (my_nodes_last) my_nodes_last->next = n; - my_nodes_last = n; - if (!my_nodes) my_nodes = n; - } -} - -inline void graph::remove_node(graph_node *n) { - { - spin_mutex::scoped_lock lock(nodelist_mutex); - __TBB_ASSERT(my_nodes && my_nodes_last, "graph::remove_node: Error: no registered nodes"); - if (n->prev) n->prev->next = n->next; - if (n->next) n->next->prev = n->prev; - if (my_nodes_last == n) my_nodes_last = n->prev; - if (my_nodes == n) my_nodes = n->next; - } - n->prev = n->next = NULL; -} - -inline void graph::reset( reset_flags f ) { - // reset context - internal::deactivate_graph(*this); - - if(my_context) my_context->reset(); - cancelled = false; - caught_exception = false; - // reset all the nodes comprising the graph - for(iterator ii = begin(); ii != end(); ++ii) { - graph_node *my_p = &(*ii); - my_p->reset_node(f); - } - // Reattach the arena. Might be useful to run the graph in a particular task_arena - // while not limiting graph lifetime to a single task_arena::execute() call. - prepare_task_arena( /*reinit=*/true ); - internal::activate_graph(*this); - // now spawn the tasks necessary to start the graph - for(task_list_type::iterator rti = my_reset_task_list.begin(); rti != my_reset_task_list.end(); ++rti) { - my_task_arena->execute(graph::spawn_functor(*(*rti))); - } - my_reset_task_list.clear(); -} - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE -inline void graph::set_name(const char *name) { - tbb::internal::fgt_graph_desc(this, name); -} -#endif - -#include "internal/_flow_graph_node_impl.h" - -//! An executable node that acts as a source, i.e. it has no predecessors -template < typename Output > -class source_node : public graph_node, public sender< Output > { -public: - //! The type of the output message, which is complete - typedef Output output_type; - - //! The type of successors of this node - typedef typename sender::successor_type successor_type; - - //Source node has no input type - typedef null_type input_type; - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - typedef typename sender::built_successors_type built_successors_type; - typedef typename sender::successor_list_type successor_list_type; -#endif - - //! Constructor for a node with a successor - template< typename Body > - source_node( graph &g, Body body, bool is_active = true ) - : graph_node(g), my_active(is_active), init_my_active(is_active), - my_body( new internal::source_body_leaf< output_type, Body>(body) ), - my_init_body( new internal::source_body_leaf< output_type, Body>(body) ), - my_reserved(false), my_has_cached_item(false) - { - my_successors.set_owner(this); - tbb::internal::fgt_node_with_body( tbb::internal::FLOW_SOURCE_NODE, &this->my_graph, - static_cast *>(this), this->my_body ); - } - - //! Copy constructor - source_node( const source_node& src ) : - graph_node(src.my_graph), sender(), - my_active(src.init_my_active), - init_my_active(src.init_my_active), my_body( src.my_init_body->clone() ), my_init_body(src.my_init_body->clone() ), - my_reserved(false), my_has_cached_item(false) - { - my_successors.set_owner(this); - tbb::internal::fgt_node_with_body( tbb::internal::FLOW_SOURCE_NODE, &this->my_graph, - static_cast *>(this), this->my_body ); - } - - //! The destructor - ~source_node() { delete my_body; delete my_init_body; } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif - - //! Add a new successor to this node - bool register_successor( successor_type &r ) __TBB_override { - spin_mutex::scoped_lock lock(my_mutex); - my_successors.register_successor(r); - if ( my_active ) - spawn_put(); - return true; - } - - //! Removes a successor from this node - bool remove_successor( successor_type &r ) __TBB_override { - spin_mutex::scoped_lock lock(my_mutex); - my_successors.remove_successor(r); - return true; - } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - - built_successors_type &built_successors() __TBB_override { return my_successors.built_successors(); } - - void internal_add_built_successor( successor_type &r) __TBB_override { - spin_mutex::scoped_lock lock(my_mutex); - my_successors.internal_add_built_successor(r); - } - - void internal_delete_built_successor( successor_type &r) __TBB_override { - spin_mutex::scoped_lock lock(my_mutex); - my_successors.internal_delete_built_successor(r); - } - - size_t successor_count() __TBB_override { - spin_mutex::scoped_lock lock(my_mutex); - return my_successors.successor_count(); - } - - void copy_successors(successor_list_type &v) __TBB_override { - spin_mutex::scoped_lock l(my_mutex); - my_successors.copy_successors(v); - } -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ - - //! Request an item from the node - bool try_get( output_type &v ) __TBB_override { - spin_mutex::scoped_lock lock(my_mutex); - if ( my_reserved ) - return false; - - if ( my_has_cached_item ) { - v = my_cached_item; - my_has_cached_item = false; - return true; - } - // we've been asked to provide an item, but we have none. enqueue a task to - // provide one. - spawn_put(); - return false; - } - - //! Reserves an item. - bool try_reserve( output_type &v ) __TBB_override { - spin_mutex::scoped_lock lock(my_mutex); - if ( my_reserved ) { - return false; - } - - if ( my_has_cached_item ) { - v = my_cached_item; - my_reserved = true; - return true; - } else { - return false; - } - } - - //! Release a reserved item. - /** true = item has been released and so remains in sender, dest must request or reserve future items */ - bool try_release( ) __TBB_override { - spin_mutex::scoped_lock lock(my_mutex); - __TBB_ASSERT( my_reserved && my_has_cached_item, "releasing non-existent reservation" ); - my_reserved = false; - if(!my_successors.empty()) - spawn_put(); - return true; - } - - //! Consumes a reserved item - bool try_consume( ) __TBB_override { - spin_mutex::scoped_lock lock(my_mutex); - __TBB_ASSERT( my_reserved && my_has_cached_item, "consuming non-existent reservation" ); - my_reserved = false; - my_has_cached_item = false; - if ( !my_successors.empty() ) { - spawn_put(); - } - return true; - } - - //! Activates a node that was created in the inactive state - void activate() { - spin_mutex::scoped_lock lock(my_mutex); - my_active = true; - if (!my_successors.empty()) - spawn_put(); - } - - template - Body copy_function_object() { - internal::source_body &body_ref = *this->my_body; - return dynamic_cast< internal::source_body_leaf & >(body_ref).get_body(); - } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - void extract( ) __TBB_override { - my_successors.built_successors().sender_extract(*this); // removes "my_owner" == this from each successor - my_active = init_my_active; - my_reserved = false; - if(my_has_cached_item) my_has_cached_item = false; - } -#endif - -protected: - - //! resets the source_node to its initial state - void reset_node( reset_flags f) __TBB_override { - my_active = init_my_active; - my_reserved =false; - if(my_has_cached_item) { - my_has_cached_item = false; - } - if(f & rf_clear_edges) my_successors.clear(); - if(f & rf_reset_bodies) { - internal::source_body *tmp = my_init_body->clone(); - delete my_body; - my_body = tmp; - } - if(my_active) - internal::add_task_to_graph_reset_list(this->my_graph, create_put_task()); - } - -private: - spin_mutex my_mutex; - bool my_active; - bool init_my_active; - internal::source_body *my_body; - internal::source_body *my_init_body; - internal::broadcast_cache< output_type > my_successors; - bool my_reserved; - bool my_has_cached_item; - output_type my_cached_item; - - // used by apply_body_bypass, can invoke body of node. - bool try_reserve_apply_body(output_type &v) { - spin_mutex::scoped_lock lock(my_mutex); - if ( my_reserved ) { - return false; - } - if ( !my_has_cached_item ) { - tbb::internal::fgt_begin_body( my_body ); - bool r = (*my_body)(my_cached_item); - tbb::internal::fgt_end_body( my_body ); - if (r) { - my_has_cached_item = true; - } - } - if ( my_has_cached_item ) { - v = my_cached_item; - my_reserved = true; - return true; - } else { - return false; - } - } - - // when resetting, and if the source_node was created with my_active == true, then - // when we reset the node we must store a task to run the node, and spawn it only - // after the reset is complete and is_active() is again true. This is why we don't - // test for is_active() here. - task* create_put_task() { - return ( new ( task::allocate_additional_child_of( *(this->my_graph.root_task()) ) ) - internal:: source_task_bypass < source_node< output_type > >( *this ) ); - } - - //! Spawns a task that applies the body - void spawn_put( ) { - if(internal::is_graph_active(this->my_graph)) { - internal::spawn_in_graph_arena(this->my_graph, *create_put_task()); - } - } - - friend class internal::source_task_bypass< source_node< output_type > >; - //! Applies the body. Returning SUCCESSFULLY_ENQUEUED okay; forward_task_bypass will handle it. - task * apply_body_bypass( ) { - output_type v; - if ( !try_reserve_apply_body(v) ) - return NULL; - - task *last_task = my_successors.try_put_task(v); - if ( last_task ) - try_consume(); - else - try_release(); - return last_task; - } -}; // class source_node - -template -struct allocate_buffer { - static const bool value = false; -}; - -template<> -struct allocate_buffer { - static const bool value = true; -}; - -//! Implements a function node that supports Input -> Output -template < typename Input, typename Output = continue_msg, typename Policy = queueing, typename Allocator=cache_aligned_allocator > -class function_node : public graph_node, public internal::function_input, public internal::function_output { -public: - typedef Input input_type; - typedef Output output_type; - typedef internal::function_input fInput_type; - typedef internal::function_input_queue input_queue_type; - typedef internal::function_output fOutput_type; - typedef typename fInput_type::predecessor_type predecessor_type; - typedef typename fOutput_type::successor_type successor_type; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - typedef typename fInput_type::predecessor_list_type predecessor_list_type; - typedef typename fOutput_type::successor_list_type successor_list_type; -#endif - using fInput_type::my_predecessors; - - //! Constructor - // input_queue_type is allocated here, but destroyed in the function_input_base. - // TODO: pass the graph_buffer_policy to the function_input_base so it can all - // be done in one place. This would be an interface-breaking change. - template< typename Body > - function_node( graph &g, size_t concurrency, Body body ) : - graph_node(g), fInput_type(g, concurrency, body, allocate_buffer::value ? - new input_queue_type( ) : NULL ) { - tbb::internal::fgt_node_with_body( tbb::internal::FLOW_FUNCTION_NODE, &this->my_graph, - static_cast *>(this), static_cast *>(this), this->my_body ); - } - - //! Copy constructor - function_node( const function_node& src ) : - graph_node(src.my_graph), - fInput_type(src, allocate_buffer::value ? new input_queue_type : NULL), - fOutput_type() { - tbb::internal::fgt_node_with_body( tbb::internal::FLOW_FUNCTION_NODE, &this->my_graph, - static_cast *>(this), static_cast *>(this), this->my_body ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - void extract( ) __TBB_override { - my_predecessors.built_predecessors().receiver_extract(*this); - successors().built_successors().sender_extract(*this); - } -#endif - -protected: - template< typename R, typename B > friend class run_and_put_task; - template friend class internal::broadcast_cache; - template friend class internal::round_robin_cache; - using fInput_type::try_put_task; - - internal::broadcast_cache &successors () __TBB_override { return fOutput_type::my_successors; } - - void reset_node(reset_flags f) __TBB_override { - fInput_type::reset_function_input(f); - // TODO: use clear() instead. - if(f & rf_clear_edges) { - successors().clear(); - my_predecessors.clear(); - } - __TBB_ASSERT(!(f & rf_clear_edges) || successors().empty(), "function_node successors not empty"); - __TBB_ASSERT(this->my_predecessors.empty(), "function_node predecessors not empty"); - } - -}; // class function_node - -//! implements a function node that supports Input -> (set of outputs) -// Output is a tuple of output types. -template < typename Input, typename Output, typename Policy = queueing, typename Allocator=cache_aligned_allocator > -class multifunction_node : - public graph_node, - public internal::multifunction_input - < - Input, - typename internal::wrap_tuple_elements< - tbb::flow::tuple_size::value, // #elements in tuple - internal::multifunction_output, // wrap this around each element - Output // the tuple providing the types - >::type, - Allocator - > { -protected: - static const int N = tbb::flow::tuple_size::value; -public: - typedef Input input_type; - typedef null_type output_type; - typedef typename internal::wrap_tuple_elements::type output_ports_type; - typedef internal::multifunction_input fInput_type; - typedef internal::function_input_queue input_queue_type; -private: - typedef typename internal::multifunction_input base_type; - using fInput_type::my_predecessors; -public: - template - multifunction_node( graph &g, size_t concurrency, Body body ) : - graph_node(g), base_type(g,concurrency, body, allocate_buffer::value ? new input_queue_type : NULL) { - tbb::internal::fgt_multioutput_node_with_body( tbb::internal::FLOW_MULTIFUNCTION_NODE, - &this->my_graph, static_cast *>(this), - this->output_ports(), this->my_body ); - } - - multifunction_node( const multifunction_node &other) : - graph_node(other.my_graph), base_type(other, allocate_buffer::value ? new input_queue_type : NULL) { - tbb::internal::fgt_multioutput_node_with_body( tbb::internal::FLOW_MULTIFUNCTION_NODE, - &this->my_graph, static_cast *>(this), - this->output_ports(), this->my_body ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_multioutput_node_desc( this, name ); - } -#endif - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - void extract( ) __TBB_override { - my_predecessors.built_predecessors().receiver_extract(*this); - base_type::extract(); - } -#endif - // all the guts are in multifunction_input... -protected: - void reset_node(reset_flags f) __TBB_override { base_type::reset(f); } -}; // multifunction_node - -//! split_node: accepts a tuple as input, forwards each element of the tuple to its -// successors. The node has unlimited concurrency, so it does not reject inputs. -template > -class split_node : public graph_node, public receiver { - static const int N = tbb::flow::tuple_size::value; - typedef receiver base_type; -public: - typedef TupleType input_type; - typedef Allocator allocator_type; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - typedef typename base_type::predecessor_type predecessor_type; - typedef typename base_type::predecessor_list_type predecessor_list_type; - typedef internal::predecessor_cache predecessor_cache_type; - typedef typename predecessor_cache_type::built_predecessors_type built_predecessors_type; -#endif - - typedef typename internal::wrap_tuple_elements< - N, // #elements in tuple - internal::multifunction_output, // wrap this around each element - TupleType // the tuple providing the types - >::type output_ports_type; - - explicit split_node(graph &g) : graph_node(g) - { - tbb::internal::fgt_multioutput_node(tbb::internal::FLOW_SPLIT_NODE, &this->my_graph, - static_cast *>(this), this->output_ports()); - } - split_node( const split_node & other) : graph_node(other.my_graph), base_type(other) - { - tbb::internal::fgt_multioutput_node(tbb::internal::FLOW_SPLIT_NODE, &this->my_graph, - static_cast *>(this), this->output_ports()); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_multioutput_node_desc( this, name ); - } -#endif - - output_ports_type &output_ports() { return my_output_ports; } - -protected: - task *try_put_task(const TupleType& t) __TBB_override { - // Sending split messages in parallel is not justified, as overheads would prevail. - // Also, we do not have successors here. So we just tell the task returned here is successful. - return internal::emit_element::emit_this(this->my_graph, t, output_ports()); - } - void reset_node(reset_flags f) __TBB_override { - if (f & rf_clear_edges) - internal::clear_element::clear_this(my_output_ports); - - __TBB_ASSERT(!(f & rf_clear_edges) || internal::clear_element::this_empty(my_output_ports), "split_node reset failed"); - } - void reset_receiver(reset_flags /*f*/) __TBB_override {} - graph& graph_reference() __TBB_override { - return my_graph; - } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES -private: //! split_node doesn't use this "predecessors" functionality; so, we have "dummies" here; - void extract() __TBB_override {} - - //! Adds to list of predecessors added by make_edge - void internal_add_built_predecessor(predecessor_type&) __TBB_override {} - - //! removes from to list of predecessors (used by remove_edge) - void internal_delete_built_predecessor(predecessor_type&) __TBB_override {} - - size_t predecessor_count() __TBB_override { return 0; } - - void copy_predecessors(predecessor_list_type&) __TBB_override {} - - built_predecessors_type &built_predecessors() __TBB_override { return my_predessors; } - - //! dummy member - built_predecessors_type my_predessors; -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ - -private: - output_ports_type my_output_ports; -}; - -//! Implements an executable node that supports continue_msg -> Output -template -class continue_node : public graph_node, public internal::continue_input, public internal::function_output { -public: - typedef continue_msg input_type; - typedef Output output_type; - typedef internal::continue_input fInput_type; - typedef internal::function_output fOutput_type; - typedef typename fInput_type::predecessor_type predecessor_type; - typedef typename fOutput_type::successor_type successor_type; - - //! Constructor for executable node with continue_msg -> Output - template - continue_node( graph &g, Body body ) : - graph_node(g), internal::continue_input( g, body ) { - tbb::internal::fgt_node_with_body( tbb::internal::FLOW_CONTINUE_NODE, &this->my_graph, - static_cast *>(this), - static_cast *>(this), this->my_body ); - } - - - //! Constructor for executable node with continue_msg -> Output - template - continue_node( graph &g, int number_of_predecessors, Body body ) : - graph_node(g), internal::continue_input( g, number_of_predecessors, body ) { - tbb::internal::fgt_node_with_body( tbb::internal::FLOW_CONTINUE_NODE, &this->my_graph, - static_cast *>(this), - static_cast *>(this), this->my_body ); - } - - //! Copy constructor - continue_node( const continue_node& src ) : - graph_node(src.my_graph), internal::continue_input(src), - internal::function_output() { - tbb::internal::fgt_node_with_body( tbb::internal::FLOW_CONTINUE_NODE, &this->my_graph, - static_cast *>(this), - static_cast *>(this), this->my_body ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - void extract() __TBB_override { - fInput_type::my_built_predecessors.receiver_extract(*this); - successors().built_successors().sender_extract(*this); - } -#endif - -protected: - template< typename R, typename B > friend class run_and_put_task; - template friend class internal::broadcast_cache; - template friend class internal::round_robin_cache; - using fInput_type::try_put_task; - internal::broadcast_cache &successors () __TBB_override { return fOutput_type::my_successors; } - - void reset_node(reset_flags f) __TBB_override { - fInput_type::reset_receiver(f); - if(f & rf_clear_edges)successors().clear(); - __TBB_ASSERT(!(f & rf_clear_edges) || successors().empty(), "continue_node not reset"); - } -}; // continue_node - -template< typename T > -class overwrite_node : public graph_node, public receiver, public sender { -public: - typedef T input_type; - typedef T output_type; - typedef typename receiver::predecessor_type predecessor_type; - typedef typename sender::successor_type successor_type; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - typedef typename receiver::built_predecessors_type built_predecessors_type; - typedef typename sender::built_successors_type built_successors_type; - typedef typename receiver::predecessor_list_type predecessor_list_type; - typedef typename sender::successor_list_type successor_list_type; -#endif - - explicit overwrite_node(graph &g) : graph_node(g), my_buffer_is_valid(false) { - my_successors.set_owner( this ); - tbb::internal::fgt_node( tbb::internal::FLOW_OVERWRITE_NODE, &this->my_graph, - static_cast *>(this), static_cast *>(this) ); - } - - //! Copy constructor; doesn't take anything from src; default won't work - overwrite_node( const overwrite_node& src ) : - graph_node(src.my_graph), receiver(), sender(), my_buffer_is_valid(false) - { - my_successors.set_owner( this ); - tbb::internal::fgt_node( tbb::internal::FLOW_OVERWRITE_NODE, &this->my_graph, - static_cast *>(this), static_cast *>(this) ); - } - - ~overwrite_node() {} - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif - - bool register_successor( successor_type &s ) __TBB_override { - spin_mutex::scoped_lock l( my_mutex ); - if (my_buffer_is_valid && internal::is_graph_active( my_graph )) { - // We have a valid value that must be forwarded immediately. - bool ret = s.try_put( my_buffer ); -#if TBB_PREVIEW_RESERVABLE_OVERWRITE_NODE - if ( ret ) { - // We add the successor that accepted our put - my_successors.register_successor( s ); - } else { - // In case of reservation a race between the moment of reservation and register_successor can appear, - // because failed reserve does not mean that register_successor is not ready to put a message immediately. - // We have some sort of infinite loop: reserving node tries to set pull state for the edge, - // but overwrite_node tries to return push state back. That is why we have to break this loop with task creation. - task *rtask = new ( task::allocate_additional_child_of( *( my_graph.root_task() ) ) ) - register_predecessor_task( *this, s ); - internal::spawn_in_graph_arena( my_graph, *rtask ); - } -#else - if ( ret || !s.register_predecessor( *this ) ) { - // We add the successor: it accepted our put or it rejected it but won't let us become a predecessor - my_successors.register_successor( s ); - } else { - // We don't add the successor: it rejected our put and we became its predecessor instead - return false; - } -#endif - } else { - // No valid value yet, just add as successor - my_successors.register_successor( s ); - } - return true; - } - - bool remove_successor( successor_type &s ) __TBB_override { - spin_mutex::scoped_lock l( my_mutex ); - my_successors.remove_successor(s); - return true; - } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - built_predecessors_type &built_predecessors() __TBB_override { return my_built_predecessors; } - built_successors_type &built_successors() __TBB_override { return my_successors.built_successors(); } - - void internal_add_built_successor( successor_type &s) __TBB_override { - spin_mutex::scoped_lock l( my_mutex ); - my_successors.internal_add_built_successor(s); - } - - void internal_delete_built_successor( successor_type &s) __TBB_override { - spin_mutex::scoped_lock l( my_mutex ); - my_successors.internal_delete_built_successor(s); - } - - size_t successor_count() __TBB_override { - spin_mutex::scoped_lock l( my_mutex ); - return my_successors.successor_count(); - } - - void copy_successors(successor_list_type &v) __TBB_override { - spin_mutex::scoped_lock l( my_mutex ); - my_successors.copy_successors(v); - } - - void internal_add_built_predecessor( predecessor_type &p) __TBB_override { - spin_mutex::scoped_lock l( my_mutex ); - my_built_predecessors.add_edge(p); - } - - void internal_delete_built_predecessor( predecessor_type &p) __TBB_override { - spin_mutex::scoped_lock l( my_mutex ); - my_built_predecessors.delete_edge(p); - } - - size_t predecessor_count() __TBB_override { - spin_mutex::scoped_lock l( my_mutex ); - return my_built_predecessors.edge_count(); - } - - void copy_predecessors( predecessor_list_type &v ) __TBB_override { - spin_mutex::scoped_lock l( my_mutex ); - my_built_predecessors.copy_edges(v); - } - - void extract() __TBB_override { - my_buffer_is_valid = false; - built_successors().sender_extract(*this); - built_predecessors().receiver_extract(*this); - } - -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ - - bool try_get( input_type &v ) __TBB_override { - spin_mutex::scoped_lock l( my_mutex ); - if ( my_buffer_is_valid ) { - v = my_buffer; - return true; - } - return false; - } - -#if TBB_PREVIEW_RESERVABLE_OVERWRITE_NODE - //! Reserves an item - bool try_reserve( T &v ) __TBB_override { - return try_get(v); - } - - //! Releases the reserved item - bool try_release() __TBB_override { return true; } - - //! Consumes the reserved item - bool try_consume() __TBB_override { return true; } -#endif - - bool is_valid() { - spin_mutex::scoped_lock l( my_mutex ); - return my_buffer_is_valid; - } - - void clear() { - spin_mutex::scoped_lock l( my_mutex ); - my_buffer_is_valid = false; - } - -protected: - - template< typename R, typename B > friend class run_and_put_task; - template friend class internal::broadcast_cache; - template friend class internal::round_robin_cache; - task * try_put_task( const input_type &v ) __TBB_override { - spin_mutex::scoped_lock l( my_mutex ); - return try_put_task_impl(v); - } - - task * try_put_task_impl(const input_type &v) { - my_buffer = v; - my_buffer_is_valid = true; - task * rtask = my_successors.try_put_task(v); - if (!rtask) rtask = SUCCESSFULLY_ENQUEUED; - return rtask; - } - - graph& graph_reference() __TBB_override { - return my_graph; - } - -#if TBB_PREVIEW_RESERVABLE_OVERWRITE_NODE - //! Breaks an infinite loop between the node reservation and register_successor call - struct register_predecessor_task : public task { - register_predecessor_task(sender& owner, receiver& succ) : - o(owner), s(succ) {}; - - tbb::task* execute() __TBB_override { - if (!s.register_predecessor(o)) { - o.register_successor(s); - } - return NULL; - } - - sender& o; - receiver& s; - }; -#endif - - spin_mutex my_mutex; - internal::broadcast_cache< input_type, null_rw_mutex > my_successors; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - internal::edge_container my_built_predecessors; -#endif - input_type my_buffer; - bool my_buffer_is_valid; - void reset_receiver(reset_flags /*f*/) __TBB_override {} - - void reset_node( reset_flags f) __TBB_override { - my_buffer_is_valid = false; - if (f&rf_clear_edges) { - my_successors.clear(); - } - } -}; // overwrite_node - -template< typename T > -class write_once_node : public overwrite_node { -public: - typedef T input_type; - typedef T output_type; - typedef overwrite_node base_type; - typedef typename receiver::predecessor_type predecessor_type; - typedef typename sender::successor_type successor_type; - - //! Constructor - explicit write_once_node(graph& g) : base_type(g) { - tbb::internal::fgt_node( tbb::internal::FLOW_WRITE_ONCE_NODE, &(this->my_graph), - static_cast *>(this), - static_cast *>(this) ); - } - - //! Copy constructor: call base class copy constructor - write_once_node( const write_once_node& src ) : base_type(src) { - tbb::internal::fgt_node( tbb::internal::FLOW_WRITE_ONCE_NODE, &(this->my_graph), - static_cast *>(this), - static_cast *>(this) ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif - -protected: - template< typename R, typename B > friend class run_and_put_task; - template friend class internal::broadcast_cache; - template friend class internal::round_robin_cache; - task *try_put_task( const T &v ) __TBB_override { - spin_mutex::scoped_lock l( this->my_mutex ); - return this->my_buffer_is_valid ? NULL : this->try_put_task_impl(v); - } -}; - -//! Forwards messages of type T to all successors -template -class broadcast_node : public graph_node, public receiver, public sender { -public: - typedef T input_type; - typedef T output_type; - typedef typename receiver::predecessor_type predecessor_type; - typedef typename sender::successor_type successor_type; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - typedef typename receiver::predecessor_list_type predecessor_list_type; - typedef typename sender::successor_list_type successor_list_type; -#endif -private: - internal::broadcast_cache my_successors; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - internal::edge_container my_built_predecessors; - spin_mutex pred_mutex; // serialize accesses on edge_container -#endif -public: - - explicit broadcast_node(graph& g) : graph_node(g) { - my_successors.set_owner( this ); - tbb::internal::fgt_node( tbb::internal::FLOW_BROADCAST_NODE, &this->my_graph, - static_cast *>(this), static_cast *>(this) ); - } - - // Copy constructor - broadcast_node( const broadcast_node& src ) : - graph_node(src.my_graph), receiver(), sender() - { - my_successors.set_owner( this ); - tbb::internal::fgt_node( tbb::internal::FLOW_BROADCAST_NODE, &this->my_graph, - static_cast *>(this), static_cast *>(this) ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif - - //! Adds a successor - bool register_successor( successor_type &r ) __TBB_override { - my_successors.register_successor( r ); - return true; - } - - //! Removes s as a successor - bool remove_successor( successor_type &r ) __TBB_override { - my_successors.remove_successor( r ); - return true; - } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - typedef typename sender::built_successors_type built_successors_type; - - built_successors_type &built_successors() __TBB_override { return my_successors.built_successors(); } - - void internal_add_built_successor(successor_type &r) __TBB_override { - my_successors.internal_add_built_successor(r); - } - - void internal_delete_built_successor(successor_type &r) __TBB_override { - my_successors.internal_delete_built_successor(r); - } - - size_t successor_count() __TBB_override { - return my_successors.successor_count(); - } - - void copy_successors(successor_list_type &v) __TBB_override { - my_successors.copy_successors(v); - } - - typedef typename receiver::built_predecessors_type built_predecessors_type; - - built_predecessors_type &built_predecessors() __TBB_override { return my_built_predecessors; } - - void internal_add_built_predecessor( predecessor_type &p) __TBB_override { - spin_mutex::scoped_lock l(pred_mutex); - my_built_predecessors.add_edge(p); - } - - void internal_delete_built_predecessor( predecessor_type &p) __TBB_override { - spin_mutex::scoped_lock l(pred_mutex); - my_built_predecessors.delete_edge(p); - } - - size_t predecessor_count() __TBB_override { - spin_mutex::scoped_lock l(pred_mutex); - return my_built_predecessors.edge_count(); - } - - void copy_predecessors(predecessor_list_type &v) __TBB_override { - spin_mutex::scoped_lock l(pred_mutex); - my_built_predecessors.copy_edges(v); - } - - void extract() __TBB_override { - my_built_predecessors.receiver_extract(*this); - my_successors.built_successors().sender_extract(*this); - } -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ - -protected: - template< typename R, typename B > friend class run_and_put_task; - template friend class internal::broadcast_cache; - template friend class internal::round_robin_cache; - //! build a task to run the successor if possible. Default is old behavior. - task *try_put_task(const T& t) __TBB_override { - task *new_task = my_successors.try_put_task(t); - if (!new_task) new_task = SUCCESSFULLY_ENQUEUED; - return new_task; - } - - graph& graph_reference() __TBB_override { - return my_graph; - } - - void reset_receiver(reset_flags /*f*/) __TBB_override {} - - void reset_node(reset_flags f) __TBB_override { - if (f&rf_clear_edges) { - my_successors.clear(); -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - my_built_predecessors.clear(); -#endif - } - __TBB_ASSERT(!(f & rf_clear_edges) || my_successors.empty(), "Error resetting broadcast_node"); - } -}; // broadcast_node - -//! Forwards messages in arbitrary order -template > -class buffer_node : public graph_node, public internal::reservable_item_buffer, public receiver, public sender { -public: - typedef T input_type; - typedef T output_type; - typedef typename receiver::predecessor_type predecessor_type; - typedef typename sender::successor_type successor_type; - typedef buffer_node class_type; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - typedef typename receiver::predecessor_list_type predecessor_list_type; - typedef typename sender::successor_list_type successor_list_type; -#endif -protected: - typedef size_t size_type; - internal::round_robin_cache< T, null_rw_mutex > my_successors; - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - internal::edge_container my_built_predecessors; -#endif - - friend class internal::forward_task_bypass< buffer_node< T, A > >; - - enum op_type {reg_succ, rem_succ, req_item, res_item, rel_res, con_res, put_item, try_fwd_task -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - , add_blt_succ, del_blt_succ, - add_blt_pred, del_blt_pred, - blt_succ_cnt, blt_pred_cnt, - blt_succ_cpy, blt_pred_cpy // create vector copies of preds and succs -#endif - }; - - // implements the aggregator_operation concept - class buffer_operation : public internal::aggregated_operation< buffer_operation > { - public: - char type; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - task * ltask; - union { - input_type *elem; - successor_type *r; - predecessor_type *p; - size_t cnt_val; - successor_list_type *svec; - predecessor_list_type *pvec; - }; -#else - T *elem; - task * ltask; - successor_type *r; -#endif - buffer_operation(const T& e, op_type t) : type(char(t)) - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - , ltask(NULL), elem(const_cast(&e)) -#else - , elem(const_cast(&e)) , ltask(NULL) -#endif - {} - buffer_operation(op_type t) : type(char(t)), ltask(NULL) {} - }; - - bool forwarder_busy; - typedef internal::aggregating_functor handler_type; - friend class internal::aggregating_functor; - internal::aggregator< handler_type, buffer_operation> my_aggregator; - - virtual void handle_operations(buffer_operation *op_list) { - handle_operations_impl(op_list, this); - } - - template - void handle_operations_impl(buffer_operation *op_list, derived_type* derived) { - __TBB_ASSERT(static_cast(derived) == this, "'this' is not a base class for derived"); - - buffer_operation *tmp = NULL; - bool try_forwarding = false; - while (op_list) { - tmp = op_list; - op_list = op_list->next; - switch (tmp->type) { - case reg_succ: internal_reg_succ(tmp); try_forwarding = true; break; - case rem_succ: internal_rem_succ(tmp); break; - case req_item: internal_pop(tmp); break; - case res_item: internal_reserve(tmp); break; - case rel_res: internal_release(tmp); try_forwarding = true; break; - case con_res: internal_consume(tmp); try_forwarding = true; break; - case put_item: try_forwarding = internal_push(tmp); break; - case try_fwd_task: internal_forward_task(tmp); break; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - // edge recording - case add_blt_succ: internal_add_built_succ(tmp); break; - case del_blt_succ: internal_del_built_succ(tmp); break; - case add_blt_pred: internal_add_built_pred(tmp); break; - case del_blt_pred: internal_del_built_pred(tmp); break; - case blt_succ_cnt: internal_succ_cnt(tmp); break; - case blt_pred_cnt: internal_pred_cnt(tmp); break; - case blt_succ_cpy: internal_copy_succs(tmp); break; - case blt_pred_cpy: internal_copy_preds(tmp); break; -#endif - } - } - - derived->order(); - - if (try_forwarding && !forwarder_busy) { - if(internal::is_graph_active(this->my_graph)) { - forwarder_busy = true; - task *new_task = new(task::allocate_additional_child_of(*(this->my_graph.root_task()))) internal:: - forward_task_bypass - < buffer_node >(*this); - // tmp should point to the last item handled by the aggregator. This is the operation - // the handling thread enqueued. So modifying that record will be okay. - // workaround for icc bug - tbb::task *z = tmp->ltask; - graph &g = this->my_graph; - tmp->ltask = combine_tasks(g, z, new_task); // in case the op generated a task - } - } - } // handle_operations - - inline task *grab_forwarding_task( buffer_operation &op_data) { - return op_data.ltask; - } - - inline bool enqueue_forwarding_task(buffer_operation &op_data) { - task *ft = grab_forwarding_task(op_data); - if(ft) { - internal::spawn_in_graph_arena(graph_reference(), *ft); - return true; - } - return false; - } - - //! This is executed by an enqueued task, the "forwarder" - virtual task *forward_task() { - buffer_operation op_data(try_fwd_task); - task *last_task = NULL; - do { - op_data.status = internal::WAIT; - op_data.ltask = NULL; - my_aggregator.execute(&op_data); - - // workaround for icc bug - tbb::task *xtask = op_data.ltask; - graph& g = this->my_graph; - last_task = combine_tasks(g, last_task, xtask); - } while (op_data.status ==internal::SUCCEEDED); - return last_task; - } - - //! Register successor - virtual void internal_reg_succ(buffer_operation *op) { - my_successors.register_successor(*(op->r)); - __TBB_store_with_release(op->status, internal::SUCCEEDED); - } - - //! Remove successor - virtual void internal_rem_succ(buffer_operation *op) { - my_successors.remove_successor(*(op->r)); - __TBB_store_with_release(op->status, internal::SUCCEEDED); - } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - typedef typename sender::built_successors_type built_successors_type; - - built_successors_type &built_successors() __TBB_override { return my_successors.built_successors(); } - - virtual void internal_add_built_succ(buffer_operation *op) { - my_successors.internal_add_built_successor(*(op->r)); - __TBB_store_with_release(op->status, internal::SUCCEEDED); - } - - virtual void internal_del_built_succ(buffer_operation *op) { - my_successors.internal_delete_built_successor(*(op->r)); - __TBB_store_with_release(op->status, internal::SUCCEEDED); - } - - typedef typename receiver::built_predecessors_type built_predecessors_type; - - built_predecessors_type &built_predecessors() __TBB_override { return my_built_predecessors; } - - virtual void internal_add_built_pred(buffer_operation *op) { - my_built_predecessors.add_edge(*(op->p)); - __TBB_store_with_release(op->status, internal::SUCCEEDED); - } - - virtual void internal_del_built_pred(buffer_operation *op) { - my_built_predecessors.delete_edge(*(op->p)); - __TBB_store_with_release(op->status, internal::SUCCEEDED); - } - - virtual void internal_succ_cnt(buffer_operation *op) { - op->cnt_val = my_successors.successor_count(); - __TBB_store_with_release(op->status, internal::SUCCEEDED); - } - - virtual void internal_pred_cnt(buffer_operation *op) { - op->cnt_val = my_built_predecessors.edge_count(); - __TBB_store_with_release(op->status, internal::SUCCEEDED); - } - - virtual void internal_copy_succs(buffer_operation *op) { - my_successors.copy_successors(*(op->svec)); - __TBB_store_with_release(op->status, internal::SUCCEEDED); - } - - virtual void internal_copy_preds(buffer_operation *op) { - my_built_predecessors.copy_edges(*(op->pvec)); - __TBB_store_with_release(op->status, internal::SUCCEEDED); - } - -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ - -private: - void order() {} - - bool is_item_valid() { - return this->my_item_valid(this->my_tail - 1); - } - - void try_put_and_add_task(task*& last_task) { - task *new_task = my_successors.try_put_task(this->back()); - if (new_task) { - // workaround for icc bug - graph& g = this->my_graph; - last_task = combine_tasks(g, last_task, new_task); - this->destroy_back(); - } - } - -protected: - //! Tries to forward valid items to successors - virtual void internal_forward_task(buffer_operation *op) { - internal_forward_task_impl(op, this); - } - - template - void internal_forward_task_impl(buffer_operation *op, derived_type* derived) { - __TBB_ASSERT(static_cast(derived) == this, "'this' is not a base class for derived"); - - if (this->my_reserved || !derived->is_item_valid()) { - __TBB_store_with_release(op->status, internal::FAILED); - this->forwarder_busy = false; - return; - } - // Try forwarding, giving each successor a chance - task * last_task = NULL; - size_type counter = my_successors.size(); - for (; counter > 0 && derived->is_item_valid(); --counter) - derived->try_put_and_add_task(last_task); - - op->ltask = last_task; // return task - if (last_task && !counter) { - __TBB_store_with_release(op->status, internal::SUCCEEDED); - } - else { - __TBB_store_with_release(op->status, internal::FAILED); - forwarder_busy = false; - } - } - - virtual bool internal_push(buffer_operation *op) { - this->push_back(*(op->elem)); - __TBB_store_with_release(op->status, internal::SUCCEEDED); - return true; - } - - virtual void internal_pop(buffer_operation *op) { - if(this->pop_back(*(op->elem))) { - __TBB_store_with_release(op->status, internal::SUCCEEDED); - } - else { - __TBB_store_with_release(op->status, internal::FAILED); - } - } - - virtual void internal_reserve(buffer_operation *op) { - if(this->reserve_front(*(op->elem))) { - __TBB_store_with_release(op->status, internal::SUCCEEDED); - } - else { - __TBB_store_with_release(op->status, internal::FAILED); - } - } - - virtual void internal_consume(buffer_operation *op) { - this->consume_front(); - __TBB_store_with_release(op->status, internal::SUCCEEDED); - } - - virtual void internal_release(buffer_operation *op) { - this->release_front(); - __TBB_store_with_release(op->status, internal::SUCCEEDED); - } - -public: - //! Constructor - explicit buffer_node( graph &g ) : graph_node(g), internal::reservable_item_buffer(), - forwarder_busy(false) { - my_successors.set_owner(this); - my_aggregator.initialize_handler(handler_type(this)); - tbb::internal::fgt_node( tbb::internal::FLOW_BUFFER_NODE, &this->my_graph, - static_cast *>(this), static_cast *>(this) ); - } - - //! Copy constructor - buffer_node( const buffer_node& src ) : graph_node(src.my_graph), - internal::reservable_item_buffer(), receiver(), sender() { - forwarder_busy = false; - my_successors.set_owner(this); - my_aggregator.initialize_handler(handler_type(this)); - tbb::internal::fgt_node( tbb::internal::FLOW_BUFFER_NODE, &this->my_graph, - static_cast *>(this), static_cast *>(this) ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif - - // - // message sender implementation - // - - //! Adds a new successor. - /** Adds successor r to the list of successors; may forward tasks. */ - bool register_successor( successor_type &r ) __TBB_override { - buffer_operation op_data(reg_succ); - op_data.r = &r; - my_aggregator.execute(&op_data); - (void)enqueue_forwarding_task(op_data); - return true; - } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - void internal_add_built_successor( successor_type &r) __TBB_override { - buffer_operation op_data(add_blt_succ); - op_data.r = &r; - my_aggregator.execute(&op_data); - } - - void internal_delete_built_successor( successor_type &r) __TBB_override { - buffer_operation op_data(del_blt_succ); - op_data.r = &r; - my_aggregator.execute(&op_data); - } - - void internal_add_built_predecessor( predecessor_type &p) __TBB_override { - buffer_operation op_data(add_blt_pred); - op_data.p = &p; - my_aggregator.execute(&op_data); - } - - void internal_delete_built_predecessor( predecessor_type &p) __TBB_override { - buffer_operation op_data(del_blt_pred); - op_data.p = &p; - my_aggregator.execute(&op_data); - } - - size_t predecessor_count() __TBB_override { - buffer_operation op_data(blt_pred_cnt); - my_aggregator.execute(&op_data); - return op_data.cnt_val; - } - - size_t successor_count() __TBB_override { - buffer_operation op_data(blt_succ_cnt); - my_aggregator.execute(&op_data); - return op_data.cnt_val; - } - - void copy_predecessors( predecessor_list_type &v ) __TBB_override { - buffer_operation op_data(blt_pred_cpy); - op_data.pvec = &v; - my_aggregator.execute(&op_data); - } - - void copy_successors( successor_list_type &v ) __TBB_override { - buffer_operation op_data(blt_succ_cpy); - op_data.svec = &v; - my_aggregator.execute(&op_data); - } - -#endif - - //! Removes a successor. - /** Removes successor r from the list of successors. - It also calls r.remove_predecessor(*this) to remove this node as a predecessor. */ - bool remove_successor( successor_type &r ) __TBB_override { - r.remove_predecessor(*this); - buffer_operation op_data(rem_succ); - op_data.r = &r; - my_aggregator.execute(&op_data); - // even though this operation does not cause a forward, if we are the handler, and - // a forward is scheduled, we may be the first to reach this point after the aggregator, - // and so should check for the task. - (void)enqueue_forwarding_task(op_data); - return true; - } - - //! Request an item from the buffer_node - /** true = v contains the returned item
- false = no item has been returned */ - bool try_get( T &v ) __TBB_override { - buffer_operation op_data(req_item); - op_data.elem = &v; - my_aggregator.execute(&op_data); - (void)enqueue_forwarding_task(op_data); - return (op_data.status==internal::SUCCEEDED); - } - - //! Reserves an item. - /** false = no item can be reserved
- true = an item is reserved */ - bool try_reserve( T &v ) __TBB_override { - buffer_operation op_data(res_item); - op_data.elem = &v; - my_aggregator.execute(&op_data); - (void)enqueue_forwarding_task(op_data); - return (op_data.status==internal::SUCCEEDED); - } - - //! Release a reserved item. - /** true = item has been released and so remains in sender */ - bool try_release() __TBB_override { - buffer_operation op_data(rel_res); - my_aggregator.execute(&op_data); - (void)enqueue_forwarding_task(op_data); - return true; - } - - //! Consumes a reserved item. - /** true = item is removed from sender and reservation removed */ - bool try_consume() __TBB_override { - buffer_operation op_data(con_res); - my_aggregator.execute(&op_data); - (void)enqueue_forwarding_task(op_data); - return true; - } - -protected: - - template< typename R, typename B > friend class run_and_put_task; - template friend class internal::broadcast_cache; - template friend class internal::round_robin_cache; - //! receive an item, return a task *if possible - task *try_put_task(const T &t) __TBB_override { - buffer_operation op_data(t, put_item); - my_aggregator.execute(&op_data); - task *ft = grab_forwarding_task(op_data); - // sequencer_nodes can return failure (if an item has been previously inserted) - // We have to spawn the returned task if our own operation fails. - - if(ft && op_data.status ==internal::FAILED) { - // we haven't succeeded queueing the item, but for some reason the - // call returned a task (if another request resulted in a successful - // forward this could happen.) Queue the task and reset the pointer. - internal::spawn_in_graph_arena(graph_reference(), *ft); ft = NULL; - } - else if(!ft && op_data.status ==internal::SUCCEEDED) { - ft = SUCCESSFULLY_ENQUEUED; - } - return ft; - } - - graph& graph_reference() __TBB_override { - return my_graph; - } - - void reset_receiver(reset_flags /*f*/) __TBB_override { } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES -public: - void extract() __TBB_override { - my_built_predecessors.receiver_extract(*this); - my_successors.built_successors().sender_extract(*this); - } -#endif - -protected: - void reset_node( reset_flags f) __TBB_override { - internal::reservable_item_buffer::reset(); - // TODO: just clear structures - if (f&rf_clear_edges) { - my_successors.clear(); -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - my_built_predecessors.clear(); -#endif - } - forwarder_busy = false; - } -}; // buffer_node - -//! Forwards messages in FIFO order -template > -class queue_node : public buffer_node { -protected: - typedef buffer_node base_type; - typedef typename base_type::size_type size_type; - typedef typename base_type::buffer_operation queue_operation; - typedef queue_node class_type; - -private: - template friend class buffer_node; - - bool is_item_valid() { - return this->my_item_valid(this->my_head); - } - - void try_put_and_add_task(task*& last_task) { - task *new_task = this->my_successors.try_put_task(this->front()); - if (new_task) { - // workaround for icc bug - graph& graph_ref = this->graph_reference(); - last_task = combine_tasks(graph_ref, last_task, new_task); - this->destroy_front(); - } - } - -protected: - void internal_forward_task(queue_operation *op) __TBB_override { - this->internal_forward_task_impl(op, this); - } - - void internal_pop(queue_operation *op) __TBB_override { - if ( this->my_reserved || !this->my_item_valid(this->my_head)){ - __TBB_store_with_release(op->status, internal::FAILED); - } - else { - this->pop_front(*(op->elem)); - __TBB_store_with_release(op->status, internal::SUCCEEDED); - } - } - void internal_reserve(queue_operation *op) __TBB_override { - if (this->my_reserved || !this->my_item_valid(this->my_head)) { - __TBB_store_with_release(op->status, internal::FAILED); - } - else { - this->reserve_front(*(op->elem)); - __TBB_store_with_release(op->status, internal::SUCCEEDED); - } - } - void internal_consume(queue_operation *op) __TBB_override { - this->consume_front(); - __TBB_store_with_release(op->status, internal::SUCCEEDED); - } - -public: - typedef T input_type; - typedef T output_type; - typedef typename receiver::predecessor_type predecessor_type; - typedef typename sender::successor_type successor_type; - - //! Constructor - explicit queue_node( graph &g ) : base_type(g) { - tbb::internal::fgt_node( tbb::internal::FLOW_QUEUE_NODE, &(this->my_graph), - static_cast *>(this), - static_cast *>(this) ); - } - - //! Copy constructor - queue_node( const queue_node& src) : base_type(src) { - tbb::internal::fgt_node( tbb::internal::FLOW_QUEUE_NODE, &(this->my_graph), - static_cast *>(this), - static_cast *>(this) ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif - -protected: - void reset_node( reset_flags f) __TBB_override { - base_type::reset_node(f); - } -}; // queue_node - -//! Forwards messages in sequence order -template< typename T, typename A=cache_aligned_allocator > -class sequencer_node : public queue_node { - internal::function_body< T, size_t > *my_sequencer; - // my_sequencer should be a benign function and must be callable - // from a parallel context. Does this mean it needn't be reset? -public: - typedef T input_type; - typedef T output_type; - typedef typename receiver::predecessor_type predecessor_type; - typedef typename sender::successor_type successor_type; - - //! Constructor - template< typename Sequencer > - sequencer_node( graph &g, const Sequencer& s ) : queue_node(g), - my_sequencer(new internal::function_body_leaf< T, size_t, Sequencer>(s) ) { - tbb::internal::fgt_node( tbb::internal::FLOW_SEQUENCER_NODE, &(this->my_graph), - static_cast *>(this), - static_cast *>(this) ); - } - - //! Copy constructor - sequencer_node( const sequencer_node& src ) : queue_node(src), - my_sequencer( src.my_sequencer->clone() ) { - tbb::internal::fgt_node( tbb::internal::FLOW_SEQUENCER_NODE, &(this->my_graph), - static_cast *>(this), - static_cast *>(this) ); - } - - //! Destructor - ~sequencer_node() { delete my_sequencer; } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif - -protected: - typedef typename buffer_node::size_type size_type; - typedef typename buffer_node::buffer_operation sequencer_operation; - -private: - bool internal_push(sequencer_operation *op) __TBB_override { - size_type tag = (*my_sequencer)(*(op->elem)); -#if !TBB_DEPRECATED_SEQUENCER_DUPLICATES - if (tag < this->my_head) { - // have already emitted a message with this tag - __TBB_store_with_release(op->status, internal::FAILED); - return false; - } -#endif - // cannot modify this->my_tail now; the buffer would be inconsistent. - size_t new_tail = (tag+1 > this->my_tail) ? tag+1 : this->my_tail; - - if (this->size(new_tail) > this->capacity()) { - this->grow_my_array(this->size(new_tail)); - } - this->my_tail = new_tail; - - const internal::op_stat res = this->place_item(tag, *(op->elem)) ? internal::SUCCEEDED : internal::FAILED; - __TBB_store_with_release(op->status, res); - return res ==internal::SUCCEEDED; - } -}; // sequencer_node - -//! Forwards messages in priority order -template< typename T, typename Compare = std::less, typename A=cache_aligned_allocator > -class priority_queue_node : public buffer_node { -public: - typedef T input_type; - typedef T output_type; - typedef buffer_node base_type; - typedef priority_queue_node class_type; - typedef typename receiver::predecessor_type predecessor_type; - typedef typename sender::successor_type successor_type; - - //! Constructor - explicit priority_queue_node( graph &g ) : buffer_node(g), mark(0) { - tbb::internal::fgt_node( tbb::internal::FLOW_PRIORITY_QUEUE_NODE, &(this->my_graph), - static_cast *>(this), - static_cast *>(this) ); - } - - //! Copy constructor - priority_queue_node( const priority_queue_node &src ) : buffer_node(src), mark(0) { - tbb::internal::fgt_node( tbb::internal::FLOW_PRIORITY_QUEUE_NODE, &(this->my_graph), - static_cast *>(this), - static_cast *>(this) ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif - -protected: - - void reset_node( reset_flags f) __TBB_override { - mark = 0; - base_type::reset_node(f); - } - - typedef typename buffer_node::size_type size_type; - typedef typename buffer_node::item_type item_type; - typedef typename buffer_node::buffer_operation prio_operation; - - //! Tries to forward valid items to successors - void internal_forward_task(prio_operation *op) __TBB_override { - this->internal_forward_task_impl(op, this); - } - - void handle_operations(prio_operation *op_list) __TBB_override { - this->handle_operations_impl(op_list, this); - } - - bool internal_push(prio_operation *op) __TBB_override { - prio_push(*(op->elem)); - __TBB_store_with_release(op->status, internal::SUCCEEDED); - return true; - } - - void internal_pop(prio_operation *op) __TBB_override { - // if empty or already reserved, don't pop - if ( this->my_reserved == true || this->my_tail == 0 ) { - __TBB_store_with_release(op->status, internal::FAILED); - return; - } - - *(op->elem) = prio(); - __TBB_store_with_release(op->status, internal::SUCCEEDED); - prio_pop(); - - } - - // pops the highest-priority item, saves copy - void internal_reserve(prio_operation *op) __TBB_override { - if (this->my_reserved == true || this->my_tail == 0) { - __TBB_store_with_release(op->status, internal::FAILED); - return; - } - this->my_reserved = true; - *(op->elem) = prio(); - reserved_item = *(op->elem); - __TBB_store_with_release(op->status, internal::SUCCEEDED); - prio_pop(); - } - - void internal_consume(prio_operation *op) __TBB_override { - __TBB_store_with_release(op->status, internal::SUCCEEDED); - this->my_reserved = false; - reserved_item = input_type(); - } - - void internal_release(prio_operation *op) __TBB_override { - __TBB_store_with_release(op->status, internal::SUCCEEDED); - prio_push(reserved_item); - this->my_reserved = false; - reserved_item = input_type(); - } - -private: - template friend class buffer_node; - - void order() { - if (mark < this->my_tail) heapify(); - __TBB_ASSERT(mark == this->my_tail, "mark unequal after heapify"); - } - - bool is_item_valid() { - return this->my_tail > 0; - } - - void try_put_and_add_task(task*& last_task) { - task * new_task = this->my_successors.try_put_task(this->prio()); - if (new_task) { - // workaround for icc bug - graph& graph_ref = this->graph_reference(); - last_task = combine_tasks(graph_ref, last_task, new_task); - prio_pop(); - } - } - -private: - Compare compare; - size_type mark; - - input_type reserved_item; - - // in case a reheap has not been done after a push, check if the mark item is higher than the 0'th item - bool prio_use_tail() { - __TBB_ASSERT(mark <= this->my_tail, "mark outside bounds before test"); - return mark < this->my_tail && compare(this->get_my_item(0), this->get_my_item(this->my_tail - 1)); - } - - // prio_push: checks that the item will fit, expand array if necessary, put at end - void prio_push(const T &src) { - if ( this->my_tail >= this->my_array_size ) - this->grow_my_array( this->my_tail + 1 ); - (void) this->place_item(this->my_tail, src); - ++(this->my_tail); - __TBB_ASSERT(mark < this->my_tail, "mark outside bounds after push"); - } - - // prio_pop: deletes highest priority item from the array, and if it is item - // 0, move last item to 0 and reheap. If end of array, just destroy and decrement tail - // and mark. Assumes the array has already been tested for emptiness; no failure. - void prio_pop() { - if (prio_use_tail()) { - // there are newly pushed elements; last one higher than top - // copy the data - this->destroy_item(this->my_tail-1); - --(this->my_tail); - __TBB_ASSERT(mark <= this->my_tail, "mark outside bounds after pop"); - return; - } - this->destroy_item(0); - if(this->my_tail > 1) { - // push the last element down heap - __TBB_ASSERT(this->my_item_valid(this->my_tail - 1), NULL); - this->move_item(0,this->my_tail - 1); - } - --(this->my_tail); - if(mark > this->my_tail) --mark; - if (this->my_tail > 1) // don't reheap for heap of size 1 - reheap(); - __TBB_ASSERT(mark <= this->my_tail, "mark outside bounds after pop"); - } - - const T& prio() { - return this->get_my_item(prio_use_tail() ? this->my_tail-1 : 0); - } - - // turn array into heap - void heapify() { - if(this->my_tail == 0) { - mark = 0; - return; - } - if (!mark) mark = 1; - for (; markmy_tail; ++mark) { // for each unheaped element - size_type cur_pos = mark; - input_type to_place; - this->fetch_item(mark,to_place); - do { // push to_place up the heap - size_type parent = (cur_pos-1)>>1; - if (!compare(this->get_my_item(parent), to_place)) - break; - this->move_item(cur_pos, parent); - cur_pos = parent; - } while( cur_pos ); - (void) this->place_item(cur_pos, to_place); - } - } - - // otherwise heapified array with new root element; rearrange to heap - void reheap() { - size_type cur_pos=0, child=1; - while (child < mark) { - size_type target = child; - if (child+1get_my_item(child), - this->get_my_item(child+1))) - ++target; - // target now has the higher priority child - if (compare(this->get_my_item(target), - this->get_my_item(cur_pos))) - break; - // swap - this->swap_items(cur_pos, target); - cur_pos = target; - child = (cur_pos<<1)+1; - } - } -}; // priority_queue_node - -//! Forwards messages only if the threshold has not been reached -/** This node forwards items until its threshold is reached. - It contains no buffering. If the downstream node rejects, the - message is dropped. */ -template< typename T > -class limiter_node : public graph_node, public receiver< T >, public sender< T > { -public: - typedef T input_type; - typedef T output_type; - typedef typename receiver::predecessor_type predecessor_type; - typedef typename sender::successor_type successor_type; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - typedef typename receiver::built_predecessors_type built_predecessors_type; - typedef typename sender::built_successors_type built_successors_type; - typedef typename receiver::predecessor_list_type predecessor_list_type; - typedef typename sender::successor_list_type successor_list_type; -#endif - //TODO: There is a lack of predefined types for its controlling "decrementer" port. It should be fixed later. - -private: - size_t my_threshold; - size_t my_count; //number of successful puts - size_t my_tries; //number of active put attempts - internal::reservable_predecessor_cache< T, spin_mutex > my_predecessors; - spin_mutex my_mutex; - internal::broadcast_cache< T > my_successors; - int init_decrement_predecessors; - - friend class internal::forward_task_bypass< limiter_node >; - - // Let decrementer call decrement_counter() - friend class internal::decrementer< limiter_node >; - - bool check_conditions() { // always called under lock - return ( my_count + my_tries < my_threshold && !my_predecessors.empty() && !my_successors.empty() ); - } - - // only returns a valid task pointer or NULL, never SUCCESSFULLY_ENQUEUED - task *forward_task() { - input_type v; - task *rval = NULL; - bool reserved = false; - { - spin_mutex::scoped_lock lock(my_mutex); - if ( check_conditions() ) - ++my_tries; - else - return NULL; - } - - //SUCCESS - // if we can reserve and can put, we consume the reservation - // we increment the count and decrement the tries - if ( (my_predecessors.try_reserve(v)) == true ){ - reserved=true; - if ( (rval = my_successors.try_put_task(v)) != NULL ){ - { - spin_mutex::scoped_lock lock(my_mutex); - ++my_count; - --my_tries; - my_predecessors.try_consume(); - if ( check_conditions() ) { - if ( internal::is_graph_active(this->my_graph) ) { - task *rtask = new ( task::allocate_additional_child_of( *(this->my_graph.root_task()) ) ) - internal::forward_task_bypass< limiter_node >( *this ); - internal::spawn_in_graph_arena(graph_reference(), *rtask); - } - } - } - return rval; - } - } - //FAILURE - //if we can't reserve, we decrement the tries - //if we can reserve but can't put, we decrement the tries and release the reservation - { - spin_mutex::scoped_lock lock(my_mutex); - --my_tries; - if (reserved) my_predecessors.try_release(); - if ( check_conditions() ) { - if ( internal::is_graph_active(this->my_graph) ) { - task *rtask = new ( task::allocate_additional_child_of( *(this->my_graph.root_task()) ) ) - internal::forward_task_bypass< limiter_node >( *this ); - __TBB_ASSERT(!rval, "Have two tasks to handle"); - return rtask; - } - } - return rval; - } - } - - void forward() { - __TBB_ASSERT(false, "Should never be called"); - return; - } - - task * decrement_counter() { - { - spin_mutex::scoped_lock lock(my_mutex); - if(my_count) --my_count; - } - return forward_task(); - } - -public: - //! The internal receiver< continue_msg > that decrements the count - internal::decrementer< limiter_node > decrement; - - //! Constructor - limiter_node(graph &g, size_t threshold, int num_decrement_predecessors=0) : - graph_node(g), my_threshold(threshold), my_count(0), my_tries(0), - init_decrement_predecessors(num_decrement_predecessors), - decrement(num_decrement_predecessors) - { - my_predecessors.set_owner(this); - my_successors.set_owner(this); - decrement.set_owner(this); - tbb::internal::fgt_node( tbb::internal::FLOW_LIMITER_NODE, &this->my_graph, - static_cast *>(this), static_cast *>(&decrement), - static_cast *>(this) ); - } - - //! Copy constructor - limiter_node( const limiter_node& src ) : - graph_node(src.my_graph), receiver(), sender(), - my_threshold(src.my_threshold), my_count(0), my_tries(0), - init_decrement_predecessors(src.init_decrement_predecessors), - decrement(src.init_decrement_predecessors) - { - my_predecessors.set_owner(this); - my_successors.set_owner(this); - decrement.set_owner(this); - tbb::internal::fgt_node( tbb::internal::FLOW_LIMITER_NODE, &this->my_graph, - static_cast *>(this), static_cast *>(&decrement), - static_cast *>(this) ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif - - //! Replace the current successor with this new successor - bool register_successor( successor_type &r ) __TBB_override { - spin_mutex::scoped_lock lock(my_mutex); - bool was_empty = my_successors.empty(); - my_successors.register_successor(r); - //spawn a forward task if this is the only successor - if ( was_empty && !my_predecessors.empty() && my_count + my_tries < my_threshold ) { - if ( internal::is_graph_active(this->my_graph) ) { - task* task = new ( task::allocate_additional_child_of( *(this->my_graph.root_task()) ) ) - internal::forward_task_bypass < limiter_node >( *this ); - internal::spawn_in_graph_arena(graph_reference(), *task); - } - } - return true; - } - - //! Removes a successor from this node - /** r.remove_predecessor(*this) is also called. */ - bool remove_successor( successor_type &r ) __TBB_override { - r.remove_predecessor(*this); - my_successors.remove_successor(r); - return true; - } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - built_successors_type &built_successors() __TBB_override { return my_successors.built_successors(); } - built_predecessors_type &built_predecessors() __TBB_override { return my_predecessors.built_predecessors(); } - - void internal_add_built_successor(successor_type &src) __TBB_override { - my_successors.internal_add_built_successor(src); - } - - void internal_delete_built_successor(successor_type &src) __TBB_override { - my_successors.internal_delete_built_successor(src); - } - - size_t successor_count() __TBB_override { return my_successors.successor_count(); } - - void copy_successors(successor_list_type &v) __TBB_override { - my_successors.copy_successors(v); - } - - void internal_add_built_predecessor(predecessor_type &src) __TBB_override { - my_predecessors.internal_add_built_predecessor(src); - } - - void internal_delete_built_predecessor(predecessor_type &src) __TBB_override { - my_predecessors.internal_delete_built_predecessor(src); - } - - size_t predecessor_count() __TBB_override { return my_predecessors.predecessor_count(); } - - void copy_predecessors(predecessor_list_type &v) __TBB_override { - my_predecessors.copy_predecessors(v); - } - - void extract() __TBB_override { - my_count = 0; - my_successors.built_successors().sender_extract(*this); - my_predecessors.built_predecessors().receiver_extract(*this); - decrement.built_predecessors().receiver_extract(decrement); - } -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ - - //! Adds src to the list of cached predecessors. - bool register_predecessor( predecessor_type &src ) __TBB_override { - spin_mutex::scoped_lock lock(my_mutex); - my_predecessors.add( src ); - if ( my_count + my_tries < my_threshold && !my_successors.empty() && internal::is_graph_active(this->my_graph) ) { - task* task = new ( task::allocate_additional_child_of( *(this->my_graph.root_task()) ) ) - internal::forward_task_bypass < limiter_node >( *this ); - internal::spawn_in_graph_arena(graph_reference(), *task); - } - return true; - } - - //! Removes src from the list of cached predecessors. - bool remove_predecessor( predecessor_type &src ) __TBB_override { - my_predecessors.remove( src ); - return true; - } - -protected: - - template< typename R, typename B > friend class run_and_put_task; - template friend class internal::broadcast_cache; - template friend class internal::round_robin_cache; - //! Puts an item to this receiver - task *try_put_task( const T &t ) __TBB_override { - { - spin_mutex::scoped_lock lock(my_mutex); - if ( my_count + my_tries >= my_threshold ) - return NULL; - else - ++my_tries; - } - - task * rtask = my_successors.try_put_task(t); - - if ( !rtask ) { // try_put_task failed. - spin_mutex::scoped_lock lock(my_mutex); - --my_tries; - if (check_conditions() && internal::is_graph_active(this->my_graph)) { - rtask = new ( task::allocate_additional_child_of( *(this->my_graph.root_task()) ) ) - internal::forward_task_bypass< limiter_node >( *this ); - } - } - else { - spin_mutex::scoped_lock lock(my_mutex); - ++my_count; - --my_tries; - } - return rtask; - } - - graph& graph_reference() __TBB_override { - return my_graph; - } - - void reset_receiver(reset_flags /*f*/) __TBB_override { - __TBB_ASSERT(false,NULL); // should never be called - } - - void reset_node( reset_flags f) __TBB_override { - my_count = 0; - if(f & rf_clear_edges) { - my_predecessors.clear(); - my_successors.clear(); - } - else - { - my_predecessors.reset( ); - } - decrement.reset_receiver(f); - } -}; // limiter_node - -#include "internal/_flow_graph_join_impl.h" - -using internal::reserving_port; -using internal::queueing_port; -using internal::key_matching_port; -using internal::input_port; -using internal::tag_value; - -template class join_node; - -template -class join_node: public internal::unfolded_join_node::value, reserving_port, OutputTuple, reserving> { -private: - static const int N = tbb::flow::tuple_size::value; - typedef typename internal::unfolded_join_node unfolded_type; -public: - typedef OutputTuple output_type; - typedef typename unfolded_type::input_ports_type input_ports_type; - explicit join_node(graph &g) : unfolded_type(g) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_JOIN_NODE_RESERVING, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - join_node(const join_node &other) : unfolded_type(other) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_JOIN_NODE_RESERVING, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif - -}; - -template -class join_node: public internal::unfolded_join_node::value, queueing_port, OutputTuple, queueing> { -private: - static const int N = tbb::flow::tuple_size::value; - typedef typename internal::unfolded_join_node unfolded_type; -public: - typedef OutputTuple output_type; - typedef typename unfolded_type::input_ports_type input_ports_type; - explicit join_node(graph &g) : unfolded_type(g) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_JOIN_NODE_QUEUEING, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - join_node(const join_node &other) : unfolded_type(other) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_JOIN_NODE_QUEUEING, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif - -}; - -// template for key_matching join_node -// tag_matching join_node is a specialization of key_matching, and is source-compatible. -template -class join_node > : public internal::unfolded_join_node::value, - key_matching_port, OutputTuple, key_matching > { -private: - static const int N = tbb::flow::tuple_size::value; - typedef typename internal::unfolded_join_node > unfolded_type; -public: - typedef OutputTuple output_type; - typedef typename unfolded_type::input_ports_type input_ports_type; - -#if __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING - join_node(graph &g) : unfolded_type(g) {} -#endif /* __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING */ - - template - join_node(graph &g, __TBB_B0 b0, __TBB_B1 b1) : unfolded_type(g, b0, b1) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_JOIN_NODE_TAG_MATCHING, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - template - join_node(graph &g, __TBB_B0 b0, __TBB_B1 b1, __TBB_B2 b2) : unfolded_type(g, b0, b1, b2) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_JOIN_NODE_TAG_MATCHING, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - template - join_node(graph &g, __TBB_B0 b0, __TBB_B1 b1, __TBB_B2 b2, __TBB_B3 b3) : unfolded_type(g, b0, b1, b2, b3) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_JOIN_NODE_TAG_MATCHING, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - template - join_node(graph &g, __TBB_B0 b0, __TBB_B1 b1, __TBB_B2 b2, __TBB_B3 b3, __TBB_B4 b4) : - unfolded_type(g, b0, b1, b2, b3, b4) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_JOIN_NODE_TAG_MATCHING, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } -#if __TBB_VARIADIC_MAX >= 6 - template - join_node(graph &g, __TBB_B0 b0, __TBB_B1 b1, __TBB_B2 b2, __TBB_B3 b3, __TBB_B4 b4, __TBB_B5 b5) : - unfolded_type(g, b0, b1, b2, b3, b4, b5) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_JOIN_NODE_TAG_MATCHING, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } -#endif -#if __TBB_VARIADIC_MAX >= 7 - template - join_node(graph &g, __TBB_B0 b0, __TBB_B1 b1, __TBB_B2 b2, __TBB_B3 b3, __TBB_B4 b4, __TBB_B5 b5, __TBB_B6 b6) : - unfolded_type(g, b0, b1, b2, b3, b4, b5, b6) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_JOIN_NODE_TAG_MATCHING, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } -#endif -#if __TBB_VARIADIC_MAX >= 8 - template - join_node(graph &g, __TBB_B0 b0, __TBB_B1 b1, __TBB_B2 b2, __TBB_B3 b3, __TBB_B4 b4, __TBB_B5 b5, __TBB_B6 b6, - __TBB_B7 b7) : unfolded_type(g, b0, b1, b2, b3, b4, b5, b6, b7) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_JOIN_NODE_TAG_MATCHING, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } -#endif -#if __TBB_VARIADIC_MAX >= 9 - template - join_node(graph &g, __TBB_B0 b0, __TBB_B1 b1, __TBB_B2 b2, __TBB_B3 b3, __TBB_B4 b4, __TBB_B5 b5, __TBB_B6 b6, - __TBB_B7 b7, __TBB_B8 b8) : unfolded_type(g, b0, b1, b2, b3, b4, b5, b6, b7, b8) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_JOIN_NODE_TAG_MATCHING, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } -#endif -#if __TBB_VARIADIC_MAX >= 10 - template - join_node(graph &g, __TBB_B0 b0, __TBB_B1 b1, __TBB_B2 b2, __TBB_B3 b3, __TBB_B4 b4, __TBB_B5 b5, __TBB_B6 b6, - __TBB_B7 b7, __TBB_B8 b8, __TBB_B9 b9) : unfolded_type(g, b0, b1, b2, b3, b4, b5, b6, b7, b8, b9) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_JOIN_NODE_TAG_MATCHING, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } -#endif - join_node(const join_node &other) : unfolded_type(other) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_JOIN_NODE_TAG_MATCHING, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif - -}; - -// indexer node -#include "internal/_flow_graph_indexer_impl.h" - -// TODO: Implement interface with variadic template or tuple -template class indexer_node; - -//indexer node specializations -template -class indexer_node : public internal::unfolded_indexer_node > { -private: - static const int N = 1; -public: - typedef tuple InputTuple; - typedef typename internal::tagged_msg output_type; - typedef typename internal::unfolded_indexer_node unfolded_type; - indexer_node(graph& g) : unfolded_type(g) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_INDEXER_NODE, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - // Copy constructor - indexer_node( const indexer_node& other ) : unfolded_type(other) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_INDEXER_NODE, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) { - tbb::internal::fgt_node_desc( this, name ); - } -#endif -}; - -template -class indexer_node : public internal::unfolded_indexer_node > { -private: - static const int N = 2; -public: - typedef tuple InputTuple; - typedef typename internal::tagged_msg output_type; - typedef typename internal::unfolded_indexer_node unfolded_type; - indexer_node(graph& g) : unfolded_type(g) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_INDEXER_NODE, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - // Copy constructor - indexer_node( const indexer_node& other ) : unfolded_type(other) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_INDEXER_NODE, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) { - tbb::internal::fgt_node_desc( this, name ); - } -#endif -}; - -template -class indexer_node : public internal::unfolded_indexer_node > { -private: - static const int N = 3; -public: - typedef tuple InputTuple; - typedef typename internal::tagged_msg output_type; - typedef typename internal::unfolded_indexer_node unfolded_type; - indexer_node(graph& g) : unfolded_type(g) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_INDEXER_NODE, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - // Copy constructor - indexer_node( const indexer_node& other ) : unfolded_type(other) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_INDEXER_NODE, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) { - tbb::internal::fgt_node_desc( this, name ); - } -#endif -}; - -template -class indexer_node : public internal::unfolded_indexer_node > { -private: - static const int N = 4; -public: - typedef tuple InputTuple; - typedef typename internal::tagged_msg output_type; - typedef typename internal::unfolded_indexer_node unfolded_type; - indexer_node(graph& g) : unfolded_type(g) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_INDEXER_NODE, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - // Copy constructor - indexer_node( const indexer_node& other ) : unfolded_type(other) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_INDEXER_NODE, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif -}; - -template -class indexer_node : public internal::unfolded_indexer_node > { -private: - static const int N = 5; -public: - typedef tuple InputTuple; - typedef typename internal::tagged_msg output_type; - typedef typename internal::unfolded_indexer_node unfolded_type; - indexer_node(graph& g) : unfolded_type(g) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_INDEXER_NODE, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - // Copy constructor - indexer_node( const indexer_node& other ) : unfolded_type(other) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_INDEXER_NODE, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif -}; - -#if __TBB_VARIADIC_MAX >= 6 -template -class indexer_node : public internal::unfolded_indexer_node > { -private: - static const int N = 6; -public: - typedef tuple InputTuple; - typedef typename internal::tagged_msg output_type; - typedef typename internal::unfolded_indexer_node unfolded_type; - indexer_node(graph& g) : unfolded_type(g) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_INDEXER_NODE, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - // Copy constructor - indexer_node( const indexer_node& other ) : unfolded_type(other) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_INDEXER_NODE, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif -}; -#endif //variadic max 6 - -#if __TBB_VARIADIC_MAX >= 7 -template -class indexer_node : public internal::unfolded_indexer_node > { -private: - static const int N = 7; -public: - typedef tuple InputTuple; - typedef typename internal::tagged_msg output_type; - typedef typename internal::unfolded_indexer_node unfolded_type; - indexer_node(graph& g) : unfolded_type(g) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_INDEXER_NODE, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - // Copy constructor - indexer_node( const indexer_node& other ) : unfolded_type(other) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_INDEXER_NODE, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif -}; -#endif //variadic max 7 - -#if __TBB_VARIADIC_MAX >= 8 -template -class indexer_node : public internal::unfolded_indexer_node > { -private: - static const int N = 8; -public: - typedef tuple InputTuple; - typedef typename internal::tagged_msg output_type; - typedef typename internal::unfolded_indexer_node unfolded_type; - indexer_node(graph& g) : unfolded_type(g) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_INDEXER_NODE, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - // Copy constructor - indexer_node( const indexer_node& other ) : unfolded_type(other) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_INDEXER_NODE, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif -}; -#endif //variadic max 8 - -#if __TBB_VARIADIC_MAX >= 9 -template -class indexer_node : public internal::unfolded_indexer_node > { -private: - static const int N = 9; -public: - typedef tuple InputTuple; - typedef typename internal::tagged_msg output_type; - typedef typename internal::unfolded_indexer_node unfolded_type; - indexer_node(graph& g) : unfolded_type(g) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_INDEXER_NODE, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - // Copy constructor - indexer_node( const indexer_node& other ) : unfolded_type(other) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_INDEXER_NODE, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif -}; -#endif //variadic max 9 - -#if __TBB_VARIADIC_MAX >= 10 -template -class indexer_node/*default*/ : public internal::unfolded_indexer_node > { -private: - static const int N = 10; -public: - typedef tuple InputTuple; - typedef typename internal::tagged_msg output_type; - typedef typename internal::unfolded_indexer_node unfolded_type; - indexer_node(graph& g) : unfolded_type(g) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_INDEXER_NODE, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - // Copy constructor - indexer_node( const indexer_node& other ) : unfolded_type(other) { - tbb::internal::fgt_multiinput_node( tbb::internal::FLOW_INDEXER_NODE, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif -}; -#endif //variadic max 10 - -#if __TBB_PREVIEW_ASYNC_MSG -inline void internal_make_edge( internal::untyped_sender &p, internal::untyped_receiver &s ) { -#else -template< typename T > -inline void internal_make_edge( sender &p, receiver &s ) { -#endif -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - s.internal_add_built_predecessor(p); - p.internal_add_built_successor(s); -#endif - p.register_successor( s ); - tbb::internal::fgt_make_edge( &p, &s ); -} - -//! Makes an edge between a single predecessor and a single successor -template< typename T > -inline void make_edge( sender &p, receiver &s ) { - internal_make_edge( p, s ); -} - -#if __TBB_PREVIEW_ASYNC_MSG -template< typename TS, typename TR, - typename = typename tbb::internal::enable_if::value - || tbb::internal::is_same_type::value>::type> -inline void make_edge( TS &p, TR &s ) { - internal_make_edge( p, s ); -} - -template< typename T > -inline void make_edge( sender &p, receiver &s ) { - internal_make_edge( p, s ); -} - -template< typename T > -inline void make_edge( sender &p, receiver &s ) { - internal_make_edge( p, s ); -} - -#endif // __TBB_PREVIEW_ASYNC_MSG - -#if __TBB_FLOW_GRAPH_CPP11_FEATURES -//Makes an edge from port 0 of a multi-output predecessor to port 0 of a multi-input successor. -template< typename T, typename V, - typename = typename T::output_ports_type, typename = typename V::input_ports_type > -inline void make_edge( T& output, V& input) { - make_edge(get<0>(output.output_ports()), get<0>(input.input_ports())); -} - -//Makes an edge from port 0 of a multi-output predecessor to a receiver. -template< typename T, typename R, - typename = typename T::output_ports_type > -inline void make_edge( T& output, receiver& input) { - make_edge(get<0>(output.output_ports()), input); -} - -//Makes an edge from a sender to port 0 of a multi-input successor. -template< typename S, typename V, - typename = typename V::input_ports_type > -inline void make_edge( sender& output, V& input) { - make_edge(output, get<0>(input.input_ports())); -} -#endif - -#if __TBB_PREVIEW_ASYNC_MSG -inline void internal_remove_edge( internal::untyped_sender &p, internal::untyped_receiver &s ) { -#else -template< typename T > -inline void internal_remove_edge( sender &p, receiver &s ) { -#endif - p.remove_successor( s ); -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - // TODO: should we try to remove p from the predecessor list of s, in case the edge is reversed? - p.internal_delete_built_successor(s); - s.internal_delete_built_predecessor(p); -#endif - tbb::internal::fgt_remove_edge( &p, &s ); -} - -//! Removes an edge between a single predecessor and a single successor -template< typename T > -inline void remove_edge( sender &p, receiver &s ) { - internal_remove_edge( p, s ); -} - -#if __TBB_PREVIEW_ASYNC_MSG -template< typename TS, typename TR, - typename = typename tbb::internal::enable_if::value - || tbb::internal::is_same_type::value>::type> -inline void remove_edge( TS &p, TR &s ) { - internal_remove_edge( p, s ); -} - -template< typename T > -inline void remove_edge( sender &p, receiver &s ) { - internal_remove_edge( p, s ); -} - -template< typename T > -inline void remove_edge( sender &p, receiver &s ) { - internal_remove_edge( p, s ); -} -#endif // __TBB_PREVIEW_ASYNC_MSG - -#if __TBB_FLOW_GRAPH_CPP11_FEATURES -//Removes an edge between port 0 of a multi-output predecessor and port 0 of a multi-input successor. -template< typename T, typename V, - typename = typename T::output_ports_type, typename = typename V::input_ports_type > -inline void remove_edge( T& output, V& input) { - remove_edge(get<0>(output.output_ports()), get<0>(input.input_ports())); -} - -//Removes an edge between port 0 of a multi-output predecessor and a receiver. -template< typename T, typename R, - typename = typename T::output_ports_type > -inline void remove_edge( T& output, receiver& input) { - remove_edge(get<0>(output.output_ports()), input); -} -//Removes an edge between a sender and port 0 of a multi-input successor. -template< typename S, typename V, - typename = typename V::input_ports_type > -inline void remove_edge( sender& output, V& input) { - remove_edge(output, get<0>(input.input_ports())); -} -#endif - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES -template -template< typename S > -void internal::edge_container::sender_extract( S &s ) { - edge_list_type e = built_edges; - for ( typename edge_list_type::iterator i = e.begin(); i != e.end(); ++i ) { - remove_edge(s, **i); - } -} - -template -template< typename R > -void internal::edge_container::receiver_extract( R &r ) { - edge_list_type e = built_edges; - for ( typename edge_list_type::iterator i = e.begin(); i != e.end(); ++i ) { - remove_edge(**i, r); - } -} -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ - -//! Returns a copy of the body from a function or continue node -template< typename Body, typename Node > -Body copy_body( Node &n ) { - return n.template copy_function_object(); -} - -#if __TBB_FLOW_GRAPH_CPP11_FEATURES - -//composite_node -template< typename InputTuple, typename OutputTuple > class composite_node; - -template< typename... InputTypes, typename... OutputTypes> -class composite_node , tbb::flow::tuple > : public graph_node{ - -public: - typedef tbb::flow::tuple< receiver&... > input_ports_type; - typedef tbb::flow::tuple< sender&... > output_ports_type; - -private: -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - const char *my_type_name; -#endif - std::unique_ptr my_input_ports; - std::unique_ptr my_output_ports; - - static const size_t NUM_INPUTS = sizeof...(InputTypes); - static const size_t NUM_OUTPUTS = sizeof...(OutputTypes); - -protected: - void reset_node(reset_flags) __TBB_override {} - -public: -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - composite_node( graph &g, const char *type_name ) : graph_node(g), my_type_name(type_name) { - tbb::internal::fgt_multiinput_multioutput_node( tbb::internal::FLOW_COMPOSITE_NODE, this, &this->my_graph ); - tbb::internal::fgt_multiinput_multioutput_node_desc( this, my_type_name ); - } -#endif - composite_node( graph &g ) : graph_node(g) { - tbb::internal::fgt_multiinput_multioutput_node( tbb::internal::FLOW_COMPOSITE_NODE, this, &this->my_graph ); - } - - template - void set_external_ports(T1&& input_ports_tuple, T2&& output_ports_tuple) { - __TBB_STATIC_ASSERT(NUM_INPUTS == tbb::flow::tuple_size::value, "number of arguments does not match number of input ports"); - __TBB_STATIC_ASSERT(NUM_OUTPUTS == tbb::flow::tuple_size::value, "number of arguments does not match number of output ports"); - my_input_ports = tbb::internal::make_unique(std::forward(input_ports_tuple)); - my_output_ports = tbb::internal::make_unique(std::forward(output_ports_tuple)); - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - tbb::internal::fgt_internal_input_alias_helper::alias_port( this, input_ports_tuple); - tbb::internal::fgt_internal_output_alias_helper::alias_port( this, output_ports_tuple); -#endif - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - template< typename... NodeTypes > - void add_visible_nodes(const NodeTypes&... n) { internal::add_nodes_impl(this, true, n...); } - - template< typename... NodeTypes > - void add_nodes(const NodeTypes&... n) { internal::add_nodes_impl(this, false, n...); } -#else - template void add_nodes(Nodes&...) { } - template void add_visible_nodes(Nodes&...) { } -#endif - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_multiinput_multioutput_node_desc( this, name ); - } -#endif - - input_ports_type& input_ports() { - __TBB_ASSERT(my_input_ports, "input ports not set, call set_external_ports to set input ports"); - return *my_input_ports; - } - - output_ports_type& output_ports() { - __TBB_ASSERT(my_output_ports, "output ports not set, call set_external_ports to set output ports"); - return *my_output_ports; - } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - void extract() __TBB_override { - __TBB_ASSERT(false, "Current composite_node implementation does not support extract"); - } -#endif -}; // class composite_node - -//composite_node with only input ports -template< typename... InputTypes> -class composite_node , tbb::flow::tuple<> > : public graph_node { -public: - typedef tbb::flow::tuple< receiver&... > input_ports_type; - -private: -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - const char *my_type_name; -#endif - std::unique_ptr my_input_ports; - static const size_t NUM_INPUTS = sizeof...(InputTypes); - -protected: - void reset_node(reset_flags) __TBB_override {} - -public: -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - composite_node( graph &g, const char *type_name = "composite_node") : graph_node(g), my_type_name(type_name) { - tbb::internal::itt_make_task_group( tbb::internal::ITT_DOMAIN_FLOW, this, tbb::internal::FLOW_NODE, &g, tbb::internal::FLOW_GRAPH, tbb::internal::FLOW_COMPOSITE_NODE ); - tbb::internal::fgt_multiinput_multioutput_node_desc( this, my_type_name ); - } -#else - composite_node( graph &g) : graph_node(g) {} -#endif - - template - void set_external_ports(T&& input_ports_tuple) { - __TBB_STATIC_ASSERT(NUM_INPUTS == tbb::flow::tuple_size::value, "number of arguments does not match number of input ports"); - - my_input_ports = tbb::internal::make_unique(std::forward(input_ports_tuple)); - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - tbb::internal::fgt_internal_input_alias_helper::alias_port( this, std::forward(input_ports_tuple)); -#endif - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - template< typename... NodeTypes > - void add_visible_nodes(const NodeTypes&... n) { internal::add_nodes_impl(this, true, n...); } - - template< typename... NodeTypes > - void add_nodes( const NodeTypes&... n) { internal::add_nodes_impl(this, false, n...); } -#else - template void add_nodes(Nodes&...) {} - template void add_visible_nodes(Nodes&...) {} -#endif - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_multiinput_multioutput_node_desc( this, name ); - } -#endif - - input_ports_type& input_ports() { - __TBB_ASSERT(my_input_ports, "input ports not set, call set_external_ports to set input ports"); - return *my_input_ports; - } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - void extract() __TBB_override { - __TBB_ASSERT(false, "Current composite_node implementation does not support extract"); - } -#endif - -}; // class composite_node - -//composite_nodes with only output_ports -template -class composite_node , tbb::flow::tuple > : public graph_node { -public: - typedef tbb::flow::tuple< sender&... > output_ports_type; - -private: -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - const char *my_type_name; -#endif - std::unique_ptr my_output_ports; - static const size_t NUM_OUTPUTS = sizeof...(OutputTypes); - -protected: - void reset_node(reset_flags) __TBB_override {} - -public: -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - composite_node( graph &g, const char *type_name = "composite_node") : graph_node(g), my_type_name(type_name) { - tbb::internal::itt_make_task_group( tbb::internal::ITT_DOMAIN_FLOW, this, tbb::internal::FLOW_NODE, &g, tbb::internal::FLOW_GRAPH, tbb::internal::FLOW_COMPOSITE_NODE ); - tbb::internal::fgt_multiinput_multioutput_node_desc( this, my_type_name ); - } -#else - composite_node( graph &g) : graph_node(g) {} -#endif - - template - void set_external_ports(T&& output_ports_tuple) { - __TBB_STATIC_ASSERT(NUM_OUTPUTS == tbb::flow::tuple_size::value, "number of arguments does not match number of output ports"); - - my_output_ports = tbb::internal::make_unique(std::forward(output_ports_tuple)); - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - tbb::internal::fgt_internal_output_alias_helper::alias_port( this, std::forward(output_ports_tuple)); -#endif - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - template - void add_visible_nodes(const NodeTypes&... n) { internal::add_nodes_impl(this, true, n...); } - - template - void add_nodes(const NodeTypes&... n) { internal::add_nodes_impl(this, false, n...); } -#else - template void add_nodes(Nodes&...) {} - template void add_visible_nodes(Nodes&...) {} -#endif - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_multiinput_multioutput_node_desc( this, name ); - } -#endif - - output_ports_type& output_ports() { - __TBB_ASSERT(my_output_ports, "output ports not set, call set_external_ports to set output ports"); - return *my_output_ports; - } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - void extract() __TBB_override { - __TBB_ASSERT(false, "Current composite_node implementation does not support extract"); - } -#endif - -}; // class composite_node - -#endif // __TBB_FLOW_GRAPH_CPP11_FEATURES - -namespace internal { - -template -class async_body_base: tbb::internal::no_assign { -public: - typedef Gateway gateway_type; - - async_body_base(gateway_type *gateway): my_gateway(gateway) { } - void set_gateway(gateway_type *gateway) { - my_gateway = gateway; - } - -protected: - gateway_type *my_gateway; -}; - -template -class async_body: public async_body_base { -public: - typedef async_body_base base_type; - typedef Gateway gateway_type; - - async_body(const Body &body, gateway_type *gateway) - : base_type(gateway), my_body(body) { } - - void operator()( const Input &v, Ports & ) { - my_body(v, *this->my_gateway); - } - - Body get_body() { return my_body; } - -private: - Body my_body; -}; - -} - -//! Implements async node -template < typename Input, typename Output, typename Policy = queueing, typename Allocator=cache_aligned_allocator > -class async_node : public multifunction_node< Input, tuple< Output >, Policy, Allocator >, public sender< Output > { - typedef multifunction_node< Input, tuple< Output >, Policy, Allocator > base_type; - typedef typename internal::multifunction_input mfn_input_type; - -public: - typedef Input input_type; - typedef Output output_type; - typedef receiver receiver_type; - typedef typename receiver_type::predecessor_type predecessor_type; - typedef typename sender::successor_type successor_type; - typedef receiver_gateway gateway_type; - typedef internal::async_body_base async_body_base_type; - typedef typename base_type::output_ports_type output_ports_type; - -private: - struct try_put_functor { - typedef internal::multifunction_output output_port_type; - output_port_type *port; - const Output *value; - bool result; - try_put_functor(output_port_type &p, const Output &v) : port(&p), value(&v), result(false) { } - void operator()() { - result = port->try_put(*value); - } - }; - - class receiver_gateway_impl: public receiver_gateway { - public: - receiver_gateway_impl(async_node* node): my_node(node) {} - void reserve_wait() __TBB_override { - tbb::internal::fgt_async_reserve(static_cast(my_node), &my_node->my_graph); - my_node->my_graph.reserve_wait(); - } - - void release_wait() __TBB_override { - my_node->my_graph.release_wait(); - tbb::internal::fgt_async_commit(static_cast(my_node), &my_node->my_graph); - } - - //! Implements gateway_type::try_put for an external activity to submit a message to FG - bool try_put(const Output &i) __TBB_override { - return my_node->try_put_impl(i); - } - - private: - async_node* my_node; - } my_gateway; - - //The substitute of 'this' for member construction, to prevent compiler warnings - async_node* self() { return this; } - - //! Implements gateway_type::try_put for an external activity to submit a message to FG - bool try_put_impl(const Output &i) { - internal::multifunction_output &port_0 = internal::output_port<0>(*this); - tbb::internal::fgt_async_try_put_begin(this, &port_0); - try_put_functor tpf(port_0, i); - internal::execute_in_graph_arena(this->my_graph, tpf); - tbb::internal::fgt_async_try_put_end(this, &port_0); - return tpf.result; - } - -public: - template - async_node( graph &g, size_t concurrency, Body body ) : - base_type( g, concurrency, internal::async_body(body, &my_gateway) ), my_gateway(self()) { - tbb::internal::fgt_multioutput_node_with_body<1>( tbb::internal::FLOW_ASYNC_NODE, - &this->my_graph, static_cast *>(this), - this->output_ports(), this->my_body ); - } - - async_node( const async_node &other ) : base_type(other), sender(), my_gateway(self()) { - static_cast(this->my_body->get_body_ptr())->set_gateway(&my_gateway); - static_cast(this->my_init_body->get_body_ptr())->set_gateway(&my_gateway); - - tbb::internal::fgt_multioutput_node_with_body<1>( tbb::internal::FLOW_ASYNC_NODE, - &this->my_graph, static_cast *>(this), - this->output_ports(), this->my_body ); - } - - gateway_type& gateway() { - return my_gateway; - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_multioutput_node_desc( this, name ); - } -#endif - - // Define sender< Output > - - //! Add a new successor to this node - bool register_successor( successor_type &r ) __TBB_override { - return internal::output_port<0>(*this).register_successor(r); - } - - //! Removes a successor from this node - bool remove_successor( successor_type &r ) __TBB_override { - return internal::output_port<0>(*this).remove_successor(r); - } - - template - Body copy_function_object() { - typedef internal::multifunction_body mfn_body_type; - typedef internal::async_body async_body_type; - mfn_body_type &body_ref = *this->my_body; - async_body_type ab = *static_cast(dynamic_cast< internal::multifunction_body_leaf & >(body_ref).get_body_ptr()); - return ab.get_body(); - } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - //! interface to record edges for traversal & deletion - typedef typename internal::edge_container built_successors_type; - typedef typename built_successors_type::edge_list_type successor_list_type; - built_successors_type &built_successors() __TBB_override { - return internal::output_port<0>(*this).built_successors(); - } - - void internal_add_built_successor( successor_type &r ) __TBB_override { - internal::output_port<0>(*this).internal_add_built_successor(r); - } - - void internal_delete_built_successor( successor_type &r ) __TBB_override { - internal::output_port<0>(*this).internal_delete_built_successor(r); - } - - void copy_successors( successor_list_type &l ) __TBB_override { - internal::output_port<0>(*this).copy_successors(l); - } - - size_t successor_count() __TBB_override { - return internal::output_port<0>(*this).successor_count(); - } -#endif - -protected: - - void reset_node( reset_flags f) __TBB_override { - base_type::reset_node(f); - } -}; - -#if __TBB_PREVIEW_STREAMING_NODE -#include "internal/_flow_graph_streaming_node.h" -#endif // __TBB_PREVIEW_STREAMING_NODE - -} // interfaceX - - using interface10::reset_flags; - using interface10::rf_reset_protocol; - using interface10::rf_reset_bodies; - using interface10::rf_clear_edges; - - using interface10::graph; - using interface10::graph_node; - using interface10::continue_msg; - - using interface10::source_node; - using interface10::function_node; - using interface10::multifunction_node; - using interface10::split_node; - using interface10::internal::output_port; - using interface10::indexer_node; - using interface10::internal::tagged_msg; - using interface10::internal::cast_to; - using interface10::internal::is_a; - using interface10::continue_node; - using interface10::overwrite_node; - using interface10::write_once_node; - using interface10::broadcast_node; - using interface10::buffer_node; - using interface10::queue_node; - using interface10::sequencer_node; - using interface10::priority_queue_node; - using interface10::limiter_node; - using namespace interface10::internal::graph_policy_namespace; - using interface10::join_node; - using interface10::input_port; - using interface10::copy_body; - using interface10::make_edge; - using interface10::remove_edge; - using interface10::internal::tag_value; -#if __TBB_FLOW_GRAPH_CPP11_FEATURES - using interface10::composite_node; -#endif - using interface10::async_node; -#if __TBB_PREVIEW_ASYNC_MSG - using interface10::async_msg; -#endif -#if __TBB_PREVIEW_STREAMING_NODE - using interface10::port_ref; - using interface10::streaming_node; -#endif // __TBB_PREVIEW_STREAMING_NODE - -} // flow -} // tbb - -#undef __TBB_PFG_RESET_ARG -#undef __TBB_COMMA - -#endif // __TBB_flow_graph_H +#include "../oneapi/tbb/flow_graph.h" diff --git a/src/tbb/include/tbb/flow_graph_abstractions.h b/src/tbb/include/tbb/flow_graph_abstractions.h index 46971e03e..cd9dc2967 100644 --- a/src/tbb/include/tbb/flow_graph_abstractions.h +++ b/src/tbb/include/tbb/flow_graph_abstractions.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,46 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ -#ifndef __TBB_flow_graph_abstractions_H -#define __TBB_flow_graph_abstractions_H - -namespace tbb { -namespace flow { -namespace interface10 { - -//! Pure virtual template classes that define interfaces for async communication -class graph_proxy { -public: - //! Inform a graph that messages may come from outside, to prevent premature graph completion - virtual void reserve_wait() = 0; - - //! Inform a graph that a previous call to reserve_wait is no longer in effect - virtual void release_wait() = 0; - - virtual ~graph_proxy() {} -}; - -template -class receiver_gateway : public graph_proxy { -public: - //! Type of inputing data into FG. - typedef Input input_type; - - //! Submit signal from an asynchronous activity to FG. - virtual bool try_put(const input_type&) = 0; -}; - -} //interfaceX - -using interface10::graph_proxy; -using interface10::receiver_gateway; - -} //flow -} //tbb -#endif +#include "../oneapi/tbb/flow_graph_abstractions.h" diff --git a/src/tbb/include/tbb/flow_graph_opencl_node.h b/src/tbb/include/tbb/flow_graph_opencl_node.h deleted file mode 100644 index e28abe785..000000000 --- a/src/tbb/include/tbb/flow_graph_opencl_node.h +++ /dev/null @@ -1,1481 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_flow_graph_opencl_node_H -#define __TBB_flow_graph_opencl_node_H - -#include "tbb/tbb_config.h" -#if __TBB_PREVIEW_OPENCL_NODE - -#include "flow_graph.h" - -#include -#include -#include -#include -#include -#include -#include - -#ifdef __APPLE__ -#include -#else -#include -#endif - -namespace tbb { -namespace flow { - -namespace interface10 { - -template -class opencl_factory; - -namespace opencl_info { -class default_opencl_factory; -} - -template -class opencl_program; - -inline void enforce_cl_retcode(cl_int err, std::string msg) { - if (err != CL_SUCCESS) { - std::cerr << msg << "; error code: " << err << std::endl; - throw msg; - } -} - -template -T event_info(cl_event e, cl_event_info i) { - T res; - enforce_cl_retcode(clGetEventInfo(e, i, sizeof(res), &res, NULL), "Failed to get OpenCL event information"); - return res; -} - -template -T device_info(cl_device_id d, cl_device_info i) { - T res; - enforce_cl_retcode(clGetDeviceInfo(d, i, sizeof(res), &res, NULL), "Failed to get OpenCL device information"); - return res; -} - -template <> -inline std::string device_info(cl_device_id d, cl_device_info i) { - size_t required; - enforce_cl_retcode(clGetDeviceInfo(d, i, 0, NULL, &required), "Failed to get OpenCL device information"); - - char *buff = (char*)alloca(required); - enforce_cl_retcode(clGetDeviceInfo(d, i, required, buff, NULL), "Failed to get OpenCL device information"); - - return buff; -} - -template -T platform_info(cl_platform_id p, cl_platform_info i) { - T res; - enforce_cl_retcode(clGetPlatformInfo(p, i, sizeof(res), &res, NULL), "Failed to get OpenCL platform information"); - return res; -} - -template <> -inline std::string platform_info(cl_platform_id p, cl_platform_info i) { - size_t required; - enforce_cl_retcode(clGetPlatformInfo(p, i, 0, NULL, &required), "Failed to get OpenCL platform information"); - - char *buff = (char*)alloca(required); - enforce_cl_retcode(clGetPlatformInfo(p, i, required, buff, NULL), "Failed to get OpenCL platform information"); - - return buff; -} - - -class opencl_device { -public: - typedef size_t device_id_type; - enum : device_id_type { - unknown = device_id_type( -2 ), - host = device_id_type( -1 ) - }; - - opencl_device() : my_device_id( unknown ), my_cl_device_id( NULL ), my_cl_command_queue( NULL ) {} - - opencl_device( cl_device_id d_id ) : my_device_id( unknown ), my_cl_device_id( d_id ), my_cl_command_queue( NULL ) {} - - opencl_device( cl_device_id cl_d_id, device_id_type device_id ) : my_device_id( device_id ), my_cl_device_id( cl_d_id ), my_cl_command_queue( NULL ) {} - - std::string platform_profile() const { - return platform_info( platform(), CL_PLATFORM_PROFILE ); - } - std::string platform_version() const { - return platform_info( platform(), CL_PLATFORM_VERSION ); - } - std::string platform_name() const { - return platform_info( platform(), CL_PLATFORM_NAME ); - } - std::string platform_vendor() const { - return platform_info( platform(), CL_PLATFORM_VENDOR ); - } - std::string platform_extensions() const { - return platform_info( platform(), CL_PLATFORM_EXTENSIONS ); - } - - template - void info( cl_device_info i, T &t ) const { - t = device_info( my_cl_device_id, i ); - } - std::string version() const { - // The version string format: OpenCL - return device_info( my_cl_device_id, CL_DEVICE_VERSION ); - } - int major_version() const { - int major; - std::sscanf( version().c_str(), "OpenCL %d", &major ); - return major; - } - int minor_version() const { - int major, minor; - std::sscanf( version().c_str(), "OpenCL %d.%d", &major, &minor ); - return minor; - } - bool out_of_order_exec_mode_on_host_present() const { -#if CL_VERSION_2_0 - if ( major_version() >= 2 ) - return (device_info( my_cl_device_id, CL_DEVICE_QUEUE_ON_HOST_PROPERTIES ) & CL_QUEUE_OUT_OF_ORDER_EXEC_MODE_ENABLE) != 0; - else -#endif /* CL_VERSION_2_0 */ - return (device_info( my_cl_device_id, CL_DEVICE_QUEUE_PROPERTIES ) & CL_QUEUE_OUT_OF_ORDER_EXEC_MODE_ENABLE) != 0; - } - bool out_of_order_exec_mode_on_device_present() const { -#if CL_VERSION_2_0 - if ( major_version() >= 2 ) - return (device_info( my_cl_device_id, CL_DEVICE_QUEUE_ON_DEVICE_PROPERTIES ) & CL_QUEUE_OUT_OF_ORDER_EXEC_MODE_ENABLE) != 0; - else -#endif /* CL_VERSION_2_0 */ - return false; - } - std::array max_work_item_sizes() const { - return device_info>( my_cl_device_id, CL_DEVICE_MAX_WORK_ITEM_SIZES ); - } - size_t max_work_group_size() const { - return device_info( my_cl_device_id, CL_DEVICE_MAX_WORK_GROUP_SIZE ); - } - bool built_in_kernel_available( const std::string& k ) const { - const std::string semi = ";"; - // Added semicolumns to force an exact match (to avoid a partial match, e.g. "add" is partly matched with "madd"). - return (semi + built_in_kernels() + semi).find( semi + k + semi ) != std::string::npos; - } - std::string built_in_kernels() const { - return device_info( my_cl_device_id, CL_DEVICE_BUILT_IN_KERNELS ); - } - std::string name() const { - return device_info( my_cl_device_id, CL_DEVICE_NAME ); - } - cl_bool available() const { - return device_info( my_cl_device_id, CL_DEVICE_AVAILABLE ); - } - cl_bool compiler_available() const { - return device_info( my_cl_device_id, CL_DEVICE_COMPILER_AVAILABLE ); - } - cl_bool linker_available() const { - return device_info( my_cl_device_id, CL_DEVICE_LINKER_AVAILABLE ); - } - bool extension_available( const std::string &ext ) const { - const std::string space = " "; - // Added space to force an exact match (to avoid a partial match, e.g. "ext" is partly matched with "ext2"). - return (space + extensions() + space).find( space + ext + space ) != std::string::npos; - } - std::string extensions() const { - return device_info( my_cl_device_id, CL_DEVICE_EXTENSIONS ); - } - - cl_device_type type() const { - return device_info( my_cl_device_id, CL_DEVICE_TYPE ); - } - - std::string vendor() const { - return device_info( my_cl_device_id, CL_DEVICE_VENDOR ); - } - - cl_uint address_bits() const { - return device_info( my_cl_device_id, CL_DEVICE_ADDRESS_BITS ); - } - - cl_device_id device_id() const { - return my_cl_device_id; - } - - cl_command_queue command_queue() const { - return my_cl_command_queue; - } - - void set_command_queue( cl_command_queue cmd_queue ) { - my_cl_command_queue = cmd_queue; - } - -private: - - cl_platform_id platform() const { - return device_info( my_cl_device_id, CL_DEVICE_PLATFORM ); - } - - device_id_type my_device_id; - cl_device_id my_cl_device_id; - cl_command_queue my_cl_command_queue; - - friend bool operator==(opencl_device d1, opencl_device d2) { return d1.my_cl_device_id == d2.my_cl_device_id; } - - template - friend class opencl_factory; - template - friend class opencl_memory; - template - friend class opencl_program; - -#if TBB_USE_ASSERT - template - friend class opencl_buffer; -#endif -}; - -class opencl_device_list { - typedef std::vector container_type; -public: - typedef container_type::iterator iterator; - typedef container_type::const_iterator const_iterator; - typedef container_type::size_type size_type; - - opencl_device_list() {} - opencl_device_list( std::initializer_list il ) : my_container( il ) {} - - void add( opencl_device d ) { my_container.push_back( d ); } - size_type size() const { return my_container.size(); } - bool empty() const { return my_container.empty(); } - iterator begin() { return my_container.begin(); } - iterator end() { return my_container.end(); } - const_iterator begin() const { return my_container.begin(); } - const_iterator end() const { return my_container.end(); } - const_iterator cbegin() const { return my_container.cbegin(); } - const_iterator cend() const { return my_container.cend(); } - -private: - container_type my_container; -}; - -namespace internal { - -// Retrieve all OpenCL devices from machine -inline opencl_device_list find_available_devices() { - opencl_device_list opencl_devices; - - cl_uint num_platforms; - enforce_cl_retcode(clGetPlatformIDs(0, NULL, &num_platforms), "clGetPlatformIDs failed"); - - std::vector platforms(num_platforms); - enforce_cl_retcode(clGetPlatformIDs(num_platforms, platforms.data(), NULL), "clGetPlatformIDs failed"); - - cl_uint num_devices; - std::vector::iterator platforms_it = platforms.begin(); - cl_uint num_all_devices = 0; - while (platforms_it != platforms.end()) { - cl_int err = clGetDeviceIDs(*platforms_it, CL_DEVICE_TYPE_ALL, 0, NULL, &num_devices); - if (err == CL_DEVICE_NOT_FOUND) { - platforms_it = platforms.erase(platforms_it); - } - else { - enforce_cl_retcode(err, "clGetDeviceIDs failed"); - num_all_devices += num_devices; - ++platforms_it; - } - } - - std::vector devices(num_all_devices); - std::vector::iterator devices_it = devices.begin(); - for (auto p = platforms.begin(); p != platforms.end(); ++p) { - enforce_cl_retcode(clGetDeviceIDs((*p), CL_DEVICE_TYPE_ALL, (cl_uint)std::distance(devices_it, devices.end()), &*devices_it, &num_devices), "clGetDeviceIDs failed"); - devices_it += num_devices; - } - - for (auto d = devices.begin(); d != devices.end(); ++d) { - opencl_devices.add(opencl_device((*d))); - } - - return opencl_devices; -} - -} // namespace internal - -// TODO: consider this namespace as public API -namespace opencl_info { - - inline const opencl_device_list& available_devices() { - // Static storage for all available OpenCL devices on machine - static const opencl_device_list my_devices = internal::find_available_devices(); - return my_devices; - } - -} // namespace opencl_info - - -class callback_base : tbb::internal::no_copy { -public: - virtual void call() = 0; - virtual ~callback_base() {} -}; - -template -class callback : public callback_base { - Callback my_callback; - T my_data; -public: - callback( Callback c, const T& t ) : my_callback( c ), my_data( t ) {} - - void call() __TBB_override { - my_callback( my_data ); - } -}; - -template -class opencl_async_msg : public async_msg { -public: - typedef T value_type; - - opencl_async_msg() : my_callback_flag_ptr( std::make_shared< tbb::atomic>() ) { - my_callback_flag_ptr->store(false); - } - - explicit opencl_async_msg( const T& data ) : my_data(data), my_callback_flag_ptr( std::make_shared>() ) { - my_callback_flag_ptr->store(false); - } - - opencl_async_msg( const T& data, cl_event event ) : my_data(data), my_event(event), my_is_event(true), my_callback_flag_ptr( std::make_shared>() ) { - my_callback_flag_ptr->store(false); - enforce_cl_retcode( clRetainEvent( my_event ), "Failed to retain an event" ); - } - - T& data( bool wait = true ) { - if ( my_is_event && wait ) { - enforce_cl_retcode( clWaitForEvents( 1, &my_event ), "Failed to wait for an event" ); - enforce_cl_retcode( clReleaseEvent( my_event ), "Failed to release an event" ); - my_is_event = false; - } - return my_data; - } - - const T& data( bool wait = true ) const { - if ( my_is_event && wait ) { - enforce_cl_retcode( clWaitForEvents( 1, &my_event ), "Failed to wait for an event" ); - enforce_cl_retcode( clReleaseEvent( my_event ), "Failed to release an event" ); - my_is_event = false; - } - return my_data; - } - - opencl_async_msg( const opencl_async_msg &dmsg ) : async_msg(dmsg), - my_data(dmsg.my_data), my_event(dmsg.my_event), my_is_event( dmsg.my_is_event ), - my_callback_flag_ptr(dmsg.my_callback_flag_ptr) - { - if ( my_is_event ) - enforce_cl_retcode( clRetainEvent( my_event ), "Failed to retain an event" ); - } - - opencl_async_msg( opencl_async_msg &&dmsg ) : async_msg(std::move(dmsg)), - my_data(std::move(dmsg.my_data)), my_event(dmsg.my_event), my_is_event(dmsg.my_is_event), - my_callback_flag_ptr( std::move(dmsg.my_callback_flag_ptr) ) - { - dmsg.my_is_event = false; - } - - opencl_async_msg& operator=(const opencl_async_msg &dmsg) { - async_msg::operator =(dmsg); - - // Release original event - if ( my_is_event ) - enforce_cl_retcode( clReleaseEvent( my_event ), "Failed to retain an event" ); - - my_data = dmsg.my_data; - my_event = dmsg.my_event; - my_is_event = dmsg.my_is_event; - - // Retain copied event - if ( my_is_event ) - enforce_cl_retcode( clRetainEvent( my_event ), "Failed to retain an event" ); - - my_callback_flag_ptr = dmsg.my_callback_flag_ptr; - return *this; - } - - ~opencl_async_msg() { - if ( my_is_event ) - enforce_cl_retcode( clReleaseEvent( my_event ), "Failed to release an event" ); - } - - cl_event const * get_event() const { return my_is_event ? &my_event : NULL; } - void set_event( cl_event e ) const { - if ( my_is_event ) { - cl_command_queue cq = event_info( my_event, CL_EVENT_COMMAND_QUEUE ); - if ( cq != event_info( e, CL_EVENT_COMMAND_QUEUE ) ) - enforce_cl_retcode( clFlush( cq ), "Failed to flush an OpenCL command queue" ); - enforce_cl_retcode( clReleaseEvent( my_event ), "Failed to release an event" ); - } - my_is_event = true; - my_event = e; - clRetainEvent( my_event ); - } - - void clear_event() const { - if ( my_is_event ) { - enforce_cl_retcode( clFlush( event_info( my_event, CL_EVENT_COMMAND_QUEUE ) ), "Failed to flush an OpenCL command queue" ); - enforce_cl_retcode( clReleaseEvent( my_event ), "Failed to release an event" ); - } - my_is_event = false; - } - - template - void register_callback( Callback c ) const { - __TBB_ASSERT( my_is_event, "The OpenCL event is not set" ); - enforce_cl_retcode( clSetEventCallback( my_event, CL_COMPLETE, register_callback_func, new callback( c, my_data ) ), "Failed to set an OpenCL callback" ); - } - - operator T&() { return data(); } - operator const T&() const { return data(); } - -protected: - // Overridden in this derived class to inform that - // async calculation chain is over - void finalize() const __TBB_override { - receive_if_memory_object(*this); - if (! my_callback_flag_ptr->fetch_and_store(true)) { - opencl_async_msg a(*this); - if (my_is_event) { - register_callback([a](const T& t) mutable { - a.set(t); - }); - } - else { - a.set(my_data); - } - } - clear_event(); - } - -private: - static void CL_CALLBACK register_callback_func( cl_event, cl_int event_command_exec_status, void *data ) { - tbb::internal::suppress_unused_warning( event_command_exec_status ); - __TBB_ASSERT( event_command_exec_status == CL_COMPLETE, NULL ); - __TBB_ASSERT( data, NULL ); - callback_base *c = static_cast(data); - c->call(); - delete c; - } - - T my_data; - mutable cl_event my_event; - mutable bool my_is_event = false; - - std::shared_ptr< tbb::atomic > my_callback_flag_ptr; -}; - -template -K key_from_message( const opencl_async_msg &dmsg ) { - using tbb::flow::key_from_message; - const T &t = dmsg.data( false ); - __TBB_STATIC_ASSERT( true, "" ); - return key_from_message( t ); -} - -template -class opencl_memory { -public: - opencl_memory() {} - opencl_memory( Factory &f ) : my_host_ptr( NULL ), my_factory( &f ), my_sending_event_present( false ) { - my_curr_device_id = my_factory->devices().begin()->my_device_id; - } - - ~opencl_memory() { - if ( my_sending_event_present ) enforce_cl_retcode( clReleaseEvent( my_sending_event ), "Failed to release an event for the OpenCL buffer" ); - enforce_cl_retcode( clReleaseMemObject( my_cl_mem ), "Failed to release an memory object" ); - } - - cl_mem get_cl_mem() const { - return my_cl_mem; - } - - void* get_host_ptr() { - if ( !my_host_ptr ) { - opencl_async_msg d = receive( NULL ); - d.data(); - __TBB_ASSERT( d.data() == my_host_ptr, NULL ); - } - return my_host_ptr; - } - - Factory *factory() const { return my_factory; } - - opencl_async_msg receive(const cl_event *e) { - opencl_async_msg d; - if (e) { - d = opencl_async_msg(my_host_ptr, *e); - } else { - d = opencl_async_msg(my_host_ptr); - } - - // Concurrent receives are prohibited so we do not worry about synchronization. - if (my_curr_device_id.load() != opencl_device::host) { - map_memory(*my_factory->devices().begin(), d); - my_curr_device_id.store(opencl_device::host); - my_host_ptr = d.data(false); - } - // Release the sending event - if (my_sending_event_present) { - enforce_cl_retcode(clReleaseEvent(my_sending_event), "Failed to release an event"); - my_sending_event_present = false; - } - return d; - } - - opencl_async_msg send(opencl_device device, const cl_event *e) { - opencl_device::device_id_type device_id = device.my_device_id; - if (!my_factory->is_same_context(my_curr_device_id.load(), device_id)) { - { - tbb::spin_mutex::scoped_lock lock(my_sending_lock); - if (!my_factory->is_same_context(my_curr_device_id.load(), device_id)) { - __TBB_ASSERT(my_host_ptr, "The buffer has not been mapped"); - opencl_async_msg d(my_host_ptr); - my_factory->enqueue_unmap_buffer(device, *this, d); - my_sending_event = *d.get_event(); - my_sending_event_present = true; - enforce_cl_retcode(clRetainEvent(my_sending_event), "Failed to retain an event"); - my_host_ptr = NULL; - my_curr_device_id.store(device_id); - } - } - __TBB_ASSERT(my_sending_event_present, NULL); - } - - // !e means that buffer has come from the host - if (!e && my_sending_event_present) e = &my_sending_event; - - __TBB_ASSERT(!my_host_ptr, "The buffer has not been unmapped"); - return e ? opencl_async_msg(NULL, *e) : opencl_async_msg(NULL); - } - - virtual void map_memory( opencl_device, opencl_async_msg & ) = 0; -protected: - cl_mem my_cl_mem; - tbb::atomic my_curr_device_id; - void* my_host_ptr; - Factory *my_factory; - - tbb::spin_mutex my_sending_lock; - bool my_sending_event_present; - cl_event my_sending_event; -}; - -template -class opencl_buffer_impl : public opencl_memory { - size_t my_size; -public: - opencl_buffer_impl( size_t size, Factory& f ) : opencl_memory( f ), my_size( size ) { - cl_int err; - this->my_cl_mem = clCreateBuffer( this->my_factory->context(), CL_MEM_ALLOC_HOST_PTR, size, NULL, &err ); - enforce_cl_retcode( err, "Failed to create an OpenCL buffer" ); - } - - // The constructor for subbuffers. - opencl_buffer_impl( cl_mem m, size_t index, size_t size, Factory& f ) : opencl_memory( f ), my_size( size ) { - cl_int err; - cl_buffer_region region = { index, size }; - this->my_cl_mem = clCreateSubBuffer( m, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err ); - enforce_cl_retcode( err, "Failed to create an OpenCL subbuffer" ); - } - - size_t size() const { - return my_size; - } - - void map_memory( opencl_device device, opencl_async_msg &dmsg ) __TBB_override { - this->my_factory->enqueue_map_buffer( device, *this, dmsg ); - } - -#if TBB_USE_ASSERT - template - friend class opencl_buffer; -#endif -}; - -enum access_type { - read_write, - write_only, - read_only -}; - -template -class opencl_subbuffer; - -template -class opencl_buffer { -public: - typedef cl_mem native_object_type; - typedef opencl_buffer memory_object_type; - typedef Factory opencl_factory_type; - - template using iterator = T*; - - template - iterator access() const { - T* ptr = (T*)my_impl->get_host_ptr(); - __TBB_ASSERT( ptr, NULL ); - return iterator( ptr ); - } - - T* data() const { return &access()[0]; } - - template - iterator begin() const { return access(); } - - template - iterator end() const { return access()+my_impl->size()/sizeof(T); } - - size_t size() const { return my_impl->size()/sizeof(T); } - - T& operator[] ( ptrdiff_t k ) { return begin()[k]; } - - opencl_buffer() {} - opencl_buffer( size_t size ); - opencl_buffer( Factory &f, size_t size ) : my_impl( std::make_shared( size*sizeof(T), f ) ) {} - - cl_mem native_object() const { - return my_impl->get_cl_mem(); - } - - const opencl_buffer& memory_object() const { - return *this; - } - - void send( opencl_device device, opencl_async_msg &dependency ) const { - __TBB_ASSERT( dependency.data( /*wait = */false ) == *this, NULL ); - opencl_async_msg d = my_impl->send( device, dependency.get_event() ); - const cl_event *e = d.get_event(); - if ( e ) dependency.set_event( *e ); - else dependency.clear_event(); - } - void receive( const opencl_async_msg &dependency ) const { - __TBB_ASSERT( dependency.data( /*wait = */false ) == *this, NULL ); - opencl_async_msg d = my_impl->receive( dependency.get_event() ); - const cl_event *e = d.get_event(); - if ( e ) dependency.set_event( *e ); - else dependency.clear_event(); - } - - opencl_subbuffer subbuffer( size_t index, size_t size ) const; -private: - // The constructor for subbuffers. - opencl_buffer( Factory &f, cl_mem m, size_t index, size_t size ) : my_impl( std::make_shared( m, index*sizeof(T), size*sizeof(T), f ) ) {} - - typedef opencl_buffer_impl impl_type; - - std::shared_ptr my_impl; - - friend bool operator==(const opencl_buffer &lhs, const opencl_buffer &rhs) { - return lhs.my_impl == rhs.my_impl; - } - - template - friend class opencl_factory; - template - friend class opencl_subbuffer; -}; - -template -class opencl_subbuffer : public opencl_buffer { - opencl_buffer my_owner; -public: - opencl_subbuffer() {} - opencl_subbuffer( const opencl_buffer &owner, size_t index, size_t size ) : - opencl_buffer( *owner.my_impl->factory(), owner.native_object(), index, size ), my_owner( owner ) {} -}; - -template -opencl_subbuffer opencl_buffer::subbuffer( size_t index, size_t size ) const { - return opencl_subbuffer( *this, index, size ); -} - - -#define is_typedef(type) \ - template \ - struct is_##type { \ - template \ - static std::true_type check( typename C::type* ); \ - template \ - static std::false_type check( ... ); \ - \ - static const bool value = decltype(check(0))::value; \ - } - -is_typedef( native_object_type ); -is_typedef( memory_object_type ); - -template -typename std::enable_if::value, typename T::native_object_type>::type get_native_object( const T &t ) { - return t.native_object(); -} - -template -typename std::enable_if::value, T>::type get_native_object( T t ) { - return t; -} - -// send_if_memory_object checks if the T type has memory_object_type and call the send method for the object. -template -typename std::enable_if::value>::type send_if_memory_object( opencl_device device, opencl_async_msg &dmsg ) { - const T &t = dmsg.data( false ); - typedef typename T::memory_object_type mem_obj_t; - mem_obj_t mem_obj = t.memory_object(); - opencl_async_msg d( mem_obj ); - if ( dmsg.get_event() ) d.set_event( *dmsg.get_event() ); - mem_obj.send( device, d ); - if ( d.get_event() ) dmsg.set_event( *d.get_event() ); -} - -template -typename std::enable_if::value>::type send_if_memory_object( opencl_device device, T &t ) { - typedef typename T::memory_object_type mem_obj_t; - mem_obj_t mem_obj = t.memory_object(); - opencl_async_msg dmsg( mem_obj ); - mem_obj.send( device, dmsg ); -} - -template -typename std::enable_if::value>::type send_if_memory_object( opencl_device, T& ) {}; - -// receive_if_memory_object checks if the T type has memory_object_type and call the receive method for the object. -template -typename std::enable_if::value>::type receive_if_memory_object( const opencl_async_msg &dmsg ) { - const T &t = dmsg.data( false ); - typedef typename T::memory_object_type mem_obj_t; - mem_obj_t mem_obj = t.memory_object(); - opencl_async_msg d( mem_obj ); - if ( dmsg.get_event() ) d.set_event( *dmsg.get_event() ); - mem_obj.receive( d ); - if ( d.get_event() ) dmsg.set_event( *d.get_event() ); -} - -template -typename std::enable_if::value>::type receive_if_memory_object( const T& ) {} - -class opencl_range { -public: - typedef size_t range_index_type; - typedef std::array nd_range_type; - - template , typename L = std::initializer_list, - typename = typename std::enable_if::type, opencl_range>::value>::type> - opencl_range(G&& global_work = std::initializer_list({ 0 }), L&& local_work = std::initializer_list({ 0, 0, 0 })) { - auto g_it = global_work.begin(); - auto l_it = local_work.begin(); - my_global_work_size = { size_t(-1), size_t(-1), size_t(-1) }; - // my_local_work_size is still uninitialized - for (int s = 0; s < 3 && g_it != global_work.end(); ++g_it, ++l_it, ++s) { - __TBB_ASSERT(l_it != local_work.end(), "global_work & local_work must have same size"); - my_global_work_size[s] = *g_it; - my_local_work_size[s] = *l_it; - } - } - - const nd_range_type& global_range() const { return my_global_work_size; } - const nd_range_type& local_range() const { return my_local_work_size; } - -private: - nd_range_type my_global_work_size; - nd_range_type my_local_work_size; -}; - -template -class opencl_factory { -public: - template using async_msg_type = opencl_async_msg>; - typedef opencl_device device_type; - - class kernel : tbb::internal::no_assign { - public: - kernel( const kernel& k ) : my_factory( k.my_factory ) { - // Clone my_cl_kernel via opencl_program - size_t ret_size = 0; - - std::vector kernel_name; - for ( size_t curr_size = 32;; curr_size <<= 1 ) { - kernel_name.resize( curr_size <<= 1 ); - enforce_cl_retcode( clGetKernelInfo( k.my_cl_kernel, CL_KERNEL_FUNCTION_NAME, curr_size, kernel_name.data(), &ret_size ), "Failed to get kernel info" ); - if ( ret_size < curr_size ) break; - } - - cl_program program; - enforce_cl_retcode( clGetKernelInfo( k.my_cl_kernel, CL_KERNEL_PROGRAM, sizeof(program), &program, &ret_size ), "Failed to get kernel info" ); - __TBB_ASSERT( ret_size == sizeof(program), NULL ); - - my_cl_kernel = opencl_program< factory_type >( my_factory, program ).get_cl_kernel( kernel_name.data() ); - } - - ~kernel() { - enforce_cl_retcode( clReleaseKernel( my_cl_kernel ), "Failed to release a kernel" ); - } - - private: - typedef opencl_factory factory_type; - - kernel( const cl_kernel& k, factory_type& f ) : my_cl_kernel( k ), my_factory( f ) {} - - // Data - cl_kernel my_cl_kernel; - factory_type& my_factory; - - template - friend class opencl_factory; - - template - friend class opencl_program; - }; - - typedef kernel kernel_type; - - // 'range_type' enables kernel_executor with range support - // it affects expectations for enqueue_kernel(.....) interface method - typedef opencl_range range_type; - - opencl_factory() {} - ~opencl_factory() { - if ( my_devices.size() ) { - for ( auto d = my_devices.begin(); d != my_devices.end(); ++d ) { - enforce_cl_retcode( clReleaseCommandQueue( (*d).my_cl_command_queue ), "Failed to release a command queue" ); - } - enforce_cl_retcode( clReleaseContext( my_cl_context ), "Failed to release a context" ); - } - } - - bool init( const opencl_device_list &device_list ) { - tbb::spin_mutex::scoped_lock lock( my_devices_mutex ); - if ( !my_devices.size() ) { - my_devices = device_list; - return true; - } - return false; - } - - -private: - template - void enqueue_map_buffer( opencl_device device, opencl_buffer_impl &buffer, opencl_async_msg& dmsg ) { - cl_event const* e1 = dmsg.get_event(); - cl_event e2; - cl_int err; - void *ptr = clEnqueueMapBuffer( device.my_cl_command_queue, buffer.get_cl_mem(), false, CL_MAP_READ | CL_MAP_WRITE, 0, buffer.size(), - e1 == NULL ? 0 : 1, e1, &e2, &err ); - enforce_cl_retcode( err, "Failed to map a buffer" ); - dmsg.data( false ) = ptr; - dmsg.set_event( e2 ); - enforce_cl_retcode( clReleaseEvent( e2 ), "Failed to release an event" ); - } - - - template - void enqueue_unmap_buffer( opencl_device device, opencl_memory &memory, opencl_async_msg& dmsg ) { - cl_event const* e1 = dmsg.get_event(); - cl_event e2; - enforce_cl_retcode( - clEnqueueUnmapMemObject( device.my_cl_command_queue, memory.get_cl_mem(), memory.get_host_ptr(), e1 == NULL ? 0 : 1, e1, &e2 ), - "Failed to unmap a buffer" ); - dmsg.set_event( e2 ); - enforce_cl_retcode( clReleaseEvent( e2 ), "Failed to release an event" ); - } - - // --------- Kernel argument & event list helpers --------- // - template - void process_one_arg( const kernel_type& kernel, std::array&, int&, int& place, const T& t ) { - auto p = get_native_object(t); - enforce_cl_retcode( clSetKernelArg(kernel.my_cl_kernel, place++, sizeof(p), &p), "Failed to set a kernel argument" ); - } - - template - void process_one_arg( const kernel_type& kernel, std::array& events, int& num_events, int& place, const opencl_async_msg& msg ) { - __TBB_ASSERT((static_cast::size_type>(num_events) < events.size()), NULL); - - const cl_event * const e = msg.get_event(); - if (e != NULL) { - events[num_events++] = *e; - } - - process_one_arg( kernel, events, num_events, place, msg.data(false) ); - } - - template - void process_arg_list( const kernel_type& kernel, std::array& events, int& num_events, int& place, const T& t, const Rest&... args ) { - process_one_arg( kernel, events, num_events, place, t ); - process_arg_list( kernel, events, num_events, place, args... ); - } - - template - void process_arg_list( const kernel_type&, std::array&, int&, int& ) {} - // ------------------------------------------- // - template - void update_one_arg( cl_event, T& ) {} - - template - void update_one_arg( cl_event e, opencl_async_msg& msg ) { - msg.set_event( e ); - } - - template - void update_arg_list( cl_event e, T& t, Rest&... args ) { - update_one_arg( e, t ); - update_arg_list( e, args... ); - } - - void update_arg_list( cl_event ) {} - // ------------------------------------------- // -public: - template - void send_kernel( opencl_device device, const kernel_type& kernel, const range_type& work_size, Args&... args ) { - std::array events; - int num_events = 0; - int place = 0; - process_arg_list( kernel, events, num_events, place, args... ); - - const cl_event e = send_kernel_impl( device, kernel.my_cl_kernel, work_size, num_events, events.data() ); - - update_arg_list(e, args...); - - // Release our own reference to cl_event - enforce_cl_retcode( clReleaseEvent(e), "Failed to release an event" ); - } - - // ------------------------------------------- // - template - void send_data(opencl_device device, T& t, Rest&... args) { - send_if_memory_object( device, t ); - send_data( device, args... ); - } - - void send_data(opencl_device) {} - // ------------------------------------------- // - -private: - cl_event send_kernel_impl( opencl_device device, const cl_kernel& kernel, - const range_type& work_size, cl_uint num_events, cl_event* event_list ) { - const typename range_type::nd_range_type g_offset = { { 0, 0, 0 } }; - const typename range_type::nd_range_type& g_size = work_size.global_range(); - const typename range_type::nd_range_type& l_size = work_size.local_range(); - cl_uint s; - for ( s = 1; s < 3 && g_size[s] != size_t(-1); ++s) {} - cl_event event; - enforce_cl_retcode( - clEnqueueNDRangeKernel( device.my_cl_command_queue, kernel, s, - g_offset.data(), g_size.data(), l_size[0] ? l_size.data() : NULL, num_events, num_events ? event_list : NULL, &event ), - "Failed to enqueue a kernel" ); - return event; - } - - // ------------------------------------------- // - template - bool get_event_from_one_arg( cl_event&, const T& ) { - return false; - } - - template - bool get_event_from_one_arg( cl_event& e, const opencl_async_msg& msg) { - cl_event const *e_ptr = msg.get_event(); - - if ( e_ptr != NULL ) { - e = *e_ptr; - return true; - } - - return false; - } - - template - bool get_event_from_args( cl_event& e, const T& t, const Rest&... args ) { - if ( get_event_from_one_arg( e, t ) ) { - return true; - } - - return get_event_from_args( e, args... ); - } - - bool get_event_from_args( cl_event& ) { - return false; - } - // ------------------------------------------- // - - struct finalize_fn : tbb::internal::no_assign { - virtual ~finalize_fn() {} - virtual void operator() () {} - }; - - template - struct finalize_fn_leaf : public finalize_fn { - Fn my_fn; - finalize_fn_leaf(Fn fn) : my_fn(fn) {} - void operator() () __TBB_override { my_fn(); } - }; - - static void CL_CALLBACK finalize_callback(cl_event, cl_int event_command_exec_status, void *data) { - tbb::internal::suppress_unused_warning(event_command_exec_status); - __TBB_ASSERT(event_command_exec_status == CL_COMPLETE, NULL); - - finalize_fn * const fn_ptr = static_cast(data); - __TBB_ASSERT(fn_ptr != NULL, "Invalid finalize function pointer"); - (*fn_ptr)(); - - // Function pointer was created by 'new' & this callback must be called once only - delete fn_ptr; - } -public: - template - void finalize( opencl_device device, FinalizeFn fn, Args&... args ) { - cl_event e; - - if ( get_event_from_args( e, args... ) ) { - enforce_cl_retcode( clSetEventCallback( e, CL_COMPLETE, finalize_callback, - new finalize_fn_leaf(fn) ), "Failed to set a callback" ); - } - - enforce_cl_retcode( clFlush( device.my_cl_command_queue ), "Failed to flush an OpenCL command queue" ); - } - - const opencl_device_list& devices() { - std::call_once( my_once_flag, &opencl_factory::init_once, this ); - return my_devices; - } - -private: - bool is_same_context( opencl_device::device_id_type d1, opencl_device::device_id_type d2 ) { - __TBB_ASSERT( d1 != opencl_device::unknown && d2 != opencl_device::unknown, NULL ); - // Currently, factory supports only one context so if the both devices are not host it means the are in the same context. - if ( d1 != opencl_device::host && d2 != opencl_device::host ) - return true; - return d1 == d2; - } -private: - opencl_factory( const opencl_factory& ); - opencl_factory& operator=(const opencl_factory&); - - cl_context context() { - std::call_once( my_once_flag, &opencl_factory::init_once, this ); - return my_cl_context; - } - - void init_once() { - { - tbb::spin_mutex::scoped_lock lock(my_devices_mutex); - if (!my_devices.size()) - my_devices = DeviceFilter()( opencl_info::available_devices() ); - } - - enforce_cl_retcode(my_devices.size() ? CL_SUCCESS : CL_INVALID_DEVICE, "No devices in the device list"); - cl_platform_id platform_id = my_devices.begin()->platform(); - for (opencl_device_list::iterator it = ++my_devices.begin(); it != my_devices.end(); ++it) - enforce_cl_retcode(it->platform() == platform_id ? CL_SUCCESS : CL_INVALID_PLATFORM, "All devices should be in the same platform"); - - std::vector cl_device_ids; - for (auto d = my_devices.begin(); d != my_devices.end(); ++d) { - cl_device_ids.push_back((*d).my_cl_device_id); - } - - cl_context_properties context_properties[3] = { CL_CONTEXT_PLATFORM, (cl_context_properties)platform_id, (cl_context_properties)NULL }; - cl_int err; - cl_context ctx = clCreateContext(context_properties, - (cl_uint)cl_device_ids.size(), - cl_device_ids.data(), - NULL, NULL, &err); - enforce_cl_retcode(err, "Failed to create context"); - my_cl_context = ctx; - - size_t device_counter = 0; - for (auto d = my_devices.begin(); d != my_devices.end(); d++) { - (*d).my_device_id = device_counter++; - cl_int err2; - cl_command_queue cq; -#if CL_VERSION_2_0 - if ((*d).major_version() >= 2) { - if ((*d).out_of_order_exec_mode_on_host_present()) { - cl_queue_properties props[] = { CL_QUEUE_PROPERTIES, CL_QUEUE_OUT_OF_ORDER_EXEC_MODE_ENABLE, 0 }; - cq = clCreateCommandQueueWithProperties(ctx, (*d).my_cl_device_id, props, &err2); - } else { - cl_queue_properties props[] = { 0 }; - cq = clCreateCommandQueueWithProperties(ctx, (*d).my_cl_device_id, props, &err2); - } - } else -#endif - { - cl_command_queue_properties props = (*d).out_of_order_exec_mode_on_host_present() ? CL_QUEUE_OUT_OF_ORDER_EXEC_MODE_ENABLE : 0; - // Suppress "declared deprecated" warning for the next line. -#if __TBB_GCC_WARNING_SUPPRESSION_PRESENT -#pragma GCC diagnostic push -// #pragma GCC diagnostic ignored "-Wdeprecated-declarations" -#endif -#if _MSC_VER || __INTEL_COMPILER -#pragma warning( push ) -#if __INTEL_COMPILER -#pragma warning (disable: 1478) -#else -#pragma warning (disable: 4996) -#endif -#endif - cq = clCreateCommandQueue(ctx, (*d).my_cl_device_id, props, &err2); -#if _MSC_VER || __INTEL_COMPILER -#pragma warning( pop ) -#endif -#if __TBB_GCC_WARNING_SUPPRESSION_PRESENT -#pragma GCC diagnostic pop -#endif - } - enforce_cl_retcode(err2, "Failed to create command queue"); - (*d).my_cl_command_queue = cq; - } - } - - std::once_flag my_once_flag; - opencl_device_list my_devices; - cl_context my_cl_context; - - tbb::spin_mutex my_devices_mutex; - - template - friend class opencl_program; - template - friend class opencl_buffer_impl; - template - friend class opencl_memory; -}; // class opencl_factory - -// TODO: consider this namespace as public API -namespace opencl_info { - -// Default types - -template -struct default_device_selector { - opencl_device operator()(Factory& f) { - __TBB_ASSERT(!f.devices().empty(), "No available devices"); - return *(f.devices().begin()); - } -}; - -struct default_device_filter { - opencl_device_list operator()(const opencl_device_list &devices) { - opencl_device_list dl; - dl.add(*devices.begin()); - return dl; - } -}; - -class default_opencl_factory : public opencl_factory < default_device_filter >, tbb::internal::no_copy { -public: - template using async_msg_type = opencl_async_msg; - - friend default_opencl_factory& default_factory(); - -private: - default_opencl_factory() = default; -}; - -inline default_opencl_factory& default_factory() { - static default_opencl_factory default_factory; - return default_factory; -} - -} // namespace opencl_info - -template -opencl_buffer::opencl_buffer( size_t size ) : my_impl( std::make_shared( size*sizeof(T), opencl_info::default_factory() ) ) {} - - -enum class opencl_program_type { - SOURCE, - PRECOMPILED, - SPIR -}; - -template -class opencl_program : tbb::internal::no_assign { -public: - typedef typename Factory::kernel_type kernel_type; - - opencl_program( Factory& factory, opencl_program_type type, const std::string& program_name ) : my_factory( factory ), my_type(type) , my_arg_str( program_name) {} - opencl_program( Factory& factory, const char* program_name ) : opencl_program( factory, std::string( program_name ) ) {} - opencl_program( Factory& factory, const std::string& program_name ) : opencl_program( factory, opencl_program_type::SOURCE, program_name ) {} - - opencl_program( opencl_program_type type, const std::string& program_name ) : opencl_program( opencl_info::default_factory(), type, program_name ) {} - opencl_program( const char* program_name ) : opencl_program( opencl_info::default_factory(), program_name ) {} - opencl_program( const std::string& program_name ) : opencl_program( opencl_info::default_factory(), program_name ) {} - opencl_program( opencl_program_type type ) : opencl_program( opencl_info::default_factory(), type ) {} - - opencl_program( const opencl_program &src ) : my_factory( src.my_factory ), my_type( src.type ), my_arg_str( src.my_arg_str ), my_cl_program( src.my_cl_program ) { - // Set my_do_once_flag to the called state. - std::call_once( my_do_once_flag, [](){} ); - } - - kernel_type get_kernel( const std::string& k ) const { - return kernel_type( get_cl_kernel(k), my_factory ); - } - -private: - opencl_program( Factory& factory, cl_program program ) : my_factory( factory ), my_cl_program( program ) { - // Set my_do_once_flag to the called state. - std::call_once( my_do_once_flag, [](){} ); - } - - cl_kernel get_cl_kernel( const std::string& k ) const { - std::call_once( my_do_once_flag, [this, &k](){ this->init( k ); } ); - cl_int err; - cl_kernel kernel = clCreateKernel( my_cl_program, k.c_str(), &err ); - enforce_cl_retcode( err, std::string( "Failed to create kernel: " ) + k ); - return kernel; - } - - class file_reader { - public: - file_reader( const std::string& filepath ) { - std::ifstream file_descriptor( filepath, std::ifstream::binary ); - if ( !file_descriptor.is_open() ) { - std::string str = std::string( "Could not open file: " ) + filepath; - std::cerr << str << std::endl; - throw str; - } - file_descriptor.seekg( 0, file_descriptor.end ); - size_t length = size_t( file_descriptor.tellg() ); - file_descriptor.seekg( 0, file_descriptor.beg ); - my_content.resize( length ); - char* begin = &*my_content.begin(); - file_descriptor.read( begin, length ); - file_descriptor.close(); - } - const char* content() { return &*my_content.cbegin(); } - size_t length() { return my_content.length(); } - private: - std::string my_content; - }; - - class opencl_program_builder { - public: - typedef void (CL_CALLBACK *cl_callback_type)(cl_program, void*); - opencl_program_builder( Factory& f, const std::string& name, cl_program program, - cl_uint num_devices, cl_device_id* device_list, - const char* options, cl_callback_type callback, - void* user_data ) { - cl_int err = clBuildProgram( program, num_devices, device_list, options, - callback, user_data ); - if( err == CL_SUCCESS ) - return; - std::string str = std::string( "Failed to build program: " ) + name; - if ( err == CL_BUILD_PROGRAM_FAILURE ) { - const opencl_device_list &devices = f.devices(); - for ( auto d = devices.begin(); d != devices.end(); ++d ) { - std::cerr << "Build log for device: " << (*d).name() << std::endl; - size_t log_size; - cl_int query_err = clGetProgramBuildInfo( - program, (*d).my_cl_device_id, CL_PROGRAM_BUILD_LOG, 0, NULL, - &log_size ); - enforce_cl_retcode( query_err, "Failed to get build log size" ); - if( log_size ) { - std::vector output; - output.resize( log_size ); - query_err = clGetProgramBuildInfo( - program, (*d).my_cl_device_id, CL_PROGRAM_BUILD_LOG, - output.size(), output.data(), NULL ); - enforce_cl_retcode( query_err, "Failed to get build output" ); - std::cerr << output.data() << std::endl; - } else { - std::cerr << "No build log available" << std::endl; - } - } - } - enforce_cl_retcode( err, str ); - } - }; - - class opencl_device_filter { - public: - template - opencl_device_filter( cl_uint& num_devices, cl_device_id* device_list, - Filter filter, const char* message ) { - for ( cl_uint i = 0; i < num_devices; ++i ) - if ( filter(device_list[i]) ) { - device_list[i--] = device_list[--num_devices]; - } - if ( !num_devices ) - enforce_cl_retcode( CL_DEVICE_NOT_AVAILABLE, message ); - } - }; - - void init( const std::string& ) const { - cl_uint num_devices; - enforce_cl_retcode( clGetContextInfo( my_factory.context(), CL_CONTEXT_NUM_DEVICES, sizeof( num_devices ), &num_devices, NULL ), - "Failed to get OpenCL context info" ); - if ( !num_devices ) - enforce_cl_retcode( CL_DEVICE_NOT_FOUND, "No supported devices found" ); - cl_device_id *device_list = (cl_device_id *)alloca( num_devices*sizeof( cl_device_id ) ); - enforce_cl_retcode( clGetContextInfo( my_factory.context(), CL_CONTEXT_DEVICES, num_devices*sizeof( cl_device_id ), device_list, NULL ), - "Failed to get OpenCL context info" ); - const char *options = NULL; - switch ( my_type ) { - case opencl_program_type::SOURCE: { - file_reader fr( my_arg_str ); - const char *s[] = { fr.content() }; - const size_t l[] = { fr.length() }; - cl_int err; - my_cl_program = clCreateProgramWithSource( my_factory.context(), 1, s, l, &err ); - enforce_cl_retcode( err, std::string( "Failed to create program: " ) + my_arg_str ); - opencl_device_filter( - num_devices, device_list, - []( const opencl_device& d ) -> bool { - return !d.compiler_available() || !d.linker_available(); - }, "No one device supports building program from sources" ); - opencl_program_builder( - my_factory, my_arg_str, my_cl_program, num_devices, device_list, - options, /*callback*/ NULL, /*user data*/NULL ); - break; - } - case opencl_program_type::SPIR: - options = "-x spir"; - case opencl_program_type::PRECOMPILED: { - file_reader fr( my_arg_str ); - std::vector s( - num_devices, reinterpret_cast(fr.content()) ); - std::vector l( num_devices, fr.length() ); - std::vector bin_statuses( num_devices, -1 ); - cl_int err; - my_cl_program = clCreateProgramWithBinary( my_factory.context(), num_devices, - device_list, l.data(), s.data(), - bin_statuses.data(), &err ); - if( err != CL_SUCCESS ) { - std::string statuses_str; - for (auto st = bin_statuses.begin(); st != bin_statuses.end(); ++st) { - statuses_str += std::to_string((*st)); - } - - enforce_cl_retcode( err, std::string( "Failed to create program, error " + std::to_string( err ) + " : " ) + my_arg_str + - std::string( ", binary_statuses = " ) + statuses_str ); - } - opencl_program_builder( - my_factory, my_arg_str, my_cl_program, num_devices, device_list, - options, /*callback*/ NULL, /*user data*/NULL ); - break; - } - default: - __TBB_ASSERT( false, "Unsupported program type" ); - } - } - - Factory& my_factory; - opencl_program_type my_type; - std::string my_arg_str; - mutable cl_program my_cl_program; - mutable std::once_flag my_do_once_flag; - - template - friend class opencl_factory; - - template - friend class opencl_factory::kernel; -}; - -template -class opencl_node; - -template -class opencl_node< tuple, JP, Factory > : public streaming_node< tuple, JP, Factory > { - typedef streaming_node < tuple, JP, Factory > base_type; -public: - typedef typename base_type::kernel_type kernel_type; - - opencl_node( graph &g, const kernel_type& kernel ) - : base_type( g, kernel, opencl_info::default_device_selector< opencl_info::default_opencl_factory >(), opencl_info::default_factory() ) - { - tbb::internal::fgt_multiinput_multioutput_node( tbb::internal::FLOW_OPENCL_NODE, this, &this->my_graph ); - } - - opencl_node( graph &g, const kernel_type& kernel, Factory &f ) - : base_type( g, kernel, opencl_info::default_device_selector (), f ) - { - tbb::internal::fgt_multiinput_multioutput_node( tbb::internal::FLOW_OPENCL_NODE, this, &this->my_graph ); - } - - template - opencl_node( graph &g, const kernel_type& kernel, DeviceSelector d, Factory &f) - : base_type( g, kernel, d, f) - { - tbb::internal::fgt_multiinput_multioutput_node( tbb::internal::FLOW_OPENCL_NODE, this, &this->my_graph ); - } -}; - -template -class opencl_node< tuple, JP > : public opencl_node < tuple, JP, opencl_info::default_opencl_factory > { - typedef opencl_node < tuple, JP, opencl_info::default_opencl_factory > base_type; -public: - typedef typename base_type::kernel_type kernel_type; - - opencl_node( graph &g, const kernel_type& kernel ) - : base_type( g, kernel, opencl_info::default_device_selector< opencl_info::default_opencl_factory >(), opencl_info::default_factory() ) - {} - - template - opencl_node( graph &g, const kernel_type& kernel, DeviceSelector d ) - : base_type( g, kernel, d, opencl_info::default_factory() ) - {} -}; - -template -class opencl_node< tuple > : public opencl_node < tuple, queueing, opencl_info::default_opencl_factory > { - typedef opencl_node < tuple, queueing, opencl_info::default_opencl_factory > base_type; -public: - typedef typename base_type::kernel_type kernel_type; - - opencl_node( graph &g, const kernel_type& kernel ) - : base_type( g, kernel, opencl_info::default_device_selector< opencl_info::default_opencl_factory >(), opencl_info::default_factory() ) - {} - - template - opencl_node( graph &g, const kernel_type& kernel, DeviceSelector d ) - : base_type( g, kernel, d, opencl_info::default_factory() ) - {} -}; - -} // namespace interface10 - -using interface10::opencl_node; -using interface10::read_only; -using interface10::read_write; -using interface10::write_only; -using interface10::opencl_buffer; -using interface10::opencl_subbuffer; -using interface10::opencl_device; -using interface10::opencl_device_list; -using interface10::opencl_program; -using interface10::opencl_program_type; -using interface10::opencl_async_msg; -using interface10::opencl_factory; -using interface10::opencl_range; - -} // namespace flow -} // namespace tbb -#endif /* __TBB_PREVIEW_OPENCL_NODE */ - -#endif // __TBB_flow_graph_opencl_node_H diff --git a/src/tbb/include/tbb/gfx_factory.h b/src/tbb/include/tbb/gfx_factory.h deleted file mode 100644 index 76ca3559a..000000000 --- a/src/tbb/include/tbb/gfx_factory.h +++ /dev/null @@ -1,359 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_flow_graph_gfx_factory_H -#define __TBB_flow_graph_gfx_factory_H - -#include "tbb/tbb_config.h" - -#if __TBB_PREVIEW_GFX_FACTORY - -#include -#include -#include -#include - -#include -#include -#include - -namespace tbb { - -namespace flow { - -namespace interface9 { - -template -class gfx_buffer; - -namespace gfx_offload { - - typedef GfxTaskId task_id_type; - - //----------------------------------------------------------------------- - // GFX errors checkers. - // For more debug output, set GFX_LOG_OFFLOAD=2 macro - //----------------------------------------------------------------------- - - // TODO: reconsider error handling approach. If exception is the right way - // then need to define and document a specific exception type. - inline void throw_gfx_exception() { - std::string msg = "GFX error occurred: " + std::to_string(_GFX_get_last_error()); - std::cerr << msg << std::endl; - throw msg; - } - - inline void check_enqueue_retcode(task_id_type err) { - if (err == 0) { - throw_gfx_exception(); - } - } - - inline void check_gfx_retcode(task_id_type err) { - if (err != GFX_SUCCESS) { - throw_gfx_exception(); - } - } - - //--------------------------------------------------------------------- - // GFX asynchronous offload and share API - //--------------------------------------------------------------------- - - // Sharing and unsharing data API - template - void share(DataType* p, SizeType n) { check_gfx_retcode(_GFX_share(p, sizeof(*p)*n)); } - template - void unshare(DataType* p) { check_gfx_retcode(_GFX_unshare(p)); } - - // Retrieving array pointer from shared gfx_buffer - // Other types remain the same - template - T* raw_data(gfx_buffer& buffer) { return buffer.data(); } - template - const T* raw_data(const gfx_buffer& buffer) { return buffer.data(); } - template - T& raw_data(T& data) { return data; } - template - const T& raw_data(const T& data) { return data; } - - // Kernel enqueuing on device with arguments - template - task_id_type run_kernel(F ptr, ArgType&... args) { - task_id_type id = _GFX_offload(ptr, raw_data(args)...); - - // Check if something during offload went wrong (ex: driver initialization failure) - gfx_offload::check_enqueue_retcode(id); - - return id; - } - - // Waiting for tasks completion - void wait_for_task(task_id_type id) { check_gfx_retcode(_GFX_wait(id)); } - -} // namespace gfx_offload - -template -class gfx_buffer { -public: - - typedef typename std::vector::iterator iterator; - typedef typename std::vector::const_iterator const_iterator; - - typedef std::size_t size_type; - - gfx_buffer() : my_vector_ptr(std::make_shared< std::vector >()) {} - gfx_buffer(size_type size) : my_vector_ptr(std::make_shared< std::vector >(size)) {} - - T* data() { return &(my_vector_ptr->front()); } - const T* data() const { return &(my_vector_ptr->front()); } - - size_type size() const { return my_vector_ptr->size(); } - - const_iterator cbegin() const { return my_vector_ptr->cbegin(); } - const_iterator cend() const { return my_vector_ptr->cend(); } - iterator begin() { return my_vector_ptr->begin(); } - iterator end() { return my_vector_ptr->end(); } - - T& operator[](size_type pos) { return (*my_vector_ptr)[pos]; } - const T& operator[](size_type pos) const { return (*my_vector_ptr)[pos]; } - -private: - std::shared_ptr< std::vector > my_vector_ptr; -}; - -template -class gfx_async_msg : public tbb::flow::async_msg { -public: - typedef gfx_offload::task_id_type kernel_id_type; - - gfx_async_msg() : my_task_id(0) {} - gfx_async_msg(const T& input_data) : my_data(input_data), my_task_id(0) {} - - T& data() { return my_data; } - const T& data() const { return my_data; } - - void set_task_id(kernel_id_type id) { my_task_id = id; } - kernel_id_type task_id() const { return my_task_id; } - -private: - T my_data; - kernel_id_type my_task_id; -}; - -class gfx_factory { -private: - - // Wrapper for GFX kernel which is just a function - class func_wrapper { - public: - - template - func_wrapper(F ptr) { my_ptr = reinterpret_cast(ptr); } - - template - void operator()(Args&&... args) {} - - operator void*() { return my_ptr; } - - private: - void* my_ptr; - }; - -public: - - // Device specific types - template using async_msg_type = gfx_async_msg; - - typedef func_wrapper kernel_type; - - // Empty device type that is needed for Factory Concept - // but is not used in gfx_factory - typedef struct {} device_type; - - typedef gfx_offload::task_id_type kernel_id_type; - - gfx_factory(tbb::flow::graph& g) : m_graph(g), current_task_id(0) {} - - // Upload data to the device - template - void send_data(device_type /*device*/, Args&... args) { - send_data_impl(args...); - } - - // Run kernel on the device - template - void send_kernel(device_type /*device*/, const kernel_type& kernel, Args&... args) { - // Get packed T data from async_msg and pass it to kernel - kernel_id_type id = gfx_offload::run_kernel(kernel, args.data()...); - - // Set id to async_msg - set_kernel_id(id, args...); - - // Extend the graph lifetime until the callback completion. - m_graph.reserve_wait(); - - // Mutex for future assignment - std::lock_guard lock(future_assignment_mutex); - - // Set callback that waits for kernel execution - callback_future = std::async(std::launch::async, &gfx_factory::callback, this, id, args...); - } - - // Finalization action after the kernel run - template - void finalize(device_type /*device*/, FinalizeFn fn, Args&... /*args*/) { - fn(); - } - - // Empty device selector. - // No way to choose a device with GFX API. - class dummy_device_selector { - public: - device_type operator()(gfx_factory& /*factory*/) { - return device_type(); - } - }; - -private: - - //--------------------------------------------------------------------- - // Callback for kernel result - //--------------------------------------------------------------------- - - template - void callback(kernel_id_type id, Args... args) { - // Waiting for specific tasks id to complete - { - std::lock_guard lock(task_wait_mutex); - if (current_task_id < id) { - gfx_offload::wait_for_task(id); - current_task_id = id; - } - } - - // Get result from device and set to async_msg (args) - receive_data(args...); - - // Data was sent to the graph, release the reference - m_graph.release_wait(); - } - - //--------------------------------------------------------------------- - // send_data() arguments processing - //--------------------------------------------------------------------- - - // GFX buffer shared data with device that will be executed on - template - void share_data(T) {} - - template - void share_data(gfx_buffer& buffer) { - gfx_offload::share(buffer.data(), buffer.size()); - } - - template - void send_arg(T) {} - - template - void send_arg(async_msg_type& msg) { - share_data(msg.data()); - } - - void send_data_impl() {} - - template - void send_data_impl(T& arg, Rest&... args) { - send_arg(arg); - send_data_impl(args...); - } - - //---------------------------------------------------------------------- - // send_kernel() arguments processing - //---------------------------------------------------------------------- - - template - void set_kernel_id_arg(kernel_id_type, T) {} - - template - void set_kernel_id_arg(kernel_id_type id, async_msg_type& msg) { - msg.set_task_id(id); - } - - void set_kernel_id(kernel_id_type) {} - - template - void set_kernel_id(kernel_id_type id, T& arg, Rest&... args) { - set_kernel_id_arg(id, arg); - set_kernel_id(id, args...); - } - - //----------------------------------------------------------------------- - // Arguments processing after kernel execution. - // Unsharing buffers and forwarding results to the graph - //----------------------------------------------------------------------- - - // After kernel execution the data should be unshared - template - void unshare_data(T) {} - - template - void unshare_data(gfx_buffer& buffer) { - gfx_offload::unshare(buffer.data()); - } - - template - void receive_arg(T) {} - - template - void receive_arg(async_msg_type& msg) { - unshare_data(msg.data()); - msg.set(msg.data()); - } - - void receive_data() {} - - template - void receive_data(T& arg, Rest&... args) { - receive_arg(arg); - receive_data(args...); - } - - //----------------------------------------------------------------------- - int current_task_id; - - std::future callback_future; - tbb::flow::graph& m_graph; - - std::mutex future_assignment_mutex; - std::mutex task_wait_mutex; -}; - -} // namespace interface9 - -using interface9::gfx_factory; -using interface9::gfx_buffer; - -} // namespace flow - -} // namespace tbb - -#endif // __TBB_PREVIEW_GFX_FACTORY - -#endif // __TBB_flow_graph_gfx_factory_H diff --git a/src/tbb/include/tbb/global_control.h b/src/tbb/include/tbb/global_control.h index fe7420203..2688996ec 100644 --- a/src/tbb/include/tbb/global_control.h +++ b/src/tbb/include/tbb/global_control.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,75 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ -#ifndef __TBB_global_control_H -#define __TBB_global_control_H - -#if !TBB_PREVIEW_GLOBAL_CONTROL && !__TBB_BUILD -#error Set TBB_PREVIEW_GLOBAL_CONTROL before including global_control.h -#endif - -#include "tbb_stddef.h" - -namespace tbb { -namespace interface9 { - -class global_control { -public: - enum parameter { - max_allowed_parallelism, - thread_stack_size, - parameter_max // insert new parameters above this point - }; - - global_control(parameter p, size_t value) : - my_value(value), my_next(NULL), my_param(p) { - __TBB_ASSERT(my_param < parameter_max, "Invalid parameter"); -#if __TBB_WIN8UI_SUPPORT - // For Windows Store* apps it's impossible to set stack size - if (p==thread_stack_size) - return; -#elif __TBB_x86_64 && (_WIN32 || _WIN64) - if (p==thread_stack_size) - __TBB_ASSERT_RELEASE((unsigned)value == value, "Stack size is limited to unsigned int range"); -#endif - if (my_param==max_allowed_parallelism) - __TBB_ASSERT_RELEASE(my_value>0, "max_allowed_parallelism cannot be 0."); - internal_create(); - } - - ~global_control() { - __TBB_ASSERT(my_param < parameter_max, "Invalid parameter. Probably the object was corrupted."); -#if __TBB_WIN8UI_SUPPORT - // For Windows Store* apps it's impossible to set stack size - if (my_param==thread_stack_size) - return; -#endif - internal_destroy(); - } - - static size_t active_value(parameter p) { - __TBB_ASSERT(p < parameter_max, "Invalid parameter"); - return active_value((int)p); - } -private: - size_t my_value; - global_control *my_next; - parameter my_param; - - void __TBB_EXPORTED_METHOD internal_create(); - void __TBB_EXPORTED_METHOD internal_destroy(); - static size_t __TBB_EXPORTED_FUNC active_value(int param); -}; -} // namespace interface9 - -using interface9::global_control; - -} // tbb - -#endif // __TBB_global_control_H +#include "../oneapi/tbb/global_control.h" diff --git a/src/tbb/include/tbb/index.html b/src/tbb/include/tbb/index.html deleted file mode 100644 index 2eddd450d..000000000 --- a/src/tbb/include/tbb/index.html +++ /dev/null @@ -1,29 +0,0 @@ - - - -

Overview

-Include files for Intel® Threading Building Blocks classes and functions. - -
Click here to see all files in the directory. - -

Directories

-
-
compat -
Include files for source level compatibility with other frameworks. -
internal -
Include files with implementation details; not for direct use. -
machine -
Include files for low-level architecture specific functionality; not for direct use. -
- -
-Up to parent directory -

-Copyright © 2005-2017 Intel Corporation. All Rights Reserved. -

-Intel is a registered trademark or trademark of Intel Corporation -or its subsidiaries in the United States and other countries. -

-* Other names and brands may be claimed as the property of others. - - diff --git a/src/tbb/include/tbb/info.h b/src/tbb/include/tbb/info.h new file mode 100644 index 000000000..02d331650 --- /dev/null +++ b/src/tbb/include/tbb/info.h @@ -0,0 +1,17 @@ +/* + Copyright (c) 2019-2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "../oneapi/tbb/info.h" diff --git a/src/tbb/include/tbb/internal/_aggregator_impl.h b/src/tbb/include/tbb/internal/_aggregator_impl.h deleted file mode 100644 index 40bbd4916..000000000 --- a/src/tbb/include/tbb/internal/_aggregator_impl.h +++ /dev/null @@ -1,184 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB__aggregator_impl_H -#define __TBB__aggregator_impl_H - -#include "../atomic.h" -#if !__TBBMALLOC_BUILD -#include "../tbb_profiling.h" -#endif - -namespace tbb { -namespace interface6 { -namespace internal { - -using namespace tbb::internal; - -//! aggregated_operation base class -template -class aggregated_operation { - public: - //! Zero value means "wait" status, all other values are "user" specified values and are defined into the scope of a class which uses "status". - uintptr_t status; - - Derived *next; - aggregated_operation() : status(0), next(NULL) {} -}; - -//! Aggregator base class -/** An aggregator for collecting operations coming from multiple sources and executing - them serially on a single thread. operation_type must be derived from - aggregated_operation. The parameter handler_type is a functor that will be passed the - list of operations and is expected to handle each operation appropriately, setting the - status of each operation to non-zero.*/ -template < typename operation_type > -class aggregator_generic { -public: - aggregator_generic() : handler_busy(false) { pending_operations = NULL; } - - //! Execute an operation - /** Places an operation into the waitlist (pending_operations), and either handles the list, - or waits for the operation to complete, or returns. - The long_life_time parameter specifies the life time of the given operation object. - Operations with long_life_time == true may be accessed after execution. - A "short" life time operation (long_life_time == false) can be destroyed - during execution, and so any access to it after it was put into the waitlist, - including status check, is invalid. As a consequence, waiting for completion - of such operation causes undefined behavior. - */ - template < typename handler_type > - void execute(operation_type *op, handler_type &handle_operations, bool long_life_time = true) { - operation_type *res; - // op->status should be read before inserting the operation into the - // aggregator waitlist since it can become invalid after executing a - // handler (if the operation has 'short' life time.) - const uintptr_t status = op->status; - - // ITT note: &(op->status) tag is used to cover accesses to this op node. This - // thread has created the operation, and now releases it so that the handler - // thread may handle the associated operation w/o triggering a race condition; - // thus this tag will be acquired just before the operation is handled in the - // handle_operations functor. - call_itt_notify(releasing, &(op->status)); - // insert the operation in the queue. - do { - // Tools may flag the following line as a race; it is a false positive: - // This is an atomic read; we don't provide itt_hide_load_word for atomics - op->next = res = pending_operations; // NOT A RACE - } while (pending_operations.compare_and_swap(op, res) != res); - if (!res) { // first in the list; handle the operations. - // ITT note: &pending_operations tag covers access to the handler_busy flag, - // which this waiting handler thread will try to set before entering - // handle_operations. - call_itt_notify(acquired, &pending_operations); - start_handle_operations(handle_operations); - // The operation with 'short' life time can already be destroyed. - if (long_life_time) - __TBB_ASSERT(op->status, NULL); - } - // not first; wait for op to be ready. - else if (!status) { // operation is blocking here. - __TBB_ASSERT(long_life_time, "Waiting for an operation object that might be destroyed during processing."); - call_itt_notify(prepare, &(op->status)); - spin_wait_while_eq(op->status, uintptr_t(0)); - itt_load_word_with_acquire(op->status); - } - } - - private: - //! An atomically updated list (aka mailbox) of pending operations - atomic pending_operations; - //! Controls thread access to handle_operations - uintptr_t handler_busy; - - //! Trigger the handling of operations when the handler is free - template < typename handler_type > - void start_handle_operations( handler_type &handle_operations ) { - operation_type *op_list; - - // ITT note: &handler_busy tag covers access to pending_operations as it is passed - // between active and waiting handlers. Below, the waiting handler waits until - // the active handler releases, and the waiting handler acquires &handler_busy as - // it becomes the active_handler. The release point is at the end of this - // function, when all operations in pending_operations have been handled by the - // owner of this aggregator. - call_itt_notify(prepare, &handler_busy); - // get the handler_busy: - // only one thread can possibly spin here at a time - spin_wait_until_eq(handler_busy, uintptr_t(0)); - call_itt_notify(acquired, &handler_busy); - // acquire fence not necessary here due to causality rule and surrounding atomics - __TBB_store_with_release(handler_busy, uintptr_t(1)); - - // ITT note: &pending_operations tag covers access to the handler_busy flag - // itself. Capturing the state of the pending_operations signifies that - // handler_busy has been set and a new active handler will now process that list's - // operations. - call_itt_notify(releasing, &pending_operations); - // grab pending_operations - op_list = pending_operations.fetch_and_store(NULL); - - // handle all the operations - handle_operations(op_list); - - // release the handler - itt_store_word_with_release(handler_busy, uintptr_t(0)); - } -}; - -template < typename handler_type, typename operation_type > -class aggregator : public aggregator_generic { - handler_type handle_operations; -public: - aggregator() {} - explicit aggregator(handler_type h) : handle_operations(h) {} - - void initialize_handler(handler_type h) { handle_operations = h; } - - void execute(operation_type *op) { - aggregator_generic::execute(op, handle_operations); - } -}; - -// the most-compatible friend declaration (vs, gcc, icc) is -// template friend class aggregating_functor; -template -class aggregating_functor { - aggregating_class *fi; -public: - aggregating_functor() {} - aggregating_functor(aggregating_class *fi_) : fi(fi_) {} - void operator()(operation_list* op_list) { fi->handle_operations(op_list); } -}; - -} // namespace internal -} // namespace interface6 - -namespace internal { - using interface6::internal::aggregated_operation; - using interface6::internal::aggregator_generic; - using interface6::internal::aggregator; - using interface6::internal::aggregating_functor; -} // namespace internal - -} // namespace tbb - -#endif // __TBB__aggregator_impl_H diff --git a/src/tbb/include/tbb/internal/_concurrent_queue_impl.h b/src/tbb/include/tbb/internal/_concurrent_queue_impl.h deleted file mode 100644 index 29e274260..000000000 --- a/src/tbb/include/tbb/internal/_concurrent_queue_impl.h +++ /dev/null @@ -1,1075 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB__concurrent_queue_impl_H -#define __TBB__concurrent_queue_impl_H - -#ifndef __TBB_concurrent_queue_H -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#include "../tbb_stddef.h" -#include "../tbb_machine.h" -#include "../atomic.h" -#include "../spin_mutex.h" -#include "../cache_aligned_allocator.h" -#include "../tbb_exception.h" -#include "../tbb_profiling.h" -#include -#include __TBB_STD_SWAP_HEADER -#include - -namespace tbb { - -#if !__TBB_TEMPLATE_FRIENDS_BROKEN - -// forward declaration -namespace strict_ppl { -template class concurrent_queue; -} - -template class concurrent_bounded_queue; - -#endif - -//! For internal use only. -namespace strict_ppl { - -//! @cond INTERNAL -namespace internal { - -using namespace tbb::internal; - -typedef size_t ticket; - -template class micro_queue ; -template class micro_queue_pop_finalizer ; -template class concurrent_queue_base_v3; -template struct concurrent_queue_rep; - -//! parts of concurrent_queue_rep that do not have references to micro_queue -/** - * For internal use only. - */ -struct concurrent_queue_rep_base : no_copy { - template friend class micro_queue; - template friend class concurrent_queue_base_v3; - -protected: - //! Approximately n_queue/golden ratio - static const size_t phi = 3; - -public: - // must be power of 2 - static const size_t n_queue = 8; - - //! Prefix on a page - struct page { - page* next; - uintptr_t mask; - }; - - atomic head_counter; - char pad1[NFS_MaxLineSize-sizeof(atomic)]; - atomic tail_counter; - char pad2[NFS_MaxLineSize-sizeof(atomic)]; - - //! Always a power of 2 - size_t items_per_page; - - //! Size of an item - size_t item_size; - - //! number of invalid entries in the queue - atomic n_invalid_entries; - - char pad3[NFS_MaxLineSize-sizeof(size_t)-sizeof(size_t)-sizeof(atomic)]; -} ; - -inline bool is_valid_page(const concurrent_queue_rep_base::page* p) { - return uintptr_t(p)>1; -} - -//! Abstract class to define interface for page allocation/deallocation -/** - * For internal use only. - */ -class concurrent_queue_page_allocator -{ - template friend class micro_queue ; - template friend class micro_queue_pop_finalizer ; -protected: - virtual ~concurrent_queue_page_allocator() {} -private: - virtual concurrent_queue_rep_base::page* allocate_page() = 0; - virtual void deallocate_page( concurrent_queue_rep_base::page* p ) = 0; -} ; - -#if _MSC_VER && !defined(__INTEL_COMPILER) -// unary minus operator applied to unsigned type, result still unsigned -#pragma warning( push ) -#pragma warning( disable: 4146 ) -#endif - -//! A queue using simple locking. -/** For efficiency, this class has no constructor. - The caller is expected to zero-initialize it. */ -template -class micro_queue : no_copy { -public: - typedef void (*item_constructor_t)(T* location, const void* src); -private: - typedef concurrent_queue_rep_base::page page; - - //! Class used to ensure exception-safety of method "pop" - class destroyer: no_copy { - T& my_value; - public: - destroyer( T& value ) : my_value(value) {} - ~destroyer() {my_value.~T();} - }; - - void copy_item( page& dst, size_t dindex, const void* src, item_constructor_t construct_item ) { - construct_item( &get_ref(dst, dindex), src ); - } - - void copy_item( page& dst, size_t dindex, const page& src, size_t sindex, - item_constructor_t construct_item ) - { - T& src_item = get_ref( const_cast(src), sindex ); - construct_item( &get_ref(dst, dindex), static_cast(&src_item) ); - } - - void assign_and_destroy_item( void* dst, page& src, size_t index ) { - T& from = get_ref(src,index); - destroyer d(from); - *static_cast(dst) = tbb::internal::move( from ); - } - - void spin_wait_until_my_turn( atomic& counter, ticket k, concurrent_queue_rep_base& rb ) const ; - -public: - friend class micro_queue_pop_finalizer; - - struct padded_page: page { - //! Not defined anywhere - exists to quiet warnings. - padded_page(); - //! Not defined anywhere - exists to quiet warnings. - void operator=( const padded_page& ); - //! Must be last field. - T last; - }; - - static T& get_ref( page& p, size_t index ) { - return (&static_cast(static_cast(&p))->last)[index]; - } - - atomic head_page; - atomic head_counter; - - atomic tail_page; - atomic tail_counter; - - spin_mutex page_mutex; - - void push( const void* item, ticket k, concurrent_queue_base_v3& base, - item_constructor_t construct_item ) ; - - bool pop( void* dst, ticket k, concurrent_queue_base_v3& base ) ; - - micro_queue& assign( const micro_queue& src, concurrent_queue_base_v3& base, - item_constructor_t construct_item ) ; - - page* make_copy( concurrent_queue_base_v3& base, const page* src_page, size_t begin_in_page, - size_t end_in_page, ticket& g_index, item_constructor_t construct_item ) ; - - void invalidate_page_and_rethrow( ticket k ) ; -}; - -template -void micro_queue::spin_wait_until_my_turn( atomic& counter, ticket k, concurrent_queue_rep_base& rb ) const { - for( atomic_backoff b(true);;b.pause() ) { - ticket c = counter; - if( c==k ) return; - else if( c&1 ) { - ++rb.n_invalid_entries; - throw_exception( eid_bad_last_alloc ); - } - } -} - -template -void micro_queue::push( const void* item, ticket k, concurrent_queue_base_v3& base, - item_constructor_t construct_item ) -{ - k &= -concurrent_queue_rep_base::n_queue; - page* p = NULL; - size_t index = modulo_power_of_two( k/concurrent_queue_rep_base::n_queue, base.my_rep->items_per_page); - if( !index ) { - __TBB_TRY { - concurrent_queue_page_allocator& pa = base; - p = pa.allocate_page(); - } __TBB_CATCH (...) { - ++base.my_rep->n_invalid_entries; - invalidate_page_and_rethrow( k ); - } - p->mask = 0; - p->next = NULL; - } - - if( tail_counter != k ) spin_wait_until_my_turn( tail_counter, k, *base.my_rep ); - call_itt_notify(acquired, &tail_counter); - - if( p ) { - spin_mutex::scoped_lock lock( page_mutex ); - page* q = tail_page; - if( is_valid_page(q) ) - q->next = p; - else - head_page = p; - tail_page = p; - } else { - p = tail_page; - } - - __TBB_TRY { - copy_item( *p, index, item, construct_item ); - // If no exception was thrown, mark item as present. - itt_hide_store_word(p->mask, p->mask | uintptr_t(1)<n_invalid_entries; - call_itt_notify(releasing, &tail_counter); - tail_counter += concurrent_queue_rep_base::n_queue; - __TBB_RETHROW(); - } -} - -template -bool micro_queue::pop( void* dst, ticket k, concurrent_queue_base_v3& base ) { - k &= -concurrent_queue_rep_base::n_queue; - if( head_counter!=k ) spin_wait_until_eq( head_counter, k ); - call_itt_notify(acquired, &head_counter); - if( tail_counter==k ) spin_wait_while_eq( tail_counter, k ); - call_itt_notify(acquired, &tail_counter); - page *p = head_page; - __TBB_ASSERT( p, NULL ); - size_t index = modulo_power_of_two( k/concurrent_queue_rep_base::n_queue, base.my_rep->items_per_page ); - bool success = false; - { - micro_queue_pop_finalizer finalizer( *this, base, k+concurrent_queue_rep_base::n_queue, index==base.my_rep->items_per_page-1 ? p : NULL ); - if( p->mask & uintptr_t(1)<n_invalid_entries; - } - } - return success; -} - -template -micro_queue& micro_queue::assign( const micro_queue& src, concurrent_queue_base_v3& base, - item_constructor_t construct_item ) -{ - head_counter = src.head_counter; - tail_counter = src.tail_counter; - - const page* srcp = src.head_page; - if( is_valid_page(srcp) ) { - ticket g_index = head_counter; - __TBB_TRY { - size_t n_items = (tail_counter-head_counter)/concurrent_queue_rep_base::n_queue; - size_t index = modulo_power_of_two( head_counter/concurrent_queue_rep_base::n_queue, base.my_rep->items_per_page ); - size_t end_in_first_page = (index+n_itemsitems_per_page)?(index+n_items):base.my_rep->items_per_page; - - head_page = make_copy( base, srcp, index, end_in_first_page, g_index, construct_item ); - page* cur_page = head_page; - - if( srcp != src.tail_page ) { - for( srcp = srcp->next; srcp!=src.tail_page; srcp=srcp->next ) { - cur_page->next = make_copy( base, srcp, 0, base.my_rep->items_per_page, g_index, construct_item ); - cur_page = cur_page->next; - } - - __TBB_ASSERT( srcp==src.tail_page, NULL ); - size_t last_index = modulo_power_of_two( tail_counter/concurrent_queue_rep_base::n_queue, base.my_rep->items_per_page ); - if( last_index==0 ) last_index = base.my_rep->items_per_page; - - cur_page->next = make_copy( base, srcp, 0, last_index, g_index, construct_item ); - cur_page = cur_page->next; - } - tail_page = cur_page; - } __TBB_CATCH (...) { - invalidate_page_and_rethrow( g_index ); - } - } else { - head_page = tail_page = NULL; - } - return *this; -} - -template -void micro_queue::invalidate_page_and_rethrow( ticket k ) { - // Append an invalid page at address 1 so that no more pushes are allowed. - page* invalid_page = (page*)uintptr_t(1); - { - spin_mutex::scoped_lock lock( page_mutex ); - itt_store_word_with_release(tail_counter, k+concurrent_queue_rep_base::n_queue+1); - page* q = tail_page; - if( is_valid_page(q) ) - q->next = invalid_page; - else - head_page = invalid_page; - tail_page = invalid_page; - } - __TBB_RETHROW(); -} - -template -concurrent_queue_rep_base::page* micro_queue::make_copy( concurrent_queue_base_v3& base, - const concurrent_queue_rep_base::page* src_page, size_t begin_in_page, size_t end_in_page, - ticket& g_index, item_constructor_t construct_item ) -{ - concurrent_queue_page_allocator& pa = base; - page* new_page = pa.allocate_page(); - new_page->next = NULL; - new_page->mask = src_page->mask; - for( ; begin_in_page!=end_in_page; ++begin_in_page, ++g_index ) - if( new_page->mask & uintptr_t(1)< -class micro_queue_pop_finalizer: no_copy { - typedef concurrent_queue_rep_base::page page; - ticket my_ticket; - micro_queue& my_queue; - page* my_page; - concurrent_queue_page_allocator& allocator; -public: - micro_queue_pop_finalizer( micro_queue& queue, concurrent_queue_base_v3& b, ticket k, page* p ) : - my_ticket(k), my_queue(queue), my_page(p), allocator(b) - {} - ~micro_queue_pop_finalizer() ; -}; - -template -micro_queue_pop_finalizer::~micro_queue_pop_finalizer() { - page* p = my_page; - if( is_valid_page(p) ) { - spin_mutex::scoped_lock lock( my_queue.page_mutex ); - page* q = p->next; - my_queue.head_page = q; - if( !is_valid_page(q) ) { - my_queue.tail_page = NULL; - } - } - itt_store_word_with_release(my_queue.head_counter, my_ticket); - if( is_valid_page(p) ) { - allocator.deallocate_page( p ); - } -} - -#if _MSC_VER && !defined(__INTEL_COMPILER) -#pragma warning( pop ) -#endif // warning 4146 is back - -template class concurrent_queue_iterator_rep ; -template class concurrent_queue_iterator_base_v3; - -//! representation of concurrent_queue_base -/** - * the class inherits from concurrent_queue_rep_base and defines an array of micro_queue's - */ -template -struct concurrent_queue_rep : public concurrent_queue_rep_base { - micro_queue array[n_queue]; - - //! Map ticket to an array index - static size_t index( ticket k ) { - return k*phi%n_queue; - } - - micro_queue& choose( ticket k ) { - // The formula here approximates LRU in a cache-oblivious way. - return array[index(k)]; - } -}; - -//! base class of concurrent_queue -/** - * The class implements the interface defined by concurrent_queue_page_allocator - * and has a pointer to an instance of concurrent_queue_rep. - */ -template -class concurrent_queue_base_v3: public concurrent_queue_page_allocator { -private: - //! Internal representation - concurrent_queue_rep* my_rep; - - friend struct concurrent_queue_rep; - friend class micro_queue; - friend class concurrent_queue_iterator_rep; - friend class concurrent_queue_iterator_base_v3; - -protected: - typedef typename concurrent_queue_rep::page page; - -private: - typedef typename micro_queue::padded_page padded_page; - typedef typename micro_queue::item_constructor_t item_constructor_t; - - virtual page *allocate_page() __TBB_override { - concurrent_queue_rep& r = *my_rep; - size_t n = sizeof(padded_page) + (r.items_per_page-1)*sizeof(T); - return reinterpret_cast(allocate_block ( n )); - } - - virtual void deallocate_page( concurrent_queue_rep_base::page *p ) __TBB_override { - concurrent_queue_rep& r = *my_rep; - size_t n = sizeof(padded_page) + (r.items_per_page-1)*sizeof(T); - deallocate_block( reinterpret_cast(p), n ); - } - - //! custom allocator - virtual void *allocate_block( size_t n ) = 0; - - //! custom de-allocator - virtual void deallocate_block( void *p, size_t n ) = 0; - -protected: - concurrent_queue_base_v3(); - - virtual ~concurrent_queue_base_v3() { -#if TBB_USE_ASSERT - size_t nq = my_rep->n_queue; - for( size_t i=0; iarray[i].tail_page==NULL, "pages were not freed properly" ); -#endif /* TBB_USE_ASSERT */ - cache_aligned_allocator >().deallocate(my_rep,1); - } - - //! Enqueue item at tail of queue - void internal_push( const void* src, item_constructor_t construct_item ) { - concurrent_queue_rep& r = *my_rep; - ticket k = r.tail_counter++; - r.choose(k).push( src, k, *this, construct_item ); - } - - //! Attempt to dequeue item from queue. - /** NULL if there was no item to dequeue. */ - bool internal_try_pop( void* dst ) ; - - //! Get size of queue; result may be invalid if queue is modified concurrently - size_t internal_size() const ; - - //! check if the queue is empty; thread safe - bool internal_empty() const ; - - //! free any remaining pages - /* note that the name may be misleading, but it remains so due to a historical accident. */ - void internal_finish_clear() ; - - //! Obsolete - void internal_throw_exception() const { - throw_exception( eid_bad_alloc ); - } - - //! copy or move internal representation - void assign( const concurrent_queue_base_v3& src, item_constructor_t construct_item ) ; - -#if __TBB_CPP11_RVALUE_REF_PRESENT - //! swap internal representation - void internal_swap( concurrent_queue_base_v3& src ) { - std::swap( my_rep, src.my_rep ); - } -#endif /* __TBB_CPP11_RVALUE_REF_PRESENT */ -}; - -template -concurrent_queue_base_v3::concurrent_queue_base_v3() { - const size_t item_size = sizeof(T); - my_rep = cache_aligned_allocator >().allocate(1); - __TBB_ASSERT( (size_t)my_rep % NFS_GetLineSize()==0, "alignment error" ); - __TBB_ASSERT( (size_t)&my_rep->head_counter % NFS_GetLineSize()==0, "alignment error" ); - __TBB_ASSERT( (size_t)&my_rep->tail_counter % NFS_GetLineSize()==0, "alignment error" ); - __TBB_ASSERT( (size_t)&my_rep->array % NFS_GetLineSize()==0, "alignment error" ); - memset(my_rep,0,sizeof(concurrent_queue_rep)); - my_rep->item_size = item_size; - my_rep->items_per_page = item_size<= 8 ? 32 : - item_size<= 16 ? 16 : - item_size<= 32 ? 8 : - item_size<= 64 ? 4 : - item_size<=128 ? 2 : - 1; -} - -template -bool concurrent_queue_base_v3::internal_try_pop( void* dst ) { - concurrent_queue_rep& r = *my_rep; - ticket k; - do { - k = r.head_counter; - for(;;) { - if( (ptrdiff_t)(r.tail_counter-k)<=0 ) { - // Queue is empty - return false; - } - // Queue had item with ticket k when we looked. Attempt to get that item. - ticket tk=k; -#if defined(_MSC_VER) && defined(_Wp64) - #pragma warning (push) - #pragma warning (disable: 4267) -#endif - k = r.head_counter.compare_and_swap( tk+1, tk ); -#if defined(_MSC_VER) && defined(_Wp64) - #pragma warning (pop) -#endif - if( k==tk ) - break; - // Another thread snatched the item, retry. - } - } while( !r.choose( k ).pop( dst, k, *this ) ); - return true; -} - -template -size_t concurrent_queue_base_v3::internal_size() const { - concurrent_queue_rep& r = *my_rep; - __TBB_ASSERT( sizeof(ptrdiff_t)<=sizeof(size_t), NULL ); - ticket hc = r.head_counter; - size_t nie = r.n_invalid_entries; - ticket tc = r.tail_counter; - __TBB_ASSERT( hc!=tc || !nie, NULL ); - ptrdiff_t sz = tc-hc-nie; - return sz<0 ? 0 : size_t(sz); -} - -template -bool concurrent_queue_base_v3::internal_empty() const { - concurrent_queue_rep& r = *my_rep; - ticket tc = r.tail_counter; - ticket hc = r.head_counter; - // if tc!=r.tail_counter, the queue was not empty at some point between the two reads. - return tc==r.tail_counter && tc==hc+r.n_invalid_entries ; -} - -template -void concurrent_queue_base_v3::internal_finish_clear() { - concurrent_queue_rep& r = *my_rep; - size_t nq = r.n_queue; - for( size_t i=0; i -void concurrent_queue_base_v3::assign( const concurrent_queue_base_v3& src, - item_constructor_t construct_item ) -{ - concurrent_queue_rep& r = *my_rep; - r.items_per_page = src.my_rep->items_per_page; - - // copy concurrent_queue_rep data - r.head_counter = src.my_rep->head_counter; - r.tail_counter = src.my_rep->tail_counter; - r.n_invalid_entries = src.my_rep->n_invalid_entries; - - // copy or move micro_queues - for( size_t i = 0; i < r.n_queue; ++i ) - r.array[i].assign( src.my_rep->array[i], *this, construct_item); - - __TBB_ASSERT( r.head_counter==src.my_rep->head_counter && r.tail_counter==src.my_rep->tail_counter, - "the source concurrent queue should not be concurrently modified." ); -} - -template class concurrent_queue_iterator; - -template -class concurrent_queue_iterator_rep: no_assign { - typedef typename micro_queue::padded_page padded_page; -public: - ticket head_counter; - const concurrent_queue_base_v3& my_queue; - typename concurrent_queue_base_v3::page* array[concurrent_queue_rep::n_queue]; - concurrent_queue_iterator_rep( const concurrent_queue_base_v3& queue ) : - head_counter(queue.my_rep->head_counter), - my_queue(queue) - { - for( size_t k=0; k::n_queue; ++k ) - array[k] = queue.my_rep->array[k].head_page; - } - - //! Set item to point to kth element. Return true if at end of queue or item is marked valid; false otherwise. - bool get_item( T*& item, size_t k ) ; -}; - -template -bool concurrent_queue_iterator_rep::get_item( T*& item, size_t k ) { - if( k==my_queue.my_rep->tail_counter ) { - item = NULL; - return true; - } else { - typename concurrent_queue_base_v3::page* p = array[concurrent_queue_rep::index(k)]; - __TBB_ASSERT(p,NULL); - size_t i = modulo_power_of_two( k/concurrent_queue_rep::n_queue, my_queue.my_rep->items_per_page ); - item = µ_queue::get_ref(*p,i); - return (p->mask & uintptr_t(1)< -class concurrent_queue_iterator_base_v3 : no_assign { - //! Represents concurrent_queue over which we are iterating. - /** NULL if one past last element in queue. */ - concurrent_queue_iterator_rep* my_rep; - - template - friend bool operator==( const concurrent_queue_iterator& i, const concurrent_queue_iterator& j ); - - template - friend bool operator!=( const concurrent_queue_iterator& i, const concurrent_queue_iterator& j ); -protected: - //! Pointer to current item - Value* my_item; - - //! Default constructor - concurrent_queue_iterator_base_v3() : my_rep(NULL), my_item(NULL) { -#if __TBB_GCC_OPTIMIZER_ORDERING_BROKEN - __TBB_compiler_fence(); -#endif - } - - //! Copy constructor - concurrent_queue_iterator_base_v3( const concurrent_queue_iterator_base_v3& i ) - : no_assign(), my_rep(NULL), my_item(NULL) { - assign(i); - } - - //! Construct iterator pointing to head of queue. - concurrent_queue_iterator_base_v3( const concurrent_queue_base_v3& queue ) ; - - //! Assignment - void assign( const concurrent_queue_iterator_base_v3& other ) ; - - //! Advance iterator one step towards tail of queue. - void advance() ; - - //! Destructor - ~concurrent_queue_iterator_base_v3() { - cache_aligned_allocator >().deallocate(my_rep, 1); - my_rep = NULL; - } -}; - -template -concurrent_queue_iterator_base_v3::concurrent_queue_iterator_base_v3( const concurrent_queue_base_v3& queue ) { - my_rep = cache_aligned_allocator >().allocate(1); - new( my_rep ) concurrent_queue_iterator_rep(queue); - size_t k = my_rep->head_counter; - if( !my_rep->get_item(my_item, k) ) advance(); -} - -template -void concurrent_queue_iterator_base_v3::assign( const concurrent_queue_iterator_base_v3& other ) { - if( my_rep!=other.my_rep ) { - if( my_rep ) { - cache_aligned_allocator >().deallocate(my_rep, 1); - my_rep = NULL; - } - if( other.my_rep ) { - my_rep = cache_aligned_allocator >().allocate(1); - new( my_rep ) concurrent_queue_iterator_rep( *other.my_rep ); - } - } - my_item = other.my_item; -} - -template -void concurrent_queue_iterator_base_v3::advance() { - __TBB_ASSERT( my_item, "attempt to increment iterator past end of queue" ); - size_t k = my_rep->head_counter; - const concurrent_queue_base_v3& queue = my_rep->my_queue; -#if TBB_USE_ASSERT - Value* tmp; - my_rep->get_item(tmp,k); - __TBB_ASSERT( my_item==tmp, NULL ); -#endif /* TBB_USE_ASSERT */ - size_t i = modulo_power_of_two( k/concurrent_queue_rep::n_queue, queue.my_rep->items_per_page ); - if( i==queue.my_rep->items_per_page-1 ) { - typename concurrent_queue_base_v3::page*& root = my_rep->array[concurrent_queue_rep::index(k)]; - root = root->next; - } - // advance k - my_rep->head_counter = ++k; - if( !my_rep->get_item(my_item, k) ) advance(); -} - -//! Similar to C++0x std::remove_cv -/** "tbb_" prefix added to avoid overload confusion with C++0x implementations. */ -template struct tbb_remove_cv {typedef T type;}; -template struct tbb_remove_cv {typedef T type;}; -template struct tbb_remove_cv {typedef T type;}; -template struct tbb_remove_cv {typedef T type;}; - -//! Meets requirements of a forward iterator for STL. -/** Value is either the T or const T type of the container. - @ingroup containers */ -template -class concurrent_queue_iterator: public concurrent_queue_iterator_base_v3::type>, - public std::iterator { -#if !__TBB_TEMPLATE_FRIENDS_BROKEN - template - friend class ::tbb::strict_ppl::concurrent_queue; -#else -public: -#endif - //! Construct iterator pointing to head of queue. - explicit concurrent_queue_iterator( const concurrent_queue_base_v3::type>& queue ) : - concurrent_queue_iterator_base_v3::type>(queue) - { - } - -public: - concurrent_queue_iterator() {} - - /** If Value==Container::value_type, then this routine is the copy constructor. - If Value==const Container::value_type, then this routine is a conversion constructor. */ - concurrent_queue_iterator( const concurrent_queue_iterator& other ) : - concurrent_queue_iterator_base_v3::type>(other) - {} - - //! Iterator assignment - concurrent_queue_iterator& operator=( const concurrent_queue_iterator& other ) { - this->assign(other); - return *this; - } - - //! Reference to current item - Value& operator*() const { - return *static_cast(this->my_item); - } - - Value* operator->() const {return &operator*();} - - //! Advance to next item in queue - concurrent_queue_iterator& operator++() { - this->advance(); - return *this; - } - - //! Post increment - Value* operator++(int) { - Value* result = &operator*(); - operator++(); - return result; - } -}; // concurrent_queue_iterator - - -template -bool operator==( const concurrent_queue_iterator& i, const concurrent_queue_iterator& j ) { - return i.my_item==j.my_item; -} - -template -bool operator!=( const concurrent_queue_iterator& i, const concurrent_queue_iterator& j ) { - return i.my_item!=j.my_item; -} - -} // namespace internal - -//! @endcond - -} // namespace strict_ppl - -//! @cond INTERNAL -namespace internal { - -class concurrent_queue_rep; -class concurrent_queue_iterator_rep; -class concurrent_queue_iterator_base_v3; -template class concurrent_queue_iterator; - -//! For internal use only. -/** Type-independent portion of concurrent_queue. - @ingroup containers */ -class concurrent_queue_base_v3: no_copy { -private: - //! Internal representation - concurrent_queue_rep* my_rep; - - friend class concurrent_queue_rep; - friend struct micro_queue; - friend class micro_queue_pop_finalizer; - friend class concurrent_queue_iterator_rep; - friend class concurrent_queue_iterator_base_v3; -protected: - //! Prefix on a page - struct page { - page* next; - uintptr_t mask; - }; - - //! Capacity of the queue - ptrdiff_t my_capacity; - - //! Always a power of 2 - size_t items_per_page; - - //! Size of an item - size_t item_size; - - enum copy_specifics { copy, move }; - -#if __TBB_PROTECTED_NESTED_CLASS_BROKEN -public: -#endif - template - struct padded_page: page { - //! Not defined anywhere - exists to quiet warnings. - padded_page(); - //! Not defined anywhere - exists to quiet warnings. - void operator=( const padded_page& ); - //! Must be last field. - T last; - }; - -private: - virtual void copy_item( page& dst, size_t index, const void* src ) = 0; - virtual void assign_and_destroy_item( void* dst, page& src, size_t index ) = 0; -protected: - __TBB_EXPORTED_METHOD concurrent_queue_base_v3( size_t item_size ); - virtual __TBB_EXPORTED_METHOD ~concurrent_queue_base_v3(); - - //! Enqueue item at tail of queue using copy operation - void __TBB_EXPORTED_METHOD internal_push( const void* src ); - - //! Dequeue item from head of queue - void __TBB_EXPORTED_METHOD internal_pop( void* dst ); - - //! Abort all pending queue operations - void __TBB_EXPORTED_METHOD internal_abort(); - - //! Attempt to enqueue item onto queue using copy operation - bool __TBB_EXPORTED_METHOD internal_push_if_not_full( const void* src ); - - //! Attempt to dequeue item from queue. - /** NULL if there was no item to dequeue. */ - bool __TBB_EXPORTED_METHOD internal_pop_if_present( void* dst ); - - //! Get size of queue - ptrdiff_t __TBB_EXPORTED_METHOD internal_size() const; - - //! Check if the queue is emtpy - bool __TBB_EXPORTED_METHOD internal_empty() const; - - //! Set the queue capacity - void __TBB_EXPORTED_METHOD internal_set_capacity( ptrdiff_t capacity, size_t element_size ); - - //! custom allocator - virtual page *allocate_page() = 0; - - //! custom de-allocator - virtual void deallocate_page( page *p ) = 0; - - //! free any remaining pages - /* note that the name may be misleading, but it remains so due to a historical accident. */ - void __TBB_EXPORTED_METHOD internal_finish_clear() ; - - //! throw an exception - void __TBB_EXPORTED_METHOD internal_throw_exception() const; - - //! copy internal representation - void __TBB_EXPORTED_METHOD assign( const concurrent_queue_base_v3& src ) ; - -#if __TBB_CPP11_RVALUE_REF_PRESENT - //! swap queues - void internal_swap( concurrent_queue_base_v3& src ) { - std::swap( my_capacity, src.my_capacity ); - std::swap( items_per_page, src.items_per_page ); - std::swap( item_size, src.item_size ); - std::swap( my_rep, src.my_rep ); - } -#endif /* __TBB_CPP11_RVALUE_REF_PRESENT */ - - //! Enqueues item at tail of queue using specified operation (copy or move) - void internal_insert_item( const void* src, copy_specifics op_type ); - - //! Attempts to enqueue at tail of queue using specified operation (copy or move) - bool internal_insert_if_not_full( const void* src, copy_specifics op_type ); - - //! Assigns one queue to another using specified operation (copy or move) - void internal_assign( const concurrent_queue_base_v3& src, copy_specifics op_type ); -private: - virtual void copy_page_item( page& dst, size_t dindex, const page& src, size_t sindex ) = 0; -}; - -//! For internal use only. -/** Backward compatible modification of concurrent_queue_base_v3 - @ingroup containers */ -class concurrent_queue_base_v8: public concurrent_queue_base_v3 { -protected: - concurrent_queue_base_v8( size_t item_sz ) : concurrent_queue_base_v3( item_sz ) {} - - //! move items - void __TBB_EXPORTED_METHOD move_content( concurrent_queue_base_v8& src ) ; - - //! Attempt to enqueue item onto queue using move operation - bool __TBB_EXPORTED_METHOD internal_push_move_if_not_full( const void* src ); - - //! Enqueue item at tail of queue using move operation - void __TBB_EXPORTED_METHOD internal_push_move( const void* src ); -private: - friend struct micro_queue; - virtual void move_page_item( page& dst, size_t dindex, const page& src, size_t sindex ) = 0; - virtual void move_item( page& dst, size_t index, const void* src ) = 0; -}; - -//! Type-independent portion of concurrent_queue_iterator. -/** @ingroup containers */ -class concurrent_queue_iterator_base_v3 { - //! concurrent_queue over which we are iterating. - /** NULL if one past last element in queue. */ - concurrent_queue_iterator_rep* my_rep; - - template - friend bool operator==( const concurrent_queue_iterator& i, const concurrent_queue_iterator& j ); - - template - friend bool operator!=( const concurrent_queue_iterator& i, const concurrent_queue_iterator& j ); - - void initialize( const concurrent_queue_base_v3& queue, size_t offset_of_data ); -protected: - //! Pointer to current item - void* my_item; - - //! Default constructor - concurrent_queue_iterator_base_v3() : my_rep(NULL), my_item(NULL) {} - - //! Copy constructor - concurrent_queue_iterator_base_v3( const concurrent_queue_iterator_base_v3& i ) : my_rep(NULL), my_item(NULL) { - assign(i); - } - - //! Obsolete entry point for constructing iterator pointing to head of queue. - /** Does not work correctly for SSE types. */ - __TBB_EXPORTED_METHOD concurrent_queue_iterator_base_v3( const concurrent_queue_base_v3& queue ); - - //! Construct iterator pointing to head of queue. - __TBB_EXPORTED_METHOD concurrent_queue_iterator_base_v3( const concurrent_queue_base_v3& queue, size_t offset_of_data ); - - //! Assignment - void __TBB_EXPORTED_METHOD assign( const concurrent_queue_iterator_base_v3& i ); - - //! Advance iterator one step towards tail of queue. - void __TBB_EXPORTED_METHOD advance(); - - //! Destructor - __TBB_EXPORTED_METHOD ~concurrent_queue_iterator_base_v3(); -}; - -typedef concurrent_queue_iterator_base_v3 concurrent_queue_iterator_base; - -//! Meets requirements of a forward iterator for STL. -/** Value is either the T or const T type of the container. - @ingroup containers */ -template -class concurrent_queue_iterator: public concurrent_queue_iterator_base, - public std::iterator { - -#if !__TBB_TEMPLATE_FRIENDS_BROKEN - template - friend class ::tbb::concurrent_bounded_queue; -#else -public: -#endif - - //! Construct iterator pointing to head of queue. - explicit concurrent_queue_iterator( const concurrent_queue_base_v3& queue ) : - concurrent_queue_iterator_base_v3(queue,__TBB_offsetof(concurrent_queue_base_v3::padded_page,last)) - { - } - -public: - concurrent_queue_iterator() {} - - /** If Value==Container::value_type, then this routine is the copy constructor. - If Value==const Container::value_type, then this routine is a conversion constructor. */ - concurrent_queue_iterator( const concurrent_queue_iterator& other ) : - concurrent_queue_iterator_base_v3(other) - {} - - //! Iterator assignment - concurrent_queue_iterator& operator=( const concurrent_queue_iterator& other ) { - assign(other); - return *this; - } - - //! Reference to current item - Value& operator*() const { - return *static_cast(my_item); - } - - Value* operator->() const {return &operator*();} - - //! Advance to next item in queue - concurrent_queue_iterator& operator++() { - advance(); - return *this; - } - - //! Post increment - Value* operator++(int) { - Value* result = &operator*(); - operator++(); - return result; - } -}; // concurrent_queue_iterator - - -template -bool operator==( const concurrent_queue_iterator& i, const concurrent_queue_iterator& j ) { - return i.my_item==j.my_item; -} - -template -bool operator!=( const concurrent_queue_iterator& i, const concurrent_queue_iterator& j ) { - return i.my_item!=j.my_item; -} - -} // namespace internal; - -//! @endcond - -} // namespace tbb - -#endif /* __TBB__concurrent_queue_impl_H */ diff --git a/src/tbb/include/tbb/internal/_concurrent_unordered_impl.h b/src/tbb/include/tbb/internal/_concurrent_unordered_impl.h deleted file mode 100644 index 2395aa084..000000000 --- a/src/tbb/include/tbb/internal/_concurrent_unordered_impl.h +++ /dev/null @@ -1,1518 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -/* Container implementations in this header are based on PPL implementations - provided by Microsoft. */ - -#ifndef __TBB__concurrent_unordered_impl_H -#define __TBB__concurrent_unordered_impl_H -#if !defined(__TBB_concurrent_unordered_map_H) && !defined(__TBB_concurrent_unordered_set_H) && !defined(__TBB_concurrent_hash_map_H) -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#include "../tbb_stddef.h" - -#include -#include // Need std::pair -#include // Need std::equal_to (in ../concurrent_unordered_*.h) -#include // For tbb_hasher -#include // Need std::memset -#include __TBB_STD_SWAP_HEADER - -#include "../atomic.h" -#include "../tbb_exception.h" -#include "../tbb_allocator.h" - -#if __TBB_INITIALIZER_LISTS_PRESENT - #include -#endif - -#include "_tbb_hash_compare_impl.h" - -namespace tbb { -namespace interface5 { -//! @cond INTERNAL -namespace internal { - -template -class split_ordered_list; -template -class concurrent_unordered_base; - -// Forward list iterators (without skipping dummy elements) -template -class flist_iterator : public std::iterator -{ - template - friend class split_ordered_list; - template - friend class concurrent_unordered_base; - template - friend class flist_iterator; - - typedef typename Solist::nodeptr_t nodeptr_t; -public: - typedef typename Solist::value_type value_type; - typedef typename Solist::difference_type difference_type; - typedef typename Solist::pointer pointer; - typedef typename Solist::reference reference; - - flist_iterator() : my_node_ptr(0) {} - flist_iterator( const flist_iterator &other ) - : my_node_ptr(other.my_node_ptr) {} - - reference operator*() const { return my_node_ptr->my_element; } - pointer operator->() const { return &**this; } - - flist_iterator& operator++() { - my_node_ptr = my_node_ptr->my_next; - return *this; - } - - flist_iterator operator++(int) { - flist_iterator tmp = *this; - ++*this; - return tmp; - } - -protected: - flist_iterator(nodeptr_t pnode) : my_node_ptr(pnode) {} - nodeptr_t get_node_ptr() const { return my_node_ptr; } - - nodeptr_t my_node_ptr; - - template - friend bool operator==( const flist_iterator &i, const flist_iterator &j ); - template - friend bool operator!=( const flist_iterator& i, const flist_iterator& j ); -}; - -template -bool operator==( const flist_iterator &i, const flist_iterator &j ) { - return i.my_node_ptr == j.my_node_ptr; -} -template -bool operator!=( const flist_iterator& i, const flist_iterator& j ) { - return i.my_node_ptr != j.my_node_ptr; -} - -// Split-order list iterators, needed to skip dummy elements -template -class solist_iterator : public flist_iterator -{ - typedef flist_iterator base_type; - typedef typename Solist::nodeptr_t nodeptr_t; - using base_type::get_node_ptr; - template - friend class split_ordered_list; - template - friend class solist_iterator; - template - friend bool operator==( const solist_iterator &i, const solist_iterator &j ); - template - friend bool operator!=( const solist_iterator& i, const solist_iterator& j ); - - const Solist *my_list_ptr; - solist_iterator(nodeptr_t pnode, const Solist *plist) : base_type(pnode), my_list_ptr(plist) {} - -public: - typedef typename Solist::value_type value_type; - typedef typename Solist::difference_type difference_type; - typedef typename Solist::pointer pointer; - typedef typename Solist::reference reference; - - solist_iterator() {} - solist_iterator(const solist_iterator &other ) - : base_type(other), my_list_ptr(other.my_list_ptr) {} - - reference operator*() const { - return this->base_type::operator*(); - } - - pointer operator->() const { - return (&**this); - } - - solist_iterator& operator++() { - do ++(*(base_type *)this); - while (get_node_ptr() != NULL && get_node_ptr()->is_dummy()); - - return (*this); - } - - solist_iterator operator++(int) { - solist_iterator tmp = *this; - do ++*this; - while (get_node_ptr() != NULL && get_node_ptr()->is_dummy()); - - return (tmp); - } -}; - -template -bool operator==( const solist_iterator &i, const solist_iterator &j ) { - return i.my_node_ptr == j.my_node_ptr && i.my_list_ptr == j.my_list_ptr; -} -template -bool operator!=( const solist_iterator& i, const solist_iterator& j ) { - return i.my_node_ptr != j.my_node_ptr || i.my_list_ptr != j.my_list_ptr; -} - -// Forward type and class definitions -typedef size_t sokey_t; - - -// Forward list in which elements are sorted in a split-order -template -class split_ordered_list -{ -public: - typedef split_ordered_list self_type; - typedef typename Allocator::template rebind::other allocator_type; - struct node; - typedef node *nodeptr_t; - - typedef typename allocator_type::size_type size_type; - typedef typename allocator_type::difference_type difference_type; - typedef typename allocator_type::pointer pointer; - typedef typename allocator_type::const_pointer const_pointer; - typedef typename allocator_type::reference reference; - typedef typename allocator_type::const_reference const_reference; - typedef typename allocator_type::value_type value_type; - - typedef solist_iterator const_iterator; - typedef solist_iterator iterator; - typedef flist_iterator raw_const_iterator; - typedef flist_iterator raw_iterator; - - // Node that holds the element in a split-ordered list - struct node : tbb::internal::no_assign - { - private: - // for compilers that try to generate default constructors though they are not needed. - node(); // VS 2008, 2010, 2012 - public: - // Initialize the node with the given order key - void init(sokey_t order_key) { - my_order_key = order_key; - my_next = NULL; - } - - // Return the order key (needed for hashing) - sokey_t get_order_key() const { // TODO: remove - return my_order_key; - } - - // Inserts the new element in the list in an atomic fashion - nodeptr_t atomic_set_next(nodeptr_t new_node, nodeptr_t current_node) - { - // Try to change the next pointer on the current element to a new element, only if it still points to the cached next - nodeptr_t exchange_node = tbb::internal::as_atomic(my_next).compare_and_swap(new_node, current_node); - - if (exchange_node == current_node) // TODO: why this branch? - { - // Operation succeeded, return the new node - return new_node; - } - else - { - // Operation failed, return the "interfering" node - return exchange_node; - } - } - - // Checks if this element in the list is a dummy, order enforcing node. Dummy nodes are used by buckets - // in the hash table to quickly index into the right subsection of the split-ordered list. - bool is_dummy() const { - return (my_order_key & 0x1) == 0; - } - - - nodeptr_t my_next; // Next element in the list - value_type my_element; // Element storage - sokey_t my_order_key; // Order key for this element - }; - - // Allocate a new node with the given order key; used to allocate dummy nodes - nodeptr_t create_node(sokey_t order_key) { - nodeptr_t pnode = my_node_allocator.allocate(1); - pnode->init(order_key); - return (pnode); - } - - // Allocate a new node with the given order key and value - template - nodeptr_t create_node(sokey_t order_key, __TBB_FORWARDING_REF(Arg) t, - /*AllowCreate=*/tbb::internal::true_type=tbb::internal::true_type()){ - nodeptr_t pnode = my_node_allocator.allocate(1); - - //TODO: use RAII scoped guard instead of explicit catch - __TBB_TRY { - new(static_cast(&pnode->my_element)) T(tbb::internal::forward(t)); - pnode->init(order_key); - } __TBB_CATCH(...) { - my_node_allocator.deallocate(pnode, 1); - __TBB_RETHROW(); - } - - return (pnode); - } - - // A helper to avoid excessive requiremens in internal_insert - template - nodeptr_t create_node(sokey_t, __TBB_FORWARDING_REF(Arg), - /*AllowCreate=*/tbb::internal::false_type){ - __TBB_ASSERT(false, "This compile-time helper should never get called"); - return nodeptr_t(); - } - - // Allocate a new node with the given parameters for constructing value - template - nodeptr_t create_node_v( __TBB_FORWARDING_REF(Args) __TBB_PARAMETER_PACK args){ - nodeptr_t pnode = my_node_allocator.allocate(1); - - //TODO: use RAII scoped guard instead of explicit catch - __TBB_TRY { - new(static_cast(&pnode->my_element)) T(__TBB_PACK_EXPANSION(tbb::internal::forward(args))); - } __TBB_CATCH(...) { - my_node_allocator.deallocate(pnode, 1); - __TBB_RETHROW(); - } - - return (pnode); - } - - split_ordered_list(allocator_type a = allocator_type()) - : my_node_allocator(a), my_element_count(0) - { - // Immediately allocate a dummy node with order key of 0. This node - // will always be the head of the list. - my_head = create_node(sokey_t(0)); - } - - ~split_ordered_list() - { - // Clear the list - clear(); - - // Remove the head element which is not cleared by clear() - nodeptr_t pnode = my_head; - my_head = NULL; - - __TBB_ASSERT(pnode != NULL && pnode->my_next == NULL, "Invalid head list node"); - - destroy_node(pnode); - } - - // Common forward list functions - - allocator_type get_allocator() const { - return (my_node_allocator); - } - - void clear() { - nodeptr_t pnext; - nodeptr_t pnode = my_head; - - __TBB_ASSERT(my_head != NULL, "Invalid head list node"); - pnext = pnode->my_next; - pnode->my_next = NULL; - pnode = pnext; - - while (pnode != NULL) - { - pnext = pnode->my_next; - destroy_node(pnode); - pnode = pnext; - } - - my_element_count = 0; - } - - // Returns a first non-dummy element in the SOL - iterator begin() { - return first_real_iterator(raw_begin()); - } - - // Returns a first non-dummy element in the SOL - const_iterator begin() const { - return first_real_iterator(raw_begin()); - } - - iterator end() { - return (iterator(0, this)); - } - - const_iterator end() const { - return (const_iterator(0, this)); - } - - const_iterator cbegin() const { - return (((const self_type *)this)->begin()); - } - - const_iterator cend() const { - return (((const self_type *)this)->end()); - } - - // Checks if the number of elements (non-dummy) is 0 - bool empty() const { - return (my_element_count == 0); - } - - // Returns the number of non-dummy elements in the list - size_type size() const { - return my_element_count; - } - - // Returns the maximum size of the list, determined by the allocator - size_type max_size() const { - return my_node_allocator.max_size(); - } - - // Swaps 'this' list with the passed in one - void swap(self_type& other) - { - if (this == &other) - { - // Nothing to do - return; - } - - std::swap(my_element_count, other.my_element_count); - std::swap(my_head, other.my_head); - } - - // Split-order list functions - - // Returns a first element in the SOL, which is always a dummy - raw_iterator raw_begin() { - return raw_iterator(my_head); - } - - // Returns a first element in the SOL, which is always a dummy - raw_const_iterator raw_begin() const { - return raw_const_iterator(my_head); - } - - raw_iterator raw_end() { - return raw_iterator(0); - } - - raw_const_iterator raw_end() const { - return raw_const_iterator(0); - } - - static sokey_t get_order_key(const raw_const_iterator& it) { - return it.get_node_ptr()->get_order_key(); - } - - static sokey_t get_safe_order_key(const raw_const_iterator& it) { - if( !it.get_node_ptr() ) return ~sokey_t(0); - return it.get_node_ptr()->get_order_key(); - } - - // Returns a public iterator version of the internal iterator. Public iterator must not - // be a dummy private iterator. - iterator get_iterator(raw_iterator it) { - __TBB_ASSERT(it.get_node_ptr() == NULL || !it.get_node_ptr()->is_dummy(), "Invalid user node (dummy)"); - return iterator(it.get_node_ptr(), this); - } - - // Returns a public iterator version of the internal iterator. Public iterator must not - // be a dummy private iterator. - const_iterator get_iterator(raw_const_iterator it) const { - __TBB_ASSERT(it.get_node_ptr() == NULL || !it.get_node_ptr()->is_dummy(), "Invalid user node (dummy)"); - return const_iterator(it.get_node_ptr(), this); - } - - // Returns a non-const version of the raw_iterator - raw_iterator get_iterator(raw_const_iterator it) { - return raw_iterator(it.get_node_ptr()); - } - - // Returns a non-const version of the iterator - static iterator get_iterator(const_iterator it) { - return iterator(it.my_node_ptr, it.my_list_ptr); - } - - // Returns a public iterator version of a first non-dummy internal iterator at or after - // the passed in internal iterator. - iterator first_real_iterator(raw_iterator it) - { - // Skip all dummy, internal only iterators - while (it != raw_end() && it.get_node_ptr()->is_dummy()) - ++it; - - return iterator(it.get_node_ptr(), this); - } - - // Returns a public iterator version of a first non-dummy internal iterator at or after - // the passed in internal iterator. - const_iterator first_real_iterator(raw_const_iterator it) const - { - // Skip all dummy, internal only iterators - while (it != raw_end() && it.get_node_ptr()->is_dummy()) - ++it; - - return const_iterator(it.get_node_ptr(), this); - } - - // Erase an element using the allocator - void destroy_node(nodeptr_t pnode) { - if (!pnode->is_dummy()) my_node_allocator.destroy(pnode); - my_node_allocator.deallocate(pnode, 1); - } - - // Try to insert a new element in the list. - // If insert fails, return the node that was inserted instead. - static nodeptr_t try_insert_atomic(nodeptr_t previous, nodeptr_t new_node, nodeptr_t current_node) { - new_node->my_next = current_node; - return previous->atomic_set_next(new_node, current_node); - } - - // Insert a new element between passed in iterators - std::pair try_insert(raw_iterator it, raw_iterator next, nodeptr_t pnode, size_type *new_count) - { - nodeptr_t inserted_node = try_insert_atomic(it.get_node_ptr(), pnode, next.get_node_ptr()); - - if (inserted_node == pnode) - { - // If the insert succeeded, check that the order is correct and increment the element count - check_range(it, next); - *new_count = tbb::internal::as_atomic(my_element_count).fetch_and_increment(); - return std::pair(iterator(pnode, this), true); - } - else - { - return std::pair(end(), false); - } - } - - // Insert a new dummy element, starting search at a parent dummy element - raw_iterator insert_dummy(raw_iterator it, sokey_t order_key) - { - raw_iterator last = raw_end(); - raw_iterator where = it; - - __TBB_ASSERT(where != last, "Invalid head node"); - - ++where; - - // Create a dummy element up front, even though it may be discarded (due to concurrent insertion) - nodeptr_t dummy_node = create_node(order_key); - - for (;;) - { - __TBB_ASSERT(it != last, "Invalid head list node"); - - // If the head iterator is at the end of the list, or past the point where this dummy - // node needs to be inserted, then try to insert it. - if (where == last || get_order_key(where) > order_key) - { - __TBB_ASSERT(get_order_key(it) < order_key, "Invalid node order in the list"); - - // Try to insert it in the right place - nodeptr_t inserted_node = try_insert_atomic(it.get_node_ptr(), dummy_node, where.get_node_ptr()); - - if (inserted_node == dummy_node) - { - // Insertion succeeded, check the list for order violations - check_range(it, where); - return raw_iterator(dummy_node); - } - else - { - // Insertion failed: either dummy node was inserted by another thread, or - // a real element was inserted at exactly the same place as dummy node. - // Proceed with the search from the previous location where order key was - // known to be larger (note: this is legal only because there is no safe - // concurrent erase operation supported). - where = it; - ++where; - continue; - } - } - else if (get_order_key(where) == order_key) - { - // Another dummy node with the same value found, discard the new one. - destroy_node(dummy_node); - return where; - } - - // Move the iterator forward - it = where; - ++where; - } - - } - - // This erase function can handle both real and dummy nodes - void erase_node(raw_iterator previous, raw_const_iterator& where) - { - nodeptr_t pnode = (where++).get_node_ptr(); - nodeptr_t prevnode = previous.get_node_ptr(); - __TBB_ASSERT(prevnode->my_next == pnode, "Erase must take consecutive iterators"); - prevnode->my_next = pnode->my_next; - - destroy_node(pnode); - } - - // Erase the element (previous node needs to be passed because this is a forward only list) - iterator erase_node(raw_iterator previous, const_iterator where) - { - raw_const_iterator it = where; - erase_node(previous, it); - my_element_count--; - - return get_iterator(first_real_iterator(it)); - } - - // Move all elements from the passed in split-ordered list to this one - void move_all(self_type& source) - { - raw_const_iterator first = source.raw_begin(); - raw_const_iterator last = source.raw_end(); - - if (first == last) - return; - - nodeptr_t previous_node = my_head; - raw_const_iterator begin_iterator = first++; - - // Move all elements one by one, including dummy ones - for (raw_const_iterator it = first; it != last;) - { - nodeptr_t pnode = it.get_node_ptr(); - - nodeptr_t dummy_node = pnode->is_dummy() ? create_node(pnode->get_order_key()) : create_node(pnode->get_order_key(), pnode->my_element); - previous_node = try_insert_atomic(previous_node, dummy_node, NULL); - __TBB_ASSERT(previous_node != NULL, "Insertion must succeed"); - raw_const_iterator where = it++; - source.erase_node(get_iterator(begin_iterator), where); - } - check_range(); - } - - -private: - //Need to setup private fields of split_ordered_list in move constructor and assignment of concurrent_unordered_base - template - friend class concurrent_unordered_base; - - // Check the list for order violations - void check_range( raw_iterator first, raw_iterator last ) - { -#if TBB_USE_ASSERT - for (raw_iterator it = first; it != last; ++it) - { - raw_iterator next = it; - ++next; - - __TBB_ASSERT(next == raw_end() || get_order_key(next) >= get_order_key(it), "!!! List order inconsistency !!!"); - } -#else - tbb::internal::suppress_unused_warning(first, last); -#endif - } - void check_range() - { -#if TBB_USE_ASSERT - check_range( raw_begin(), raw_end() ); -#endif - } - - typename allocator_type::template rebind::other my_node_allocator; // allocator object for nodes - size_type my_element_count; // Total item count, not counting dummy nodes - nodeptr_t my_head; // pointer to head node -}; - -#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) -#pragma warning(push) -#pragma warning(disable: 4127) // warning C4127: conditional expression is constant -#endif - -template -class concurrent_unordered_base : public Traits -{ -protected: - // Type definitions - typedef concurrent_unordered_base self_type; - typedef typename Traits::value_type value_type; - typedef typename Traits::key_type key_type; - typedef typename Traits::hash_compare hash_compare; - typedef typename Traits::allocator_type allocator_type; - typedef typename hash_compare::hasher hasher; - typedef typename hash_compare::key_equal key_equal; - typedef typename allocator_type::pointer pointer; - typedef typename allocator_type::const_pointer const_pointer; - typedef typename allocator_type::reference reference; - typedef typename allocator_type::const_reference const_reference; - typedef typename allocator_type::size_type size_type; - typedef typename allocator_type::difference_type difference_type; - typedef split_ordered_list solist_t; - typedef typename solist_t::nodeptr_t nodeptr_t; - // Iterators that walk the entire split-order list, including dummy nodes - typedef typename solist_t::raw_iterator raw_iterator; - typedef typename solist_t::raw_const_iterator raw_const_iterator; - typedef typename solist_t::iterator iterator; // TODO: restore const iterator for unordered_sets - typedef typename solist_t::const_iterator const_iterator; - typedef iterator local_iterator; - typedef const_iterator const_local_iterator; - using Traits::my_hash_compare; - using Traits::get_key; - using Traits::allow_multimapping; - - static const size_type initial_bucket_number = 8; // Initial number of buckets -private: - typedef std::pair pairii_t; - typedef std::pair paircc_t; - - static size_type const pointers_per_table = sizeof(size_type) * 8; // One bucket segment per bit - static const size_type initial_bucket_load = 4; // Initial maximum number of elements per bucket - - struct call_internal_clear_on_exit{ - concurrent_unordered_base* my_instance; - call_internal_clear_on_exit(concurrent_unordered_base* instance) : my_instance(instance) {} - void dismiss(){ my_instance = NULL;} - ~call_internal_clear_on_exit(){ - if (my_instance){ - my_instance->internal_clear(); - } - } - }; -protected: - // Constructors/Destructors - concurrent_unordered_base(size_type n_of_buckets = initial_bucket_number, - const hash_compare& hc = hash_compare(), const allocator_type& a = allocator_type()) - : Traits(hc), my_solist(a), - my_allocator(a), my_maximum_bucket_size((float) initial_bucket_load) - { - if( n_of_buckets == 0) ++n_of_buckets; - my_number_of_buckets = size_type(1)<<__TBB_Log2((uintptr_t)n_of_buckets*2-1); // round up to power of 2 - internal_init(); - } - - concurrent_unordered_base(const concurrent_unordered_base& right, const allocator_type& a) - : Traits(right.my_hash_compare), my_solist(a), my_allocator(a) - { - internal_init(); - internal_copy(right); - } - - concurrent_unordered_base(const concurrent_unordered_base& right) - : Traits(right.my_hash_compare), my_solist(right.get_allocator()), my_allocator(right.get_allocator()) - { - //FIXME:exception safety seems to be broken here - internal_init(); - internal_copy(right); - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - concurrent_unordered_base(concurrent_unordered_base&& right) - : Traits(right.my_hash_compare), my_solist(right.get_allocator()), my_allocator(right.get_allocator()) - { - internal_init(); - swap(right); - } - - concurrent_unordered_base(concurrent_unordered_base&& right, const allocator_type& a) - : Traits(right.my_hash_compare), my_solist(a), my_allocator(a) - { - call_internal_clear_on_exit clear_buckets_on_exception(this); - - internal_init(); - if (a == right.get_allocator()){ - this->swap(right); - }else{ - my_maximum_bucket_size = right.my_maximum_bucket_size; - my_number_of_buckets = right.my_number_of_buckets; - my_solist.my_element_count = right.my_solist.my_element_count; - - if (! right.my_solist.empty()){ - nodeptr_t previous_node = my_solist.my_head; - - // Move all elements one by one, including dummy ones - for (raw_const_iterator it = ++(right.my_solist.raw_begin()), last = right.my_solist.raw_end(); it != last; ++it) - { - const nodeptr_t pnode = it.get_node_ptr(); - nodeptr_t node; - if (pnode->is_dummy()) { - node = my_solist.create_node(pnode->get_order_key()); - size_type bucket = __TBB_ReverseBits(pnode->get_order_key()) % my_number_of_buckets; - set_bucket(bucket, node); - }else{ - node = my_solist.create_node(pnode->get_order_key(), std::move(pnode->my_element)); - } - - previous_node = my_solist.try_insert_atomic(previous_node, node, NULL); - __TBB_ASSERT(previous_node != NULL, "Insertion of node failed. Concurrent inserts in constructor ?"); - } - my_solist.check_range(); - } - } - - clear_buckets_on_exception.dismiss(); - } - -#endif // __TBB_CPP11_RVALUE_REF_PRESENT - - concurrent_unordered_base& operator=(const concurrent_unordered_base& right) { - if (this != &right) - internal_copy(right); - return (*this); - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - concurrent_unordered_base& operator=(concurrent_unordered_base&& other) - { - if(this != &other){ - typedef typename tbb::internal::allocator_traits::propagate_on_container_move_assignment pocma_t; - if(pocma_t::value || this->my_allocator == other.my_allocator) { - concurrent_unordered_base trash (std::move(*this)); - swap(other); - if (pocma_t::value) { - using std::swap; - //TODO: swapping allocators here may be a problem, replace with single direction moving - swap(this->my_solist.my_node_allocator, other.my_solist.my_node_allocator); - swap(this->my_allocator, other.my_allocator); - } - } else { - concurrent_unordered_base moved_copy(std::move(other),this->my_allocator); - this->swap(moved_copy); - } - } - return *this; - } - -#endif // __TBB_CPP11_RVALUE_REF_PRESENT - -#if __TBB_INITIALIZER_LISTS_PRESENT - //! assignment operator from initializer_list - concurrent_unordered_base& operator=(std::initializer_list il) - { - this->clear(); - this->insert(il.begin(),il.end()); - return (*this); - } -#endif // __TBB_INITIALIZER_LISTS_PRESENT - - - ~concurrent_unordered_base() { - // Delete all node segments - internal_clear(); - } - -public: - allocator_type get_allocator() const { - return my_solist.get_allocator(); - } - - // Size and capacity function - bool empty() const { - return my_solist.empty(); - } - - size_type size() const { - return my_solist.size(); - } - - size_type max_size() const { - return my_solist.max_size(); - } - - // Iterators - iterator begin() { - return my_solist.begin(); - } - - const_iterator begin() const { - return my_solist.begin(); - } - - iterator end() { - return my_solist.end(); - } - - const_iterator end() const { - return my_solist.end(); - } - - const_iterator cbegin() const { - return my_solist.cbegin(); - } - - const_iterator cend() const { - return my_solist.cend(); - } - - // Parallel traversal support - class const_range_type : tbb::internal::no_assign { - const concurrent_unordered_base &my_table; - raw_const_iterator my_begin_node; - raw_const_iterator my_end_node; - mutable raw_const_iterator my_midpoint_node; - public: - //! Type for size of a range - typedef typename concurrent_unordered_base::size_type size_type; - typedef typename concurrent_unordered_base::value_type value_type; - typedef typename concurrent_unordered_base::reference reference; - typedef typename concurrent_unordered_base::difference_type difference_type; - typedef typename concurrent_unordered_base::const_iterator iterator; - - //! True if range is empty. - bool empty() const {return my_begin_node == my_end_node;} - - //! True if range can be partitioned into two subranges. - bool is_divisible() const { - return my_midpoint_node != my_end_node; - } - //! Split range. - const_range_type( const_range_type &r, split ) : - my_table(r.my_table), my_end_node(r.my_end_node) - { - r.my_end_node = my_begin_node = r.my_midpoint_node; - __TBB_ASSERT( !empty(), "Splitting despite the range is not divisible" ); - __TBB_ASSERT( !r.empty(), "Splitting despite the range is not divisible" ); - set_midpoint(); - r.set_midpoint(); - } - //! Init range with container and grainsize specified - const_range_type( const concurrent_unordered_base &a_table ) : - my_table(a_table), my_begin_node(a_table.my_solist.begin()), - my_end_node(a_table.my_solist.end()) - { - set_midpoint(); - } - iterator begin() const { return my_table.my_solist.get_iterator(my_begin_node); } - iterator end() const { return my_table.my_solist.get_iterator(my_end_node); } - //! The grain size for this range. - size_type grainsize() const { return 1; } - - //! Set my_midpoint_node to point approximately half way between my_begin_node and my_end_node. - void set_midpoint() const { - if( my_begin_node == my_end_node ) // not divisible - my_midpoint_node = my_end_node; - else { - sokey_t begin_key = solist_t::get_safe_order_key(my_begin_node); - sokey_t end_key = solist_t::get_safe_order_key(my_end_node); - size_t mid_bucket = __TBB_ReverseBits( begin_key + (end_key-begin_key)/2 ) % my_table.my_number_of_buckets; - while ( !my_table.is_initialized(mid_bucket) ) mid_bucket = my_table.get_parent(mid_bucket); - if(__TBB_ReverseBits(mid_bucket) > begin_key) { - // found a dummy_node between begin and end - my_midpoint_node = my_table.my_solist.first_real_iterator(my_table.get_bucket( mid_bucket )); - } - else { - // didn't find a dummy node between begin and end. - my_midpoint_node = my_end_node; - } -#if TBB_USE_ASSERT - { - sokey_t mid_key = solist_t::get_safe_order_key(my_midpoint_node); - __TBB_ASSERT( begin_key < mid_key, "my_begin_node is after my_midpoint_node" ); - __TBB_ASSERT( mid_key <= end_key, "my_midpoint_node is after my_end_node" ); - } -#endif // TBB_USE_ASSERT - } - } - }; - - class range_type : public const_range_type { - public: - typedef typename concurrent_unordered_base::iterator iterator; - //! Split range. - range_type( range_type &r, split ) : const_range_type( r, split() ) {} - //! Init range with container and grainsize specified - range_type( const concurrent_unordered_base &a_table ) : const_range_type(a_table) {} - - iterator begin() const { return solist_t::get_iterator( const_range_type::begin() ); } - iterator end() const { return solist_t::get_iterator( const_range_type::end() ); } - }; - - range_type range() { - return range_type( *this ); - } - - const_range_type range() const { - return const_range_type( *this ); - } - - // Modifiers - std::pair insert(const value_type& value) { - return internal_insert(value); - } - - iterator insert(const_iterator, const value_type& value) { - // Ignore hint - return insert(value).first; - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - std::pair insert(value_type&& value) { - return internal_insert(std::move(value)); - } - - iterator insert(const_iterator, value_type&& value) { - // Ignore hint - return insert(std::move(value)).first; - } - -#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT - template - std::pair emplace(Args&&... args) { - nodeptr_t pnode = my_solist.create_node_v(tbb::internal::forward(args)...); - const sokey_t hashed_element_key = (sokey_t) my_hash_compare(get_key(pnode->my_element)); - const sokey_t order_key = split_order_key_regular(hashed_element_key); - pnode->init(order_key); - - return internal_insert(pnode->my_element, pnode); - } - - template - iterator emplace_hint(const_iterator, Args&&... args) { - // Ignore hint - return emplace(tbb::internal::forward(args)...).first; - } - -#endif // __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT -#endif // __TBB_CPP11_RVALUE_REF_PRESENT - - template - void insert(Iterator first, Iterator last) { - for (Iterator it = first; it != last; ++it) - insert(*it); - } - -#if __TBB_INITIALIZER_LISTS_PRESENT - //! Insert initializer list - void insert(std::initializer_list il) { - insert(il.begin(), il.end()); - } -#endif - - iterator unsafe_erase(const_iterator where) { - return internal_erase(where); - } - - iterator unsafe_erase(const_iterator first, const_iterator last) { - while (first != last) - unsafe_erase(first++); - return my_solist.get_iterator(first); - } - - size_type unsafe_erase(const key_type& key) { - pairii_t where = equal_range(key); - size_type item_count = internal_distance(where.first, where.second); - unsafe_erase(where.first, where.second); - return item_count; - } - - void swap(concurrent_unordered_base& right) { - if (this != &right) { - std::swap(my_hash_compare, right.my_hash_compare); // TODO: check what ADL meant here - my_solist.swap(right.my_solist); - internal_swap_buckets(right); - std::swap(my_number_of_buckets, right.my_number_of_buckets); - std::swap(my_maximum_bucket_size, right.my_maximum_bucket_size); - } - } - - // Observers - hasher hash_function() const { - return my_hash_compare.my_hash_object; - } - - key_equal key_eq() const { - return my_hash_compare.my_key_compare_object; - } - - void clear() { - // Clear list - my_solist.clear(); - - // Clear buckets - internal_clear(); - - // Initialize bucket 0 - __TBB_ASSERT(my_buckets[0] == NULL, NULL); - raw_iterator dummy_node = my_solist.raw_begin(); - set_bucket(0, dummy_node); - } - - // Lookup - iterator find(const key_type& key) { - return internal_find(key); - } - - const_iterator find(const key_type& key) const { - return const_cast(this)->internal_find(key); - } - - size_type count(const key_type& key) const { - if(allow_multimapping) { - paircc_t answer = equal_range(key); - size_type item_count = internal_distance(answer.first, answer.second); - return item_count; - } else { - return const_cast(this)->internal_find(key) == end()?0:1; - } - } - - std::pair equal_range(const key_type& key) { - return internal_equal_range(key); - } - - std::pair equal_range(const key_type& key) const { - return const_cast(this)->internal_equal_range(key); - } - - // Bucket interface - for debugging - size_type unsafe_bucket_count() const { - return my_number_of_buckets; - } - - size_type unsafe_max_bucket_count() const { - return segment_size(pointers_per_table-1); - } - - size_type unsafe_bucket_size(size_type bucket) { - size_type item_count = 0; - if (is_initialized(bucket)) { - raw_iterator it = get_bucket(bucket); - ++it; - for (; it != my_solist.raw_end() && !it.get_node_ptr()->is_dummy(); ++it) - ++item_count; - } - return item_count; - } - - size_type unsafe_bucket(const key_type& key) const { - sokey_t order_key = (sokey_t) my_hash_compare(key); - size_type bucket = order_key % my_number_of_buckets; - return bucket; - } - - // If the bucket is initialized, return a first non-dummy element in it - local_iterator unsafe_begin(size_type bucket) { - if (!is_initialized(bucket)) - return end(); - - raw_iterator it = get_bucket(bucket); - return my_solist.first_real_iterator(it); - } - - // If the bucket is initialized, return a first non-dummy element in it - const_local_iterator unsafe_begin(size_type bucket) const - { - if (!is_initialized(bucket)) - return end(); - - raw_const_iterator it = get_bucket(bucket); - return my_solist.first_real_iterator(it); - } - - // @REVIEW: Takes O(n) - // Returns the iterator after the last non-dummy element in the bucket - local_iterator unsafe_end(size_type bucket) - { - if (!is_initialized(bucket)) - return end(); - - raw_iterator it = get_bucket(bucket); - - // Find the end of the bucket, denoted by the dummy element - do ++it; - while(it != my_solist.raw_end() && !it.get_node_ptr()->is_dummy()); - - // Return the first real element past the end of the bucket - return my_solist.first_real_iterator(it); - } - - // @REVIEW: Takes O(n) - // Returns the iterator after the last non-dummy element in the bucket - const_local_iterator unsafe_end(size_type bucket) const - { - if (!is_initialized(bucket)) - return end(); - - raw_const_iterator it = get_bucket(bucket); - - // Find the end of the bucket, denoted by the dummy element - do ++it; - while(it != my_solist.raw_end() && !it.get_node_ptr()->is_dummy()); - - // Return the first real element past the end of the bucket - return my_solist.first_real_iterator(it); - } - - const_local_iterator unsafe_cbegin(size_type bucket) const { - return ((const self_type *) this)->unsafe_begin(bucket); - } - - const_local_iterator unsafe_cend(size_type bucket) const { - return ((const self_type *) this)->unsafe_end(bucket); - } - - // Hash policy - float load_factor() const { - return (float) size() / (float) unsafe_bucket_count(); - } - - float max_load_factor() const { - return my_maximum_bucket_size; - } - - void max_load_factor(float newmax) { - if (newmax != newmax || newmax < 0) - tbb::internal::throw_exception(tbb::internal::eid_invalid_load_factor); - my_maximum_bucket_size = newmax; - } - - // This function is a noop, because the underlying split-ordered list - // is already sorted, so an increase in the bucket number will be - // reflected next time this bucket is touched. - void rehash(size_type buckets) { - size_type current_buckets = my_number_of_buckets; - if (current_buckets >= buckets) - return; - my_number_of_buckets = size_type(1)<<__TBB_Log2((uintptr_t)buckets*2-1); // round up to power of 2 - } - -private: - - // Initialize the hash and keep the first bucket open - void internal_init() { - // Initialize the array of segment pointers - memset(my_buckets, 0, sizeof(my_buckets)); - - // Initialize bucket 0 - raw_iterator dummy_node = my_solist.raw_begin(); - set_bucket(0, dummy_node); - } - - void internal_clear() { - for (size_type index = 0; index < pointers_per_table; ++index) { - if (my_buckets[index] != NULL) { - size_type sz = segment_size(index); - for (size_type index2 = 0; index2 < sz; ++index2) - my_allocator.destroy(&my_buckets[index][index2]); - my_allocator.deallocate(my_buckets[index], sz); - my_buckets[index] = 0; - } - } - } - - void internal_copy(const self_type& right) { - clear(); - - my_maximum_bucket_size = right.my_maximum_bucket_size; - my_number_of_buckets = right.my_number_of_buckets; - - __TBB_TRY { - insert(right.begin(), right.end()); - my_hash_compare = right.my_hash_compare; - } __TBB_CATCH(...) { - my_solist.clear(); - __TBB_RETHROW(); - } - } - - void internal_swap_buckets(concurrent_unordered_base& right) - { - // Swap all node segments - for (size_type index = 0; index < pointers_per_table; ++index) - { - raw_iterator * iterator_pointer = my_buckets[index]; - my_buckets[index] = right.my_buckets[index]; - right.my_buckets[index] = iterator_pointer; - } - } - - //TODO: why not use std::distance? - // Hash APIs - static size_type internal_distance(const_iterator first, const_iterator last) - { - size_type num = 0; - - for (const_iterator it = first; it != last; ++it) - ++num; - - return num; - } - - // Insert an element in the hash given its value - template - std::pair internal_insert(__TBB_FORWARDING_REF(ValueType) value, nodeptr_t pnode = NULL) - { - const key_type *pkey = &get_key(value); - sokey_t hash_key = (sokey_t) my_hash_compare(*pkey); - size_type new_count = 0; - sokey_t order_key = split_order_key_regular(hash_key); - raw_iterator previous = prepare_bucket(hash_key); - raw_iterator last = my_solist.raw_end(); - __TBB_ASSERT(previous != last, "Invalid head node"); - - // First node is a dummy node - for (raw_iterator where = previous;;) - { - ++where; - if (where == last || solist_t::get_order_key(where) > order_key || - // if multimapped, stop at the first item equal to us. - (allow_multimapping && solist_t::get_order_key(where) == order_key && - !my_hash_compare(get_key(*where), *pkey))) // TODO: fix negation - { - if (!pnode) { - pnode = my_solist.create_node(order_key, tbb::internal::forward(value), AllowCreate()); - // If the value was moved, the known reference to key might be invalid - pkey = &get_key(pnode->my_element); - } - - // Try to insert 'pnode' between 'previous' and 'where' - std::pair result = my_solist.try_insert(previous, where, pnode, &new_count); - - if (result.second) - { - // Insertion succeeded, adjust the table size, if needed - adjust_table_size(new_count, my_number_of_buckets); - return result; - } - else - { - // Insertion failed: either the same node was inserted by another thread, or - // another element was inserted at exactly the same place as this node. - // Proceed with the search from the previous location where order key was - // known to be larger (note: this is legal only because there is no safe - // concurrent erase operation supported). - where = previous; - continue; - } - } - else if (!allow_multimapping && solist_t::get_order_key(where) == order_key && - !my_hash_compare(get_key(*where), *pkey)) // TODO: fix negation - { // Element already in the list, return it - if (pnode) - my_solist.destroy_node(pnode); - return std::pair(my_solist.get_iterator(where), false); - } - // Move the iterator forward - previous = where; - } - } - - // Find the element in the split-ordered list - iterator internal_find(const key_type& key) - { - sokey_t hash_key = (sokey_t) my_hash_compare(key); - sokey_t order_key = split_order_key_regular(hash_key); - raw_iterator last = my_solist.raw_end(); - - for (raw_iterator it = prepare_bucket(hash_key); it != last; ++it) - { - if (solist_t::get_order_key(it) > order_key) - { - // If the order key is smaller than the current order key, the element - // is not in the hash. - return end(); - } - else if (solist_t::get_order_key(it) == order_key) - { - // The fact that order keys match does not mean that the element is found. - // Key function comparison has to be performed to check whether this is the - // right element. If not, keep searching while order key is the same. - if (!my_hash_compare(get_key(*it), key)) // TODO: fix negation - return my_solist.get_iterator(it); - } - } - - return end(); - } - - // Erase an element from the list. This is not a concurrency safe function. - iterator internal_erase(const_iterator it) - { - sokey_t hash_key = (sokey_t) my_hash_compare(get_key(*it)); - raw_iterator previous = prepare_bucket(hash_key); - raw_iterator last = my_solist.raw_end(); - __TBB_ASSERT(previous != last, "Invalid head node"); - - // First node is a dummy node - for (raw_iterator where = previous; ; previous = where) { - ++where; - if (where == last) - return end(); - else if (my_solist.get_iterator(where) == it) - return my_solist.erase_node(previous, it); - } - } - - // Return the [begin, end) pair of iterators with the same key values. - // This operation makes sense only if mapping is many-to-one. - pairii_t internal_equal_range(const key_type& key) - { - sokey_t hash_key = (sokey_t) my_hash_compare(key); - sokey_t order_key = split_order_key_regular(hash_key); - raw_iterator end_it = my_solist.raw_end(); - - for (raw_iterator it = prepare_bucket(hash_key); it != end_it; ++it) - { - if (solist_t::get_order_key(it) > order_key) - { - // There is no element with the given key - return pairii_t(end(), end()); - } - else if (solist_t::get_order_key(it) == order_key && - !my_hash_compare(get_key(*it), key)) // TODO: fix negation; also below - { - iterator first = my_solist.get_iterator(it); - iterator last = first; - do ++last; while( allow_multimapping && last != end() && !my_hash_compare(get_key(*last), key) ); - return pairii_t(first, last); - } - } - - return pairii_t(end(), end()); - } - - // Bucket APIs - void init_bucket(size_type bucket) - { - // Bucket 0 has no parent. - __TBB_ASSERT( bucket != 0, "The first bucket must always be initialized"); - - size_type parent_bucket = get_parent(bucket); - - // All parent_bucket buckets have to be initialized before this bucket is - if (!is_initialized(parent_bucket)) - init_bucket(parent_bucket); - - raw_iterator parent = get_bucket(parent_bucket); - - // Create a dummy first node in this bucket - raw_iterator dummy_node = my_solist.insert_dummy(parent, split_order_key_dummy(bucket)); - set_bucket(bucket, dummy_node); - } - - void adjust_table_size(size_type total_elements, size_type current_size) - { - // Grow the table by a factor of 2 if possible and needed - if ( ((float) total_elements / (float) current_size) > my_maximum_bucket_size ) - { - // Double the size of the hash only if size has not changed in between loads - my_number_of_buckets.compare_and_swap(2u*current_size, current_size); - //Simple "my_number_of_buckets.compare_and_swap( current_size<<1, current_size );" does not work for VC8 - //due to overzealous compiler warnings in /Wp64 mode - } - } - - size_type get_parent(size_type bucket) const - { - // Unsets bucket's most significant turned-on bit - size_type msb = __TBB_Log2((uintptr_t)bucket); - return bucket & ~(size_type(1) << msb); - } - - - // Dynamic sized array (segments) - //! @return segment index of given index in the array - static size_type segment_index_of( size_type index ) { - return size_type( __TBB_Log2( uintptr_t(index|1) ) ); - } - - //! @return the first array index of given segment - static size_type segment_base( size_type k ) { - return (size_type(1)< my_number_of_buckets; // Current table size - solist_t my_solist; // List where all the elements are kept - typename allocator_type::template rebind::other my_allocator; // Allocator object for segments - float my_maximum_bucket_size; // Maximum size of the bucket - atomic my_buckets[pointers_per_table]; // The segment table -}; -#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) -#pragma warning(pop) // warning 4127 is back -#endif - -} // namespace internal -//! @endcond -} // namespace interface5 -} // namespace tbb -#endif // __TBB__concurrent_unordered_impl_H diff --git a/src/tbb/include/tbb/internal/_flow_graph_async_msg_impl.h b/src/tbb/include/tbb/internal/_flow_graph_async_msg_impl.h deleted file mode 100644 index 794960db3..000000000 --- a/src/tbb/include/tbb/internal/_flow_graph_async_msg_impl.h +++ /dev/null @@ -1,157 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB__flow_graph_async_msg_impl_H -#define __TBB__flow_graph_async_msg_impl_H - -#ifndef __TBB_flow_graph_H -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -namespace internal { - -template -class async_storage { -public: - typedef receiver async_storage_client; - - async_storage() : my_graph(nullptr) { - my_data_ready.store(false); - } - - ~async_storage() { - // Release reference to the graph if async_storage - // was destructed before set() call - if (my_graph) { - my_graph->release_wait(); - my_graph = nullptr; - } - } - - template - async_storage(C&& data) : my_graph(nullptr), my_data( std::forward(data) ) { - using namespace tbb::internal; - __TBB_STATIC_ASSERT( (is_same_type::type, typename strip::type>::value), "incoming type must be T" ); - - my_data_ready.store(true); - } - - template - bool set(C&& data) { - using namespace tbb::internal; - __TBB_STATIC_ASSERT( (is_same_type::type, typename strip::type>::value), "incoming type must be T" ); - - { - tbb::spin_mutex::scoped_lock locker(my_mutex); - - if (my_data_ready.load()) { - __TBB_ASSERT(false, "double set() call"); - return false; - } - - my_data = std::forward(data); - my_data_ready.store(true); - } - - // Thread sync is on my_data_ready flag - for (typename subscriber_list_type::iterator it = my_clients.begin(); it != my_clients.end(); ++it) { - (*it)->try_put(my_data); - } - - // Data was sent, release reference to the graph - if (my_graph) { - my_graph->release_wait(); - my_graph = nullptr; - } - - return true; - } - - task* subscribe(async_storage_client& client, graph& g) { - if (! my_data_ready.load()) - { - tbb::spin_mutex::scoped_lock locker(my_mutex); - - if (! my_data_ready.load()) { -#if TBB_USE_ASSERT - for (typename subscriber_list_type::iterator it = my_clients.begin(); it != my_clients.end(); ++it) { - __TBB_ASSERT(*it != &client, "unexpected double subscription"); - } -#endif // TBB_USE_ASSERT - - // Increase graph lifetime - my_graph = &g; - my_graph->reserve_wait(); - - // Subscribe - my_clients.push_back(&client); - return SUCCESSFULLY_ENQUEUED; - } - } - - __TBB_ASSERT(my_data_ready.load(), "data is NOT ready"); - return client.try_put_task(my_data); - } - -private: - graph* my_graph; - tbb::spin_mutex my_mutex; - tbb::atomic my_data_ready; - T my_data; - typedef std::vector subscriber_list_type; - subscriber_list_type my_clients; -}; - -} // namespace internal - -template -class async_msg { - template< typename > friend class receiver; - template< typename, typename > friend struct internal::async_helpers; -public: - typedef T async_msg_data_type; - - async_msg() : my_storage(std::make_shared< internal::async_storage >()) {} - - async_msg(const T& t) : my_storage(std::make_shared< internal::async_storage >(t)) {} - - async_msg(T&& t) : my_storage(std::make_shared< internal::async_storage >( std::move(t) )) {} - - virtual ~async_msg() {} - - void set(const T& t) { - my_storage->set(t); - } - - void set(T&& t) { - my_storage->set( std::move(t) ); - } - -protected: - // Can be overridden in derived class to inform that - // async calculation chain is over - virtual void finalize() const {} - -private: - typedef std::shared_ptr< internal::async_storage > async_storage_ptr; - async_storage_ptr my_storage; -}; - -#endif // __TBB__flow_graph_async_msg_impl_H diff --git a/src/tbb/include/tbb/internal/_flow_graph_body_impl.h b/src/tbb/include/tbb/internal/_flow_graph_body_impl.h deleted file mode 100644 index 54f37411d..000000000 --- a/src/tbb/include/tbb/internal/_flow_graph_body_impl.h +++ /dev/null @@ -1,320 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB__flow_graph_body_impl_H -#define __TBB__flow_graph_body_impl_H - -#ifndef __TBB_flow_graph_H -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -// included in namespace tbb::flow::interfaceX (in flow_graph.h) - -namespace internal { - -typedef tbb::internal::uint64_t tag_value; - -using tbb::internal::strip; - -namespace graph_policy_namespace { - - struct rejecting { }; - struct reserving { }; - struct queueing { }; - - // K == type of field used for key-matching. Each tag-matching port will be provided - // functor that, given an object accepted by the port, will return the - /// field of type K being used for matching. - template::type > > - struct key_matching { - typedef K key_type; - typedef typename strip::type base_key_type; - typedef KHash hash_compare_type; - }; - - // old tag_matching join's new specifier - typedef key_matching tag_matching; - -} // namespace graph_policy_namespace - -// -------------- function_body containers ---------------------- - -//! A functor that takes no input and generates a value of type Output -template< typename Output > -class source_body : tbb::internal::no_assign { -public: - virtual ~source_body() {} - virtual bool operator()(Output &output) = 0; - virtual source_body* clone() = 0; -}; - -//! The leaf for source_body -template< typename Output, typename Body> -class source_body_leaf : public source_body { -public: - source_body_leaf( const Body &_body ) : body(_body) { } - bool operator()(Output &output) __TBB_override { return body( output ); } - source_body_leaf* clone() __TBB_override { - return new source_body_leaf< Output, Body >(body); - } - Body get_body() { return body; } -private: - Body body; -}; - -//! A functor that takes an Input and generates an Output -template< typename Input, typename Output > -class function_body : tbb::internal::no_assign { -public: - virtual ~function_body() {} - virtual Output operator()(const Input &input) = 0; - virtual function_body* clone() = 0; -}; - -//! the leaf for function_body -template -class function_body_leaf : public function_body< Input, Output > { -public: - function_body_leaf( const B &_body ) : body(_body) { } - Output operator()(const Input &i) __TBB_override { return body(i); } - B get_body() { return body; } - function_body_leaf* clone() __TBB_override { - return new function_body_leaf< Input, Output, B >(body); - } -private: - B body; -}; - -//! the leaf for function_body specialized for Input and output of continue_msg -template -class function_body_leaf< continue_msg, continue_msg, B> : public function_body< continue_msg, continue_msg > { -public: - function_body_leaf( const B &_body ) : body(_body) { } - continue_msg operator()( const continue_msg &i ) __TBB_override { - body(i); - return i; - } - B get_body() { return body; } - function_body_leaf* clone() __TBB_override { - return new function_body_leaf< continue_msg, continue_msg, B >(body); - } -private: - B body; -}; - -//! the leaf for function_body specialized for Output of continue_msg -template -class function_body_leaf< Input, continue_msg, B> : public function_body< Input, continue_msg > { -public: - function_body_leaf( const B &_body ) : body(_body) { } - continue_msg operator()(const Input &i) __TBB_override { - body(i); - return continue_msg(); - } - B get_body() { return body; } - function_body_leaf* clone() __TBB_override { - return new function_body_leaf< Input, continue_msg, B >(body); - } -private: - B body; -}; - -//! the leaf for function_body specialized for Input of continue_msg -template -class function_body_leaf< continue_msg, Output, B > : public function_body< continue_msg, Output > { -public: - function_body_leaf( const B &_body ) : body(_body) { } - Output operator()(const continue_msg &i) __TBB_override { - return body(i); - } - B get_body() { return body; } - function_body_leaf* clone() __TBB_override { - return new function_body_leaf< continue_msg, Output, B >(body); - } -private: - B body; -}; - -//! function_body that takes an Input and a set of output ports -template -class multifunction_body : tbb::internal::no_assign { -public: - virtual ~multifunction_body () {} - virtual void operator()(const Input &/* input*/, OutputSet &/*oset*/) = 0; - virtual multifunction_body* clone() = 0; - virtual void* get_body_ptr() = 0; -}; - -//! leaf for multifunction. OutputSet can be a std::tuple or a vector. -template -class multifunction_body_leaf : public multifunction_body { -public: - multifunction_body_leaf(const B &_body) : body(_body) { } - void operator()(const Input &input, OutputSet &oset) __TBB_override { - body(input, oset); // body may explicitly put() to one or more of oset. - } - void* get_body_ptr() __TBB_override { return &body; } - multifunction_body_leaf* clone() __TBB_override { - return new multifunction_body_leaf(body); - } - -private: - B body; -}; - -// ------ function bodies for hash_buffers and key-matching joins. - -template -class type_to_key_function_body : tbb::internal::no_assign { - public: - virtual ~type_to_key_function_body() {} - virtual Output operator()(const Input &input) = 0; // returns an Output - virtual type_to_key_function_body* clone() = 0; -}; - -// specialization for ref output -template -class type_to_key_function_body : tbb::internal::no_assign { - public: - virtual ~type_to_key_function_body() {} - virtual const Output & operator()(const Input &input) = 0; // returns a const Output& - virtual type_to_key_function_body* clone() = 0; -}; - -template -class type_to_key_function_body_leaf : public type_to_key_function_body { -public: - type_to_key_function_body_leaf( const B &_body ) : body(_body) { } - Output operator()(const Input &i) __TBB_override { return body(i); } - B get_body() { return body; } - type_to_key_function_body_leaf* clone() __TBB_override { - return new type_to_key_function_body_leaf< Input, Output, B>(body); - } -private: - B body; -}; - -template -class type_to_key_function_body_leaf : public type_to_key_function_body< Input, Output&> { -public: - type_to_key_function_body_leaf( const B &_body ) : body(_body) { } - const Output& operator()(const Input &i) __TBB_override { - return body(i); - } - B get_body() { return body; } - type_to_key_function_body_leaf* clone() __TBB_override { - return new type_to_key_function_body_leaf< Input, Output&, B>(body); - } -private: - B body; -}; - -// --------------------------- end of function_body containers ------------------------ - -// --------------------------- node task bodies --------------------------------------- - -//! A task that calls a node's forward_task function -template< typename NodeType > -class forward_task_bypass : public task { - - NodeType &my_node; - -public: - - forward_task_bypass( NodeType &n ) : my_node(n) {} - - task *execute() __TBB_override { - task * new_task = my_node.forward_task(); - if (new_task == SUCCESSFULLY_ENQUEUED) new_task = NULL; - return new_task; - } -}; - -//! A task that calls a node's apply_body_bypass function, passing in an input of type Input -// return the task* unless it is SUCCESSFULLY_ENQUEUED, in which case return NULL -template< typename NodeType, typename Input > -class apply_body_task_bypass : public task { - - NodeType &my_node; - Input my_input; - -public: - - apply_body_task_bypass( NodeType &n, const Input &i ) : my_node(n), my_input(i) {} - - task *execute() __TBB_override { - task * next_task = my_node.apply_body_bypass( my_input ); - if(next_task == SUCCESSFULLY_ENQUEUED) next_task = NULL; - return next_task; - } -}; - -//! A task that calls a node's apply_body_bypass function with no input -template< typename NodeType > -class source_task_bypass : public task { - - NodeType &my_node; - -public: - - source_task_bypass( NodeType &n ) : my_node(n) {} - - task *execute() __TBB_override { - task *new_task = my_node.apply_body_bypass( ); - if(new_task == SUCCESSFULLY_ENQUEUED) return NULL; - return new_task; - } -}; - -// ------------------------ end of node task bodies ----------------------------------- - -//! An empty functor that takes an Input and returns a default constructed Output -template< typename Input, typename Output > -struct empty_body { - Output operator()( const Input & ) const { return Output(); } -}; - -template -class decrementer : public continue_receiver, tbb::internal::no_copy { - - T *my_node; - - task *execute() __TBB_override { - return my_node->decrement_counter(); - } - -protected: - - graph& graph_reference() __TBB_override { - return my_node->my_graph; - } - -public: - - typedef continue_msg input_type; - typedef continue_msg output_type; - decrementer( int number_of_predecessors = 0 ) : continue_receiver( number_of_predecessors ) { } - void set_owner( T *node ) { my_node = node; } -}; - -} // namespace internal - -#endif // __TBB__flow_graph_body_impl_H - diff --git a/src/tbb/include/tbb/internal/_flow_graph_cache_impl.h b/src/tbb/include/tbb/internal/_flow_graph_cache_impl.h deleted file mode 100644 index 7fac84e8a..000000000 --- a/src/tbb/include/tbb/internal/_flow_graph_cache_impl.h +++ /dev/null @@ -1,562 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB__flow_graph_cache_impl_H -#define __TBB__flow_graph_cache_impl_H - -#ifndef __TBB_flow_graph_H -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -// included in namespace tbb::flow::interfaceX (in flow_graph.h) - -namespace internal { - -//! A node_cache maintains a std::queue of elements of type T. Each operation is protected by a lock. -template< typename T, typename M=spin_mutex > -class node_cache { - public: - - typedef size_t size_type; - - bool empty() { - typename mutex_type::scoped_lock lock( my_mutex ); - return internal_empty(); - } - - void add( T &n ) { - typename mutex_type::scoped_lock lock( my_mutex ); - internal_push(n); - } - - void remove( T &n ) { - typename mutex_type::scoped_lock lock( my_mutex ); - for ( size_t i = internal_size(); i != 0; --i ) { - T &s = internal_pop(); - if ( &s == &n ) return; // only remove one predecessor per request - internal_push(s); - } - } - - void clear() { - while( !my_q.empty()) (void)my_q.pop(); -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - my_built_predecessors.clear(); -#endif - } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - typedef edge_container built_predecessors_type; - built_predecessors_type &built_predecessors() { return my_built_predecessors; } - - typedef typename edge_container::edge_list_type predecessor_list_type; - void internal_add_built_predecessor( T &n ) { - typename mutex_type::scoped_lock lock( my_mutex ); - my_built_predecessors.add_edge(n); - } - - void internal_delete_built_predecessor( T &n ) { - typename mutex_type::scoped_lock lock( my_mutex ); - my_built_predecessors.delete_edge(n); - } - - void copy_predecessors( predecessor_list_type &v) { - typename mutex_type::scoped_lock lock( my_mutex ); - my_built_predecessors.copy_edges(v); - } - - size_t predecessor_count() { - typename mutex_type::scoped_lock lock(my_mutex); - return (size_t)(my_built_predecessors.edge_count()); - } -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ - -protected: - - typedef M mutex_type; - mutex_type my_mutex; - std::queue< T * > my_q; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - built_predecessors_type my_built_predecessors; -#endif - - // Assumes lock is held - inline bool internal_empty( ) { - return my_q.empty(); - } - - // Assumes lock is held - inline size_type internal_size( ) { - return my_q.size(); - } - - // Assumes lock is held - inline void internal_push( T &n ) { - my_q.push(&n); - } - - // Assumes lock is held - inline T &internal_pop() { - T *v = my_q.front(); - my_q.pop(); - return *v; - } - -}; - -//! A cache of predecessors that only supports try_get -template< typename T, typename M=spin_mutex > -#if __TBB_PREVIEW_ASYNC_MSG -// TODO: make predecessor_cache type T-independent when async_msg becomes regular feature -class predecessor_cache : public node_cache< untyped_sender, M > { -#else -class predecessor_cache : public node_cache< sender, M > { -#endif // __TBB_PREVIEW_ASYNC_MSG -public: - typedef M mutex_type; - typedef T output_type; -#if __TBB_PREVIEW_ASYNC_MSG - typedef untyped_sender predecessor_type; - typedef untyped_receiver successor_type; -#else - typedef sender predecessor_type; - typedef receiver successor_type; -#endif // __TBB_PREVIEW_ASYNC_MSG - - predecessor_cache( ) : my_owner( NULL ) { } - - void set_owner( successor_type *owner ) { my_owner = owner; } - - bool get_item( output_type &v ) { - - bool msg = false; - - do { - predecessor_type *src; - { - typename mutex_type::scoped_lock lock(this->my_mutex); - if ( this->internal_empty() ) { - break; - } - src = &this->internal_pop(); - } - - // Try to get from this sender - msg = src->try_get( v ); - - if (msg == false) { - // Relinquish ownership of the edge - if (my_owner) - src->register_successor( *my_owner ); - } else { - // Retain ownership of the edge - this->add(*src); - } - } while ( msg == false ); - return msg; - } - - // If we are removing arcs (rf_clear_edges), call clear() rather than reset(). - void reset() { - if (my_owner) { - for(;;) { - predecessor_type *src; - { - if (this->internal_empty()) break; - src = &this->internal_pop(); - } - src->register_successor( *my_owner ); - } - } - } - -protected: - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - using node_cache< predecessor_type, M >::my_built_predecessors; -#endif - successor_type *my_owner; -}; - -//! An cache of predecessors that supports requests and reservations -// TODO: make reservable_predecessor_cache type T-independent when async_msg becomes regular feature -template< typename T, typename M=spin_mutex > -class reservable_predecessor_cache : public predecessor_cache< T, M > { -public: - typedef M mutex_type; - typedef T output_type; -#if __TBB_PREVIEW_ASYNC_MSG - typedef untyped_sender predecessor_type; - typedef untyped_receiver successor_type; -#else - typedef sender predecessor_type; - typedef receiver successor_type; -#endif // __TBB_PREVIEW_ASYNC_MSG - - reservable_predecessor_cache( ) : reserved_src(NULL) { } - - bool - try_reserve( output_type &v ) { - bool msg = false; - - do { - { - typename mutex_type::scoped_lock lock(this->my_mutex); - if ( reserved_src || this->internal_empty() ) - return false; - - reserved_src = &this->internal_pop(); - } - - // Try to get from this sender - msg = reserved_src->try_reserve( v ); - - if (msg == false) { - typename mutex_type::scoped_lock lock(this->my_mutex); - // Relinquish ownership of the edge - reserved_src->register_successor( *this->my_owner ); - reserved_src = NULL; - } else { - // Retain ownership of the edge - this->add( *reserved_src ); - } - } while ( msg == false ); - - return msg; - } - - bool - try_release( ) { - reserved_src->try_release( ); - reserved_src = NULL; - return true; - } - - bool - try_consume( ) { - reserved_src->try_consume( ); - reserved_src = NULL; - return true; - } - - void reset( ) { - reserved_src = NULL; - predecessor_cache::reset( ); - } - - void clear() { - reserved_src = NULL; - predecessor_cache::clear(); - } - -private: - predecessor_type *reserved_src; -}; - - -//! An abstract cache of successors -// TODO: make successor_cache type T-independent when async_msg becomes regular feature -template -class successor_cache : tbb::internal::no_copy { -protected: - - typedef M mutex_type; - mutex_type my_mutex; - -#if __TBB_PREVIEW_ASYNC_MSG - typedef untyped_receiver successor_type; - typedef untyped_receiver *pointer_type; - typedef untyped_sender owner_type; -#else - typedef receiver successor_type; - typedef receiver *pointer_type; - typedef sender owner_type; -#endif // __TBB_PREVIEW_ASYNC_MSG - typedef std::list< pointer_type > successors_type; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - edge_container my_built_successors; -#endif - successors_type my_successors; - - owner_type *my_owner; - -public: -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - typedef typename edge_container::edge_list_type successor_list_type; - - edge_container &built_successors() { return my_built_successors; } - - void internal_add_built_successor( successor_type &r) { - typename mutex_type::scoped_lock l(my_mutex, true); - my_built_successors.add_edge( r ); - } - - void internal_delete_built_successor( successor_type &r) { - typename mutex_type::scoped_lock l(my_mutex, true); - my_built_successors.delete_edge(r); - } - - void copy_successors( successor_list_type &v) { - typename mutex_type::scoped_lock l(my_mutex, false); - my_built_successors.copy_edges(v); - } - - size_t successor_count() { - typename mutex_type::scoped_lock l(my_mutex,false); - return my_built_successors.edge_count(); - } - -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ - - successor_cache( ) : my_owner(NULL) {} - - void set_owner( owner_type *owner ) { my_owner = owner; } - - virtual ~successor_cache() {} - - void register_successor( successor_type &r ) { - typename mutex_type::scoped_lock l(my_mutex, true); - my_successors.push_back( &r ); - } - - void remove_successor( successor_type &r ) { - typename mutex_type::scoped_lock l(my_mutex, true); - for ( typename successors_type::iterator i = my_successors.begin(); - i != my_successors.end(); ++i ) { - if ( *i == & r ) { - my_successors.erase(i); - break; - } - } - } - - bool empty() { - typename mutex_type::scoped_lock l(my_mutex, false); - return my_successors.empty(); - } - - void clear() { - my_successors.clear(); -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - my_built_successors.clear(); -#endif - } - -#if !__TBB_PREVIEW_ASYNC_MSG - virtual task * try_put_task( const T &t ) = 0; -#endif // __TBB_PREVIEW_ASYNC_MSG - }; // successor_cache - -//! An abstract cache of successors, specialized to continue_msg -template<> -class successor_cache< continue_msg > : tbb::internal::no_copy { -protected: - - typedef spin_rw_mutex mutex_type; - mutex_type my_mutex; - -#if __TBB_PREVIEW_ASYNC_MSG - typedef untyped_receiver successor_type; - typedef untyped_receiver *pointer_type; -#else - typedef receiver successor_type; - typedef receiver *pointer_type; -#endif // __TBB_PREVIEW_ASYNC_MSG - typedef std::list< pointer_type > successors_type; - successors_type my_successors; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - edge_container my_built_successors; - typedef edge_container::edge_list_type successor_list_type; -#endif - - sender *my_owner; - -public: - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - - edge_container &built_successors() { return my_built_successors; } - - void internal_add_built_successor( successor_type &r) { - mutex_type::scoped_lock l(my_mutex, true); - my_built_successors.add_edge( r ); - } - - void internal_delete_built_successor( successor_type &r) { - mutex_type::scoped_lock l(my_mutex, true); - my_built_successors.delete_edge(r); - } - - void copy_successors( successor_list_type &v) { - mutex_type::scoped_lock l(my_mutex, false); - my_built_successors.copy_edges(v); - } - - size_t successor_count() { - mutex_type::scoped_lock l(my_mutex,false); - return my_built_successors.edge_count(); - } - -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ - - successor_cache( ) : my_owner(NULL) {} - - void set_owner( sender *owner ) { my_owner = owner; } - - virtual ~successor_cache() {} - - void register_successor( successor_type &r ) { - mutex_type::scoped_lock l(my_mutex, true); - my_successors.push_back( &r ); - if ( my_owner && r.is_continue_receiver() ) { - r.register_predecessor( *my_owner ); - } - } - - void remove_successor( successor_type &r ) { - mutex_type::scoped_lock l(my_mutex, true); - for ( successors_type::iterator i = my_successors.begin(); - i != my_successors.end(); ++i ) { - if ( *i == & r ) { - // TODO: Check if we need to test for continue_receiver before - // removing from r. - if ( my_owner ) - r.remove_predecessor( *my_owner ); - my_successors.erase(i); - break; - } - } - } - - bool empty() { - mutex_type::scoped_lock l(my_mutex, false); - return my_successors.empty(); - } - - void clear() { - my_successors.clear(); -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - my_built_successors.clear(); -#endif - } - -#if !__TBB_PREVIEW_ASYNC_MSG - virtual task * try_put_task( const continue_msg &t ) = 0; -#endif // __TBB_PREVIEW_ASYNC_MSG - -}; // successor_cache< continue_msg > - -//! A cache of successors that are broadcast to -// TODO: make broadcast_cache type T-independent when async_msg becomes regular feature -template -class broadcast_cache : public successor_cache { - typedef M mutex_type; - typedef typename successor_cache::successors_type successors_type; - -public: - - broadcast_cache( ) {} - - // as above, but call try_put_task instead, and return the last task we received (if any) -#if __TBB_PREVIEW_ASYNC_MSG - template - task * try_put_task( const X &t ) { -#else - task * try_put_task( const T &t ) __TBB_override { -#endif // __TBB_PREVIEW_ASYNC_MSG - task * last_task = NULL; - bool upgraded = true; - typename mutex_type::scoped_lock l(this->my_mutex, upgraded); - typename successors_type::iterator i = this->my_successors.begin(); - while ( i != this->my_successors.end() ) { - task *new_task = (*i)->try_put_task(t); - // workaround for icc bug - graph& graph_ref = (*i)->graph_reference(); - last_task = combine_tasks(graph_ref, last_task, new_task); // enqueue if necessary - if(new_task) { - ++i; - } - else { // failed - if ( (*i)->register_predecessor(*this->my_owner) ) { - if (!upgraded) { - l.upgrade_to_writer(); - upgraded = true; - } - i = this->my_successors.erase(i); - } else { - ++i; - } - } - } - return last_task; - } - -}; - -//! A cache of successors that are put in a round-robin fashion -// TODO: make round_robin_cache type T-independent when async_msg becomes regular feature -template -class round_robin_cache : public successor_cache { - typedef size_t size_type; - typedef M mutex_type; - typedef typename successor_cache::successors_type successors_type; - -public: - - round_robin_cache( ) {} - - size_type size() { - typename mutex_type::scoped_lock l(this->my_mutex, false); - return this->my_successors.size(); - } - -#if __TBB_PREVIEW_ASYNC_MSG - template - task * try_put_task( const X &t ) { -#else - task *try_put_task( const T &t ) __TBB_override { -#endif // __TBB_PREVIEW_ASYNC_MSG - bool upgraded = true; - typename mutex_type::scoped_lock l(this->my_mutex, upgraded); - typename successors_type::iterator i = this->my_successors.begin(); - while ( i != this->my_successors.end() ) { - task *new_task = (*i)->try_put_task(t); - if ( new_task ) { - return new_task; - } else { - if ( (*i)->register_predecessor(*this->my_owner) ) { - if (!upgraded) { - l.upgrade_to_writer(); - upgraded = true; - } - i = this->my_successors.erase(i); - } - else { - ++i; - } - } - } - return NULL; - } -}; - -} // namespace internal - -#endif // __TBB__flow_graph_cache_impl_H diff --git a/src/tbb/include/tbb/internal/_flow_graph_impl.h b/src/tbb/include/tbb/internal/_flow_graph_impl.h deleted file mode 100644 index ffdd57be8..000000000 --- a/src/tbb/include/tbb/internal/_flow_graph_impl.h +++ /dev/null @@ -1,423 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_flow_graph_impl_H -#define __TBB_flow_graph_impl_H - -#include "../tbb_stddef.h" -#include "../task.h" -#include "../task_arena.h" -#include "../flow_graph_abstractions.h" - -#include - -#if TBB_DEPRECATED_FLOW_ENQUEUE -#define FLOW_SPAWN(a) tbb::task::enqueue((a)) -#else -#define FLOW_SPAWN(a) tbb::task::spawn((a)) -#endif - -namespace tbb { -namespace flow { - -namespace internal { -static tbb::task * const SUCCESSFULLY_ENQUEUED = (task *)-1; -} - -namespace interface10 { - -using tbb::flow::internal::SUCCESSFULLY_ENQUEUED; - -class graph; -class graph_node; - -template -class graph_iterator { - friend class graph; - friend class graph_node; -public: - typedef size_t size_type; - typedef GraphNodeType value_type; - typedef GraphNodeType* pointer; - typedef GraphNodeType& reference; - typedef const GraphNodeType& const_reference; - typedef std::forward_iterator_tag iterator_category; - - //! Default constructor - graph_iterator() : my_graph(NULL), current_node(NULL) {} - - //! Copy constructor - graph_iterator(const graph_iterator& other) : - my_graph(other.my_graph), current_node(other.current_node) - {} - - //! Assignment - graph_iterator& operator=(const graph_iterator& other) { - if (this != &other) { - my_graph = other.my_graph; - current_node = other.current_node; - } - return *this; - } - - //! Dereference - reference operator*() const; - - //! Dereference - pointer operator->() const; - - //! Equality - bool operator==(const graph_iterator& other) const { - return ((my_graph == other.my_graph) && (current_node == other.current_node)); - } - - //! Inequality - bool operator!=(const graph_iterator& other) const { return !(operator==(other)); } - - //! Pre-increment - graph_iterator& operator++() { - internal_forward(); - return *this; - } - - //! Post-increment - graph_iterator operator++(int) { - graph_iterator result = *this; - operator++(); - return result; - } - -private: - // the graph over which we are iterating - GraphContainerType *my_graph; - // pointer into my_graph's my_nodes list - pointer current_node; - - //! Private initializing constructor for begin() and end() iterators - graph_iterator(GraphContainerType *g, bool begin); - void internal_forward(); -}; // class graph_iterator - -// flags to modify the behavior of the graph reset(). Can be combined. -enum reset_flags { - rf_reset_protocol = 0, - rf_reset_bodies = 1 << 0, // delete the current node body, reset to a copy of the initial node body. - rf_clear_edges = 1 << 1 // delete edges -}; - -namespace internal { - -void activate_graph(graph& g); -void deactivate_graph(graph& g); -bool is_graph_active(graph& g); -void spawn_in_graph_arena(graph& g, tbb::task& arena_task); -void add_task_to_graph_reset_list(graph& g, tbb::task *tp); -template void execute_in_graph_arena(graph& g, F& f); - -} - -//! The graph class -/** This class serves as a handle to the graph */ -class graph : tbb::internal::no_copy, public tbb::flow::graph_proxy { - friend class graph_node; - - template< typename Body > - class run_task : public task { - public: - run_task(Body& body) : my_body(body) {} - tbb::task *execute() __TBB_override { - my_body(); - return NULL; - } - private: - Body my_body; - }; - - template< typename Receiver, typename Body > - class run_and_put_task : public task { - public: - run_and_put_task(Receiver &r, Body& body) : my_receiver(r), my_body(body) {} - tbb::task *execute() __TBB_override { - tbb::task *res = my_receiver.try_put_task(my_body()); - if (res == SUCCESSFULLY_ENQUEUED) res = NULL; - return res; - } - private: - Receiver &my_receiver; - Body my_body; - }; - typedef std::list task_list_type; - - class wait_functor { - tbb::task* graph_root_task; - public: - wait_functor(tbb::task* t) : graph_root_task(t) {} - void operator()() const { graph_root_task->wait_for_all(); } - }; - - //! A functor that spawns a task - class spawn_functor : tbb::internal::no_assign { - tbb::task& spawn_task; - public: - spawn_functor(tbb::task& t) : spawn_task(t) {} - void operator()() const { - FLOW_SPAWN(spawn_task); - } - }; - - void prepare_task_arena(bool reinit = false) { - if (reinit) { - __TBB_ASSERT(my_task_arena, "task arena is NULL"); - my_task_arena->terminate(); - my_task_arena->initialize(tbb::task_arena::attach()); - } - else { - __TBB_ASSERT(my_task_arena == NULL, "task arena is not NULL"); - my_task_arena = new tbb::task_arena(tbb::task_arena::attach()); - } - if (!my_task_arena->is_active()) // failed to attach - my_task_arena->initialize(); // create a new, default-initialized arena - __TBB_ASSERT(my_task_arena->is_active(), "task arena is not active"); - } - -public: - //! Constructs a graph with isolated task_group_context - graph(); - - //! Constructs a graph with use_this_context as context - explicit graph(tbb::task_group_context& use_this_context); - - //! Destroys the graph. - /** Calls wait_for_all, then destroys the root task and context. */ - ~graph() { - wait_for_all(); - my_root_task->set_ref_count(0); - tbb::task::destroy(*my_root_task); - if (own_context) delete my_context; - delete my_task_arena; - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name(const char *name); -#endif - - void increment_wait_count() { - reserve_wait(); - } - - void decrement_wait_count() { - release_wait(); - } - - //! Used to register that an external entity may still interact with the graph. - /** The graph will not return from wait_for_all until a matching number of decrement_wait_count calls - is made. */ - void reserve_wait() __TBB_override; - - //! Deregisters an external entity that may have interacted with the graph. - /** The graph will not return from wait_for_all until all the number of decrement_wait_count calls - matches the number of increment_wait_count calls. */ - void release_wait() __TBB_override; - - //! Spawns a task that runs a body and puts its output to a specific receiver - /** The task is spawned as a child of the graph. This is useful for running tasks - that need to block a wait_for_all() on the graph. For example a one-off source. */ - template< typename Receiver, typename Body > - void run(Receiver &r, Body body) { - if (internal::is_graph_active(*this)) { - task* rtask = new (task::allocate_additional_child_of(*root_task())) - run_and_put_task< Receiver, Body >(r, body); - my_task_arena->execute(spawn_functor(*rtask)); - } - } - - //! Spawns a task that runs a function object - /** The task is spawned as a child of the graph. This is useful for running tasks - that need to block a wait_for_all() on the graph. For example a one-off source. */ - template< typename Body > - void run(Body body) { - if (internal::is_graph_active(*this)) { - task* rtask = new (task::allocate_additional_child_of(*root_task())) run_task< Body >(body); - my_task_arena->execute(spawn_functor(*rtask)); - } - } - - //! Wait until graph is idle and decrement_wait_count calls equals increment_wait_count calls. - /** The waiting thread will go off and steal work while it is block in the wait_for_all. */ - void wait_for_all() { - cancelled = false; - caught_exception = false; - if (my_root_task) { -#if TBB_USE_EXCEPTIONS - try { -#endif - my_task_arena->execute(wait_functor(my_root_task)); - cancelled = my_context->is_group_execution_cancelled(); -#if TBB_USE_EXCEPTIONS - } - catch (...) { - my_root_task->set_ref_count(1); - my_context->reset(); - caught_exception = true; - cancelled = true; - throw; - } -#endif - // TODO: the "if" condition below is just a work-around to support the concurrent wait - // mode. The cancellation and exception mechanisms are still broken in this mode. - // Consider using task group not to re-implement the same functionality. - if (!(my_context->traits() & tbb::task_group_context::concurrent_wait)) { - my_context->reset(); // consistent with behavior in catch() - my_root_task->set_ref_count(1); - } - } - } - - //! Returns the root task of the graph - tbb::task * root_task() { - return my_root_task; - } - - // ITERATORS - template - friend class graph_iterator; - - // Graph iterator typedefs - typedef graph_iterator iterator; - typedef graph_iterator const_iterator; - - // Graph iterator constructors - //! start iterator - iterator begin() { return iterator(this, true); } - //! end iterator - iterator end() { return iterator(this, false); } - //! start const iterator - const_iterator begin() const { return const_iterator(this, true); } - //! end const iterator - const_iterator end() const { return const_iterator(this, false); } - //! start const iterator - const_iterator cbegin() const { return const_iterator(this, true); } - //! end const iterator - const_iterator cend() const { return const_iterator(this, false); } - - //! return status of graph execution - bool is_cancelled() { return cancelled; } - bool exception_thrown() { return caught_exception; } - - // thread-unsafe state reset. - void reset(reset_flags f = rf_reset_protocol); - -private: - tbb::task *my_root_task; - tbb::task_group_context *my_context; - bool own_context; - bool cancelled; - bool caught_exception; - bool my_is_active; - task_list_type my_reset_task_list; - - graph_node *my_nodes, *my_nodes_last; - - tbb::spin_mutex nodelist_mutex; - void register_node(graph_node *n); - void remove_node(graph_node *n); - - tbb::task_arena* my_task_arena; - - friend void internal::activate_graph(graph& g); - friend void internal::deactivate_graph(graph& g); - friend bool internal::is_graph_active(graph& g); - friend void internal::spawn_in_graph_arena(graph& g, tbb::task& arena_task); - friend void internal::add_task_to_graph_reset_list(graph& g, tbb::task *tp); - template friend void internal::execute_in_graph_arena(graph& g, F& f); - - friend class tbb::interface7::internal::task_arena_base; - -}; // class graph - -//! The base of all graph nodes. -class graph_node : tbb::internal::no_copy { - friend class graph; - template - friend class graph_iterator; -protected: - graph& my_graph; - graph_node *next, *prev; -public: - explicit graph_node(graph& g) : my_graph(g) { - my_graph.register_node(this); - } - virtual ~graph_node() { - my_graph.remove_node(this); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - virtual void set_name(const char *name) = 0; -#endif - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - virtual void extract() = 0; -#endif - -protected: - // performs the reset on an individual node. - virtual void reset_node(reset_flags f = rf_reset_protocol) = 0; -}; // class graph_node - -namespace internal { - -inline void activate_graph(graph& g) { - g.my_is_active = true; -} - -inline void deactivate_graph(graph& g) { - g.my_is_active = false; -} - -inline bool is_graph_active(graph& g) { - return g.my_is_active; -} - -//! Executes custom functor inside graph arena -template -inline void execute_in_graph_arena(graph& g, F& f) { - if (is_graph_active(g)) { - __TBB_ASSERT(g.my_task_arena && g.my_task_arena->is_active(), NULL); - g.my_task_arena->execute(f); - } -} - -//! Spawns a task inside graph arena -inline void spawn_in_graph_arena(graph& g, tbb::task& arena_task) { - graph::spawn_functor s_fn(arena_task); - execute_in_graph_arena(g, s_fn); -} - -inline void add_task_to_graph_reset_list(graph& g, tbb::task *tp) { - g.my_reset_task_list.push_back(tp); -} - -} // namespace internal - -} // namespace interface10 -} // namespace flow -} // namespace tbb - -#endif // __TBB_flow_graph_impl_H diff --git a/src/tbb/include/tbb/internal/_flow_graph_indexer_impl.h b/src/tbb/include/tbb/internal/_flow_graph_indexer_impl.h deleted file mode 100644 index c8d4fd4e2..000000000 --- a/src/tbb/include/tbb/internal/_flow_graph_indexer_impl.h +++ /dev/null @@ -1,484 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB__flow_graph_indexer_impl_H -#define __TBB__flow_graph_indexer_impl_H - -#ifndef __TBB_flow_graph_H -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#include "_flow_graph_types_impl.h" - -namespace internal { - - // Output of the indexer_node is a tbb::flow::tagged_msg, and will be of - // the form tagged_msg - // where the value of tag will indicate which result was put to the - // successor. - - template - task* do_try_put(const T &v, void *p) { - typename IndexerNodeBaseType::output_type o(K, v); - return reinterpret_cast(p)->try_put_task(&o); - } - - template - struct indexer_helper { - template - static inline void set_indexer_node_pointer(PortTuple &my_input, IndexerNodeBaseType *p, graph& g) { - typedef typename tuple_element::type T; - task *(*indexer_node_put_task)(const T&, void *) = do_try_put; - tbb::flow::get(my_input).set_up(p, indexer_node_put_task, g); - indexer_helper::template set_indexer_node_pointer(my_input, p, g); - } - template - static inline void reset_inputs(InputTuple &my_input, reset_flags f) { - indexer_helper::reset_inputs(my_input, f); - tbb::flow::get(my_input).reset_receiver(f); - } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - template - static inline void extract(InputTuple &my_input) { - indexer_helper::extract(my_input); - tbb::flow::get(my_input).extract_receiver(); - } -#endif - }; - - template - struct indexer_helper { - template - static inline void set_indexer_node_pointer(PortTuple &my_input, IndexerNodeBaseType *p, graph& g) { - typedef typename tuple_element<0, TupleTypes>::type T; - task *(*indexer_node_put_task)(const T&, void *) = do_try_put; - tbb::flow::get<0>(my_input).set_up(p, indexer_node_put_task, g); - } - template - static inline void reset_inputs(InputTuple &my_input, reset_flags f) { - tbb::flow::get<0>(my_input).reset_receiver(f); - } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - template - static inline void extract(InputTuple &my_input) { - tbb::flow::get<0>(my_input).extract_receiver(); - } -#endif - }; - - template - class indexer_input_port : public receiver { - private: - void* my_indexer_ptr; - typedef task* (* forward_function_ptr)(T const &, void* ); - forward_function_ptr my_try_put_task; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - spin_mutex my_pred_mutex; - typedef typename receiver::built_predecessors_type built_predecessors_type; - built_predecessors_type my_built_predecessors; -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ - graph* my_graph; - public: -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - indexer_input_port() : my_pred_mutex(), my_graph(NULL) {} - indexer_input_port( const indexer_input_port & other) : receiver(), my_pred_mutex(), my_graph(other.my_graph) { - } -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ - void set_up(void* p, forward_function_ptr f, graph& g) { - my_indexer_ptr = p; - my_try_put_task = f; - my_graph = &g; - } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - typedef typename receiver::predecessor_list_type predecessor_list_type; - typedef typename receiver::predecessor_type predecessor_type; - - built_predecessors_type &built_predecessors() __TBB_override { return my_built_predecessors; } - - size_t predecessor_count() __TBB_override { - spin_mutex::scoped_lock l(my_pred_mutex); - return my_built_predecessors.edge_count(); - } - void internal_add_built_predecessor(predecessor_type &p) __TBB_override { - spin_mutex::scoped_lock l(my_pred_mutex); - my_built_predecessors.add_edge(p); - } - void internal_delete_built_predecessor(predecessor_type &p) __TBB_override { - spin_mutex::scoped_lock l(my_pred_mutex); - my_built_predecessors.delete_edge(p); - } - void copy_predecessors( predecessor_list_type &v) __TBB_override { - spin_mutex::scoped_lock l(my_pred_mutex); - my_built_predecessors.copy_edges(v); - } -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ - protected: - template< typename R, typename B > friend class run_and_put_task; - template friend class internal::broadcast_cache; - template friend class internal::round_robin_cache; - task *try_put_task(const T &v) __TBB_override { - return my_try_put_task(v, my_indexer_ptr); - } - - graph& graph_reference() __TBB_override { - return *my_graph; - } - - public: -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - void reset_receiver(reset_flags f) __TBB_override { if(f&rf_clear_edges) my_built_predecessors.clear(); } -#else - void reset_receiver(reset_flags /*f*/) __TBB_override { } -#endif - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - void extract_receiver() { my_built_predecessors.receiver_extract(*this); } -#endif - }; - - template - class indexer_node_FE { - public: - static const int N = tbb::flow::tuple_size::value; - typedef OutputType output_type; - typedef InputTuple input_type; - - // Some versions of Intel(R) C++ Compiler fail to generate an implicit constructor for the class which has std::tuple as a member. - indexer_node_FE() : my_inputs() {} - - input_type &input_ports() { return my_inputs; } - protected: - input_type my_inputs; - }; - - //! indexer_node_base - template - class indexer_node_base : public graph_node, public indexer_node_FE, - public sender { - protected: - using graph_node::my_graph; - public: - static const size_t N = tbb::flow::tuple_size::value; - typedef OutputType output_type; - typedef StructTypes tuple_types; - typedef typename sender::successor_type successor_type; - typedef indexer_node_FE input_ports_type; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - typedef typename sender::built_successors_type built_successors_type; - typedef typename sender::successor_list_type successor_list_type; -#endif - - private: - // ----------- Aggregator ------------ - enum op_type { reg_succ, rem_succ, try__put_task -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - , add_blt_succ, del_blt_succ, - blt_succ_cnt, blt_succ_cpy -#endif - }; - typedef indexer_node_base class_type; - - class indexer_node_base_operation : public aggregated_operation { - public: - char type; - union { - output_type const *my_arg; - successor_type *my_succ; - task *bypass_t; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - size_t cnt_val; - successor_list_type *succv; -#endif - }; - indexer_node_base_operation(const output_type* e, op_type t) : - type(char(t)), my_arg(e) {} - indexer_node_base_operation(const successor_type &s, op_type t) : type(char(t)), - my_succ(const_cast(&s)) {} - indexer_node_base_operation(op_type t) : type(char(t)) {} - }; - - typedef internal::aggregating_functor handler_type; - friend class internal::aggregating_functor; - aggregator my_aggregator; - - void handle_operations(indexer_node_base_operation* op_list) { - indexer_node_base_operation *current; - while(op_list) { - current = op_list; - op_list = op_list->next; - switch(current->type) { - - case reg_succ: - my_successors.register_successor(*(current->my_succ)); - __TBB_store_with_release(current->status, SUCCEEDED); - break; - - case rem_succ: - my_successors.remove_successor(*(current->my_succ)); - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case try__put_task: { - current->bypass_t = my_successors.try_put_task(*(current->my_arg)); - __TBB_store_with_release(current->status, SUCCEEDED); // return of try_put_task actual return value - } - break; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - case add_blt_succ: - my_successors.internal_add_built_successor(*(current->my_succ)); - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case del_blt_succ: - my_successors.internal_delete_built_successor(*(current->my_succ)); - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case blt_succ_cnt: - current->cnt_val = my_successors.successor_count(); - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case blt_succ_cpy: - my_successors.copy_successors(*(current->succv)); - __TBB_store_with_release(current->status, SUCCEEDED); - break; -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ - } - } - } - // ---------- end aggregator ----------- - public: - indexer_node_base(graph& g) : graph_node(g), input_ports_type() { - indexer_helper::set_indexer_node_pointer(this->my_inputs, this, g); - my_successors.set_owner(this); - my_aggregator.initialize_handler(handler_type(this)); - } - - indexer_node_base(const indexer_node_base& other) : graph_node(other.my_graph), input_ports_type(), sender() { - indexer_helper::set_indexer_node_pointer(this->my_inputs, this, other.my_graph); - my_successors.set_owner(this); - my_aggregator.initialize_handler(handler_type(this)); - } - - bool register_successor(successor_type &r) __TBB_override { - indexer_node_base_operation op_data(r, reg_succ); - my_aggregator.execute(&op_data); - return op_data.status == SUCCEEDED; - } - - bool remove_successor( successor_type &r) __TBB_override { - indexer_node_base_operation op_data(r, rem_succ); - my_aggregator.execute(&op_data); - return op_data.status == SUCCEEDED; - } - - task * try_put_task(output_type const *v) { // not a virtual method in this class - indexer_node_base_operation op_data(v, try__put_task); - my_aggregator.execute(&op_data); - return op_data.bypass_t; - } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - - built_successors_type &built_successors() __TBB_override { return my_successors.built_successors(); } - - void internal_add_built_successor( successor_type &r) __TBB_override { - indexer_node_base_operation op_data(r, add_blt_succ); - my_aggregator.execute(&op_data); - } - - void internal_delete_built_successor( successor_type &r) __TBB_override { - indexer_node_base_operation op_data(r, del_blt_succ); - my_aggregator.execute(&op_data); - } - - size_t successor_count() __TBB_override { - indexer_node_base_operation op_data(blt_succ_cnt); - my_aggregator.execute(&op_data); - return op_data.cnt_val; - } - - void copy_successors( successor_list_type &v) __TBB_override { - indexer_node_base_operation op_data(blt_succ_cpy); - op_data.succv = &v; - my_aggregator.execute(&op_data); - } - void extract() __TBB_override { - my_successors.built_successors().sender_extract(*this); - indexer_helper::extract(this->my_inputs); - } -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ - protected: - void reset_node(reset_flags f) __TBB_override { - if(f & rf_clear_edges) { - my_successors.clear(); - indexer_helper::reset_inputs(this->my_inputs,f); - } - } - - private: - broadcast_cache my_successors; - }; //indexer_node_base - - - template struct input_types; - - template - struct input_types<1, InputTuple> { - typedef typename tuple_element<0, InputTuple>::type first_type; - typedef typename internal::tagged_msg type; - }; - - template - struct input_types<2, InputTuple> { - typedef typename tuple_element<0, InputTuple>::type first_type; - typedef typename tuple_element<1, InputTuple>::type second_type; - typedef typename internal::tagged_msg type; - }; - - template - struct input_types<3, InputTuple> { - typedef typename tuple_element<0, InputTuple>::type first_type; - typedef typename tuple_element<1, InputTuple>::type second_type; - typedef typename tuple_element<2, InputTuple>::type third_type; - typedef typename internal::tagged_msg type; - }; - - template - struct input_types<4, InputTuple> { - typedef typename tuple_element<0, InputTuple>::type first_type; - typedef typename tuple_element<1, InputTuple>::type second_type; - typedef typename tuple_element<2, InputTuple>::type third_type; - typedef typename tuple_element<3, InputTuple>::type fourth_type; - typedef typename internal::tagged_msg type; - }; - - template - struct input_types<5, InputTuple> { - typedef typename tuple_element<0, InputTuple>::type first_type; - typedef typename tuple_element<1, InputTuple>::type second_type; - typedef typename tuple_element<2, InputTuple>::type third_type; - typedef typename tuple_element<3, InputTuple>::type fourth_type; - typedef typename tuple_element<4, InputTuple>::type fifth_type; - typedef typename internal::tagged_msg type; - }; - - template - struct input_types<6, InputTuple> { - typedef typename tuple_element<0, InputTuple>::type first_type; - typedef typename tuple_element<1, InputTuple>::type second_type; - typedef typename tuple_element<2, InputTuple>::type third_type; - typedef typename tuple_element<3, InputTuple>::type fourth_type; - typedef typename tuple_element<4, InputTuple>::type fifth_type; - typedef typename tuple_element<5, InputTuple>::type sixth_type; - typedef typename internal::tagged_msg type; - }; - - template - struct input_types<7, InputTuple> { - typedef typename tuple_element<0, InputTuple>::type first_type; - typedef typename tuple_element<1, InputTuple>::type second_type; - typedef typename tuple_element<2, InputTuple>::type third_type; - typedef typename tuple_element<3, InputTuple>::type fourth_type; - typedef typename tuple_element<4, InputTuple>::type fifth_type; - typedef typename tuple_element<5, InputTuple>::type sixth_type; - typedef typename tuple_element<6, InputTuple>::type seventh_type; - typedef typename internal::tagged_msg type; - }; - - - template - struct input_types<8, InputTuple> { - typedef typename tuple_element<0, InputTuple>::type first_type; - typedef typename tuple_element<1, InputTuple>::type second_type; - typedef typename tuple_element<2, InputTuple>::type third_type; - typedef typename tuple_element<3, InputTuple>::type fourth_type; - typedef typename tuple_element<4, InputTuple>::type fifth_type; - typedef typename tuple_element<5, InputTuple>::type sixth_type; - typedef typename tuple_element<6, InputTuple>::type seventh_type; - typedef typename tuple_element<7, InputTuple>::type eighth_type; - typedef typename internal::tagged_msg type; - }; - - - template - struct input_types<9, InputTuple> { - typedef typename tuple_element<0, InputTuple>::type first_type; - typedef typename tuple_element<1, InputTuple>::type second_type; - typedef typename tuple_element<2, InputTuple>::type third_type; - typedef typename tuple_element<3, InputTuple>::type fourth_type; - typedef typename tuple_element<4, InputTuple>::type fifth_type; - typedef typename tuple_element<5, InputTuple>::type sixth_type; - typedef typename tuple_element<6, InputTuple>::type seventh_type; - typedef typename tuple_element<7, InputTuple>::type eighth_type; - typedef typename tuple_element<8, InputTuple>::type nineth_type; - typedef typename internal::tagged_msg type; - }; - - template - struct input_types<10, InputTuple> { - typedef typename tuple_element<0, InputTuple>::type first_type; - typedef typename tuple_element<1, InputTuple>::type second_type; - typedef typename tuple_element<2, InputTuple>::type third_type; - typedef typename tuple_element<3, InputTuple>::type fourth_type; - typedef typename tuple_element<4, InputTuple>::type fifth_type; - typedef typename tuple_element<5, InputTuple>::type sixth_type; - typedef typename tuple_element<6, InputTuple>::type seventh_type; - typedef typename tuple_element<7, InputTuple>::type eighth_type; - typedef typename tuple_element<8, InputTuple>::type nineth_type; - typedef typename tuple_element<9, InputTuple>::type tenth_type; - typedef typename internal::tagged_msg type; - }; - - // type generators - template - struct indexer_types : public input_types::value, OutputTuple> { - static const int N = tbb::flow::tuple_size::value; - typedef typename input_types::type output_type; - typedef typename wrap_tuple_elements::type input_ports_type; - typedef internal::indexer_node_FE indexer_FE_type; - typedef internal::indexer_node_base indexer_base_type; - }; - - template - class unfolded_indexer_node : public indexer_types::indexer_base_type { - public: - typedef typename indexer_types::input_ports_type input_ports_type; - typedef OutputTuple tuple_types; - typedef typename indexer_types::output_type output_type; - private: - typedef typename indexer_types::indexer_base_type base_type; - public: - unfolded_indexer_node(graph& g) : base_type(g) {} - unfolded_indexer_node(const unfolded_indexer_node &other) : base_type(other) {} - }; - -} /* namespace internal */ - -#endif /* __TBB__flow_graph_indexer_impl_H */ diff --git a/src/tbb/include/tbb/internal/_flow_graph_item_buffer_impl.h b/src/tbb/include/tbb/internal/_flow_graph_item_buffer_impl.h deleted file mode 100644 index 85d2686dc..000000000 --- a/src/tbb/include/tbb/internal/_flow_graph_item_buffer_impl.h +++ /dev/null @@ -1,288 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB__flow_graph_item_buffer_impl_H -#define __TBB__flow_graph_item_buffer_impl_H - -#ifndef __TBB_flow_graph_H -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#include "tbb/internal/_flow_graph_types_impl.h" // for aligned_pair - -// in namespace tbb::flow::interfaceX (included in _flow_graph_node_impl.h) - - //! Expandable buffer of items. The possible operations are push, pop, - //* tests for empty and so forth. No mutual exclusion is built in. - //* objects are constructed into and explicitly-destroyed. get_my_item gives - // a read-only reference to the item in the buffer. set_my_item may be called - // with either an empty or occupied slot. - - using internal::aligned_pair; - using internal::alignment_of; - -namespace internal { - - template > - class item_buffer { - public: - typedef T item_type; - enum buffer_item_state { no_item=0, has_item=1, reserved_item=2 }; - protected: - typedef size_t size_type; - typedef typename aligned_pair::type buffer_item_type; - typedef typename A::template rebind::other allocator_type; - - buffer_item_type *my_array; - size_type my_array_size; - static const size_type initial_buffer_size = 4; - size_type my_head; - size_type my_tail; - - bool buffer_empty() const { return my_head == my_tail; } - - buffer_item_type &item(size_type i) { - __TBB_ASSERT(!(size_type(&(my_array[i&(my_array_size-1)].second))%alignment_of::value),NULL); - __TBB_ASSERT(!(size_type(&(my_array[i&(my_array_size-1)].first))%alignment_of::value), NULL); - return my_array[i & (my_array_size - 1) ]; - } - - const buffer_item_type &item(size_type i) const { - __TBB_ASSERT(!(size_type(&(my_array[i&(my_array_size-1)].second))%alignment_of::value), NULL); - __TBB_ASSERT(!(size_type(&(my_array[i&(my_array_size-1)].first))%alignment_of::value), NULL); - return my_array[i & (my_array_size-1)]; - } - - bool my_item_valid(size_type i) const { return (i < my_tail) && (i >= my_head) && (item(i).second != no_item); } - bool my_item_reserved(size_type i) const { return item(i).second == reserved_item; } - - // object management in buffer - const item_type &get_my_item(size_t i) const { - __TBB_ASSERT(my_item_valid(i),"attempt to get invalid item"); - item_type *itm = (tbb::internal::punned_cast(&(item(i).first))); - return *(const item_type *)itm; - } - - // may be called with an empty slot or a slot that has already been constructed into. - void set_my_item(size_t i, const item_type &o) { - if(item(i).second != no_item) { - destroy_item(i); - } - new(&(item(i).first)) item_type(o); - item(i).second = has_item; - } - - // destructively-fetch an object from the buffer - void fetch_item(size_t i, item_type &o) { - __TBB_ASSERT(my_item_valid(i), "Trying to fetch an empty slot"); - o = get_my_item(i); // could have std::move assign semantics - destroy_item(i); - } - - // move an existing item from one slot to another. The moved-to slot must be unoccupied, - // the moved-from slot must exist and not be reserved. The after, from will be empty, - // to will be occupied but not reserved - void move_item(size_t to, size_t from) { - __TBB_ASSERT(!my_item_valid(to), "Trying to move to a non-empty slot"); - __TBB_ASSERT(my_item_valid(from), "Trying to move from an empty slot"); - set_my_item(to, get_my_item(from)); // could have std::move semantics - destroy_item(from); - - } - - // put an item in an empty slot. Return true if successful, else false - bool place_item(size_t here, const item_type &me) { -#if !TBB_DEPRECATED_SEQUENCER_DUPLICATES - if(my_item_valid(here)) return false; -#endif - set_my_item(here, me); - return true; - } - - // could be implemented with std::move semantics - void swap_items(size_t i, size_t j) { - __TBB_ASSERT(my_item_valid(i) && my_item_valid(j), "attempt to swap invalid item(s)"); - item_type temp = get_my_item(i); - set_my_item(i, get_my_item(j)); - set_my_item(j, temp); - } - - void destroy_item(size_type i) { - __TBB_ASSERT(my_item_valid(i), "destruction of invalid item"); - (tbb::internal::punned_cast(&(item(i).first)))->~item_type(); - item(i).second = no_item; - } - - // returns the front element - const item_type& front() const - { - __TBB_ASSERT(my_item_valid(my_head), "attempt to fetch head non-item"); - return get_my_item(my_head); - } - - // returns the back element - const item_type& back() const - { - __TBB_ASSERT(my_item_valid(my_tail - 1), "attempt to fetch head non-item"); - return get_my_item(my_tail - 1); - } - - // following methods are for reservation of the front of a bufffer. - void reserve_item(size_type i) { __TBB_ASSERT(my_item_valid(i) && !my_item_reserved(i), "item cannot be reserved"); item(i).second = reserved_item; } - void release_item(size_type i) { __TBB_ASSERT(my_item_reserved(i), "item is not reserved"); item(i).second = has_item; } - - void destroy_front() { destroy_item(my_head); ++my_head; } - void destroy_back() { destroy_item(my_tail-1); --my_tail; } - - // we have to be able to test against a new tail value without changing my_tail - // grow_array doesn't work if we change my_tail when the old array is too small - size_type size(size_t new_tail = 0) { return (new_tail ? new_tail : my_tail) - my_head; } - size_type capacity() { return my_array_size; } - // sequencer_node does not use this method, so we don't - // need a version that passes in the new_tail value. - bool buffer_full() { return size() >= capacity(); } - - //! Grows the internal array. - void grow_my_array( size_t minimum_size ) { - // test that we haven't made the structure inconsistent. - __TBB_ASSERT(capacity() >= my_tail - my_head, "total items exceed capacity"); - size_type new_size = my_array_size ? 2*my_array_size : initial_buffer_size; - while( new_sizeback(); - destroy_back(); - return true; - } - - bool pop_front(item_type &v) { - if(!my_item_valid(my_head)) { - return false; - } - v = this->front(); - destroy_front(); - return true; - } - - // This is used both for reset and for grow_my_array. In the case of grow_my_array - // we want to retain the values of the head and tail. - void clean_up_buffer(bool reset_pointers) { - if (my_array) { - for( size_type i=my_head; i > - class reservable_item_buffer : public item_buffer { - protected: - using item_buffer::my_item_valid; - using item_buffer::my_head; - - public: - reservable_item_buffer() : item_buffer(), my_reserved(false) {} - void reset() {my_reserved = false; item_buffer::reset(); } - protected: - - bool reserve_front(T &v) { - if(my_reserved || !my_item_valid(this->my_head)) return false; - my_reserved = true; - // reserving the head - v = this->front(); - this->reserve_item(this->my_head); - return true; - } - - void consume_front() { - __TBB_ASSERT(my_reserved, "Attempt to consume a non-reserved item"); - this->destroy_front(); - my_reserved = false; - } - - void release_front() { - __TBB_ASSERT(my_reserved, "Attempt to release a non-reserved item"); - this->release_item(this->my_head); - my_reserved = false; - } - - bool my_reserved; - }; - -} // namespace internal - -#endif // __TBB__flow_graph_item_buffer_impl_H diff --git a/src/tbb/include/tbb/internal/_flow_graph_join_impl.h b/src/tbb/include/tbb/internal/_flow_graph_join_impl.h deleted file mode 100644 index 8fc32a9a0..000000000 --- a/src/tbb/include/tbb/internal/_flow_graph_join_impl.h +++ /dev/null @@ -1,2003 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB__flow_graph_join_impl_H -#define __TBB__flow_graph_join_impl_H - -#ifndef __TBB_flow_graph_H -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -namespace internal { - - struct forwarding_base : tbb::internal::no_assign { - forwarding_base(graph &g) : graph_ref(g) {} - virtual ~forwarding_base() {} - // decrement_port_count may create a forwarding task. If we cannot handle the task - // ourselves, ask decrement_port_count to deal with it. - virtual task * decrement_port_count(bool handle_task) = 0; - virtual void increment_port_count() = 0; - // moved here so input ports can queue tasks - graph& graph_ref; - }; - - // specialization that lets us keep a copy of the current_key for building results. - // KeyType can be a reference type. - template - struct matching_forwarding_base : public forwarding_base { - typedef typename tbb::internal::strip::type current_key_type; - matching_forwarding_base(graph &g) : forwarding_base(g) { } - virtual task * increment_key_count(current_key_type const & /*t*/, bool /*handle_task*/) = 0; // {return NULL;} - current_key_type current_key; // so ports can refer to FE's desired items - }; - - template< int N > - struct join_helper { - - template< typename TupleType, typename PortType > - static inline void set_join_node_pointer(TupleType &my_input, PortType *port) { - tbb::flow::get( my_input ).set_join_node_pointer(port); - join_helper::set_join_node_pointer( my_input, port ); - } - template< typename TupleType > - static inline void consume_reservations( TupleType &my_input ) { - tbb::flow::get( my_input ).consume(); - join_helper::consume_reservations( my_input ); - } - - template< typename TupleType > - static inline void release_my_reservation( TupleType &my_input ) { - tbb::flow::get( my_input ).release(); - } - - template - static inline void release_reservations( TupleType &my_input) { - join_helper::release_reservations(my_input); - release_my_reservation(my_input); - } - - template< typename InputTuple, typename OutputTuple > - static inline bool reserve( InputTuple &my_input, OutputTuple &out) { - if ( !tbb::flow::get( my_input ).reserve( tbb::flow::get( out ) ) ) return false; - if ( !join_helper::reserve( my_input, out ) ) { - release_my_reservation( my_input ); - return false; - } - return true; - } - - template - static inline bool get_my_item( InputTuple &my_input, OutputTuple &out) { - bool res = tbb::flow::get(my_input).get_item(tbb::flow::get(out) ); // may fail - return join_helper::get_my_item(my_input, out) && res; // do get on other inputs before returning - } - - template - static inline bool get_items(InputTuple &my_input, OutputTuple &out) { - return get_my_item(my_input, out); - } - - template - static inline void reset_my_port(InputTuple &my_input) { - join_helper::reset_my_port(my_input); - tbb::flow::get(my_input).reset_port(); - } - - template - static inline void reset_ports(InputTuple& my_input) { - reset_my_port(my_input); - } - - template - static inline void set_key_functors(InputTuple &my_input, KeyFuncTuple &my_key_funcs) { - tbb::flow::get(my_input).set_my_key_func(tbb::flow::get(my_key_funcs)); - tbb::flow::get(my_key_funcs) = NULL; - join_helper::set_key_functors(my_input, my_key_funcs); - } - - template< typename KeyFuncTuple> - static inline void copy_key_functors(KeyFuncTuple &my_inputs, KeyFuncTuple &other_inputs) { - if(tbb::flow::get(other_inputs).get_my_key_func()) { - tbb::flow::get(my_inputs).set_my_key_func(tbb::flow::get(other_inputs).get_my_key_func()->clone()); - } - join_helper::copy_key_functors(my_inputs, other_inputs); - } - - template - static inline void reset_inputs(InputTuple &my_input, reset_flags f) { - join_helper::reset_inputs(my_input, f); - tbb::flow::get(my_input).reset_receiver(f); - } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - template - static inline void extract_inputs(InputTuple &my_input) { - join_helper::extract_inputs(my_input); - tbb::flow::get(my_input).extract_receiver(); - } -#endif - }; // join_helper - - template< > - struct join_helper<1> { - - template< typename TupleType, typename PortType > - static inline void set_join_node_pointer(TupleType &my_input, PortType *port) { - tbb::flow::get<0>( my_input ).set_join_node_pointer(port); - } - - template< typename TupleType > - static inline void consume_reservations( TupleType &my_input ) { - tbb::flow::get<0>( my_input ).consume(); - } - - template< typename TupleType > - static inline void release_my_reservation( TupleType &my_input ) { - tbb::flow::get<0>( my_input ).release(); - } - - template - static inline void release_reservations( TupleType &my_input) { - release_my_reservation(my_input); - } - - template< typename InputTuple, typename OutputTuple > - static inline bool reserve( InputTuple &my_input, OutputTuple &out) { - return tbb::flow::get<0>( my_input ).reserve( tbb::flow::get<0>( out ) ); - } - - template - static inline bool get_my_item( InputTuple &my_input, OutputTuple &out) { - return tbb::flow::get<0>(my_input).get_item(tbb::flow::get<0>(out)); - } - - template - static inline bool get_items(InputTuple &my_input, OutputTuple &out) { - return get_my_item(my_input, out); - } - - template - static inline void reset_my_port(InputTuple &my_input) { - tbb::flow::get<0>(my_input).reset_port(); - } - - template - static inline void reset_ports(InputTuple& my_input) { - reset_my_port(my_input); - } - - template - static inline void set_key_functors(InputTuple &my_input, KeyFuncTuple &my_key_funcs) { - tbb::flow::get<0>(my_input).set_my_key_func(tbb::flow::get<0>(my_key_funcs)); - tbb::flow::get<0>(my_key_funcs) = NULL; - } - - template< typename KeyFuncTuple> - static inline void copy_key_functors(KeyFuncTuple &my_inputs, KeyFuncTuple &other_inputs) { - if(tbb::flow::get<0>(other_inputs).get_my_key_func()) { - tbb::flow::get<0>(my_inputs).set_my_key_func(tbb::flow::get<0>(other_inputs).get_my_key_func()->clone()); - } - } - template - static inline void reset_inputs(InputTuple &my_input, reset_flags f) { - tbb::flow::get<0>(my_input).reset_receiver(f); - } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - template - static inline void extract_inputs(InputTuple &my_input) { - tbb::flow::get<0>(my_input).extract_receiver(); - } -#endif - }; // join_helper<1> - - //! The two-phase join port - template< typename T > - class reserving_port : public receiver { - public: - typedef T input_type; - typedef typename receiver::predecessor_type predecessor_type; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - typedef typename receiver::predecessor_list_type predecessor_list_type; - typedef typename receiver::built_predecessors_type built_predecessors_type; -#endif - private: - // ----------- Aggregator ------------ - enum op_type { reg_pred, rem_pred, res_item, rel_res, con_res -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - , add_blt_pred, del_blt_pred, blt_pred_cnt, blt_pred_cpy -#endif - }; - enum op_stat {WAIT=0, SUCCEEDED, FAILED}; - typedef reserving_port class_type; - - class reserving_port_operation : public aggregated_operation { - public: - char type; - union { - T *my_arg; - predecessor_type *my_pred; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - size_t cnt_val; - predecessor_list_type *plist; -#endif - }; - reserving_port_operation(const T& e, op_type t) : - type(char(t)), my_arg(const_cast(&e)) {} - reserving_port_operation(const predecessor_type &s, op_type t) : type(char(t)), - my_pred(const_cast(&s)) {} - reserving_port_operation(op_type t) : type(char(t)) {} - }; - - typedef internal::aggregating_functor handler_type; - friend class internal::aggregating_functor; - aggregator my_aggregator; - - void handle_operations(reserving_port_operation* op_list) { - reserving_port_operation *current; - bool no_predecessors; - while(op_list) { - current = op_list; - op_list = op_list->next; - switch(current->type) { - case reg_pred: - no_predecessors = my_predecessors.empty(); - my_predecessors.add(*(current->my_pred)); - if ( no_predecessors ) { - (void) my_join->decrement_port_count(true); // may try to forward - } - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case rem_pred: - my_predecessors.remove(*(current->my_pred)); - if(my_predecessors.empty()) my_join->increment_port_count(); - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case res_item: - if ( reserved ) { - __TBB_store_with_release(current->status, FAILED); - } - else if ( my_predecessors.try_reserve( *(current->my_arg) ) ) { - reserved = true; - __TBB_store_with_release(current->status, SUCCEEDED); - } else { - if ( my_predecessors.empty() ) { - my_join->increment_port_count(); - } - __TBB_store_with_release(current->status, FAILED); - } - break; - case rel_res: - reserved = false; - my_predecessors.try_release( ); - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case con_res: - reserved = false; - my_predecessors.try_consume( ); - __TBB_store_with_release(current->status, SUCCEEDED); - break; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - case add_blt_pred: - my_predecessors.internal_add_built_predecessor(*(current->my_pred)); - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case del_blt_pred: - my_predecessors.internal_delete_built_predecessor(*(current->my_pred)); - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case blt_pred_cnt: - current->cnt_val = my_predecessors.predecessor_count(); - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case blt_pred_cpy: - my_predecessors.copy_predecessors(*(current->plist)); - __TBB_store_with_release(current->status, SUCCEEDED); - break; -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ - } - } - } - - protected: - template< typename R, typename B > friend class run_and_put_task; - template friend class internal::broadcast_cache; - template friend class internal::round_robin_cache; - task *try_put_task( const T & ) __TBB_override { - return NULL; - } - - graph& graph_reference() __TBB_override { - return my_join->graph_ref; - } - - public: - - //! Constructor - reserving_port() : reserved(false) { - my_join = NULL; - my_predecessors.set_owner( this ); - my_aggregator.initialize_handler(handler_type(this)); - } - - // copy constructor - reserving_port(const reserving_port& /* other */) : receiver() { - reserved = false; - my_join = NULL; - my_predecessors.set_owner( this ); - my_aggregator.initialize_handler(handler_type(this)); - } - - void set_join_node_pointer(forwarding_base *join) { - my_join = join; - } - - //! Add a predecessor - bool register_predecessor( predecessor_type &src ) __TBB_override { - reserving_port_operation op_data(src, reg_pred); - my_aggregator.execute(&op_data); - return op_data.status == SUCCEEDED; - } - - //! Remove a predecessor - bool remove_predecessor( predecessor_type &src ) __TBB_override { - reserving_port_operation op_data(src, rem_pred); - my_aggregator.execute(&op_data); - return op_data.status == SUCCEEDED; - } - - //! Reserve an item from the port - bool reserve( T &v ) { - reserving_port_operation op_data(v, res_item); - my_aggregator.execute(&op_data); - return op_data.status == SUCCEEDED; - } - - //! Release the port - void release( ) { - reserving_port_operation op_data(rel_res); - my_aggregator.execute(&op_data); - } - - //! Complete use of the port - void consume( ) { - reserving_port_operation op_data(con_res); - my_aggregator.execute(&op_data); - } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - built_predecessors_type &built_predecessors() __TBB_override { return my_predecessors.built_predecessors(); } - void internal_add_built_predecessor(predecessor_type &src) __TBB_override { - reserving_port_operation op_data(src, add_blt_pred); - my_aggregator.execute(&op_data); - } - - void internal_delete_built_predecessor(predecessor_type &src) __TBB_override { - reserving_port_operation op_data(src, del_blt_pred); - my_aggregator.execute(&op_data); - } - - size_t predecessor_count() __TBB_override { - reserving_port_operation op_data(blt_pred_cnt); - my_aggregator.execute(&op_data); - return op_data.cnt_val; - } - - void copy_predecessors(predecessor_list_type &l) __TBB_override { - reserving_port_operation op_data(blt_pred_cpy); - op_data.plist = &l; - my_aggregator.execute(&op_data); - } - - void extract_receiver() { - my_predecessors.built_predecessors().receiver_extract(*this); - } - -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ - - void reset_receiver( reset_flags f) __TBB_override { - if(f & rf_clear_edges) my_predecessors.clear(); - else - my_predecessors.reset(); - reserved = false; - __TBB_ASSERT(!(f&rf_clear_edges) || my_predecessors.empty(), "port edges not removed"); - } - - private: - forwarding_base *my_join; - reservable_predecessor_cache< T, null_mutex > my_predecessors; - bool reserved; - }; // reserving_port - - //! queueing join_port - template - class queueing_port : public receiver, public item_buffer { - public: - typedef T input_type; - typedef typename receiver::predecessor_type predecessor_type; - typedef queueing_port class_type; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - typedef typename receiver::built_predecessors_type built_predecessors_type; - typedef typename receiver::predecessor_list_type predecessor_list_type; -#endif - - // ----------- Aggregator ------------ - private: - enum op_type { get__item, res_port, try__put_task -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - , add_blt_pred, del_blt_pred, blt_pred_cnt, blt_pred_cpy -#endif - }; - enum op_stat {WAIT=0, SUCCEEDED, FAILED}; - - class queueing_port_operation : public aggregated_operation { - public: - char type; - T my_val; - T *my_arg; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - predecessor_type *pred; - size_t cnt_val; - predecessor_list_type *plist; -#endif - task * bypass_t; - // constructor for value parameter - queueing_port_operation(const T& e, op_type t) : - type(char(t)), my_val(e) - , bypass_t(NULL) - {} - // constructor for pointer parameter - queueing_port_operation(const T* p, op_type t) : - type(char(t)), my_arg(const_cast(p)) - , bypass_t(NULL) - {} - // constructor with no parameter - queueing_port_operation(op_type t) : type(char(t)) - , bypass_t(NULL) - {} - }; - - typedef internal::aggregating_functor handler_type; - friend class internal::aggregating_functor; - aggregator my_aggregator; - - void handle_operations(queueing_port_operation* op_list) { - queueing_port_operation *current; - bool was_empty; - while(op_list) { - current = op_list; - op_list = op_list->next; - switch(current->type) { - case try__put_task: { - task *rtask = NULL; - was_empty = this->buffer_empty(); - this->push_back(current->my_val); - if (was_empty) rtask = my_join->decrement_port_count(false); - else - rtask = SUCCESSFULLY_ENQUEUED; - current->bypass_t = rtask; - __TBB_store_with_release(current->status, SUCCEEDED); - } - break; - case get__item: - if(!this->buffer_empty()) { - *(current->my_arg) = this->front(); - __TBB_store_with_release(current->status, SUCCEEDED); - } - else { - __TBB_store_with_release(current->status, FAILED); - } - break; - case res_port: - __TBB_ASSERT(this->my_item_valid(this->my_head), "No item to reset"); - this->destroy_front(); - if(this->my_item_valid(this->my_head)) { - (void)my_join->decrement_port_count(true); - } - __TBB_store_with_release(current->status, SUCCEEDED); - break; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - case add_blt_pred: - my_built_predecessors.add_edge(*(current->pred)); - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case del_blt_pred: - my_built_predecessors.delete_edge(*(current->pred)); - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case blt_pred_cnt: - current->cnt_val = my_built_predecessors.edge_count(); - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case blt_pred_cpy: - my_built_predecessors.copy_edges(*(current->plist)); - __TBB_store_with_release(current->status, SUCCEEDED); - break; -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ - } - } - } - // ------------ End Aggregator --------------- - - protected: - template< typename R, typename B > friend class run_and_put_task; - template friend class internal::broadcast_cache; - template friend class internal::round_robin_cache; - task *try_put_task(const T &v) __TBB_override { - queueing_port_operation op_data(v, try__put_task); - my_aggregator.execute(&op_data); - __TBB_ASSERT(op_data.status == SUCCEEDED || !op_data.bypass_t, "inconsistent return from aggregator"); - if(!op_data.bypass_t) return SUCCESSFULLY_ENQUEUED; - return op_data.bypass_t; - } - - graph& graph_reference() __TBB_override { - return my_join->graph_ref; - } - - public: - - //! Constructor - queueing_port() : item_buffer() { - my_join = NULL; - my_aggregator.initialize_handler(handler_type(this)); - } - - //! copy constructor - queueing_port(const queueing_port& /* other */) : receiver(), item_buffer() { - my_join = NULL; - my_aggregator.initialize_handler(handler_type(this)); - } - - //! record parent for tallying available items - void set_join_node_pointer(forwarding_base *join) { - my_join = join; - } - - bool get_item( T &v ) { - queueing_port_operation op_data(&v, get__item); - my_aggregator.execute(&op_data); - return op_data.status == SUCCEEDED; - } - - // reset_port is called when item is accepted by successor, but - // is initiated by join_node. - void reset_port() { - queueing_port_operation op_data(res_port); - my_aggregator.execute(&op_data); - return; - } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - built_predecessors_type &built_predecessors() __TBB_override { return my_built_predecessors; } - - void internal_add_built_predecessor(predecessor_type &p) __TBB_override { - queueing_port_operation op_data(add_blt_pred); - op_data.pred = &p; - my_aggregator.execute(&op_data); - } - - void internal_delete_built_predecessor(predecessor_type &p) __TBB_override { - queueing_port_operation op_data(del_blt_pred); - op_data.pred = &p; - my_aggregator.execute(&op_data); - } - - size_t predecessor_count() __TBB_override { - queueing_port_operation op_data(blt_pred_cnt); - my_aggregator.execute(&op_data); - return op_data.cnt_val; - } - - void copy_predecessors(predecessor_list_type &l) __TBB_override { - queueing_port_operation op_data(blt_pred_cpy); - op_data.plist = &l; - my_aggregator.execute(&op_data); - } - - void extract_receiver() { - item_buffer::reset(); - my_built_predecessors.receiver_extract(*this); - } -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ - - void reset_receiver(reset_flags f) __TBB_override { - tbb::internal::suppress_unused_warning(f); - item_buffer::reset(); -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - if (f & rf_clear_edges) - my_built_predecessors.clear(); -#endif - } - - private: - forwarding_base *my_join; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - edge_container my_built_predecessors; -#endif - }; // queueing_port - -#include "_flow_graph_tagged_buffer_impl.h" - - template - struct count_element { - K my_key; - size_t my_value; - }; - - // method to access the key in the counting table - // the ref has already been removed from K - template< typename K > - struct key_to_count_functor { - typedef count_element table_item_type; - const K& operator()(const table_item_type& v) { return v.my_key; } - }; - - // the ports can have only one template parameter. We wrap the types needed in - // a traits type - template< class TraitsType > - class key_matching_port : - public receiver, - public hash_buffer< typename TraitsType::K, typename TraitsType::T, typename TraitsType::TtoK, - typename TraitsType::KHash > { - public: - typedef TraitsType traits; - typedef key_matching_port class_type; - typedef typename TraitsType::T input_type; - typedef typename TraitsType::K key_type; - typedef typename tbb::internal::strip::type noref_key_type; - typedef typename receiver::predecessor_type predecessor_type; - typedef typename TraitsType::TtoK type_to_key_func_type; - typedef typename TraitsType::KHash hash_compare_type; - typedef hash_buffer< key_type, input_type, type_to_key_func_type, hash_compare_type > buffer_type; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - typedef typename receiver::built_predecessors_type built_predecessors_type; - typedef typename receiver::predecessor_list_type predecessor_list_type; -#endif - private: -// ----------- Aggregator ------------ - private: - enum op_type { try__put, get__item, res_port -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - , add_blt_pred, del_blt_pred, blt_pred_cnt, blt_pred_cpy -#endif - }; - enum op_stat {WAIT=0, SUCCEEDED, FAILED}; - - class key_matching_port_operation : public aggregated_operation { - public: - char type; - input_type my_val; - input_type *my_arg; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - predecessor_type *pred; - size_t cnt_val; - predecessor_list_type *plist; -#endif - // constructor for value parameter - key_matching_port_operation(const input_type& e, op_type t) : - type(char(t)), my_val(e) {} - // constructor for pointer parameter - key_matching_port_operation(const input_type* p, op_type t) : - type(char(t)), my_arg(const_cast(p)) {} - // constructor with no parameter - key_matching_port_operation(op_type t) : type(char(t)) {} - }; - - typedef internal::aggregating_functor handler_type; - friend class internal::aggregating_functor; - aggregator my_aggregator; - - void handle_operations(key_matching_port_operation* op_list) { - key_matching_port_operation *current; - while(op_list) { - current = op_list; - op_list = op_list->next; - switch(current->type) { - case try__put: { - bool was_inserted = this->insert_with_key(current->my_val); - // return failure if a duplicate insertion occurs - __TBB_store_with_release(current->status, was_inserted ? SUCCEEDED : FAILED); - } - break; - case get__item: - // use current_key from FE for item - if(!this->find_with_key(my_join->current_key, *(current->my_arg))) { - __TBB_ASSERT(false, "Failed to find item corresponding to current_key."); - } - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case res_port: - // use current_key from FE for item - this->delete_with_key(my_join->current_key); - __TBB_store_with_release(current->status, SUCCEEDED); - break; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - case add_blt_pred: - my_built_predecessors.add_edge(*(current->pred)); - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case del_blt_pred: - my_built_predecessors.delete_edge(*(current->pred)); - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case blt_pred_cnt: - current->cnt_val = my_built_predecessors.edge_count(); - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case blt_pred_cpy: - my_built_predecessors.copy_edges(*(current->plist)); - __TBB_store_with_release(current->status, SUCCEEDED); - break; -#endif - } - } - } -// ------------ End Aggregator --------------- - protected: - template< typename R, typename B > friend class run_and_put_task; - template friend class internal::broadcast_cache; - template friend class internal::round_robin_cache; - task *try_put_task(const input_type& v) __TBB_override { - key_matching_port_operation op_data(v, try__put); - task *rtask = NULL; - my_aggregator.execute(&op_data); - if(op_data.status == SUCCEEDED) { - rtask = my_join->increment_key_count((*(this->get_key_func()))(v), false); // may spawn - // rtask has to reflect the return status of the try_put - if(!rtask) rtask = SUCCESSFULLY_ENQUEUED; - } - return rtask; - } - - graph& graph_reference() __TBB_override { - return my_join->graph_ref; - } - - public: - - key_matching_port() : receiver(), buffer_type() { - my_join = NULL; - my_aggregator.initialize_handler(handler_type(this)); - } - - // copy constructor - key_matching_port(const key_matching_port& /*other*/) : receiver(), buffer_type() { - my_join = NULL; - my_aggregator.initialize_handler(handler_type(this)); - } - - ~key_matching_port() { } - - void set_join_node_pointer(forwarding_base *join) { - my_join = dynamic_cast*>(join); - } - - void set_my_key_func(type_to_key_func_type *f) { this->set_key_func(f); } - - type_to_key_func_type* get_my_key_func() { return this->get_key_func(); } - - bool get_item( input_type &v ) { - // aggregator uses current_key from FE for Key - key_matching_port_operation op_data(&v, get__item); - my_aggregator.execute(&op_data); - return op_data.status == SUCCEEDED; - } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - built_predecessors_type &built_predecessors() __TBB_override { return my_built_predecessors; } - - void internal_add_built_predecessor(predecessor_type &p) __TBB_override { - key_matching_port_operation op_data(add_blt_pred); - op_data.pred = &p; - my_aggregator.execute(&op_data); - } - - void internal_delete_built_predecessor(predecessor_type &p) __TBB_override { - key_matching_port_operation op_data(del_blt_pred); - op_data.pred = &p; - my_aggregator.execute(&op_data); - } - - size_t predecessor_count() __TBB_override { - key_matching_port_operation op_data(blt_pred_cnt); - my_aggregator.execute(&op_data); - return op_data.cnt_val; - } - - void copy_predecessors(predecessor_list_type &l) __TBB_override { - key_matching_port_operation op_data(blt_pred_cpy); - op_data.plist = &l; - my_aggregator.execute(&op_data); - } -#endif - - // reset_port is called when item is accepted by successor, but - // is initiated by join_node. - void reset_port() { - key_matching_port_operation op_data(res_port); - my_aggregator.execute(&op_data); - return; - } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - void extract_receiver() { - buffer_type::reset(); - my_built_predecessors.receiver_extract(*this); - } -#endif - void reset_receiver(reset_flags f ) __TBB_override { - tbb::internal::suppress_unused_warning(f); - buffer_type::reset(); -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - if (f & rf_clear_edges) - my_built_predecessors.clear(); -#endif - } - - private: - // my_join forwarding base used to count number of inputs that - // received key. - matching_forwarding_base *my_join; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - edge_container my_built_predecessors; -#endif - }; // key_matching_port - - using namespace graph_policy_namespace; - - template - class join_node_base; - - //! join_node_FE : implements input port policy - template - class join_node_FE; - - template - class join_node_FE : public forwarding_base { - public: - static const int N = tbb::flow::tuple_size::value; - typedef OutputTuple output_type; - typedef InputTuple input_type; - typedef join_node_base base_node_type; // for forwarding - - join_node_FE(graph &g) : forwarding_base(g), my_node(NULL) { - ports_with_no_inputs = N; - join_helper::set_join_node_pointer(my_inputs, this); - } - - join_node_FE(const join_node_FE& other) : forwarding_base((other.forwarding_base::graph_ref)), my_node(NULL) { - ports_with_no_inputs = N; - join_helper::set_join_node_pointer(my_inputs, this); - } - - void set_my_node(base_node_type *new_my_node) { my_node = new_my_node; } - - void increment_port_count() __TBB_override { - ++ports_with_no_inputs; - } - - // if all input_ports have predecessors, spawn forward to try and consume tuples - task * decrement_port_count(bool handle_task) __TBB_override { - if(ports_with_no_inputs.fetch_and_decrement() == 1) { - if(internal::is_graph_active(this->graph_ref)) { - task *rtask = new ( task::allocate_additional_child_of( *(this->graph_ref.root_task()) ) ) - forward_task_bypass(*my_node); - if(!handle_task) return rtask; - internal::spawn_in_graph_arena(this->graph_ref, *rtask); - } - } - return NULL; - } - - input_type &input_ports() { return my_inputs; } - - protected: - - void reset( reset_flags f) { - // called outside of parallel contexts - ports_with_no_inputs = N; - join_helper::reset_inputs(my_inputs, f); - } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - void extract( ) { - // called outside of parallel contexts - ports_with_no_inputs = N; - join_helper::extract_inputs(my_inputs); - } -#endif - - // all methods on input ports should be called under mutual exclusion from join_node_base. - - bool tuple_build_may_succeed() { - return !ports_with_no_inputs; - } - - bool try_to_make_tuple(output_type &out) { - if(ports_with_no_inputs) return false; - return join_helper::reserve(my_inputs, out); - } - - void tuple_accepted() { - join_helper::consume_reservations(my_inputs); - } - void tuple_rejected() { - join_helper::release_reservations(my_inputs); - } - - input_type my_inputs; - base_node_type *my_node; - atomic ports_with_no_inputs; - }; // join_node_FE - - template - class join_node_FE : public forwarding_base { - public: - static const int N = tbb::flow::tuple_size::value; - typedef OutputTuple output_type; - typedef InputTuple input_type; - typedef join_node_base base_node_type; // for forwarding - - join_node_FE(graph &g) : forwarding_base(g), my_node(NULL) { - ports_with_no_items = N; - join_helper::set_join_node_pointer(my_inputs, this); - } - - join_node_FE(const join_node_FE& other) : forwarding_base((other.forwarding_base::graph_ref)), my_node(NULL) { - ports_with_no_items = N; - join_helper::set_join_node_pointer(my_inputs, this); - } - - // needed for forwarding - void set_my_node(base_node_type *new_my_node) { my_node = new_my_node; } - - void reset_port_count() { - ports_with_no_items = N; - } - - // if all input_ports have items, spawn forward to try and consume tuples - task * decrement_port_count(bool handle_task) __TBB_override - { - if(ports_with_no_items.fetch_and_decrement() == 1) { - if(internal::is_graph_active(this->graph_ref)) { - task *rtask = new ( task::allocate_additional_child_of( *(this->graph_ref.root_task()) ) ) - forward_task_bypass (*my_node); - if(!handle_task) return rtask; - internal::spawn_in_graph_arena(this->graph_ref, *rtask); - } - } - return NULL; - } - - void increment_port_count() __TBB_override { __TBB_ASSERT(false, NULL); } // should never be called - - input_type &input_ports() { return my_inputs; } - - protected: - - void reset( reset_flags f) { - reset_port_count(); - join_helper::reset_inputs(my_inputs, f ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - void extract() { - reset_port_count(); - join_helper::extract_inputs(my_inputs); - } -#endif - // all methods on input ports should be called under mutual exclusion from join_node_base. - - bool tuple_build_may_succeed() { - return !ports_with_no_items; - } - - bool try_to_make_tuple(output_type &out) { - if(ports_with_no_items) return false; - return join_helper::get_items(my_inputs, out); - } - - void tuple_accepted() { - reset_port_count(); - join_helper::reset_ports(my_inputs); - } - void tuple_rejected() { - // nothing to do. - } - - input_type my_inputs; - base_node_type *my_node; - atomic ports_with_no_items; - }; // join_node_FE - - // key_matching join front-end. - template - class join_node_FE, InputTuple, OutputTuple> : public matching_forwarding_base, - // buffer of key value counts - public hash_buffer< // typedefed below to key_to_count_buffer_type - typename tbb::internal::strip::type&, // force ref type on K - count_element::type>, - internal::type_to_key_function_body< - count_element::type>, - typename tbb::internal::strip::type& >, - KHash >, - // buffer of output items - public item_buffer { - public: - static const int N = tbb::flow::tuple_size::value; - typedef OutputTuple output_type; - typedef InputTuple input_type; - typedef K key_type; - typedef typename tbb::internal::strip::type unref_key_type; - typedef KHash key_hash_compare; - // must use K without ref. - typedef count_element count_element_type; - // method that lets us refer to the key of this type. - typedef key_to_count_functor key_to_count_func; - typedef internal::type_to_key_function_body< count_element_type, unref_key_type&> TtoK_function_body_type; - typedef internal::type_to_key_function_body_leaf TtoK_function_body_leaf_type; - // this is the type of the special table that keeps track of the number of discrete - // elements corresponding to each key that we've seen. - typedef hash_buffer< unref_key_type&, count_element_type, TtoK_function_body_type, key_hash_compare > - key_to_count_buffer_type; - typedef item_buffer output_buffer_type; - typedef join_node_base, InputTuple, OutputTuple> base_node_type; // for forwarding - typedef matching_forwarding_base forwarding_base_type; - -// ----------- Aggregator ------------ - // the aggregator is only needed to serialize the access to the hash table. - // and the output_buffer_type base class - private: - enum op_type { res_count, inc_count, may_succeed, try_make }; - enum op_stat {WAIT=0, SUCCEEDED, FAILED}; - typedef join_node_FE, InputTuple, OutputTuple> class_type; - - class key_matching_FE_operation : public aggregated_operation { - public: - char type; - unref_key_type my_val; - output_type* my_output; - task *bypass_t; - bool enqueue_task; - // constructor for value parameter - key_matching_FE_operation(const unref_key_type& e , bool q_task , op_type t) : type(char(t)), my_val(e), - my_output(NULL), bypass_t(NULL), enqueue_task(q_task) {} - key_matching_FE_operation(output_type *p, op_type t) : type(char(t)), my_output(p), bypass_t(NULL), - enqueue_task(true) {} - // constructor with no parameter - key_matching_FE_operation(op_type t) : type(char(t)), my_output(NULL), bypass_t(NULL), enqueue_task(true) {} - }; - - typedef internal::aggregating_functor handler_type; - friend class internal::aggregating_functor; - aggregator my_aggregator; - - // called from aggregator, so serialized - // returns a task pointer if the a task would have been enqueued but we asked that - // it be returned. Otherwise returns NULL. - task * fill_output_buffer(unref_key_type &t, bool should_enqueue, bool handle_task) { - output_type l_out; - task *rtask = NULL; - bool do_fwd = should_enqueue && this->buffer_empty() && internal::is_graph_active(this->graph_ref); - this->current_key = t; - this->delete_with_key(this->current_key); // remove the key - if(join_helper::get_items(my_inputs, l_out)) { // <== call back - this->push_back(l_out); - if(do_fwd) { // we enqueue if receiving an item from predecessor, not if successor asks for item - rtask = new ( task::allocate_additional_child_of( *(this->graph_ref.root_task()) ) ) - forward_task_bypass(*my_node); - if(handle_task) { - internal::spawn_in_graph_arena(this->graph_ref, *rtask); - rtask = NULL; - } - do_fwd = false; - } - // retire the input values - join_helper::reset_ports(my_inputs); // <== call back - } - else { - __TBB_ASSERT(false, "should have had something to push"); - } - return rtask; - } - - void handle_operations(key_matching_FE_operation* op_list) { - key_matching_FE_operation *current; - while(op_list) { - current = op_list; - op_list = op_list->next; - switch(current->type) { - case res_count: // called from BE - { - this->destroy_front(); - __TBB_store_with_release(current->status, SUCCEEDED); - } - break; - case inc_count: { // called from input ports - count_element_type *p = 0; - unref_key_type &t = current->my_val; - bool do_enqueue = current->enqueue_task; - if(!(this->find_ref_with_key(t,p))) { - count_element_type ev; - ev.my_key = t; - ev.my_value = 0; - this->insert_with_key(ev); - if(!(this->find_ref_with_key(t,p))) { - __TBB_ASSERT(false, "should find key after inserting it"); - } - } - if(++(p->my_value) == size_t(N)) { - task *rtask = fill_output_buffer(t, true, do_enqueue); - __TBB_ASSERT(!rtask || !do_enqueue, "task should not be returned"); - current->bypass_t = rtask; - } - } - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case may_succeed: // called from BE - __TBB_store_with_release(current->status, this->buffer_empty() ? FAILED : SUCCEEDED); - break; - case try_make: // called from BE - if(this->buffer_empty()) { - __TBB_store_with_release(current->status, FAILED); - } - else { - *(current->my_output) = this->front(); - __TBB_store_with_release(current->status, SUCCEEDED); - } - break; - } - } - } -// ------------ End Aggregator --------------- - - public: - template - join_node_FE(graph &g, FunctionTuple &TtoK_funcs) : forwarding_base_type(g), my_node(NULL) { - join_helper::set_join_node_pointer(my_inputs, this); - join_helper::set_key_functors(my_inputs, TtoK_funcs); - my_aggregator.initialize_handler(handler_type(this)); - TtoK_function_body_type *cfb = new TtoK_function_body_leaf_type(key_to_count_func()); - this->set_key_func(cfb); - } - - join_node_FE(const join_node_FE& other) : forwarding_base_type((other.forwarding_base_type::graph_ref)), key_to_count_buffer_type(), - output_buffer_type() { - my_node = NULL; - join_helper::set_join_node_pointer(my_inputs, this); - join_helper::copy_key_functors(my_inputs, const_cast(other.my_inputs)); - my_aggregator.initialize_handler(handler_type(this)); - TtoK_function_body_type *cfb = new TtoK_function_body_leaf_type(key_to_count_func()); - this->set_key_func(cfb); - } - - // needed for forwarding - void set_my_node(base_node_type *new_my_node) { my_node = new_my_node; } - - void reset_port_count() { // called from BE - key_matching_FE_operation op_data(res_count); - my_aggregator.execute(&op_data); - return; - } - - // if all input_ports have items, spawn forward to try and consume tuples - // return a task if we are asked and did create one. - task *increment_key_count(unref_key_type const & t, bool handle_task) __TBB_override { // called from input_ports - key_matching_FE_operation op_data(t, handle_task, inc_count); - my_aggregator.execute(&op_data); - return op_data.bypass_t; - } - - task *decrement_port_count(bool /*handle_task*/) __TBB_override { __TBB_ASSERT(false, NULL); return NULL; } - - void increment_port_count() __TBB_override { __TBB_ASSERT(false, NULL); } // should never be called - - input_type &input_ports() { return my_inputs; } - - protected: - - void reset( reset_flags f ) { - // called outside of parallel contexts - join_helper::reset_inputs(my_inputs, f); - - key_to_count_buffer_type::reset(); - output_buffer_type::reset(); - } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - void extract() { - // called outside of parallel contexts - join_helper::extract_inputs(my_inputs); - key_to_count_buffer_type::reset(); // have to reset the tag counts - output_buffer_type::reset(); // also the queue of outputs - // my_node->current_tag = NO_TAG; - } -#endif - // all methods on input ports should be called under mutual exclusion from join_node_base. - - bool tuple_build_may_succeed() { // called from back-end - key_matching_FE_operation op_data(may_succeed); - my_aggregator.execute(&op_data); - return op_data.status == SUCCEEDED; - } - - // cannot lock while calling back to input_ports. current_key will only be set - // and reset under the aggregator, so it will remain consistent. - bool try_to_make_tuple(output_type &out) { - key_matching_FE_operation op_data(&out,try_make); - my_aggregator.execute(&op_data); - return op_data.status == SUCCEEDED; - } - - void tuple_accepted() { - reset_port_count(); // reset current_key after ports reset. - } - - void tuple_rejected() { - // nothing to do. - } - - input_type my_inputs; // input ports - base_node_type *my_node; - }; // join_node_FE, InputTuple, OutputTuple> - - //! join_node_base - template - class join_node_base : public graph_node, public join_node_FE, - public sender { - protected: - using graph_node::my_graph; - public: - typedef OutputTuple output_type; - - typedef typename sender::successor_type successor_type; - typedef join_node_FE input_ports_type; - using input_ports_type::tuple_build_may_succeed; - using input_ports_type::try_to_make_tuple; - using input_ports_type::tuple_accepted; - using input_ports_type::tuple_rejected; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - typedef typename sender::built_successors_type built_successors_type; - typedef typename sender::successor_list_type successor_list_type; -#endif - - private: - // ----------- Aggregator ------------ - enum op_type { reg_succ, rem_succ, try__get, do_fwrd, do_fwrd_bypass -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - , add_blt_succ, del_blt_succ, blt_succ_cnt, blt_succ_cpy -#endif - }; - enum op_stat {WAIT=0, SUCCEEDED, FAILED}; - typedef join_node_base class_type; - - class join_node_base_operation : public aggregated_operation { - public: - char type; - union { - output_type *my_arg; - successor_type *my_succ; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - size_t cnt_val; - successor_list_type *slist; -#endif - }; - task *bypass_t; - join_node_base_operation(const output_type& e, op_type t) : type(char(t)), - my_arg(const_cast(&e)), bypass_t(NULL) {} - join_node_base_operation(const successor_type &s, op_type t) : type(char(t)), - my_succ(const_cast(&s)), bypass_t(NULL) {} - join_node_base_operation(op_type t) : type(char(t)), bypass_t(NULL) {} - }; - - typedef internal::aggregating_functor handler_type; - friend class internal::aggregating_functor; - bool forwarder_busy; - aggregator my_aggregator; - - void handle_operations(join_node_base_operation* op_list) { - join_node_base_operation *current; - while(op_list) { - current = op_list; - op_list = op_list->next; - switch(current->type) { - case reg_succ: { - my_successors.register_successor(*(current->my_succ)); - if(tuple_build_may_succeed() && !forwarder_busy && internal::is_graph_active(my_graph)) { - task *rtask = new ( task::allocate_additional_child_of(*(my_graph.root_task())) ) - forward_task_bypass - >(*this); - internal::spawn_in_graph_arena(my_graph, *rtask); - forwarder_busy = true; - } - __TBB_store_with_release(current->status, SUCCEEDED); - } - break; - case rem_succ: - my_successors.remove_successor(*(current->my_succ)); - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case try__get: - if(tuple_build_may_succeed()) { - if(try_to_make_tuple(*(current->my_arg))) { - tuple_accepted(); - __TBB_store_with_release(current->status, SUCCEEDED); - } - else __TBB_store_with_release(current->status, FAILED); - } - else __TBB_store_with_release(current->status, FAILED); - break; - case do_fwrd_bypass: { - bool build_succeeded; - task *last_task = NULL; - output_type out; - if(tuple_build_may_succeed()) { // checks output queue of FE - do { - build_succeeded = try_to_make_tuple(out); // fetch front_end of queue - if(build_succeeded) { - task *new_task = my_successors.try_put_task(out); - last_task = combine_tasks(my_graph, last_task, new_task); - if(new_task) { - tuple_accepted(); - } - else { - tuple_rejected(); - build_succeeded = false; - } - } - } while(build_succeeded); - } - current->bypass_t = last_task; - __TBB_store_with_release(current->status, SUCCEEDED); - forwarder_busy = false; - } - break; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - case add_blt_succ: - my_successors.internal_add_built_successor(*(current->my_succ)); - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case del_blt_succ: - my_successors.internal_delete_built_successor(*(current->my_succ)); - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case blt_succ_cnt: - current->cnt_val = my_successors.successor_count(); - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case blt_succ_cpy: - my_successors.copy_successors(*(current->slist)); - __TBB_store_with_release(current->status, SUCCEEDED); - break; -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ - } - } - } - // ---------- end aggregator ----------- - public: - join_node_base(graph &g) : graph_node(g), input_ports_type(g), forwarder_busy(false) { - my_successors.set_owner(this); - input_ports_type::set_my_node(this); - my_aggregator.initialize_handler(handler_type(this)); - } - - join_node_base(const join_node_base& other) : - graph_node(other.graph_node::my_graph), input_ports_type(other), - sender(), forwarder_busy(false), my_successors() { - my_successors.set_owner(this); - input_ports_type::set_my_node(this); - my_aggregator.initialize_handler(handler_type(this)); - } - - template - join_node_base(graph &g, FunctionTuple f) : graph_node(g), input_ports_type(g, f), forwarder_busy(false) { - my_successors.set_owner(this); - input_ports_type::set_my_node(this); - my_aggregator.initialize_handler(handler_type(this)); - } - - bool register_successor(successor_type &r) __TBB_override { - join_node_base_operation op_data(r, reg_succ); - my_aggregator.execute(&op_data); - return op_data.status == SUCCEEDED; - } - - bool remove_successor( successor_type &r) __TBB_override { - join_node_base_operation op_data(r, rem_succ); - my_aggregator.execute(&op_data); - return op_data.status == SUCCEEDED; - } - - bool try_get( output_type &v) __TBB_override { - join_node_base_operation op_data(v, try__get); - my_aggregator.execute(&op_data); - return op_data.status == SUCCEEDED; - } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - built_successors_type &built_successors() __TBB_override { return my_successors.built_successors(); } - - void internal_add_built_successor( successor_type &r) __TBB_override { - join_node_base_operation op_data(r, add_blt_succ); - my_aggregator.execute(&op_data); - } - - void internal_delete_built_successor( successor_type &r) __TBB_override { - join_node_base_operation op_data(r, del_blt_succ); - my_aggregator.execute(&op_data); - } - - size_t successor_count() __TBB_override { - join_node_base_operation op_data(blt_succ_cnt); - my_aggregator.execute(&op_data); - return op_data.cnt_val; - } - - void copy_successors(successor_list_type &l) __TBB_override { - join_node_base_operation op_data(blt_succ_cpy); - op_data.slist = &l; - my_aggregator.execute(&op_data); - } -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - void extract() __TBB_override { - input_ports_type::extract(); - my_successors.built_successors().sender_extract(*this); - } -#endif - - protected: - - void reset_node(reset_flags f) __TBB_override { - input_ports_type::reset(f); - if(f & rf_clear_edges) my_successors.clear(); - } - - private: - broadcast_cache my_successors; - - friend class forward_task_bypass< join_node_base >; - task *forward_task() { - join_node_base_operation op_data(do_fwrd_bypass); - my_aggregator.execute(&op_data); - return op_data.bypass_t; - } - - }; // join_node_base - - // join base class type generator - template class PT, typename OutputTuple, typename JP> - struct join_base { - typedef typename internal::join_node_base::type, OutputTuple> type; - }; - - template - struct join_base > { - typedef key_matching key_traits_type; - typedef K key_type; - typedef KHash key_hash_compare; - typedef typename internal::join_node_base< key_traits_type, - // ports type - typename wrap_key_tuple_elements::type, - OutputTuple > type; - }; - - //! unfolded_join_node : passes input_ports_type to join_node_base. We build the input port type - // using tuple_element. The class PT is the port type (reserving_port, queueing_port, key_matching_port) - // and should match the typename. - - template class PT, typename OutputTuple, typename JP> - class unfolded_join_node : public join_base::type { - public: - typedef typename wrap_tuple_elements::type input_ports_type; - typedef OutputTuple output_type; - private: - typedef join_node_base base_type; - public: - unfolded_join_node(graph &g) : base_type(g) {} - unfolded_join_node(const unfolded_join_node &other) : base_type(other) {} - }; - -#if __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING - template - struct key_from_message_body { - K operator()(const T& t) const { - using tbb::flow::key_from_message; - return key_from_message(t); - } - }; - // Adds const to reference type - template - struct key_from_message_body { - const K& operator()(const T& t) const { - using tbb::flow::key_from_message; - return key_from_message(t); - } - }; -#endif /* __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING */ - // key_matching unfolded_join_node. This must be a separate specialization because the constructors - // differ. - - template - class unfolded_join_node<2,key_matching_port,OutputTuple,key_matching > : public - join_base<2,key_matching_port,OutputTuple,key_matching >::type { - typedef typename tbb::flow::tuple_element<0, OutputTuple>::type T0; - typedef typename tbb::flow::tuple_element<1, OutputTuple>::type T1; - public: - typedef typename wrap_key_tuple_elements<2,key_matching_port,key_matching,OutputTuple>::type input_ports_type; - typedef OutputTuple output_type; - private: - typedef join_node_base, input_ports_type, output_type > base_type; - typedef typename internal::type_to_key_function_body *f0_p; - typedef typename internal::type_to_key_function_body *f1_p; - typedef typename tbb::flow::tuple< f0_p, f1_p > func_initializer_type; - public: -#if __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING - unfolded_join_node(graph &g) : base_type(g, - func_initializer_type( - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()) - ) ) { - } -#endif /* __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING */ - template - unfolded_join_node(graph &g, Body0 body0, Body1 body1) : base_type(g, - func_initializer_type( - new internal::type_to_key_function_body_leaf(body0), - new internal::type_to_key_function_body_leaf(body1) - ) ) { - __TBB_STATIC_ASSERT(tbb::flow::tuple_size::value == 2, "wrong number of body initializers"); - } - unfolded_join_node(const unfolded_join_node &other) : base_type(other) {} - }; - - template - class unfolded_join_node<3,key_matching_port,OutputTuple,key_matching > : public - join_base<3,key_matching_port,OutputTuple,key_matching >::type { - typedef typename tbb::flow::tuple_element<0, OutputTuple>::type T0; - typedef typename tbb::flow::tuple_element<1, OutputTuple>::type T1; - typedef typename tbb::flow::tuple_element<2, OutputTuple>::type T2; - public: - typedef typename wrap_key_tuple_elements<3,key_matching_port,key_matching,OutputTuple>::type input_ports_type; - typedef OutputTuple output_type; - private: - typedef join_node_base, input_ports_type, output_type > base_type; - typedef typename internal::type_to_key_function_body *f0_p; - typedef typename internal::type_to_key_function_body *f1_p; - typedef typename internal::type_to_key_function_body *f2_p; - typedef typename tbb::flow::tuple< f0_p, f1_p, f2_p > func_initializer_type; - public: -#if __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING - unfolded_join_node(graph &g) : base_type(g, - func_initializer_type( - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()) - ) ) { - } -#endif /* __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING */ - template - unfolded_join_node(graph &g, Body0 body0, Body1 body1, Body2 body2) : base_type(g, - func_initializer_type( - new internal::type_to_key_function_body_leaf(body0), - new internal::type_to_key_function_body_leaf(body1), - new internal::type_to_key_function_body_leaf(body2) - ) ) { - __TBB_STATIC_ASSERT(tbb::flow::tuple_size::value == 3, "wrong number of body initializers"); - } - unfolded_join_node(const unfolded_join_node &other) : base_type(other) {} - }; - - template - class unfolded_join_node<4,key_matching_port,OutputTuple,key_matching > : public - join_base<4,key_matching_port,OutputTuple,key_matching >::type { - typedef typename tbb::flow::tuple_element<0, OutputTuple>::type T0; - typedef typename tbb::flow::tuple_element<1, OutputTuple>::type T1; - typedef typename tbb::flow::tuple_element<2, OutputTuple>::type T2; - typedef typename tbb::flow::tuple_element<3, OutputTuple>::type T3; - public: - typedef typename wrap_key_tuple_elements<4,key_matching_port,key_matching,OutputTuple>::type input_ports_type; - typedef OutputTuple output_type; - private: - typedef join_node_base, input_ports_type, output_type > base_type; - typedef typename internal::type_to_key_function_body *f0_p; - typedef typename internal::type_to_key_function_body *f1_p; - typedef typename internal::type_to_key_function_body *f2_p; - typedef typename internal::type_to_key_function_body *f3_p; - typedef typename tbb::flow::tuple< f0_p, f1_p, f2_p, f3_p > func_initializer_type; - public: -#if __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING - unfolded_join_node(graph &g) : base_type(g, - func_initializer_type( - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()) - ) ) { - } -#endif /* __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING */ - template - unfolded_join_node(graph &g, Body0 body0, Body1 body1, Body2 body2, Body3 body3) : base_type(g, - func_initializer_type( - new internal::type_to_key_function_body_leaf(body0), - new internal::type_to_key_function_body_leaf(body1), - new internal::type_to_key_function_body_leaf(body2), - new internal::type_to_key_function_body_leaf(body3) - ) ) { - __TBB_STATIC_ASSERT(tbb::flow::tuple_size::value == 4, "wrong number of body initializers"); - } - unfolded_join_node(const unfolded_join_node &other) : base_type(other) {} - }; - - template - class unfolded_join_node<5,key_matching_port,OutputTuple,key_matching > : public - join_base<5,key_matching_port,OutputTuple,key_matching >::type { - typedef typename tbb::flow::tuple_element<0, OutputTuple>::type T0; - typedef typename tbb::flow::tuple_element<1, OutputTuple>::type T1; - typedef typename tbb::flow::tuple_element<2, OutputTuple>::type T2; - typedef typename tbb::flow::tuple_element<3, OutputTuple>::type T3; - typedef typename tbb::flow::tuple_element<4, OutputTuple>::type T4; - public: - typedef typename wrap_key_tuple_elements<5,key_matching_port,key_matching,OutputTuple>::type input_ports_type; - typedef OutputTuple output_type; - private: - typedef join_node_base , input_ports_type, output_type > base_type; - typedef typename internal::type_to_key_function_body *f0_p; - typedef typename internal::type_to_key_function_body *f1_p; - typedef typename internal::type_to_key_function_body *f2_p; - typedef typename internal::type_to_key_function_body *f3_p; - typedef typename internal::type_to_key_function_body *f4_p; - typedef typename tbb::flow::tuple< f0_p, f1_p, f2_p, f3_p, f4_p > func_initializer_type; - public: -#if __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING - unfolded_join_node(graph &g) : base_type(g, - func_initializer_type( - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()) - ) ) { - } -#endif /* __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING */ - template - unfolded_join_node(graph &g, Body0 body0, Body1 body1, Body2 body2, Body3 body3, Body4 body4) : base_type(g, - func_initializer_type( - new internal::type_to_key_function_body_leaf(body0), - new internal::type_to_key_function_body_leaf(body1), - new internal::type_to_key_function_body_leaf(body2), - new internal::type_to_key_function_body_leaf(body3), - new internal::type_to_key_function_body_leaf(body4) - ) ) { - __TBB_STATIC_ASSERT(tbb::flow::tuple_size::value == 5, "wrong number of body initializers"); - } - unfolded_join_node(const unfolded_join_node &other) : base_type(other) {} - }; - -#if __TBB_VARIADIC_MAX >= 6 - template - class unfolded_join_node<6,key_matching_port,OutputTuple,key_matching > : public - join_base<6,key_matching_port,OutputTuple,key_matching >::type { - typedef typename tbb::flow::tuple_element<0, OutputTuple>::type T0; - typedef typename tbb::flow::tuple_element<1, OutputTuple>::type T1; - typedef typename tbb::flow::tuple_element<2, OutputTuple>::type T2; - typedef typename tbb::flow::tuple_element<3, OutputTuple>::type T3; - typedef typename tbb::flow::tuple_element<4, OutputTuple>::type T4; - typedef typename tbb::flow::tuple_element<5, OutputTuple>::type T5; - public: - typedef typename wrap_key_tuple_elements<6,key_matching_port,key_matching,OutputTuple>::type input_ports_type; - typedef OutputTuple output_type; - private: - typedef join_node_base , input_ports_type, output_type > base_type; - typedef typename internal::type_to_key_function_body *f0_p; - typedef typename internal::type_to_key_function_body *f1_p; - typedef typename internal::type_to_key_function_body *f2_p; - typedef typename internal::type_to_key_function_body *f3_p; - typedef typename internal::type_to_key_function_body *f4_p; - typedef typename internal::type_to_key_function_body *f5_p; - typedef typename tbb::flow::tuple< f0_p, f1_p, f2_p, f3_p, f4_p, f5_p > func_initializer_type; - public: -#if __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING - unfolded_join_node(graph &g) : base_type(g, - func_initializer_type( - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()) - ) ) { - } -#endif /* __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING */ - template - unfolded_join_node(graph &g, Body0 body0, Body1 body1, Body2 body2, Body3 body3, Body4 body4, Body5 body5) - : base_type(g, func_initializer_type( - new internal::type_to_key_function_body_leaf(body0), - new internal::type_to_key_function_body_leaf(body1), - new internal::type_to_key_function_body_leaf(body2), - new internal::type_to_key_function_body_leaf(body3), - new internal::type_to_key_function_body_leaf(body4), - new internal::type_to_key_function_body_leaf(body5) - ) ) { - __TBB_STATIC_ASSERT(tbb::flow::tuple_size::value == 6, "wrong number of body initializers"); - } - unfolded_join_node(const unfolded_join_node &other) : base_type(other) {} - }; -#endif - -#if __TBB_VARIADIC_MAX >= 7 - template - class unfolded_join_node<7,key_matching_port,OutputTuple,key_matching > : public - join_base<7,key_matching_port,OutputTuple,key_matching >::type { - typedef typename tbb::flow::tuple_element<0, OutputTuple>::type T0; - typedef typename tbb::flow::tuple_element<1, OutputTuple>::type T1; - typedef typename tbb::flow::tuple_element<2, OutputTuple>::type T2; - typedef typename tbb::flow::tuple_element<3, OutputTuple>::type T3; - typedef typename tbb::flow::tuple_element<4, OutputTuple>::type T4; - typedef typename tbb::flow::tuple_element<5, OutputTuple>::type T5; - typedef typename tbb::flow::tuple_element<6, OutputTuple>::type T6; - public: - typedef typename wrap_key_tuple_elements<7,key_matching_port,key_matching,OutputTuple>::type input_ports_type; - typedef OutputTuple output_type; - private: - typedef join_node_base , input_ports_type, output_type > base_type; - typedef typename internal::type_to_key_function_body *f0_p; - typedef typename internal::type_to_key_function_body *f1_p; - typedef typename internal::type_to_key_function_body *f2_p; - typedef typename internal::type_to_key_function_body *f3_p; - typedef typename internal::type_to_key_function_body *f4_p; - typedef typename internal::type_to_key_function_body *f5_p; - typedef typename internal::type_to_key_function_body *f6_p; - typedef typename tbb::flow::tuple< f0_p, f1_p, f2_p, f3_p, f4_p, f5_p, f6_p > func_initializer_type; - public: -#if __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING - unfolded_join_node(graph &g) : base_type(g, - func_initializer_type( - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()) - ) ) { - } -#endif /* __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING */ - template - unfolded_join_node(graph &g, Body0 body0, Body1 body1, Body2 body2, Body3 body3, Body4 body4, - Body5 body5, Body6 body6) : base_type(g, func_initializer_type( - new internal::type_to_key_function_body_leaf(body0), - new internal::type_to_key_function_body_leaf(body1), - new internal::type_to_key_function_body_leaf(body2), - new internal::type_to_key_function_body_leaf(body3), - new internal::type_to_key_function_body_leaf(body4), - new internal::type_to_key_function_body_leaf(body5), - new internal::type_to_key_function_body_leaf(body6) - ) ) { - __TBB_STATIC_ASSERT(tbb::flow::tuple_size::value == 7, "wrong number of body initializers"); - } - unfolded_join_node(const unfolded_join_node &other) : base_type(other) {} - }; -#endif - -#if __TBB_VARIADIC_MAX >= 8 - template - class unfolded_join_node<8,key_matching_port,OutputTuple,key_matching > : public - join_base<8,key_matching_port,OutputTuple,key_matching >::type { - typedef typename tbb::flow::tuple_element<0, OutputTuple>::type T0; - typedef typename tbb::flow::tuple_element<1, OutputTuple>::type T1; - typedef typename tbb::flow::tuple_element<2, OutputTuple>::type T2; - typedef typename tbb::flow::tuple_element<3, OutputTuple>::type T3; - typedef typename tbb::flow::tuple_element<4, OutputTuple>::type T4; - typedef typename tbb::flow::tuple_element<5, OutputTuple>::type T5; - typedef typename tbb::flow::tuple_element<6, OutputTuple>::type T6; - typedef typename tbb::flow::tuple_element<7, OutputTuple>::type T7; - public: - typedef typename wrap_key_tuple_elements<8,key_matching_port,key_matching,OutputTuple>::type input_ports_type; - typedef OutputTuple output_type; - private: - typedef join_node_base , input_ports_type, output_type > base_type; - typedef typename internal::type_to_key_function_body *f0_p; - typedef typename internal::type_to_key_function_body *f1_p; - typedef typename internal::type_to_key_function_body *f2_p; - typedef typename internal::type_to_key_function_body *f3_p; - typedef typename internal::type_to_key_function_body *f4_p; - typedef typename internal::type_to_key_function_body *f5_p; - typedef typename internal::type_to_key_function_body *f6_p; - typedef typename internal::type_to_key_function_body *f7_p; - typedef typename tbb::flow::tuple< f0_p, f1_p, f2_p, f3_p, f4_p, f5_p, f6_p, f7_p > func_initializer_type; - public: -#if __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING - unfolded_join_node(graph &g) : base_type(g, - func_initializer_type( - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()) - ) ) { - } -#endif /* __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING */ - template - unfolded_join_node(graph &g, Body0 body0, Body1 body1, Body2 body2, Body3 body3, Body4 body4, - Body5 body5, Body6 body6, Body7 body7) : base_type(g, func_initializer_type( - new internal::type_to_key_function_body_leaf(body0), - new internal::type_to_key_function_body_leaf(body1), - new internal::type_to_key_function_body_leaf(body2), - new internal::type_to_key_function_body_leaf(body3), - new internal::type_to_key_function_body_leaf(body4), - new internal::type_to_key_function_body_leaf(body5), - new internal::type_to_key_function_body_leaf(body6), - new internal::type_to_key_function_body_leaf(body7) - ) ) { - __TBB_STATIC_ASSERT(tbb::flow::tuple_size::value == 8, "wrong number of body initializers"); - } - unfolded_join_node(const unfolded_join_node &other) : base_type(other) {} - }; -#endif - -#if __TBB_VARIADIC_MAX >= 9 - template - class unfolded_join_node<9,key_matching_port,OutputTuple,key_matching > : public - join_base<9,key_matching_port,OutputTuple,key_matching >::type { - typedef typename tbb::flow::tuple_element<0, OutputTuple>::type T0; - typedef typename tbb::flow::tuple_element<1, OutputTuple>::type T1; - typedef typename tbb::flow::tuple_element<2, OutputTuple>::type T2; - typedef typename tbb::flow::tuple_element<3, OutputTuple>::type T3; - typedef typename tbb::flow::tuple_element<4, OutputTuple>::type T4; - typedef typename tbb::flow::tuple_element<5, OutputTuple>::type T5; - typedef typename tbb::flow::tuple_element<6, OutputTuple>::type T6; - typedef typename tbb::flow::tuple_element<7, OutputTuple>::type T7; - typedef typename tbb::flow::tuple_element<8, OutputTuple>::type T8; - public: - typedef typename wrap_key_tuple_elements<9,key_matching_port,key_matching,OutputTuple>::type input_ports_type; - typedef OutputTuple output_type; - private: - typedef join_node_base , input_ports_type, output_type > base_type; - typedef typename internal::type_to_key_function_body *f0_p; - typedef typename internal::type_to_key_function_body *f1_p; - typedef typename internal::type_to_key_function_body *f2_p; - typedef typename internal::type_to_key_function_body *f3_p; - typedef typename internal::type_to_key_function_body *f4_p; - typedef typename internal::type_to_key_function_body *f5_p; - typedef typename internal::type_to_key_function_body *f6_p; - typedef typename internal::type_to_key_function_body *f7_p; - typedef typename internal::type_to_key_function_body *f8_p; - typedef typename tbb::flow::tuple< f0_p, f1_p, f2_p, f3_p, f4_p, f5_p, f6_p, f7_p, f8_p > func_initializer_type; - public: -#if __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING - unfolded_join_node(graph &g) : base_type(g, - func_initializer_type( - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()) - ) ) { - } -#endif /* __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING */ - template - unfolded_join_node(graph &g, Body0 body0, Body1 body1, Body2 body2, Body3 body3, Body4 body4, - Body5 body5, Body6 body6, Body7 body7, Body8 body8) : base_type(g, func_initializer_type( - new internal::type_to_key_function_body_leaf(body0), - new internal::type_to_key_function_body_leaf(body1), - new internal::type_to_key_function_body_leaf(body2), - new internal::type_to_key_function_body_leaf(body3), - new internal::type_to_key_function_body_leaf(body4), - new internal::type_to_key_function_body_leaf(body5), - new internal::type_to_key_function_body_leaf(body6), - new internal::type_to_key_function_body_leaf(body7), - new internal::type_to_key_function_body_leaf(body8) - ) ) { - __TBB_STATIC_ASSERT(tbb::flow::tuple_size::value == 9, "wrong number of body initializers"); - } - unfolded_join_node(const unfolded_join_node &other) : base_type(other) {} - }; -#endif - -#if __TBB_VARIADIC_MAX >= 10 - template - class unfolded_join_node<10,key_matching_port,OutputTuple,key_matching > : public - join_base<10,key_matching_port,OutputTuple,key_matching >::type { - typedef typename tbb::flow::tuple_element<0, OutputTuple>::type T0; - typedef typename tbb::flow::tuple_element<1, OutputTuple>::type T1; - typedef typename tbb::flow::tuple_element<2, OutputTuple>::type T2; - typedef typename tbb::flow::tuple_element<3, OutputTuple>::type T3; - typedef typename tbb::flow::tuple_element<4, OutputTuple>::type T4; - typedef typename tbb::flow::tuple_element<5, OutputTuple>::type T5; - typedef typename tbb::flow::tuple_element<6, OutputTuple>::type T6; - typedef typename tbb::flow::tuple_element<7, OutputTuple>::type T7; - typedef typename tbb::flow::tuple_element<8, OutputTuple>::type T8; - typedef typename tbb::flow::tuple_element<9, OutputTuple>::type T9; - public: - typedef typename wrap_key_tuple_elements<10,key_matching_port,key_matching,OutputTuple>::type input_ports_type; - typedef OutputTuple output_type; - private: - typedef join_node_base , input_ports_type, output_type > base_type; - typedef typename internal::type_to_key_function_body *f0_p; - typedef typename internal::type_to_key_function_body *f1_p; - typedef typename internal::type_to_key_function_body *f2_p; - typedef typename internal::type_to_key_function_body *f3_p; - typedef typename internal::type_to_key_function_body *f4_p; - typedef typename internal::type_to_key_function_body *f5_p; - typedef typename internal::type_to_key_function_body *f6_p; - typedef typename internal::type_to_key_function_body *f7_p; - typedef typename internal::type_to_key_function_body *f8_p; - typedef typename internal::type_to_key_function_body *f9_p; - typedef typename tbb::flow::tuple< f0_p, f1_p, f2_p, f3_p, f4_p, f5_p, f6_p, f7_p, f8_p, f9_p > func_initializer_type; - public: -#if __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING - unfolded_join_node(graph &g) : base_type(g, - func_initializer_type( - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()) - ) ) { - } -#endif /* __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING */ - template - unfolded_join_node(graph &g, Body0 body0, Body1 body1, Body2 body2, Body3 body3, Body4 body4, - Body5 body5, Body6 body6, Body7 body7, Body8 body8, Body9 body9) : base_type(g, func_initializer_type( - new internal::type_to_key_function_body_leaf(body0), - new internal::type_to_key_function_body_leaf(body1), - new internal::type_to_key_function_body_leaf(body2), - new internal::type_to_key_function_body_leaf(body3), - new internal::type_to_key_function_body_leaf(body4), - new internal::type_to_key_function_body_leaf(body5), - new internal::type_to_key_function_body_leaf(body6), - new internal::type_to_key_function_body_leaf(body7), - new internal::type_to_key_function_body_leaf(body8), - new internal::type_to_key_function_body_leaf(body9) - ) ) { - __TBB_STATIC_ASSERT(tbb::flow::tuple_size::value == 10, "wrong number of body initializers"); - } - unfolded_join_node(const unfolded_join_node &other) : base_type(other) {} - }; -#endif - - //! templated function to refer to input ports of the join node - template - typename tbb::flow::tuple_element::type &input_port(JNT &jn) { - return tbb::flow::get(jn.input_ports()); - } - -} -#endif // __TBB__flow_graph_join_impl_H - diff --git a/src/tbb/include/tbb/internal/_flow_graph_node_impl.h b/src/tbb/include/tbb/internal/_flow_graph_node_impl.h deleted file mode 100644 index 65a55b2de..000000000 --- a/src/tbb/include/tbb/internal/_flow_graph_node_impl.h +++ /dev/null @@ -1,829 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB__flow_graph_node_impl_H -#define __TBB__flow_graph_node_impl_H - -#ifndef __TBB_flow_graph_H -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#include "_flow_graph_item_buffer_impl.h" - -//! @cond INTERNAL -namespace internal { - - using tbb::internal::aggregated_operation; - using tbb::internal::aggregating_functor; - using tbb::internal::aggregator; - - template< typename T, typename A > - class function_input_queue : public item_buffer { - public: - bool empty() const { - return this->buffer_empty(); - } - - const T& front() const { - return this->item_buffer::front(); - } - - bool pop( T& t ) { - return this->pop_front( t ); - } - - void pop() { - this->destroy_front(); - } - - bool push( T& t ) { - return this->push_back( t ); - } - }; - - //! Input and scheduling for a function node that takes a type Input as input - // The only up-ref is apply_body_impl, which should implement the function - // call and any handling of the result. - template< typename Input, typename A, typename ImplType > - class function_input_base : public receiver, tbb::internal::no_assign { - enum op_type {reg_pred, rem_pred, app_body, try_fwd, tryput_bypass, app_body_bypass -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - , add_blt_pred, del_blt_pred, - blt_pred_cnt, blt_pred_cpy // create vector copies of preds and succs -#endif - }; - typedef function_input_base class_type; - - public: - - //! The input type of this receiver - typedef Input input_type; - typedef typename receiver::predecessor_type predecessor_type; - typedef predecessor_cache predecessor_cache_type; - typedef function_input_queue input_queue_type; - typedef typename A::template rebind< input_queue_type >::other queue_allocator_type; - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - typedef typename predecessor_cache_type::built_predecessors_type built_predecessors_type; - typedef typename receiver::predecessor_list_type predecessor_list_type; -#endif - - //! Constructor for function_input_base - function_input_base( graph &g, size_t max_concurrency, input_queue_type *q = NULL) - : my_graph_ref(g), my_max_concurrency(max_concurrency), my_concurrency(0), - my_queue(q), forwarder_busy(false) { - my_predecessors.set_owner(this); - my_aggregator.initialize_handler(handler_type(this)); - } - - //! Copy constructor - function_input_base( const function_input_base& src, input_queue_type *q = NULL) : - receiver(), tbb::internal::no_assign(), - my_graph_ref(src.my_graph_ref), my_max_concurrency(src.my_max_concurrency), - my_concurrency(0), my_queue(q), forwarder_busy(false) - { - my_predecessors.set_owner(this); - my_aggregator.initialize_handler(handler_type(this)); - } - - //! Destructor - // The queue is allocated by the constructor for {multi}function_node. - // TODO: pass the graph_buffer_policy to the base so it can allocate the queue instead. - // This would be an interface-breaking change. - virtual ~function_input_base() { - if ( my_queue ) delete my_queue; - } - - //! Put to the node, returning a task if available - task * try_put_task( const input_type &t ) __TBB_override { - if ( my_max_concurrency == 0 ) { - return create_body_task( t ); - } else { - operation_type op_data(t, tryput_bypass); - my_aggregator.execute(&op_data); - if(op_data.status == internal::SUCCEEDED) { - return op_data.bypass_t; - } - return NULL; - } - } - - //! Adds src to the list of cached predecessors. - bool register_predecessor( predecessor_type &src ) __TBB_override { - operation_type op_data(reg_pred); - op_data.r = &src; - my_aggregator.execute(&op_data); - return true; - } - - //! Removes src from the list of cached predecessors. - bool remove_predecessor( predecessor_type &src ) __TBB_override { - operation_type op_data(rem_pred); - op_data.r = &src; - my_aggregator.execute(&op_data); - return true; - } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - //! Adds to list of predecessors added by make_edge - void internal_add_built_predecessor( predecessor_type &src) __TBB_override { - operation_type op_data(add_blt_pred); - op_data.r = &src; - my_aggregator.execute(&op_data); - } - - //! removes from to list of predecessors (used by remove_edge) - void internal_delete_built_predecessor( predecessor_type &src) __TBB_override { - operation_type op_data(del_blt_pred); - op_data.r = &src; - my_aggregator.execute(&op_data); - } - - size_t predecessor_count() __TBB_override { - operation_type op_data(blt_pred_cnt); - my_aggregator.execute(&op_data); - return op_data.cnt_val; - } - - void copy_predecessors(predecessor_list_type &v) __TBB_override { - operation_type op_data(blt_pred_cpy); - op_data.predv = &v; - my_aggregator.execute(&op_data); - } - - built_predecessors_type &built_predecessors() __TBB_override { - return my_predecessors.built_predecessors(); - } -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ - - protected: - - void reset_function_input_base( reset_flags f) { - my_concurrency = 0; - if(my_queue) { - my_queue->reset(); - } - reset_receiver(f); - forwarder_busy = false; - } - - graph& my_graph_ref; - const size_t my_max_concurrency; - size_t my_concurrency; - input_queue_type *my_queue; - predecessor_cache my_predecessors; - - void reset_receiver( reset_flags f) __TBB_override { - if( f & rf_clear_edges) my_predecessors.clear(); - else - my_predecessors.reset(); - __TBB_ASSERT(!(f & rf_clear_edges) || my_predecessors.empty(), "function_input_base reset failed"); - } - - graph& graph_reference() __TBB_override { - return my_graph_ref; - } - - private: - - friend class apply_body_task_bypass< class_type, input_type >; - friend class forward_task_bypass< class_type >; - - class operation_type : public aggregated_operation< operation_type > { - public: - char type; - union { - input_type *elem; - predecessor_type *r; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - size_t cnt_val; - predecessor_list_type *predv; -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ - }; - tbb::task *bypass_t; - operation_type(const input_type& e, op_type t) : - type(char(t)), elem(const_cast(&e)) {} - operation_type(op_type t) : type(char(t)), r(NULL) {} - }; - - bool forwarder_busy; - typedef internal::aggregating_functor handler_type; - friend class internal::aggregating_functor; - aggregator< handler_type, operation_type > my_aggregator; - - task* create_and_spawn_task(bool spawn) { - task* new_task = NULL; - if(my_queue) { - if(!my_queue->empty()) { - ++my_concurrency; - new_task = create_body_task(my_queue->front()); - - my_queue->pop(); - } - } - else { - input_type i; - if(my_predecessors.get_item(i)) { - ++my_concurrency; - new_task = create_body_task(i); - } - } - //! Spawns a task that applies a body - // task == NULL => g.reset(), which shouldn't occur in concurrent context - if(spawn && new_task) { - internal::spawn_in_graph_arena(graph_reference(), *new_task); - new_task = SUCCESSFULLY_ENQUEUED; - } - - return new_task; - } - void handle_operations(operation_type *op_list) { - operation_type *tmp; - while (op_list) { - tmp = op_list; - op_list = op_list->next; - switch (tmp->type) { - case reg_pred: - my_predecessors.add(*(tmp->r)); - __TBB_store_with_release(tmp->status, SUCCEEDED); - if (!forwarder_busy) { - forwarder_busy = true; - spawn_forward_task(); - } - break; - case rem_pred: - my_predecessors.remove(*(tmp->r)); - __TBB_store_with_release(tmp->status, SUCCEEDED); - break; - case app_body: - __TBB_ASSERT(my_max_concurrency != 0, NULL); - --my_concurrency; - __TBB_store_with_release(tmp->status, SUCCEEDED); - if (my_concurrencybypass_t = NULL; - __TBB_ASSERT(my_max_concurrency != 0, NULL); - --my_concurrency; - if(my_concurrencybypass_t = create_and_spawn_task(/*spawn=*/false); - - __TBB_store_with_release(tmp->status, SUCCEEDED); - } - break; - case tryput_bypass: internal_try_put_task(tmp); break; - case try_fwd: internal_forward(tmp); break; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - case add_blt_pred: { - my_predecessors.internal_add_built_predecessor(*(tmp->r)); - __TBB_store_with_release(tmp->status, SUCCEEDED); - } - break; - case del_blt_pred: - my_predecessors.internal_delete_built_predecessor(*(tmp->r)); - __TBB_store_with_release(tmp->status, SUCCEEDED); - break; - case blt_pred_cnt: - tmp->cnt_val = my_predecessors.predecessor_count(); - __TBB_store_with_release(tmp->status, SUCCEEDED); - break; - case blt_pred_cpy: - my_predecessors.copy_predecessors( *(tmp->predv) ); - __TBB_store_with_release(tmp->status, SUCCEEDED); - break; -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ - } - } - } - - //! Put to the node, but return the task instead of enqueueing it - void internal_try_put_task(operation_type *op) { - __TBB_ASSERT(my_max_concurrency != 0, NULL); - if (my_concurrency < my_max_concurrency) { - ++my_concurrency; - task * new_task = create_body_task(*(op->elem)); - op->bypass_t = new_task; - __TBB_store_with_release(op->status, SUCCEEDED); - } else if ( my_queue && my_queue->push(*(op->elem)) ) { - op->bypass_t = SUCCESSFULLY_ENQUEUED; - __TBB_store_with_release(op->status, SUCCEEDED); - } else { - op->bypass_t = NULL; - __TBB_store_with_release(op->status, FAILED); - } - } - - //! Tries to spawn bodies if available and if concurrency allows - void internal_forward(operation_type *op) { - op->bypass_t = NULL; - if (my_concurrency < my_max_concurrency || !my_max_concurrency) - op->bypass_t = create_and_spawn_task(/*spawn=*/false); - if(op->bypass_t) - __TBB_store_with_release(op->status, SUCCEEDED); - else { - forwarder_busy = false; - __TBB_store_with_release(op->status, FAILED); - } - } - - //! Applies the body to the provided input - // then decides if more work is available - task * apply_body_bypass( input_type &i ) { - task * new_task = static_cast(this)->apply_body_impl_bypass(i); - if ( my_max_concurrency != 0 ) { - operation_type op_data(app_body_bypass); // tries to pop an item or get_item, enqueues another apply_body - my_aggregator.execute(&op_data); - // workaround for icc bug - tbb::task *ttask = op_data.bypass_t; - new_task = combine_tasks(my_graph_ref, new_task, ttask); - } - return new_task; - } - - //! allocates a task to apply a body - inline task * create_body_task( const input_type &input ) { - - return (internal::is_graph_active(my_graph_ref)) ? - new(task::allocate_additional_child_of(*(my_graph_ref.root_task()))) - apply_body_task_bypass < class_type, input_type >(*this, input) : - NULL; - } - - //! This is executed by an enqueued task, the "forwarder" - task *forward_task() { - operation_type op_data(try_fwd); - task *rval = NULL; - do { - op_data.status = WAIT; - my_aggregator.execute(&op_data); - if(op_data.status == SUCCEEDED) { - // workaround for icc bug - tbb::task *ttask = op_data.bypass_t; - rval = combine_tasks(my_graph_ref, rval, ttask); - } - } while (op_data.status == SUCCEEDED); - return rval; - } - - inline task *create_forward_task() { - return (internal::is_graph_active(my_graph_ref)) ? - new(task::allocate_additional_child_of(*(my_graph_ref.root_task()))) forward_task_bypass< class_type >(*this) : - NULL; - } - - //! Spawns a task that calls forward() - inline void spawn_forward_task() { - task* tp = create_forward_task(); - if(tp) { - internal::spawn_in_graph_arena(graph_reference(), *tp); - } - } - }; // function_input_base - - //! Implements methods for a function node that takes a type Input as input and sends - // a type Output to its successors. - template< typename Input, typename Output, typename A> - class function_input : public function_input_base > { - public: - typedef Input input_type; - typedef Output output_type; - typedef function_body function_body_type; - typedef function_input my_class; - typedef function_input_base base_type; - typedef function_input_queue input_queue_type; - - // constructor - template - function_input( graph &g, size_t max_concurrency, Body& body, input_queue_type *q = NULL ) : - base_type(g, max_concurrency, q), - my_body( new internal::function_body_leaf< input_type, output_type, Body>(body) ), - my_init_body( new internal::function_body_leaf< input_type, output_type, Body>(body) ) { - } - - //! Copy constructor - function_input( const function_input& src, input_queue_type *q = NULL ) : - base_type(src, q), - my_body( src.my_init_body->clone() ), - my_init_body(src.my_init_body->clone() ) { - } - - ~function_input() { - delete my_body; - delete my_init_body; - } - - template< typename Body > - Body copy_function_object() { - function_body_type &body_ref = *this->my_body; - return dynamic_cast< internal::function_body_leaf & >(body_ref).get_body(); - } - - task * apply_body_impl_bypass( const input_type &i) { -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - // There is an extra copied needed to capture the - // body execution without the try_put - tbb::internal::fgt_begin_body( my_body ); - output_type v = (*my_body)(i); - tbb::internal::fgt_end_body( my_body ); - task * new_task = successors().try_put_task( v ); -#else - task * new_task = successors().try_put_task( (*my_body)(i) ); -#endif - return new_task; - } - - protected: - - void reset_function_input(reset_flags f) { - base_type::reset_function_input_base(f); - if(f & rf_reset_bodies) { - function_body_type *tmp = my_init_body->clone(); - delete my_body; - my_body = tmp; - } - } - - function_body_type *my_body; - function_body_type *my_init_body; - virtual broadcast_cache &successors() = 0; - - }; // function_input - - - // helper templates to clear the successor edges of the output ports of an multifunction_node - template struct clear_element { - template static void clear_this(P &p) { - (void)tbb::flow::get(p).successors().clear(); - clear_element::clear_this(p); - } - template static bool this_empty(P &p) { - if(tbb::flow::get(p).successors().empty()) - return clear_element::this_empty(p); - return false; - } - }; - - template<> struct clear_element<1> { - template static void clear_this(P &p) { - (void)tbb::flow::get<0>(p).successors().clear(); - } - template static bool this_empty(P &p) { - return tbb::flow::get<0>(p).successors().empty(); - } - }; - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - // helper templates to extract the output ports of an multifunction_node from graph - template struct extract_element { - template static void extract_this(P &p) { - (void)tbb::flow::get(p).successors().built_successors().sender_extract(tbb::flow::get(p)); - extract_element::extract_this(p); - } - }; - - template<> struct extract_element<1> { - template static void extract_this(P &p) { - (void)tbb::flow::get<0>(p).successors().built_successors().sender_extract(tbb::flow::get<0>(p)); - } - }; -#endif - - //! Implements methods for a function node that takes a type Input as input - // and has a tuple of output ports specified. - template< typename Input, typename OutputPortSet, typename A> - class multifunction_input : public function_input_base > { - public: - static const int N = tbb::flow::tuple_size::value; - typedef Input input_type; - typedef OutputPortSet output_ports_type; - typedef multifunction_body multifunction_body_type; - typedef multifunction_input my_class; - typedef function_input_base base_type; - typedef function_input_queue input_queue_type; - - // constructor - template - multifunction_input( - graph &g, - size_t max_concurrency, - Body& body, - input_queue_type *q = NULL ) : - base_type(g, max_concurrency, q), - my_body( new internal::multifunction_body_leaf(body) ), - my_init_body( new internal::multifunction_body_leaf(body) ) { - } - - //! Copy constructor - multifunction_input( const multifunction_input& src, input_queue_type *q = NULL ) : - base_type(src, q), - my_body( src.my_init_body->clone() ), - my_init_body(src.my_init_body->clone() ) { - } - - ~multifunction_input() { - delete my_body; - delete my_init_body; - } - - template< typename Body > - Body copy_function_object() { - multifunction_body_type &body_ref = *this->my_body; - return *static_cast(dynamic_cast< internal::multifunction_body_leaf & >(body_ref).get_body_ptr()); - } - - // for multifunction nodes we do not have a single successor as such. So we just tell - // the task we were successful. - task * apply_body_impl_bypass( const input_type &i) { - tbb::internal::fgt_begin_body( my_body ); - (*my_body)(i, my_output_ports); - tbb::internal::fgt_end_body( my_body ); - task * new_task = SUCCESSFULLY_ENQUEUED; - return new_task; - } - - output_ports_type &output_ports(){ return my_output_ports; } - - protected: -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - void extract() { - extract_element::extract_this(my_output_ports); - } -#endif - - void reset(reset_flags f) { - base_type::reset_function_input_base(f); - if(f & rf_clear_edges)clear_element::clear_this(my_output_ports); - if(f & rf_reset_bodies) { - multifunction_body_type *tmp = my_init_body->clone(); - delete my_body; - my_body = tmp; - } - __TBB_ASSERT(!(f & rf_clear_edges) || clear_element::this_empty(my_output_ports), "multifunction_node reset failed"); - } - - multifunction_body_type *my_body; - multifunction_body_type *my_init_body; - output_ports_type my_output_ports; - - }; // multifunction_input - - // template to refer to an output port of a multifunction_node - template - typename tbb::flow::tuple_element::type &output_port(MOP &op) { - return tbb::flow::get(op.output_ports()); - } - - inline void check_task_and_spawn(graph& g, task* t) { - if (t && t != SUCCESSFULLY_ENQUEUED) { - internal::spawn_in_graph_arena(g, *t); - } - } - - // helper structs for split_node - template - struct emit_element { - template - static task* emit_this(graph& g, const T &t, P &p) { - // TODO: consider to collect all the tasks in task_list and spawn them all at once - task* last_task = tbb::flow::get(p).try_put_task(tbb::flow::get(t)); - check_task_and_spawn(g, last_task); - return emit_element::emit_this(g,t,p); - } - }; - - template<> - struct emit_element<1> { - template - static task* emit_this(graph& g, const T &t, P &p) { - task* last_task = tbb::flow::get<0>(p).try_put_task(tbb::flow::get<0>(t)); - check_task_and_spawn(g, last_task); - return SUCCESSFULLY_ENQUEUED; - } - }; - - //! Implements methods for an executable node that takes continue_msg as input - template< typename Output > - class continue_input : public continue_receiver { - public: - - //! The input type of this receiver - typedef continue_msg input_type; - - //! The output type of this receiver - typedef Output output_type; - typedef function_body function_body_type; - - template< typename Body > - continue_input( graph &g, Body& body ) - : my_graph_ref(g), - my_body( new internal::function_body_leaf< input_type, output_type, Body>(body) ), - my_init_body( new internal::function_body_leaf< input_type, output_type, Body>(body) ) { } - - template< typename Body > - continue_input( graph &g, int number_of_predecessors, Body& body ) - : continue_receiver( number_of_predecessors ), my_graph_ref(g), - my_body( new internal::function_body_leaf< input_type, output_type, Body>(body) ), - my_init_body( new internal::function_body_leaf< input_type, output_type, Body>(body) ) - { } - - continue_input( const continue_input& src ) : continue_receiver(src), - my_graph_ref(src.my_graph_ref), - my_body( src.my_init_body->clone() ), - my_init_body( src.my_init_body->clone() ) {} - - ~continue_input() { - delete my_body; - delete my_init_body; - } - - template< typename Body > - Body copy_function_object() { - function_body_type &body_ref = *my_body; - return dynamic_cast< internal::function_body_leaf & >(body_ref).get_body(); - } - - void reset_receiver( reset_flags f) __TBB_override { - continue_receiver::reset_receiver(f); - if(f & rf_reset_bodies) { - function_body_type *tmp = my_init_body->clone(); - delete my_body; - my_body = tmp; - } - } - - protected: - - graph& my_graph_ref; - function_body_type *my_body; - function_body_type *my_init_body; - - virtual broadcast_cache &successors() = 0; - - friend class apply_body_task_bypass< continue_input< Output >, continue_msg >; - - //! Applies the body to the provided input - task *apply_body_bypass( input_type ) { -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - // There is an extra copied needed to capture the - // body execution without the try_put - tbb::internal::fgt_begin_body( my_body ); - output_type v = (*my_body)( continue_msg() ); - tbb::internal::fgt_end_body( my_body ); - return successors().try_put_task( v ); -#else - return successors().try_put_task( (*my_body)( continue_msg() ) ); -#endif - } - - //! Spawns a task that applies the body - task *execute( ) __TBB_override { - return (internal::is_graph_active(my_graph_ref)) ? - new ( task::allocate_additional_child_of( *(my_graph_ref.root_task()) ) ) - apply_body_task_bypass< continue_input< Output >, continue_msg >( *this, continue_msg() ) : - NULL; - } - - graph& graph_reference() __TBB_override { - return my_graph_ref; - } - - }; // continue_input - - //! Implements methods for both executable and function nodes that puts Output to its successors - template< typename Output > - class function_output : public sender { - public: - - template friend struct clear_element; - typedef Output output_type; - typedef typename sender::successor_type successor_type; - typedef broadcast_cache broadcast_cache_type; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - typedef typename sender::built_successors_type built_successors_type; - typedef typename sender::successor_list_type successor_list_type; -#endif - - function_output() { my_successors.set_owner(this); } - function_output(const function_output & /*other*/) : sender() { - my_successors.set_owner(this); - } - - //! Adds a new successor to this node - bool register_successor( successor_type &r ) __TBB_override { - successors().register_successor( r ); - return true; - } - - //! Removes a successor from this node - bool remove_successor( successor_type &r ) __TBB_override { - successors().remove_successor( r ); - return true; - } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - built_successors_type &built_successors() __TBB_override { return successors().built_successors(); } - - - void internal_add_built_successor( successor_type &r) __TBB_override { - successors().internal_add_built_successor( r ); - } - - void internal_delete_built_successor( successor_type &r) __TBB_override { - successors().internal_delete_built_successor( r ); - } - - size_t successor_count() __TBB_override { - return successors().successor_count(); - } - - void copy_successors( successor_list_type &v) __TBB_override { - successors().copy_successors(v); - } -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ - - // for multifunction_node. The function_body that implements - // the node will have an input and an output tuple of ports. To put - // an item to a successor, the body should - // - // get(output_ports).try_put(output_value); - // - // if task pointer is returned will always spawn and return true, else - // return value will be bool returned from successors.try_put. - task *try_put_task(const output_type &i) { // not a virtual method in this class - return my_successors.try_put_task(i); - } - - broadcast_cache_type &successors() { return my_successors; } - protected: - broadcast_cache_type my_successors; - - }; // function_output - - template< typename Output > - class multifunction_output : public function_output { - public: - typedef Output output_type; - typedef function_output base_type; - using base_type::my_successors; - - multifunction_output() : base_type() {my_successors.set_owner(this);} - multifunction_output( const multifunction_output &/*other*/) : base_type() { my_successors.set_owner(this); } - - bool try_put(const output_type &i) { - task *res = try_put_task(i); - if(!res) return false; - if(res != SUCCESSFULLY_ENQUEUED) { - FLOW_SPAWN(*res); // TODO: Spawn task inside arena - } - return true; - } - - protected: - - task* try_put_task(const output_type &i) { - return my_successors.try_put_task(i); - } - - template friend struct emit_element; - - }; // multifunction_output - -//composite_node -#if TBB_PREVIEW_FLOW_GRAPH_TRACE && __TBB_FLOW_GRAPH_CPP11_FEATURES - template - void add_nodes_impl(CompositeType*, bool) {} - - template< typename CompositeType, typename NodeType1, typename... NodeTypes > - void add_nodes_impl(CompositeType *c_node, bool visible, const NodeType1& n1, const NodeTypes&... n) { - void *addr = const_cast(&n1); - - if(visible) - tbb::internal::itt_relation_add( tbb::internal::ITT_DOMAIN_FLOW, c_node, tbb::internal::FLOW_NODE, tbb::internal::__itt_relation_is_parent_of, addr, tbb::internal::FLOW_NODE ); - else - tbb::internal::itt_relation_add( tbb::internal::ITT_DOMAIN_FLOW, addr, tbb::internal::FLOW_NODE, tbb::internal::__itt_relation_is_child_of, c_node, tbb::internal::FLOW_NODE ); - add_nodes_impl(c_node, visible, n...); - } -#endif - -} // internal - -#endif // __TBB__flow_graph_node_impl_H diff --git a/src/tbb/include/tbb/internal/_flow_graph_streaming_node.h b/src/tbb/include/tbb/internal/_flow_graph_streaming_node.h deleted file mode 100644 index 493f76e80..000000000 --- a/src/tbb/include/tbb/internal/_flow_graph_streaming_node.h +++ /dev/null @@ -1,745 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_flow_graph_streaming_H -#define __TBB_flow_graph_streaming_H - -#ifndef __TBB_flow_graph_H -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#if __TBB_PREVIEW_STREAMING_NODE - -// Included in namespace tbb::flow::interfaceX (in flow_graph.h) - -namespace internal { - -template -struct port_ref_impl { - // "+1" since the port_ref range is a closed interval (includes its endpoints). - static const int size = N2 - N1 + 1; -}; - -} // internal - -// The purpose of the port_ref_impl is the pretty syntax: the deduction of a compile-time constant is processed from the return type. -// So it is possible to use this helper without parentheses, e.g. "port_ref<0>". -template -internal::port_ref_impl port_ref() { - return internal::port_ref_impl(); -}; - -namespace internal { - -template -struct num_arguments { - static const int value = 1; -}; - -template -struct num_arguments(*)()> { - static const int value = port_ref_impl::size; -}; - -template -struct num_arguments> { - static const int value = port_ref_impl::size; -}; - -template -void ignore_return_values( Args&&... ) {} - -template -T or_return_values( T&& t ) { return t; } -template -T or_return_values( T&& t, Rest&&... rest ) { - return t | or_return_values( std::forward(rest)... ); -} - -template -struct key_from_policy { - typedef size_t type; - typedef std::false_type is_key_matching; -}; - -template -struct key_from_policy< key_matching > { - typedef Key type; - typedef std::true_type is_key_matching; -}; - -template -struct key_from_policy< key_matching > { - typedef const Key &type; - typedef std::true_type is_key_matching; -}; - -template -class streaming_device_with_key { - Device my_device; - typename std::decay::type my_key; -public: - // TODO: investigate why default constructor is required - streaming_device_with_key() {} - streaming_device_with_key( const Device& d, Key k ) : my_device( d ), my_key( k ) {} - Key key() const { return my_key; } - const Device& device() const { return my_device; } -}; - -// --------- Kernel argument helpers --------- // -template -struct is_port_ref_impl { - typedef std::false_type type; -}; - -template -struct is_port_ref_impl< port_ref_impl > { - typedef std::true_type type; -}; - -template -struct is_port_ref_impl< port_ref_impl( * )() > { - typedef std::true_type type; -}; - -template -struct is_port_ref { - typedef typename is_port_ref_impl< typename tbb::internal::strip::type >::type type; -}; - -template -struct convert_and_call_impl; - -template -struct convert_and_call_impl { - static const size_t my_delta = 1; // Index 0 contains device - - template - static void doit(F& f, Tuple& t, A1& a1, Args1&... args1, Args2&... args2) { - convert_and_call_impl::doit_impl(typename is_port_ref::type(), f, t, a1, args1..., args2...); - } - template - static void doit_impl(std::false_type, F& f, Tuple& t, A1& a1, Args1&... args1, Args2&... args2) { - convert_and_call_impl::doit(f, t, args1..., args2..., a1); - } - template - static void doit_impl(std::true_type x, F& f, Tuple& t, port_ref_impl, Args1&... args1, Args2&... args2) { - convert_and_call_impl, Args1...>::doit_impl(x, f, t, port_ref(), args1..., - args2..., std::get(t)); - } - template - static void doit_impl(std::true_type, F& f, Tuple& t, port_ref_impl, Args1&... args1, Args2&... args2) { - convert_and_call_impl::doit(f, t, args1..., args2..., std::get(t)); - } - - template - static void doit_impl(std::true_type x, F& f, Tuple& t, port_ref_impl(* fn)(), Args1&... args1, Args2&... args2) { - doit_impl(x, f, t, fn(), args1..., args2...); - } - template - static void doit_impl(std::true_type x, F& f, Tuple& t, port_ref_impl(* fn)(), Args1&... args1, Args2&... args2) { - doit_impl(x, f, t, fn(), args1..., args2...); - } -}; - -template <> -struct convert_and_call_impl<> { - template - static void doit(F& f, Tuple&, Args2&... args2) { - f(args2...); - } -}; -// ------------------------------------------- // - -template -struct streaming_node_traits { - // Do not use 'using' instead of 'struct' because Microsoft Visual C++ 12.0 fails to compile. - template - struct async_msg_type { - typedef typename StreamFactory::template async_msg_type type; - }; - - typedef tuple< typename async_msg_type::type... > input_tuple; - typedef input_tuple output_tuple; - typedef tuple< streaming_device_with_key< typename StreamFactory::device_type, typename key_from_policy::type >, - typename async_msg_type::type... > kernel_input_tuple; - - // indexer_node parameters pack expansion workaround for VS2013 for streaming_node - typedef indexer_node< typename async_msg_type::type... > indexer_node_type; -}; - -// Default empty implementation -template -class kernel_executor_helper { - typedef typename StreamFactory::device_type device_type; - typedef typename StreamFactory::kernel_type kernel_type; - typedef KernelInputTuple kernel_input_tuple; -protected: - template - void enqueue_kernel_impl( kernel_input_tuple&, StreamFactory& factory, device_type device, const kernel_type& kernel, Args&... args ) const { - factory.send_kernel( device, kernel, args... ); - } -}; - -// Implementation for StreamFactory supporting range -template -class kernel_executor_helper::type > { - typedef typename StreamFactory::device_type device_type; - typedef typename StreamFactory::kernel_type kernel_type; - typedef KernelInputTuple kernel_input_tuple; - - typedef typename StreamFactory::range_type range_type; - - // Container for randge. It can contain either port references or real range. - struct range_wrapper { - virtual range_type get_range( const kernel_input_tuple &ip ) const = 0; - virtual range_wrapper *clone() const = 0; - virtual ~range_wrapper() {} - }; - - struct range_value : public range_wrapper { - range_value( const range_type& value ) : my_value(value) {} - - range_value( range_type&& value ) : my_value(std::move(value)) {} - - range_type get_range( const kernel_input_tuple & ) const __TBB_override { - return my_value; - } - - range_wrapper *clone() const __TBB_override { - return new range_value(my_value); - } - private: - range_type my_value; - }; - - template - struct range_mapper : public range_wrapper { - range_mapper() {} - - range_type get_range( const kernel_input_tuple &ip ) const __TBB_override { - // "+1" since get<0>(ip) is StreamFactory::device. - return get(ip).data(false); - } - - range_wrapper *clone() const __TBB_override { - return new range_mapper; - } - }; - -protected: - template - void enqueue_kernel_impl( kernel_input_tuple& ip, StreamFactory& factory, device_type device, const kernel_type& kernel, Args&... args ) const { - __TBB_ASSERT(my_range_wrapper, "Range is not set. Call set_range() before running streaming_node."); - factory.send_kernel( device, kernel, my_range_wrapper->get_range(ip), args... ); - } - -public: - kernel_executor_helper() : my_range_wrapper(NULL) {} - - kernel_executor_helper(const kernel_executor_helper& executor) : my_range_wrapper(executor.my_range_wrapper ? executor.my_range_wrapper->clone() : NULL) {} - - kernel_executor_helper(kernel_executor_helper&& executor) : my_range_wrapper(executor.my_range_wrapper) { - // Set moving holder mappers to NULL to prevent double deallocation - executor.my_range_wrapper = NULL; - } - - ~kernel_executor_helper() { - if (my_range_wrapper) delete my_range_wrapper; - } - - void set_range(const range_type& work_size) { - my_range_wrapper = new range_value(work_size); - } - - void set_range(range_type&& work_size) { - my_range_wrapper = new range_value(std::move(work_size)); - } - - template - void set_range(port_ref_impl) { - my_range_wrapper = new range_mapper; - } - - template - void set_range(port_ref_impl(*)()) { - my_range_wrapper = new range_mapper; - } - -private: - range_wrapper* my_range_wrapper; -}; - -} // internal - -/* -/---------------------------------------- streaming_node ------------------------------------\ -| | -| /--------------\ /----------------------\ /-----------\ /----------------------\ | -| | | | (device_with_key) O---O | | | | -| | | | | | | | | | -O---O indexer_node O---O device_selector_node O---O join_node O---O kernel_node O---O -| | | | (multifunction_node) | | | | (multifunction_node) | | -O---O | | O---O | | O---O -| \--------------/ \----------------------/ \-----------/ \----------------------/ | -| | -\--------------------------------------------------------------------------------------------/ -*/ -template -class streaming_node; - -template -class streaming_node< tuple, JP, StreamFactory > - : public composite_node < typename internal::streaming_node_traits::input_tuple, - typename internal::streaming_node_traits::output_tuple > - , public internal::kernel_executor_helper< StreamFactory, typename internal::streaming_node_traits::kernel_input_tuple > -{ - typedef typename internal::streaming_node_traits::input_tuple input_tuple; - typedef typename internal::streaming_node_traits::output_tuple output_tuple; - typedef typename internal::key_from_policy::type key_type; -protected: - typedef typename StreamFactory::device_type device_type; - typedef typename StreamFactory::kernel_type kernel_type; -private: - typedef internal::streaming_device_with_key device_with_key_type; - typedef composite_node base_type; - static const size_t NUM_INPUTS = tuple_size::value; - static const size_t NUM_OUTPUTS = tuple_size::value; - - typedef typename internal::make_sequence::type input_sequence; - typedef typename internal::make_sequence::type output_sequence; - - typedef typename internal::streaming_node_traits::indexer_node_type indexer_node_type; - typedef typename indexer_node_type::output_type indexer_node_output_type; - typedef typename internal::streaming_node_traits::kernel_input_tuple kernel_input_tuple; - typedef multifunction_node device_selector_node; - typedef multifunction_node kernel_multifunction_node; - - template - typename base_type::input_ports_type get_input_ports( internal::sequence ) { - return std::tie( internal::input_port( my_indexer_node )... ); - } - - template - typename base_type::output_ports_type get_output_ports( internal::sequence ) { - return std::tie( internal::output_port( my_kernel_node )... ); - } - - typename base_type::input_ports_type get_input_ports() { - return get_input_ports( input_sequence() ); - } - - typename base_type::output_ports_type get_output_ports() { - return get_output_ports( output_sequence() ); - } - - template - int make_Nth_edge() { - make_edge( internal::output_port( my_device_selector_node ), internal::input_port( my_join_node ) ); - return 0; - } - - template - void make_edges( internal::sequence ) { - make_edge( my_indexer_node, my_device_selector_node ); - make_edge( my_device_selector_node, my_join_node ); - internal::ignore_return_values( make_Nth_edge()... ); - make_edge( my_join_node, my_kernel_node ); - } - - void make_edges() { - make_edges( input_sequence() ); - } - - class device_selector_base { - public: - virtual void operator()( const indexer_node_output_type &v, typename device_selector_node::output_ports_type &op ) = 0; - virtual device_selector_base *clone( streaming_node &n ) const = 0; - virtual ~device_selector_base() {} - }; - - template - class device_selector : public device_selector_base, tbb::internal::no_assign { - public: - device_selector( UserFunctor uf, streaming_node &n, StreamFactory &f ) - : my_dispatch_funcs( create_dispatch_funcs( input_sequence() ) ) - , my_user_functor( uf ), my_node(n), my_factory( f ) - { - my_port_epoches.fill( 0 ); - } - - void operator()( const indexer_node_output_type &v, typename device_selector_node::output_ports_type &op ) __TBB_override { - (this->*my_dispatch_funcs[ v.tag() ])( my_port_epoches[ v.tag() ], v, op ); - __TBB_ASSERT( (tbb::internal::is_same_type::is_key_matching, std::false_type>::value) - || my_port_epoches[v.tag()] == 0, "Epoch is changed when key matching is requested" ); - } - - device_selector_base *clone( streaming_node &n ) const __TBB_override { - return new device_selector( my_user_functor, n, my_factory ); - } - private: - typedef void(device_selector::*send_and_put_fn_type)(size_t &, const indexer_node_output_type &, typename device_selector_node::output_ports_type &); - typedef std::array < send_and_put_fn_type, NUM_INPUTS > dispatch_funcs_type; - - template - static dispatch_funcs_type create_dispatch_funcs( internal::sequence ) { - dispatch_funcs_type dispatch = { { &device_selector::send_and_put_impl... } }; - return dispatch; - } - - template - key_type get_key( std::false_type, const T &, size_t &epoch ) { - __TBB_STATIC_ASSERT( (tbb::internal::is_same_type::value), "" ); - return epoch++; - } - - template - key_type get_key( std::true_type, const T &t, size_t &/*epoch*/ ) { - using tbb::flow::key_from_message; - return key_from_message( t ); - } - - template - void send_and_put_impl( size_t &epoch, const indexer_node_output_type &v, typename device_selector_node::output_ports_type &op ) { - typedef typename tuple_element::type::output_type elem_type; - elem_type e = internal::cast_to( v ); - device_type device = get_device( get_key( typename internal::key_from_policy::is_key_matching(), e, epoch ), get<0>( op ) ); - my_factory.send_data( device, e ); - get( op ).try_put( e ); - } - - template< typename DevicePort > - device_type get_device( key_type key, DevicePort& dp ) { - typename std::unordered_map::type, epoch_desc>::iterator it = my_devices.find( key ); - if ( it == my_devices.end() ) { - device_type d = my_user_functor( my_factory ); - std::tie( it, std::ignore ) = my_devices.insert( std::make_pair( key, d ) ); - bool res = dp.try_put( device_with_key_type( d, key ) ); - __TBB_ASSERT_EX( res, NULL ); - my_node.notify_new_device( d ); - } - epoch_desc &e = it->second; - device_type d = e.my_device; - if ( ++e.my_request_number == NUM_INPUTS ) my_devices.erase( it ); - return d; - } - - struct epoch_desc { - epoch_desc(device_type d ) : my_device( d ), my_request_number( 0 ) {} - device_type my_device; - size_t my_request_number; - }; - - std::unordered_map::type, epoch_desc> my_devices; - std::array my_port_epoches; - dispatch_funcs_type my_dispatch_funcs; - UserFunctor my_user_functor; - streaming_node &my_node; - StreamFactory &my_factory; - }; - - class device_selector_body { - public: - device_selector_body( device_selector_base *d ) : my_device_selector( d ) {} - - void operator()( const indexer_node_output_type &v, typename device_selector_node::output_ports_type &op ) { - (*my_device_selector)(v, op); - } - private: - device_selector_base *my_device_selector; - }; - - class args_storage_base : tbb::internal::no_copy { - public: - typedef typename kernel_multifunction_node::output_ports_type output_ports_type; - - virtual void enqueue( kernel_input_tuple &ip, output_ports_type &op, const streaming_node &n ) = 0; - virtual void send( device_type d ) = 0; - virtual args_storage_base *clone() const = 0; - virtual ~args_storage_base () {} - - protected: - args_storage_base( const kernel_type& kernel, StreamFactory &f ) - : my_kernel( kernel ), my_factory( f ) - {} - - args_storage_base( const args_storage_base &k ) - : my_kernel( k.my_kernel ), my_factory( k.my_factory ) - {} - - const kernel_type my_kernel; - StreamFactory &my_factory; - }; - - template - class args_storage : public args_storage_base { - typedef typename args_storage_base::output_ports_type output_ports_type; - - // ---------- Update events helpers ---------- // - template - bool do_try_put( const kernel_input_tuple& ip, output_ports_type &op ) const { - const auto& t = get( ip ); - auto &port = get( op ); - return port.try_put( t ); - } - - template - bool do_try_put( const kernel_input_tuple& ip, output_ports_type &op, internal::sequence ) const { - return internal::or_return_values( do_try_put( ip, op )... ); - } - - // ------------------------------------------- // - class run_kernel_func : tbb::internal::no_assign { - public: - run_kernel_func( kernel_input_tuple &ip, const streaming_node &node, const args_storage& storage ) - : my_kernel_func( ip, node, storage, get<0>(ip).device() ) {} - - // It is immpossible to use Args... because a function pointer cannot be casted to a function reference implicitly. - // Allow the compiler to deduce types for function pointers automatically. - template - void operator()( FnArgs&... args ) { - internal::convert_and_call_impl::doit( my_kernel_func, my_kernel_func.my_ip, args... ); - } - private: - struct kernel_func : tbb::internal::no_copy { - kernel_input_tuple &my_ip; - const streaming_node &my_node; - const args_storage& my_storage; - device_type my_device; - - kernel_func( kernel_input_tuple &ip, const streaming_node &node, const args_storage& storage, device_type device ) - : my_ip( ip ), my_node( node ), my_storage( storage ), my_device( device ) - {} - - template - void operator()( FnArgs&... args ) { - my_node.enqueue_kernel( my_ip, my_storage.my_factory, my_device, my_storage.my_kernel, args... ); - } - } my_kernel_func; - }; - - template - class run_finalize_func : tbb::internal::no_assign { - public: - run_finalize_func( kernel_input_tuple &ip, StreamFactory &factory, FinalizeFn fn ) - : my_ip( ip ), my_finalize_func( factory, get<0>(ip).device(), fn ) {} - - // It is immpossible to use Args... because a function pointer cannot be casted to a function reference implicitly. - // Allow the compiler to deduce types for function pointers automatically. - template - void operator()( FnArgs&... args ) { - internal::convert_and_call_impl::doit( my_finalize_func, my_ip, args... ); - } - private: - kernel_input_tuple &my_ip; - - struct finalize_func : tbb::internal::no_assign { - StreamFactory &my_factory; - device_type my_device; - FinalizeFn my_fn; - - finalize_func( StreamFactory &factory, device_type device, FinalizeFn fn ) - : my_factory(factory), my_device(device), my_fn(fn) {} - - template - void operator()( FnArgs&... args ) { - my_factory.finalize( my_device, my_fn, args... ); - } - } my_finalize_func; - }; - - template - static run_finalize_func make_run_finalize_func( kernel_input_tuple &ip, StreamFactory &factory, FinalizeFn fn ) { - return run_finalize_func( ip, factory, fn ); - } - - class send_func : tbb::internal::no_assign { - public: - send_func( StreamFactory &factory, device_type d ) - : my_factory(factory), my_device( d ) {} - - template - void operator()( FnArgs&... args ) { - my_factory.send_data( my_device, args... ); - } - private: - StreamFactory &my_factory; - device_type my_device; - }; - - public: - args_storage( const kernel_type& kernel, StreamFactory &f, Args&&... args ) - : args_storage_base( kernel, f ) - , my_args_pack( std::forward(args)... ) - {} - - args_storage( const args_storage &k ) : args_storage_base( k ), my_args_pack( k.my_args_pack ) {} - - args_storage( const args_storage_base &k, Args&&... args ) : args_storage_base( k ), my_args_pack( std::forward(args)... ) {} - - void enqueue( kernel_input_tuple &ip, output_ports_type &op, const streaming_node &n ) __TBB_override { - // Make const qualified args_pack (from non-const) - const args_pack_type& const_args_pack = my_args_pack; - // factory.enqure_kernel() gets - // - 'ip' tuple elements by reference and updates it (and 'ip') with dependencies - // - arguments (from my_args_pack) by const-reference via const_args_pack - tbb::internal::call( run_kernel_func( ip, n, *this ), const_args_pack ); - - if (! do_try_put( ip, op, input_sequence() ) ) { - graph& g = n.my_graph; - // No one message was passed to successors so set a callback to extend the graph lifetime until the kernel completion. - g.increment_wait_count(); - - // factory.finalize() gets - // - 'ip' tuple elements by reference, so 'ip' might be changed - // - arguments (from my_args_pack) by const-reference via const_args_pack - tbb::internal::call( make_run_finalize_func(ip, this->my_factory, [&g] { - g.decrement_wait_count(); - }), const_args_pack ); - } - } - - void send( device_type d ) __TBB_override { - // factory.send() gets arguments by reference and updates these arguments with dependencies - // (it gets but usually ignores port_ref-s) - tbb::internal::call( send_func( this->my_factory, d ), my_args_pack ); - } - - args_storage_base *clone() const __TBB_override { - // Create new args_storage with copying constructor. - return new args_storage( *this ); - } - - private: - typedef tbb::internal::stored_pack args_pack_type; - args_pack_type my_args_pack; - }; - - // Body for kernel_multifunction_node. - class kernel_body : tbb::internal::no_assign { - public: - kernel_body( const streaming_node &node ) : my_node( node ) {} - - void operator()( kernel_input_tuple ip, typename args_storage_base::output_ports_type &op ) { - __TBB_ASSERT( (my_node.my_args_storage != NULL), "No arguments storage" ); - // 'ip' is passed by value to create local copy for updating inside enqueue_kernel() - my_node.my_args_storage->enqueue( ip, op, my_node ); - } - private: - const streaming_node &my_node; - }; - - template ::type > - struct wrap_to_async { - typedef T type; // Keep port_ref as it is - }; - - template - struct wrap_to_async { - typedef typename StreamFactory::template async_msg_type< typename tbb::internal::strip::type > type; - }; - - template - args_storage_base *make_args_storage(const args_storage_base& storage, Args&&... args) const { - // In this variadic template convert all simple types 'T' into 'async_msg_type' - return new args_storage(storage, std::forward(args)...); - } - - void notify_new_device( device_type d ) { - my_args_storage->send( d ); - } - - template - void enqueue_kernel( kernel_input_tuple& ip, StreamFactory& factory, device_type device, const kernel_type& kernel, Args&... args ) const { - this->enqueue_kernel_impl( ip, factory, device, kernel, args... ); - } - -public: - template - streaming_node( graph &g, const kernel_type& kernel, DeviceSelector d, StreamFactory &f ) - : base_type( g ) - , my_indexer_node( g ) - , my_device_selector( new device_selector( d, *this, f ) ) - , my_device_selector_node( g, serial, device_selector_body( my_device_selector ) ) - , my_join_node( g ) - , my_kernel_node( g, serial, kernel_body( *this ) ) - // By default, streaming_node maps all its ports to the kernel arguments on a one-to-one basis. - , my_args_storage( make_args_storage( args_storage<>(kernel, f), port_ref<0, NUM_INPUTS - 1>() ) ) - { - base_type::set_external_ports( get_input_ports(), get_output_ports() ); - make_edges(); - } - - streaming_node( const streaming_node &node ) - : base_type( node.my_graph ) - , my_indexer_node( node.my_indexer_node ) - , my_device_selector( node.my_device_selector->clone( *this ) ) - , my_device_selector_node( node.my_graph, serial, device_selector_body( my_device_selector ) ) - , my_join_node( node.my_join_node ) - , my_kernel_node( node.my_graph, serial, kernel_body( *this ) ) - , my_args_storage( node.my_args_storage->clone() ) - { - base_type::set_external_ports( get_input_ports(), get_output_ports() ); - make_edges(); - } - - streaming_node( streaming_node &&node ) - : base_type( node.my_graph ) - , my_indexer_node( std::move( node.my_indexer_node ) ) - , my_device_selector( node.my_device_selector->clone(*this) ) - , my_device_selector_node( node.my_graph, serial, device_selector_body( my_device_selector ) ) - , my_join_node( std::move( node.my_join_node ) ) - , my_kernel_node( node.my_graph, serial, kernel_body( *this ) ) - , my_args_storage( node.my_args_storage ) - { - base_type::set_external_ports( get_input_ports(), get_output_ports() ); - make_edges(); - // Set moving node mappers to NULL to prevent double deallocation. - node.my_args_storage = NULL; - } - - ~streaming_node() { - if ( my_args_storage ) delete my_args_storage; - if ( my_device_selector ) delete my_device_selector; - } - - template - void set_args( Args&&... args ) { - // Copy the base class of args_storage and create new storage for "Args...". - args_storage_base * const new_args_storage = make_args_storage( *my_args_storage, typename wrap_to_async::type(std::forward(args))...); - delete my_args_storage; - my_args_storage = new_args_storage; - } - -protected: - void reset_node( reset_flags = rf_reset_protocol ) __TBB_override { __TBB_ASSERT( false, "Not implemented yet" ); } - -private: - indexer_node_type my_indexer_node; - device_selector_base *my_device_selector; - device_selector_node my_device_selector_node; - join_node my_join_node; - kernel_multifunction_node my_kernel_node; - - args_storage_base *my_args_storage; -}; - -#endif // __TBB_PREVIEW_STREAMING_NODE -#endif // __TBB_flow_graph_streaming_H diff --git a/src/tbb/include/tbb/internal/_flow_graph_tagged_buffer_impl.h b/src/tbb/include/tbb/internal/_flow_graph_tagged_buffer_impl.h deleted file mode 100644 index 46755fe07..000000000 --- a/src/tbb/include/tbb/internal/_flow_graph_tagged_buffer_impl.h +++ /dev/null @@ -1,253 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -// a hash table buffer that can expand, and can support as many deletions as -// additions, list-based, with elements of list held in array (for destruction -// management), multiplicative hashing (like ets). No synchronization built-in. -// - -#ifndef __TBB__flow_graph_hash_buffer_impl_H -#define __TBB__flow_graph_hash_buffer_impl_H - -#ifndef __TBB_flow_graph_H -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -// included in namespace tbb::flow::interfaceX::internal - -// elements in the table are a simple list; we need pointer to next element to -// traverse the chain -template -struct buffer_element_type { - // the second parameter below is void * because we can't forward-declare the type - // itself, so we just reinterpret_cast below. - typedef typename aligned_pair::type type; -}; - -template - < - typename Key, // type of key within ValueType - typename ValueType, - typename ValueToKey, // abstract method that returns "const Key" or "const Key&" given ValueType - typename HashCompare, // has hash and equal - typename Allocator=tbb::cache_aligned_allocator< typename aligned_pair::type > - > -class hash_buffer : public HashCompare { -public: - static const size_t INITIAL_SIZE = 8; // initial size of the hash pointer table - typedef ValueType value_type; - typedef typename buffer_element_type< value_type >::type element_type; - typedef value_type *pointer_type; - typedef element_type *list_array_type; // array we manage manually - typedef list_array_type *pointer_array_type; - typedef typename Allocator::template rebind::other pointer_array_allocator_type; - typedef typename Allocator::template rebind::other elements_array_allocator; - typedef typename tbb::internal::strip::type Knoref; - -private: - ValueToKey *my_key; - size_t my_size; - size_t nelements; - pointer_array_type pointer_array; // pointer_array[my_size] - list_array_type elements_array; // elements_array[my_size / 2] - element_type* free_list; - - size_t mask() { return my_size - 1; } - - void set_up_free_list( element_type **p_free_list, list_array_type la, size_t sz) { - for(size_t i=0; i < sz - 1; ++i ) { // construct free list - la[i].second = &(la[i+1]); - } - la[sz-1].second = NULL; - *p_free_list = (element_type *)&(la[0]); - } - - // cleanup for exceptions - struct DoCleanup { - pointer_array_type *my_pa; - list_array_type *my_elements; - size_t my_size; - - DoCleanup(pointer_array_type &pa, list_array_type &my_els, size_t sz) : - my_pa(&pa), my_elements(&my_els), my_size(sz) { } - ~DoCleanup() { - if(my_pa) { - size_t dont_care = 0; - internal_free_buffer(*my_pa, *my_elements, my_size, dont_care); - } - } - }; - - // exception-safety requires we do all the potentially-throwing operations first - void grow_array() { - size_t new_size = my_size*2; - size_t new_nelements = nelements; // internal_free_buffer zeroes this - list_array_type new_elements_array = NULL; - pointer_array_type new_pointer_array = NULL; - list_array_type new_free_list = NULL; - { - DoCleanup my_cleanup(new_pointer_array, new_elements_array, new_size); - new_elements_array = elements_array_allocator().allocate(my_size); - new_pointer_array = pointer_array_allocator_type().allocate(new_size); - for(size_t i=0; i < new_size; ++i) new_pointer_array[i] = NULL; - set_up_free_list(&new_free_list, new_elements_array, my_size ); - - for(size_t i=0; i < my_size; ++i) { - for( element_type* op = pointer_array[i]; op; op = (element_type *)(op->second)) { - value_type *ov = reinterpret_cast(&(op->first)); - // could have std::move semantics - internal_insert_with_key(new_pointer_array, new_size, new_free_list, *ov); - } - } - my_cleanup.my_pa = NULL; - my_cleanup.my_elements = NULL; - } - - internal_free_buffer(pointer_array, elements_array, my_size, nelements); - free_list = new_free_list; - pointer_array = new_pointer_array; - elements_array = new_elements_array; - my_size = new_size; - nelements = new_nelements; - } - - // v should have perfect forwarding if std::move implemented. - // we use this method to move elements in grow_array, so can't use class fields - void internal_insert_with_key( element_type **p_pointer_array, size_t p_sz, list_array_type &p_free_list, - const value_type &v) { - size_t l_mask = p_sz-1; - __TBB_ASSERT(my_key, "Error: value-to-key functor not provided"); - size_t h = this->hash((*my_key)(v)) & l_mask; - __TBB_ASSERT(p_free_list, "Error: free list not set up."); - element_type* my_elem = p_free_list; p_free_list = (element_type *)(p_free_list->second); - (void) new(&(my_elem->first)) value_type(v); - my_elem->second = p_pointer_array[h]; - p_pointer_array[h] = my_elem; - } - - void internal_initialize_buffer() { - pointer_array = pointer_array_allocator_type().allocate(my_size); - for(size_t i = 0; i < my_size; ++i) pointer_array[i] = NULL; - elements_array = elements_array_allocator().allocate(my_size / 2); - set_up_free_list(&free_list, elements_array, my_size / 2); - } - - // made static so an enclosed class can use to properly dispose of the internals - static void internal_free_buffer( pointer_array_type &pa, list_array_type &el, size_t &sz, size_t &ne ) { - if(pa) { - for(size_t i = 0; i < sz; ++i ) { - element_type *p_next; - for( element_type *p = pa[i]; p; p = p_next) { - p_next = (element_type *)p->second; - internal::punned_cast(&(p->first))->~value_type(); - } - } - pointer_array_allocator_type().deallocate(pa, sz); - pa = NULL; - } - // Separate test (if allocation of pa throws, el may be allocated. - // but no elements will be constructed.) - if(el) { - elements_array_allocator().deallocate(el, sz / 2); - el = NULL; - } - sz = INITIAL_SIZE; - ne = 0; - } - -public: - hash_buffer() : my_key(NULL), my_size(INITIAL_SIZE), nelements(0) { - internal_initialize_buffer(); - } - - ~hash_buffer() { - internal_free_buffer(pointer_array, elements_array, my_size, nelements); - if(my_key) delete my_key; - } - - void reset() { - internal_free_buffer(pointer_array, elements_array, my_size, nelements); - internal_initialize_buffer(); - } - - // Take ownership of func object allocated with new. - // This method is only used internally, so can't be misused by user. - void set_key_func(ValueToKey *vtk) { my_key = vtk; } - // pointer is used to clone() - ValueToKey* get_key_func() { return my_key; } - - bool insert_with_key(const value_type &v) { - pointer_type p = NULL; - __TBB_ASSERT(my_key, "Error: value-to-key functor not provided"); - if(find_ref_with_key((*my_key)(v), p)) { - p->~value_type(); - (void) new(p) value_type(v); // copy-construct into the space - return false; - } - ++nelements; - if(nelements*2 > my_size) grow_array(); - internal_insert_with_key(pointer_array, my_size, free_list, v); - return true; - } - - // returns true and sets v to array element if found, else returns false. - bool find_ref_with_key(const Knoref& k, pointer_type &v) { - size_t i = this->hash(k) & mask(); - for(element_type* p = pointer_array[i]; p; p = (element_type *)(p->second)) { - pointer_type pv = reinterpret_cast(&(p->first)); - __TBB_ASSERT(my_key, "Error: value-to-key functor not provided"); - if(this->equal((*my_key)(*pv), k)) { - v = pv; - return true; - } - } - return false; - } - - bool find_with_key( const Knoref& k, value_type &v) { - value_type *p; - if(find_ref_with_key(k, p)) { - v = *p; - return true; - } - else - return false; - } - - void delete_with_key(const Knoref& k) { - size_t h = this->hash(k) & mask(); - element_type* prev = NULL; - for(element_type* p = pointer_array[h]; p; prev = p, p = (element_type *)(p->second)) { - value_type *vp = reinterpret_cast(&(p->first)); - __TBB_ASSERT(my_key, "Error: value-to-key functor not provided"); - if(this->equal((*my_key)(*vp), k)) { - vp->~value_type(); - if(prev) prev->second = p->second; - else pointer_array[h] = (element_type *)(p->second); - p->second = free_list; - free_list = p; - --nelements; - return; - } - } - __TBB_ASSERT(false, "key not found for delete"); - } -}; -#endif // __TBB__flow_graph_hash_buffer_impl_H diff --git a/src/tbb/include/tbb/internal/_flow_graph_trace_impl.h b/src/tbb/include/tbb/internal/_flow_graph_trace_impl.h deleted file mode 100644 index 111810c3b..000000000 --- a/src/tbb/include/tbb/internal/_flow_graph_trace_impl.h +++ /dev/null @@ -1,304 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef _FGT_GRAPH_TRACE_IMPL_H -#define _FGT_GRAPH_TRACE_IMPL_H - -#include "../tbb_profiling.h" - -namespace tbb { - namespace internal { - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - -static inline void fgt_internal_alias_input_port( void *node, void *p, string_index name_index ) { - itt_make_task_group( ITT_DOMAIN_FLOW, p, FLOW_INPUT_PORT, node, FLOW_NODE, name_index ); - itt_relation_add( ITT_DOMAIN_FLOW, node, FLOW_NODE, __itt_relation_is_parent_of, p, FLOW_INPUT_PORT ); -} - -static inline void fgt_internal_alias_output_port( void *node, void *p, string_index name_index ) { - itt_make_task_group( ITT_DOMAIN_FLOW, p, FLOW_OUTPUT_PORT, node, FLOW_NODE, name_index ); - itt_relation_add( ITT_DOMAIN_FLOW, node, FLOW_NODE, __itt_relation_is_parent_of, p, FLOW_OUTPUT_PORT ); -} - -template -void alias_input_port(void *node, tbb::flow::receiver* port, string_index name_index) { - // TODO: Make fgt_internal_alias_input_port a function template? - fgt_internal_alias_input_port( node, port, name_index); -} - -template < typename PortsTuple, int N > -struct fgt_internal_input_alias_helper { - static void alias_port( void *node, PortsTuple &ports ) { - alias_input_port( node, &(tbb::flow::get(ports)), static_cast(FLOW_INPUT_PORT_0 + N - 1) ); - fgt_internal_input_alias_helper::alias_port( node, ports ); - } -}; - -template < typename PortsTuple > -struct fgt_internal_input_alias_helper { - static void alias_port( void * /* node */, PortsTuple & /* ports */ ) { } -}; - -template -void alias_output_port(void *node, tbb::flow::sender* port, string_index name_index) { - // TODO: Make fgt_internal_alias_output_port a function template? - fgt_internal_alias_output_port( node, static_cast(port), name_index); -} - -template < typename PortsTuple, int N > -struct fgt_internal_output_alias_helper { - static void alias_port( void *node, PortsTuple &ports ) { - alias_output_port( node, &(tbb::flow::get(ports)), static_cast(FLOW_OUTPUT_PORT_0 + N - 1) ); - fgt_internal_output_alias_helper::alias_port( node, ports ); - } -}; - -template < typename PortsTuple > -struct fgt_internal_output_alias_helper { - static void alias_port( void * /*node*/, PortsTuple &/*ports*/ ) { - } -}; - -static inline void fgt_internal_create_input_port( void *node, void *p, string_index name_index ) { - itt_make_task_group( ITT_DOMAIN_FLOW, p, FLOW_INPUT_PORT, node, FLOW_NODE, name_index ); -} - -static inline void fgt_internal_create_output_port( void *node, void *p, string_index name_index ) { - itt_make_task_group( ITT_DOMAIN_FLOW, p, FLOW_OUTPUT_PORT, node, FLOW_NODE, name_index ); -} - -template -void register_input_port(void *node, tbb::flow::receiver* port, string_index name_index) { - // TODO: Make fgt_internal_create_input_port a function template? - fgt_internal_create_input_port( node, port, name_index); -} - -template < typename PortsTuple, int N > -struct fgt_internal_input_helper { - static void register_port( void *node, PortsTuple &ports ) { - register_input_port( node, &(tbb::flow::get(ports)), static_cast(FLOW_INPUT_PORT_0 + N - 1) ); - fgt_internal_input_helper::register_port( node, ports ); - } -}; - -template < typename PortsTuple > -struct fgt_internal_input_helper { - static void register_port( void *node, PortsTuple &ports ) { - register_input_port( node, &(tbb::flow::get<0>(ports)), FLOW_INPUT_PORT_0 ); - } -}; - -template -void register_output_port(void *node, tbb::flow::sender* port, string_index name_index) { - // TODO: Make fgt_internal_create_output_port a function template? - fgt_internal_create_output_port( node, static_cast(port), name_index); -} - -template < typename PortsTuple, int N > -struct fgt_internal_output_helper { - static void register_port( void *node, PortsTuple &ports ) { - register_output_port( node, &(tbb::flow::get(ports)), static_cast(FLOW_OUTPUT_PORT_0 + N - 1) ); - fgt_internal_output_helper::register_port( node, ports ); - } -}; - -template < typename PortsTuple > -struct fgt_internal_output_helper { - static void register_port( void *node, PortsTuple &ports ) { - register_output_port( node, &(tbb::flow::get<0>(ports)), FLOW_OUTPUT_PORT_0 ); - } -}; - -template< typename NodeType > -void fgt_multioutput_node_desc( const NodeType *node, const char *desc ) { - void *addr = (void *)( static_cast< tbb::flow::receiver< typename NodeType::input_type > * >(const_cast< NodeType *>(node)) ); - itt_metadata_str_add( ITT_DOMAIN_FLOW, addr, FLOW_NODE, FLOW_OBJECT_NAME, desc ); -} - -template< typename NodeType > -void fgt_multiinput_multioutput_node_desc( const NodeType *node, const char *desc ) { - void *addr = const_cast(node); - itt_metadata_str_add( ITT_DOMAIN_FLOW, addr, FLOW_NODE, FLOW_OBJECT_NAME, desc ); -} - -template< typename NodeType > -static inline void fgt_node_desc( const NodeType *node, const char *desc ) { - void *addr = (void *)( static_cast< tbb::flow::sender< typename NodeType::output_type > * >(const_cast< NodeType *>(node)) ); - itt_metadata_str_add( ITT_DOMAIN_FLOW, addr, FLOW_NODE, FLOW_OBJECT_NAME, desc ); -} - -static inline void fgt_graph_desc( void *g, const char *desc ) { - itt_metadata_str_add( ITT_DOMAIN_FLOW, g, FLOW_GRAPH, FLOW_OBJECT_NAME, desc ); -} - -static inline void fgt_body( void *node, void *body ) { - itt_relation_add( ITT_DOMAIN_FLOW, body, FLOW_BODY, __itt_relation_is_child_of, node, FLOW_NODE ); -} - -template< int N, typename PortsTuple > -static inline void fgt_multioutput_node( string_index t, void *g, void *input_port, PortsTuple &ports ) { - itt_make_task_group( ITT_DOMAIN_FLOW, input_port, FLOW_NODE, g, FLOW_GRAPH, t ); - fgt_internal_create_input_port( input_port, input_port, FLOW_INPUT_PORT_0 ); - fgt_internal_output_helper::register_port( input_port, ports ); -} - -template< int N, typename PortsTuple > -static inline void fgt_multioutput_node_with_body( string_index t, void *g, void *input_port, PortsTuple &ports, void *body ) { - itt_make_task_group( ITT_DOMAIN_FLOW, input_port, FLOW_NODE, g, FLOW_GRAPH, t ); - fgt_internal_create_input_port( input_port, input_port, FLOW_INPUT_PORT_0 ); - fgt_internal_output_helper::register_port( input_port, ports ); - fgt_body( input_port, body ); -} - -template< int N, typename PortsTuple > -static inline void fgt_multiinput_node( string_index t, void *g, PortsTuple &ports, void *output_port) { - itt_make_task_group( ITT_DOMAIN_FLOW, output_port, FLOW_NODE, g, FLOW_GRAPH, t ); - fgt_internal_create_output_port( output_port, output_port, FLOW_OUTPUT_PORT_0 ); - fgt_internal_input_helper::register_port( output_port, ports ); -} - -static inline void fgt_multiinput_multioutput_node( string_index t, void *n, void *g ) { - itt_make_task_group( ITT_DOMAIN_FLOW, n, FLOW_NODE, g, FLOW_GRAPH, t ); -} - -static inline void fgt_node( string_index t, void *g, void *output_port ) { - itt_make_task_group( ITT_DOMAIN_FLOW, output_port, FLOW_NODE, g, FLOW_GRAPH, t ); - fgt_internal_create_output_port( output_port, output_port, FLOW_OUTPUT_PORT_0 ); -} - -static inline void fgt_node_with_body( string_index t, void *g, void *output_port, void *body ) { - itt_make_task_group( ITT_DOMAIN_FLOW, output_port, FLOW_NODE, g, FLOW_GRAPH, t ); - fgt_internal_create_output_port( output_port, output_port, FLOW_OUTPUT_PORT_0 ); - fgt_body( output_port, body ); -} - - -static inline void fgt_node( string_index t, void *g, void *input_port, void *output_port ) { - fgt_node( t, g, output_port ); - fgt_internal_create_input_port( output_port, input_port, FLOW_INPUT_PORT_0 ); -} - -static inline void fgt_node_with_body( string_index t, void *g, void *input_port, void *output_port, void *body ) { - fgt_node_with_body( t, g, output_port, body ); - fgt_internal_create_input_port( output_port, input_port, FLOW_INPUT_PORT_0 ); -} - - -static inline void fgt_node( string_index t, void *g, void *input_port, void *decrement_port, void *output_port ) { - fgt_node( t, g, input_port, output_port ); - fgt_internal_create_input_port( output_port, decrement_port, FLOW_INPUT_PORT_1 ); -} - -static inline void fgt_make_edge( void *output_port, void *input_port ) { - itt_relation_add( ITT_DOMAIN_FLOW, output_port, FLOW_OUTPUT_PORT, __itt_relation_is_predecessor_to, input_port, FLOW_INPUT_PORT); -} - -static inline void fgt_remove_edge( void *output_port, void *input_port ) { - itt_relation_add( ITT_DOMAIN_FLOW, output_port, FLOW_OUTPUT_PORT, __itt_relation_is_sibling_of, input_port, FLOW_INPUT_PORT); -} - -static inline void fgt_graph( void *g ) { - itt_make_task_group( ITT_DOMAIN_FLOW, g, FLOW_GRAPH, NULL, FLOW_NULL, FLOW_GRAPH ); -} - -static inline void fgt_begin_body( void *body ) { - itt_task_begin( ITT_DOMAIN_FLOW, body, FLOW_BODY, NULL, FLOW_NULL, FLOW_BODY ); -} - -static inline void fgt_end_body( void * ) { - itt_task_end( ITT_DOMAIN_FLOW ); -} - -static inline void fgt_async_try_put_begin( void *node, void *port ) { - itt_task_begin( ITT_DOMAIN_FLOW, port, FLOW_OUTPUT_PORT, node, FLOW_NODE, FLOW_OUTPUT_PORT ); -} - -static inline void fgt_async_try_put_end( void *, void * ) { - itt_task_end( ITT_DOMAIN_FLOW ); -} - -static inline void fgt_async_reserve( void *node, void *graph ) { - itt_region_begin( ITT_DOMAIN_FLOW, node, FLOW_NODE, graph, FLOW_GRAPH, FLOW_NULL ); -} - -static inline void fgt_async_commit( void *node, void *graph ) { - itt_region_end( ITT_DOMAIN_FLOW, node, FLOW_NODE ); -} - -static inline void fgt_reserve_wait( void *graph ) { - itt_region_begin( ITT_DOMAIN_FLOW, graph, FLOW_GRAPH, NULL, FLOW_NULL, FLOW_NULL ); -} - -static inline void fgt_release_wait( void *graph ) { - itt_region_end( ITT_DOMAIN_FLOW, graph, FLOW_GRAPH ); -} - -#else // TBB_PREVIEW_FLOW_GRAPH_TRACE - -static inline void fgt_graph( void * /*g*/ ) { } - -template< typename NodeType > -static inline void fgt_multioutput_node_desc( const NodeType * /*node*/, const char * /*desc*/ ) { } - -template< typename NodeType > -static inline void fgt_node_desc( const NodeType * /*node*/, const char * /*desc*/ ) { } - -static inline void fgt_graph_desc( void * /*g*/, const char * /*desc*/ ) { } - -static inline void fgt_body( void * /*node*/, void * /*body*/ ) { } - -template< int N, typename PortsTuple > -static inline void fgt_multioutput_node( string_index /*t*/, void * /*g*/, void * /*input_port*/, PortsTuple & /*ports*/ ) { } - -template< int N, typename PortsTuple > -static inline void fgt_multioutput_node_with_body( string_index /*t*/, void * /*g*/, void * /*input_port*/, PortsTuple & /*ports*/, void * /*body*/ ) { } - -template< int N, typename PortsTuple > -static inline void fgt_multiinput_node( string_index /*t*/, void * /*g*/, PortsTuple & /*ports*/, void * /*output_port*/ ) { } - -static inline void fgt_multiinput_multioutput_node( string_index /*t*/, void * /*node*/, void * /*graph*/ ) { } - -static inline void fgt_node( string_index /*t*/, void * /*g*/, void * /*output_port*/ ) { } -static inline void fgt_node( string_index /*t*/, void * /*g*/, void * /*input_port*/, void * /*output_port*/ ) { } -static inline void fgt_node( string_index /*t*/, void * /*g*/, void * /*input_port*/, void * /*decrement_port*/, void * /*output_port*/ ) { } - -static inline void fgt_node_with_body( string_index /*t*/, void * /*g*/, void * /*output_port*/, void * /*body*/ ) { } -static inline void fgt_node_with_body( string_index /*t*/, void * /*g*/, void * /*input_port*/, void * /*output_port*/, void * /*body*/ ) { } - -static inline void fgt_make_edge( void * /*output_port*/, void * /*input_port*/ ) { } -static inline void fgt_remove_edge( void * /*output_port*/, void * /*input_port*/ ) { } - -static inline void fgt_begin_body( void * /*body*/ ) { } -static inline void fgt_end_body( void * /*body*/) { } - -static inline void fgt_async_try_put_begin( void * /*node*/, void * /*port*/ ) { } -static inline void fgt_async_try_put_end( void * /*node*/ , void * /*port*/ ) { } -static inline void fgt_async_reserve( void * /*node*/, void * /*graph*/ ) { } -static inline void fgt_async_commit( void * /*node*/, void * /*graph*/ ) { } -static inline void fgt_reserve_wait( void * /*graph*/ ) { } -static inline void fgt_release_wait( void * /*graph*/ ) { } - -#endif // TBB_PREVIEW_FLOW_GRAPH_TRACE - - } // namespace internal -} // namespace tbb - -#endif diff --git a/src/tbb/include/tbb/internal/_flow_graph_types_impl.h b/src/tbb/include/tbb/internal/_flow_graph_types_impl.h deleted file mode 100644 index 73b5f5479..000000000 --- a/src/tbb/include/tbb/internal/_flow_graph_types_impl.h +++ /dev/null @@ -1,713 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB__flow_graph_types_impl_H -#define __TBB__flow_graph_types_impl_H - -#ifndef __TBB_flow_graph_H -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -// included in namespace tbb::flow::interfaceX - -namespace internal { - - // the change to key_matching (adding a K and KHash template parameter, making it a class) - // means we have to pass this data to the key_matching_port. All the ports have only one - // template parameter, so we have to wrap the following types in a trait: - // - // . K == key_type - // . KHash == hash and compare for Key - // . TtoK == function_body that given an object of T, returns its K - // . T == type accepted by port, and stored in the hash table - // - // The port will have an additional parameter on node construction, which is a function_body - // that accepts a const T& and returns a K which is the field in T which is its K. - template - struct KeyTrait { - typedef Kp K; - typedef Tp T; - typedef internal::type_to_key_function_body TtoK; - typedef KHashp KHash; - }; - -// wrap each element of a tuple in a template, and make a tuple of the result. - template class PT, typename TypeTuple> - struct wrap_tuple_elements; - - // A wrapper that generates the traits needed for each port of a key-matching join, - // and the type of the tuple of input ports. - template class PT, typename KeyTraits, typename TypeTuple> - struct wrap_key_tuple_elements; - - template class PT, typename TypeTuple> - struct wrap_tuple_elements<1, PT, TypeTuple> { - typedef typename tbb::flow::tuple< - PT::type> > - type; - }; - - template class PT, typename KeyTraits, typename TypeTuple> - struct wrap_key_tuple_elements<1, PT, KeyTraits, TypeTuple > { - typedef typename KeyTraits::key_type K; - typedef typename KeyTraits::hash_compare_type KHash; - typedef KeyTrait::type> KeyTrait0; - typedef typename tbb::flow::tuple< PT > type; - }; - - template class PT, typename TypeTuple> - struct wrap_tuple_elements<2, PT, TypeTuple> { - typedef typename tbb::flow::tuple< - PT::type>, - PT::type> > - type; - }; - - template class PT, typename KeyTraits, typename TypeTuple> - struct wrap_key_tuple_elements<2, PT, KeyTraits, TypeTuple> { - typedef typename KeyTraits::key_type K; - typedef typename KeyTraits::hash_compare_type KHash; - typedef KeyTrait::type> KeyTrait0; - typedef KeyTrait::type> KeyTrait1; - typedef typename tbb::flow::tuple< PT, PT > type; - }; - - template class PT, typename TypeTuple> - struct wrap_tuple_elements<3, PT, TypeTuple> { - typedef typename tbb::flow::tuple< - PT::type>, - PT::type>, - PT::type> > - type; - }; - - template class PT, typename KeyTraits, typename TypeTuple> - struct wrap_key_tuple_elements<3, PT, KeyTraits, TypeTuple> { - typedef typename KeyTraits::key_type K; - typedef typename KeyTraits::hash_compare_type KHash; - typedef KeyTrait::type> KeyTrait0; - typedef KeyTrait::type> KeyTrait1; - typedef KeyTrait::type> KeyTrait2; - typedef typename tbb::flow::tuple< PT, PT, PT > type; - }; - - template class PT, typename TypeTuple> - struct wrap_tuple_elements<4, PT, TypeTuple> { - typedef typename tbb::flow::tuple< - PT::type>, - PT::type>, - PT::type>, - PT::type> > - type; - }; - - template class PT, typename KeyTraits, typename TypeTuple> - struct wrap_key_tuple_elements<4, PT, KeyTraits, TypeTuple> { - typedef typename KeyTraits::key_type K; - typedef typename KeyTraits::hash_compare_type KHash; - typedef KeyTrait::type> KeyTrait0; - typedef KeyTrait::type> KeyTrait1; - typedef KeyTrait::type> KeyTrait2; - typedef KeyTrait::type> KeyTrait3; - typedef typename tbb::flow::tuple< PT, PT, PT, - PT > type; - }; - - template class PT, typename TypeTuple> - struct wrap_tuple_elements<5, PT, TypeTuple> { - typedef typename tbb::flow::tuple< - PT::type>, - PT::type>, - PT::type>, - PT::type>, - PT::type> > - type; - }; - - template class PT, typename KeyTraits, typename TypeTuple> - struct wrap_key_tuple_elements<5, PT, KeyTraits, TypeTuple> { - typedef typename KeyTraits::key_type K; - typedef typename KeyTraits::hash_compare_type KHash; - typedef KeyTrait::type> KeyTrait0; - typedef KeyTrait::type> KeyTrait1; - typedef KeyTrait::type> KeyTrait2; - typedef KeyTrait::type> KeyTrait3; - typedef KeyTrait::type> KeyTrait4; - typedef typename tbb::flow::tuple< PT, PT, PT, - PT, PT > type; - }; - -#if __TBB_VARIADIC_MAX >= 6 - template class PT, typename TypeTuple> - struct wrap_tuple_elements<6, PT, TypeTuple> { - typedef typename tbb::flow::tuple< - PT::type>, - PT::type>, - PT::type>, - PT::type>, - PT::type>, - PT::type> > - type; - }; - - template class PT, typename KeyTraits, typename TypeTuple> - struct wrap_key_tuple_elements<6, PT, KeyTraits, TypeTuple> { - typedef typename KeyTraits::key_type K; - typedef typename KeyTraits::hash_compare_type KHash; - typedef KeyTrait::type> KeyTrait0; - typedef KeyTrait::type> KeyTrait1; - typedef KeyTrait::type> KeyTrait2; - typedef KeyTrait::type> KeyTrait3; - typedef KeyTrait::type> KeyTrait4; - typedef KeyTrait::type> KeyTrait5; - typedef typename tbb::flow::tuple< PT, PT, PT, PT, - PT, PT > type; - }; -#endif - -#if __TBB_VARIADIC_MAX >= 7 - template class PT, typename TypeTuple> - struct wrap_tuple_elements<7, PT, TypeTuple> { - typedef typename tbb::flow::tuple< - PT::type>, - PT::type>, - PT::type>, - PT::type>, - PT::type>, - PT::type>, - PT::type> > - type; - }; - - template class PT, typename KeyTraits, typename TypeTuple> - struct wrap_key_tuple_elements<7, PT, KeyTraits, TypeTuple> { - typedef typename KeyTraits::key_type K; - typedef typename KeyTraits::hash_compare_type KHash; - typedef KeyTrait::type> KeyTrait0; - typedef KeyTrait::type> KeyTrait1; - typedef KeyTrait::type> KeyTrait2; - typedef KeyTrait::type> KeyTrait3; - typedef KeyTrait::type> KeyTrait4; - typedef KeyTrait::type> KeyTrait5; - typedef KeyTrait::type> KeyTrait6; - typedef typename tbb::flow::tuple< PT, PT, PT, PT, - PT, PT, PT > type; - }; -#endif - -#if __TBB_VARIADIC_MAX >= 8 - template class PT, typename TypeTuple> - struct wrap_tuple_elements<8, PT, TypeTuple> { - typedef typename tbb::flow::tuple< - PT::type>, - PT::type>, - PT::type>, - PT::type>, - PT::type>, - PT::type>, - PT::type>, - PT::type> > - type; - }; - - template class PT, typename KeyTraits, typename TypeTuple> - struct wrap_key_tuple_elements<8, PT, KeyTraits, TypeTuple> { - typedef typename KeyTraits::key_type K; - typedef typename KeyTraits::hash_compare_type KHash; - typedef KeyTrait::type> KeyTrait0; - typedef KeyTrait::type> KeyTrait1; - typedef KeyTrait::type> KeyTrait2; - typedef KeyTrait::type> KeyTrait3; - typedef KeyTrait::type> KeyTrait4; - typedef KeyTrait::type> KeyTrait5; - typedef KeyTrait::type> KeyTrait6; - typedef KeyTrait::type> KeyTrait7; - typedef typename tbb::flow::tuple< PT, PT, PT, PT, - PT, PT, PT, PT > type; - }; -#endif - -#if __TBB_VARIADIC_MAX >= 9 - template class PT, typename TypeTuple> - struct wrap_tuple_elements<9, PT, TypeTuple> { - typedef typename tbb::flow::tuple< - PT::type>, - PT::type>, - PT::type>, - PT::type>, - PT::type>, - PT::type>, - PT::type>, - PT::type>, - PT::type> > - type; - }; - - template class PT, typename KeyTraits, typename TypeTuple> - struct wrap_key_tuple_elements<9, PT, KeyTraits, TypeTuple> { - typedef typename KeyTraits::key_type K; - typedef typename KeyTraits::hash_compare_type KHash; - typedef KeyTrait::type> KeyTrait0; - typedef KeyTrait::type> KeyTrait1; - typedef KeyTrait::type> KeyTrait2; - typedef KeyTrait::type> KeyTrait3; - typedef KeyTrait::type> KeyTrait4; - typedef KeyTrait::type> KeyTrait5; - typedef KeyTrait::type> KeyTrait6; - typedef KeyTrait::type> KeyTrait7; - typedef KeyTrait::type> KeyTrait8; - typedef typename tbb::flow::tuple< PT, PT, PT, PT, - PT, PT, PT, PT, PT > type; - }; -#endif - -#if __TBB_VARIADIC_MAX >= 10 - template class PT, typename TypeTuple> - struct wrap_tuple_elements<10, PT, TypeTuple> { - typedef typename tbb::flow::tuple< - PT::type>, - PT::type>, - PT::type>, - PT::type>, - PT::type>, - PT::type>, - PT::type>, - PT::type>, - PT::type>, - PT::type> > - type; - }; - - template class PT, typename KeyTraits, typename TypeTuple> - struct wrap_key_tuple_elements<10, PT, KeyTraits, TypeTuple> { - typedef typename KeyTraits::key_type K; - typedef typename KeyTraits::hash_compare_type KHash; - typedef KeyTrait::type> KeyTrait0; - typedef KeyTrait::type> KeyTrait1; - typedef KeyTrait::type> KeyTrait2; - typedef KeyTrait::type> KeyTrait3; - typedef KeyTrait::type> KeyTrait4; - typedef KeyTrait::type> KeyTrait5; - typedef KeyTrait::type> KeyTrait6; - typedef KeyTrait::type> KeyTrait7; - typedef KeyTrait::type> KeyTrait8; - typedef KeyTrait::type> KeyTrait9; - typedef typename tbb::flow::tuple< PT, PT, PT, PT, - PT, PT, PT, PT, PT, - PT > type; - }; -#endif - -#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT - template< int... S > class sequence {}; - - template< int N, int... S > - struct make_sequence : make_sequence < N - 1, N - 1, S... > {}; - - template< int... S > - struct make_sequence < 0, S... > { - typedef sequence type; - }; -#endif /* __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT */ - -#if __TBB_INITIALIZER_LISTS_PRESENT - // Until C++14 std::initializer_list does not guarantee life time of contained objects. - template - class initializer_list_wrapper { - public: - typedef T value_type; - typedef const T& reference; - typedef const T& const_reference; - typedef size_t size_type; - - typedef T* iterator; - typedef const T* const_iterator; - - initializer_list_wrapper( std::initializer_list il ) __TBB_NOEXCEPT( true ) : my_begin( static_cast(malloc( il.size()*sizeof( T ) )) ) { - iterator dst = my_begin; - for ( typename std::initializer_list::const_iterator src = il.begin(); src != il.end(); ++src ) - new (dst++) T( *src ); - my_end = dst; - } - - initializer_list_wrapper( const initializer_list_wrapper& ilw ) __TBB_NOEXCEPT( true ) : my_begin( static_cast(malloc( ilw.size()*sizeof( T ) )) ) { - iterator dst = my_begin; - for ( typename std::initializer_list::const_iterator src = ilw.begin(); src != ilw.end(); ++src ) - new (dst++) T( *src ); - my_end = dst; - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - initializer_list_wrapper( initializer_list_wrapper&& ilw ) __TBB_NOEXCEPT( true ) : my_begin( ilw.my_begin ), my_end( ilw.my_end ) { - ilw.my_begin = ilw.my_end = NULL; - } -#endif /* __TBB_CPP11_RVALUE_REF_PRESENT */ - - ~initializer_list_wrapper() { - if ( my_begin ) - free( my_begin ); - } - - const_iterator begin() const __TBB_NOEXCEPT(true) { return my_begin; } - const_iterator end() const __TBB_NOEXCEPT(true) { return my_end; } - size_t size() const __TBB_NOEXCEPT(true) { return (size_t)(my_end - my_begin); } - - private: - iterator my_begin; - iterator my_end; - }; -#endif /* __TBB_INITIALIZER_LISTS_PRESENT */ - -//! type mimicking std::pair but with trailing fill to ensure each element of an array -//* will have the correct alignment - template - struct type_plus_align { - char first[sizeof(T1)]; - T2 second; - char fill1[REM]; - }; - - template - struct type_plus_align { - char first[sizeof(T1)]; - T2 second; - }; - - template struct alignment_of { - typedef struct { char t; U padded; } test_alignment; - static const size_t value = sizeof(test_alignment) - sizeof(U); - }; - - // T1, T2 are actual types stored. The space defined for T1 in the type returned - // is a char array of the correct size. Type T2 should be trivially-constructible, - // T1 must be explicitly managed. - template - struct aligned_pair { - static const size_t t1_align = alignment_of::value; - static const size_t t2_align = alignment_of::value; - typedef type_plus_align just_pair; - static const size_t max_align = t1_align < t2_align ? t2_align : t1_align; - static const size_t extra_bytes = sizeof(just_pair) % max_align; - static const size_t remainder = extra_bytes ? max_align - extra_bytes : 0; - public: - typedef type_plus_align type; - }; // aligned_pair - -// support for variant type -// type we use when we're not storing a value -struct default_constructed { }; - -// type which contains another type, tests for what type is contained, and references to it. -// internal::Wrapper -// void CopyTo( void *newSpace) : builds a Wrapper copy of itself in newSpace - -// struct to allow us to copy and test the type of objects -struct WrapperBase { - virtual ~WrapperBase() {} - virtual void CopyTo(void* /*newSpace*/) const { } -}; - -// Wrapper contains a T, with the ability to test what T is. The Wrapper can be -// constructed from a T, can be copy-constructed from another Wrapper, and can be -// examined via value(), but not modified. -template -struct Wrapper: public WrapperBase { - typedef T value_type; - typedef T* pointer_type; -private: - T value_space; -public: - const value_type &value() const { return value_space; } - -private: - Wrapper(); - - // on exception will ensure the Wrapper will contain only a trivially-constructed object - struct _unwind_space { - pointer_type space; - _unwind_space(pointer_type p) : space(p) {} - ~_unwind_space() { - if(space) (void) new (space) Wrapper(default_constructed()); - } - }; -public: - explicit Wrapper( const T& other ) : value_space(other) { } - explicit Wrapper(const Wrapper& other) : value_space(other.value_space) { } - - void CopyTo(void* newSpace) const __TBB_override { - _unwind_space guard((pointer_type)newSpace); - (void) new(newSpace) Wrapper(value_space); - guard.space = NULL; - } - ~Wrapper() { } -}; - -// specialization for array objects -template -struct Wrapper : public WrapperBase { - typedef T value_type; - typedef T* pointer_type; - // space must be untyped. - typedef T ArrayType[N]; -private: - // The space is not of type T[N] because when copy-constructing, it would be - // default-initialized and then copied to in some fashion, resulting in two - // constructions and one destruction per element. If the type is char[ ], we - // placement new into each element, resulting in one construction per element. - static const size_t space_size = sizeof(ArrayType) / sizeof(char); - char value_space[space_size]; - - - // on exception will ensure the already-built objects will be destructed - // (the value_space is a char array, so it is already trivially-destructible.) - struct _unwind_class { - pointer_type space; - int already_built; - _unwind_class(pointer_type p) : space(p), already_built(0) {} - ~_unwind_class() { - if(space) { - for(size_t i = already_built; i > 0 ; --i ) space[i-1].~value_type(); - (void) new(space) Wrapper(default_constructed()); - } - } - }; -public: - const ArrayType &value() const { - char *vp = const_cast(value_space); - return reinterpret_cast(*vp); - } - -private: - Wrapper(); -public: - // have to explicitly construct because other decays to a const value_type* - explicit Wrapper(const ArrayType& other) { - _unwind_class guard((pointer_type)value_space); - pointer_type vp = reinterpret_cast(&value_space); - for(size_t i = 0; i < N; ++i ) { - (void) new(vp++) value_type(other[i]); - ++(guard.already_built); - } - guard.space = NULL; - } - explicit Wrapper(const Wrapper& other) : WrapperBase() { - // we have to do the heavy lifting to copy contents - _unwind_class guard((pointer_type)value_space); - pointer_type dp = reinterpret_cast(value_space); - pointer_type sp = reinterpret_cast(const_cast(other.value_space)); - for(size_t i = 0; i < N; ++i, ++dp, ++sp) { - (void) new(dp) value_type(*sp); - ++(guard.already_built); - } - guard.space = NULL; - } - - void CopyTo(void* newSpace) const __TBB_override { - (void) new(newSpace) Wrapper(*this); // exceptions handled in copy constructor - } - - ~Wrapper() { - // have to destroy explicitly in reverse order - pointer_type vp = reinterpret_cast(&value_space); - for(size_t i = N; i > 0 ; --i ) vp[i-1].~value_type(); - } -}; - -// given a tuple, return the type of the element that has the maximum alignment requirement. -// Given a tuple and that type, return the number of elements of the object with the max -// alignment requirement that is at least as big as the largest object in the tuple. - -template struct pick_one; -template struct pick_one { typedef T1 type; }; -template struct pick_one { typedef T2 type; }; - -template< template class Selector, typename T1, typename T2 > -struct pick_max { - typedef typename pick_one< (Selector::value > Selector::value), T1, T2 >::type type; -}; - -template struct size_of { static const int value = sizeof(T); }; - -template< size_t N, class Tuple, template class Selector > struct pick_tuple_max { - typedef typename pick_tuple_max::type LeftMaxType; - typedef typename tbb::flow::tuple_element::type ThisType; - typedef typename pick_max::type type; -}; - -template< class Tuple, template class Selector > struct pick_tuple_max<0, Tuple, Selector> { - typedef typename tbb::flow::tuple_element<0, Tuple>::type type; -}; - -// is the specified type included in a tuple? -template -struct is_element_of { - typedef typename tbb::flow::tuple_element::type T_i; - static const bool value = tbb::internal::is_same_type::value || is_element_of::value; -}; - -template -struct is_element_of { - typedef typename tbb::flow::tuple_element<0, Tuple>::type T_i; - static const bool value = tbb::internal::is_same_type::value; -}; - -// allow the construction of types that are listed tuple. If a disallowed type -// construction is written, a method involving this type is created. The -// type has no definition, so a syntax error is generated. -template struct ERROR_Type_Not_allowed_In_Tagged_Msg_Not_Member_Of_Tuple; - -template struct do_if; -template -struct do_if { - static void construct(void *mySpace, const T& x) { - (void) new(mySpace) Wrapper(x); - } -}; -template -struct do_if { - static void construct(void * /*mySpace*/, const T& x) { - // This method is instantiated when the type T does not match any of the - // element types in the Tuple in variant. - ERROR_Type_Not_allowed_In_Tagged_Msg_Not_Member_Of_Tuple::bad_type(x); - } -}; - -// Tuple tells us the allowed types that variant can hold. It determines the alignment of the space in -// Wrapper, and how big Wrapper is. -// -// the object can only be tested for type, and a read-only reference can be fetched by cast_to(). - -using tbb::internal::punned_cast; -struct tagged_null_type {}; -template -class tagged_msg { - typedef tbb::flow::tuple= 6 - , T5 - #endif - #if __TBB_VARIADIC_MAX >= 7 - , T6 - #endif - #if __TBB_VARIADIC_MAX >= 8 - , T7 - #endif - #if __TBB_VARIADIC_MAX >= 9 - , T8 - #endif - #if __TBB_VARIADIC_MAX >= 10 - , T9 - #endif - > Tuple; - -private: - class variant { - static const size_t N = tbb::flow::tuple_size::value; - typedef typename pick_tuple_max::type AlignType; - typedef typename pick_tuple_max::type MaxSizeType; - static const size_t MaxNBytes = (sizeof(Wrapper)+sizeof(AlignType)-1); - static const size_t MaxNElements = MaxNBytes/sizeof(AlignType); - typedef typename tbb::aligned_space SpaceType; - SpaceType my_space; - static const size_t MaxSize = sizeof(SpaceType); - - public: - variant() { (void) new(&my_space) Wrapper(default_constructed()); } - - template - variant( const T& x ) { - do_if::value>::construct(&my_space,x); - } - - variant(const variant& other) { - const WrapperBase * h = punned_cast(&(other.my_space)); - h->CopyTo(&my_space); - } - - // assignment must destroy and re-create the Wrapper type, as there is no way - // to create a Wrapper-to-Wrapper assign even if we find they agree in type. - void operator=( const variant& rhs ) { - if(&rhs != this) { - WrapperBase *h = punned_cast(&my_space); - h->~WrapperBase(); - const WrapperBase *ch = punned_cast(&(rhs.my_space)); - ch->CopyTo(&my_space); - } - } - - template - const U& variant_cast_to() const { - const Wrapper *h = dynamic_cast*>(punned_cast(&my_space)); - if(!h) { - tbb::internal::throw_exception(tbb::internal::eid_bad_tagged_msg_cast); - } - return h->value(); - } - template - bool variant_is_a() const { return dynamic_cast*>(punned_cast(&my_space)) != NULL; } - - bool variant_is_default_constructed() const {return variant_is_a();} - - ~variant() { - WrapperBase *h = punned_cast(&my_space); - h->~WrapperBase(); - } - }; //class variant - - TagType my_tag; - variant my_msg; - -public: - tagged_msg(): my_tag(TagType(~0)), my_msg(){} - - template - tagged_msg(T const &index, R const &value) : my_tag(index), my_msg(value) {} - - #if __TBB_CONST_REF_TO_ARRAY_TEMPLATE_PARAM_BROKEN - template - tagged_msg(T const &index, R (&value)[N]) : my_tag(index), my_msg(value) {} - #endif - - void set_tag(TagType const &index) {my_tag = index;} - TagType tag() const {return my_tag;} - - template - const V& cast_to() const {return my_msg.template variant_cast_to();} - - template - bool is_a() const {return my_msg.template variant_is_a();} - - bool is_default_constructed() const {return my_msg.variant_is_default_constructed();} -}; //class tagged_msg - -// template to simplify cast and test for tagged_msg in template contexts -template -const V& cast_to(T const &t) { return t.template cast_to(); } - -template -bool is_a(T const &t) { return t.template is_a(); } - -enum op_stat { WAIT = 0, SUCCEEDED, FAILED }; - -} // namespace internal - -#endif /* __TBB__flow_graph_types_impl_H */ diff --git a/src/tbb/include/tbb/internal/_mutex_padding.h b/src/tbb/include/tbb/internal/_mutex_padding.h deleted file mode 100644 index 6c1d9b59e..000000000 --- a/src/tbb/include/tbb/internal/_mutex_padding.h +++ /dev/null @@ -1,102 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_mutex_padding_H -#define __TBB_mutex_padding_H - -// wrapper for padding mutexes to be alone on a cache line, without requiring they be allocated -// from a pool. Because we allow them to be defined anywhere they must be two cache lines in size. - - -namespace tbb { -namespace interface7 { -namespace internal { - -static const size_t cache_line_size = 64; - -// Pad a mutex to occupy a number of full cache lines sufficient to avoid false sharing -// with other data; space overhead is up to 2*cache_line_size-1. -template class padded_mutex; - -template -class padded_mutex : tbb::internal::mutex_copy_deprecated_and_disabled { - typedef long pad_type; - pad_type my_pad[((sizeof(Mutex)+cache_line_size-1)/cache_line_size+1)*cache_line_size/sizeof(pad_type)]; - - Mutex *impl() { return (Mutex *)((uintptr_t(this)|(cache_line_size-1))+1);} - -public: - static const bool is_rw_mutex = Mutex::is_rw_mutex; - static const bool is_recursive_mutex = Mutex::is_recursive_mutex; - static const bool is_fair_mutex = Mutex::is_fair_mutex; - - padded_mutex() { new(impl()) Mutex(); } - ~padded_mutex() { impl()->~Mutex(); } - - //! Represents acquisition of a mutex. - class scoped_lock : tbb::internal::no_copy { - typename Mutex::scoped_lock my_scoped_lock; - public: - scoped_lock() : my_scoped_lock() {} - scoped_lock( padded_mutex& m ) : my_scoped_lock(*m.impl()) { } - ~scoped_lock() { } - - void acquire( padded_mutex& m ) { my_scoped_lock.acquire(*m.impl()); } - bool try_acquire( padded_mutex& m ) { return my_scoped_lock.try_acquire(*m.impl()); } - void release() { my_scoped_lock.release(); } - }; -}; - -template -class padded_mutex : tbb::internal::mutex_copy_deprecated_and_disabled { - typedef long pad_type; - pad_type my_pad[((sizeof(Mutex)+cache_line_size-1)/cache_line_size+1)*cache_line_size/sizeof(pad_type)]; - - Mutex *impl() { return (Mutex *)((uintptr_t(this)|(cache_line_size-1))+1);} - -public: - static const bool is_rw_mutex = Mutex::is_rw_mutex; - static const bool is_recursive_mutex = Mutex::is_recursive_mutex; - static const bool is_fair_mutex = Mutex::is_fair_mutex; - - padded_mutex() { new(impl()) Mutex(); } - ~padded_mutex() { impl()->~Mutex(); } - - //! Represents acquisition of a mutex. - class scoped_lock : tbb::internal::no_copy { - typename Mutex::scoped_lock my_scoped_lock; - public: - scoped_lock() : my_scoped_lock() {} - scoped_lock( padded_mutex& m, bool write = true ) : my_scoped_lock(*m.impl(),write) { } - ~scoped_lock() { } - - void acquire( padded_mutex& m, bool write = true ) { my_scoped_lock.acquire(*m.impl(),write); } - bool try_acquire( padded_mutex& m, bool write = true ) { return my_scoped_lock.try_acquire(*m.impl(),write); } - bool upgrade_to_writer() { return my_scoped_lock.upgrade_to_writer(); } - bool downgrade_to_reader() { return my_scoped_lock.downgrade_to_reader(); } - void release() { my_scoped_lock.release(); } - }; -}; - -} // namespace internal -} // namespace interface7 -} // namespace tbb - -#endif /* __TBB_mutex_padding_H */ diff --git a/src/tbb/include/tbb/internal/_range_iterator.h b/src/tbb/include/tbb/internal/_range_iterator.h deleted file mode 100644 index 5ebc42e89..000000000 --- a/src/tbb/include/tbb/internal/_range_iterator.h +++ /dev/null @@ -1,70 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_range_iterator_H -#define __TBB_range_iterator_H - -#include "../tbb_stddef.h" - -#if __TBB_CPP11_STD_BEGIN_END_PRESENT && __TBB_CPP11_AUTO_PRESENT && __TBB_CPP11_DECLTYPE_PRESENT - #include -#endif - -namespace tbb { - // iterators to first and last elements of container - namespace internal { - -#if __TBB_CPP11_STD_BEGIN_END_PRESENT && __TBB_CPP11_AUTO_PRESENT && __TBB_CPP11_DECLTYPE_PRESENT - using std::begin; - using std::end; - template - auto first(Container& c)-> decltype(begin(c)) {return begin(c);} - - template - auto first(const Container& c)-> decltype(begin(c)) {return begin(c);} - - template - auto last(Container& c)-> decltype(begin(c)) {return end(c);} - - template - auto last(const Container& c)-> decltype(begin(c)) {return end(c);} -#else - template - typename Container::iterator first(Container& c) {return c.begin();} - - template - typename Container::const_iterator first(const Container& c) {return c.begin();} - - template - typename Container::iterator last(Container& c) {return c.end();} - - template - typename Container::const_iterator last(const Container& c) {return c.end();} -#endif - - template - T* first(T (&arr) [size]) {return arr;} - - template - T* last(T (&arr) [size]) {return arr + size;} - } //namespace internal -} //namespace tbb - -#endif // __TBB_range_iterator_H diff --git a/src/tbb/include/tbb/internal/_tbb_hash_compare_impl.h b/src/tbb/include/tbb/internal/_tbb_hash_compare_impl.h deleted file mode 100644 index 6381e2dc8..000000000 --- a/src/tbb/include/tbb/internal/_tbb_hash_compare_impl.h +++ /dev/null @@ -1,109 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -// must be included outside namespaces. -#ifndef __TBB_tbb_hash_compare_impl_H -#define __TBB_tbb_hash_compare_impl_H - -#include - -namespace tbb { -namespace interface5 { -namespace internal { - -// Template class for hash compare -template -class hash_compare -{ -public: - typedef Hasher hasher; - typedef Key_equality key_equal; - - hash_compare() {} - - hash_compare(Hasher a_hasher) : my_hash_object(a_hasher) {} - - hash_compare(Hasher a_hasher, Key_equality a_keyeq) : my_hash_object(a_hasher), my_key_compare_object(a_keyeq) {} - - size_t operator()(const Key& key) const { - return ((size_t)my_hash_object(key)); - } - - bool operator()(const Key& key1, const Key& key2) const { - // TODO: get rid of the result invertion - return (!my_key_compare_object(key1, key2)); - } - - Hasher my_hash_object; // The hash object - Key_equality my_key_compare_object; // The equality comparator object -}; - -//! Hash multiplier -static const size_t hash_multiplier = tbb::internal::select_size_t_constant<2654435769U, 11400714819323198485ULL>::value; - -} // namespace internal - -//! Hasher functions -template -inline size_t tbb_hasher( const T& t ) { - return static_cast( t ) * internal::hash_multiplier; -} -template -inline size_t tbb_hasher( P* ptr ) { - size_t const h = reinterpret_cast( ptr ); - return (h >> 3) ^ h; -} -template -inline size_t tbb_hasher( const std::basic_string& s ) { - size_t h = 0; - for( const E* c = s.c_str(); *c; ++c ) - h = static_cast(*c) ^ (h * internal::hash_multiplier); - return h; -} -template -inline size_t tbb_hasher( const std::pair& p ) { - return tbb_hasher(p.first) ^ tbb_hasher(p.second); -} - -} // namespace interface5 -using interface5::tbb_hasher; - -// Template class for hash compare -template -class tbb_hash -{ -public: - tbb_hash() {} - - size_t operator()(const Key& key) const - { - return tbb_hasher(key); - } -}; - -//! hash_compare that is default argument for concurrent_hash_map -template -struct tbb_hash_compare { - static size_t hash( const Key& a ) { return tbb_hasher(a); } - static bool equal( const Key& a, const Key& b ) { return a == b; } -}; - -} // namespace tbb -#endif /* __TBB_tbb_hash_compare_impl_H */ diff --git a/src/tbb/include/tbb/internal/_tbb_strings.h b/src/tbb/include/tbb/internal/_tbb_strings.h deleted file mode 100644 index 5c836e642..000000000 --- a/src/tbb/include/tbb/internal/_tbb_strings.h +++ /dev/null @@ -1,71 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -TBB_STRING_RESOURCE(FLOW_BROADCAST_NODE, "broadcast_node") -TBB_STRING_RESOURCE(FLOW_BUFFER_NODE, "buffer_node") -TBB_STRING_RESOURCE(FLOW_CONTINUE_NODE, "continue_node") -TBB_STRING_RESOURCE(FLOW_FUNCTION_NODE, "function_node") -TBB_STRING_RESOURCE(FLOW_JOIN_NODE_QUEUEING, "join_node (queueing)") -TBB_STRING_RESOURCE(FLOW_JOIN_NODE_RESERVING, "join_node (reserving)") -TBB_STRING_RESOURCE(FLOW_JOIN_NODE_TAG_MATCHING, "join_node (tag_matching)") -TBB_STRING_RESOURCE(FLOW_LIMITER_NODE, "limiter_node") -TBB_STRING_RESOURCE(FLOW_MULTIFUNCTION_NODE, "multifunction_node") -TBB_STRING_RESOURCE(FLOW_OR_NODE, "or_node") //no longer in use, kept for backward compatibility -TBB_STRING_RESOURCE(FLOW_OVERWRITE_NODE, "overwrite_node") -TBB_STRING_RESOURCE(FLOW_PRIORITY_QUEUE_NODE, "priority_queue_node") -TBB_STRING_RESOURCE(FLOW_QUEUE_NODE, "queue_node") -TBB_STRING_RESOURCE(FLOW_SEQUENCER_NODE, "sequencer_node") -TBB_STRING_RESOURCE(FLOW_SOURCE_NODE, "source_node") -TBB_STRING_RESOURCE(FLOW_SPLIT_NODE, "split_node") -TBB_STRING_RESOURCE(FLOW_WRITE_ONCE_NODE, "write_once_node") -TBB_STRING_RESOURCE(FLOW_BODY, "body") -TBB_STRING_RESOURCE(FLOW_GRAPH, "graph") -TBB_STRING_RESOURCE(FLOW_NODE, "node") -TBB_STRING_RESOURCE(FLOW_INPUT_PORT, "input_port") -TBB_STRING_RESOURCE(FLOW_INPUT_PORT_0, "input_port_0") -TBB_STRING_RESOURCE(FLOW_INPUT_PORT_1, "input_port_1") -TBB_STRING_RESOURCE(FLOW_INPUT_PORT_2, "input_port_2") -TBB_STRING_RESOURCE(FLOW_INPUT_PORT_3, "input_port_3") -TBB_STRING_RESOURCE(FLOW_INPUT_PORT_4, "input_port_4") -TBB_STRING_RESOURCE(FLOW_INPUT_PORT_5, "input_port_5") -TBB_STRING_RESOURCE(FLOW_INPUT_PORT_6, "input_port_6") -TBB_STRING_RESOURCE(FLOW_INPUT_PORT_7, "input_port_7") -TBB_STRING_RESOURCE(FLOW_INPUT_PORT_8, "input_port_8") -TBB_STRING_RESOURCE(FLOW_INPUT_PORT_9, "input_port_9") -TBB_STRING_RESOURCE(FLOW_OUTPUT_PORT, "output_port") -TBB_STRING_RESOURCE(FLOW_OUTPUT_PORT_0, "output_port_0") -TBB_STRING_RESOURCE(FLOW_OUTPUT_PORT_1, "output_port_1") -TBB_STRING_RESOURCE(FLOW_OUTPUT_PORT_2, "output_port_2") -TBB_STRING_RESOURCE(FLOW_OUTPUT_PORT_3, "output_port_3") -TBB_STRING_RESOURCE(FLOW_OUTPUT_PORT_4, "output_port_4") -TBB_STRING_RESOURCE(FLOW_OUTPUT_PORT_5, "output_port_5") -TBB_STRING_RESOURCE(FLOW_OUTPUT_PORT_6, "output_port_6") -TBB_STRING_RESOURCE(FLOW_OUTPUT_PORT_7, "output_port_7") -TBB_STRING_RESOURCE(FLOW_OUTPUT_PORT_8, "output_port_8") -TBB_STRING_RESOURCE(FLOW_OUTPUT_PORT_9, "output_port_9") -TBB_STRING_RESOURCE(FLOW_OBJECT_NAME, "object_name") -TBB_STRING_RESOURCE(FLOW_NULL, "null") -TBB_STRING_RESOURCE(FLOW_INDEXER_NODE, "indexer_node") -TBB_STRING_RESOURCE(FLOW_COMPOSITE_NODE, "composite_node") -TBB_STRING_RESOURCE(FLOW_ASYNC_NODE, "async_node") -TBB_STRING_RESOURCE(FLOW_OPENCL_NODE, "opencl_node") -// TODO: Drop following string prefix "tbb_" here and in FGA's collector -TBB_STRING_RESOURCE(FGT_ALGORITHM, "tbb_algorithm") -TBB_STRING_RESOURCE(FGT_PARALLEL_FOR, "tbb_parallel_for") diff --git a/src/tbb/include/tbb/internal/_tbb_trace_impl.h b/src/tbb/include/tbb/internal/_tbb_trace_impl.h deleted file mode 100644 index e7eb5784f..000000000 --- a/src/tbb/include/tbb/internal/_tbb_trace_impl.h +++ /dev/null @@ -1,60 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef _FGT_TBB_TRACE_IMPL_H -#define _FGT_TBB_TRACE_IMPL_H - -#include "../tbb_profiling.h" - -namespace tbb { - namespace internal { - -#if TBB_PREVIEW_ALGORITHM_TRACE - - static inline void fgt_algorithm( string_index t, void *algorithm, void *parent ) { - itt_make_task_group( ITT_DOMAIN_FLOW, algorithm, FGT_ALGORITHM, parent, FGT_ALGORITHM, t ); - } - static inline void fgt_begin_algorithm( string_index t, void *algorithm ) { - itt_task_begin( ITT_DOMAIN_FLOW, algorithm, FGT_ALGORITHM, NULL, FLOW_NULL, t ); - } - static inline void fgt_end_algorithm( void * ) { - itt_task_end( ITT_DOMAIN_FLOW ); - } - static inline void fgt_alg_begin_body( string_index t, void *body, void *algorithm ) { - itt_task_begin( ITT_DOMAIN_FLOW, body, FLOW_BODY, algorithm, FGT_ALGORITHM, t ); - } - static inline void fgt_alg_end_body( void * ) { - itt_task_end( ITT_DOMAIN_FLOW ); - } - -#else // TBB_PREVIEW_ALGORITHM_TRACE - - static inline void fgt_algorithm( string_index /*t*/, void * /*algorithm*/, void * /*parent*/ ) { } - static inline void fgt_begin_algorithm( string_index /*t*/, void * /*algorithm*/ ) { } - static inline void fgt_end_algorithm( void * ) { } - static inline void fgt_alg_begin_body( string_index /*t*/, void * /*body*/, void * /*algorithm*/ ) { } - static inline void fgt_alg_end_body( void * ) { } - -#endif // TBB_PREVIEW_ALGORITHM_TRACEE - - } // namespace internal -} // namespace tbb - -#endif diff --git a/src/tbb/include/tbb/internal/_tbb_windef.h b/src/tbb/include/tbb/internal/_tbb_windef.h deleted file mode 100644 index e798dee49..000000000 --- a/src/tbb/include/tbb/internal/_tbb_windef.h +++ /dev/null @@ -1,73 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_tbb_windef_H -#error Do not #include this internal file directly; use public TBB headers instead. -#endif /* __TBB_tbb_windef_H */ - -// Check that the target Windows version has all API calls requried for TBB. -// Do not increase the version in condition beyond 0x0500 without prior discussion! -#if defined(_WIN32_WINNT) && _WIN32_WINNT<0x0501 -#error TBB is unable to run on old Windows versions; _WIN32_WINNT must be 0x0501 or greater. -#endif - -#if !defined(_MT) -#error TBB requires linkage with multithreaded C/C++ runtime library. \ - Choose multithreaded DLL runtime in project settings, or use /MD[d] compiler switch. -#endif - -// Workaround for the problem with MVSC headers failing to define namespace std -namespace std { - using ::size_t; using ::ptrdiff_t; -} - -#define __TBB_STRING_AUX(x) #x -#define __TBB_STRING(x) __TBB_STRING_AUX(x) - -// Default setting of TBB_USE_DEBUG -#ifdef TBB_USE_DEBUG -# if TBB_USE_DEBUG -# if !defined(_DEBUG) -# pragma message(__FILE__ "(" __TBB_STRING(__LINE__) ") : Warning: Recommend using /MDd if compiling with TBB_USE_DEBUG!=0") -# endif -# else -# if defined(_DEBUG) -# pragma message(__FILE__ "(" __TBB_STRING(__LINE__) ") : Warning: Recommend using /MD if compiling with TBB_USE_DEBUG==0") -# endif -# endif -#endif - -#if (__TBB_BUILD || __TBBMALLOC_BUILD) && !defined(__TBB_NO_IMPLICIT_LINKAGE) -#define __TBB_NO_IMPLICIT_LINKAGE 1 -#endif - -#if _MSC_VER - #if !__TBB_NO_IMPLICIT_LINKAGE - #ifdef __TBB_LIB_NAME - #pragma comment(lib, __TBB_STRING(__TBB_LIB_NAME)) - #else - #ifdef _DEBUG - #pragma comment(lib, "tbb_debug.lib") - #else - #pragma comment(lib, "tbb.lib") - #endif - #endif - #endif -#endif diff --git a/src/tbb/include/tbb/internal/_template_helpers.h b/src/tbb/include/tbb/internal/_template_helpers.h deleted file mode 100644 index 1e0abbe87..000000000 --- a/src/tbb/include/tbb/internal/_template_helpers.h +++ /dev/null @@ -1,158 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_template_helpers_H -#define __TBB_template_helpers_H - -#include -#include - -namespace tbb { namespace internal { - -//! Enables one or the other code branches -template struct enable_if {}; -template struct enable_if { typedef T type; }; - -//! Strips its template type argument from cv- and ref-qualifiers -template struct strip { typedef T type; }; -template struct strip { typedef T type; }; -template struct strip { typedef T type; }; -template struct strip { typedef T type; }; -template struct strip { typedef T type; }; -template struct strip { typedef T type; }; -template struct strip { typedef T type; }; -template struct strip { typedef T type; }; -//! Specialization for function pointers -template struct strip { typedef T(*type)(); }; -#if __TBB_CPP11_RVALUE_REF_PRESENT -template struct strip { typedef T type; }; -template struct strip { typedef T type; }; -template struct strip { typedef T type; }; -template struct strip { typedef T type; }; -#endif -//! Specialization for arrays converts to a corresponding pointer -template struct strip { typedef T* type; }; -template struct strip { typedef const T* type; }; -template struct strip { typedef volatile T* type; }; -template struct strip { typedef const volatile T* type; }; - -//! Detects whether two given types are the same -template struct is_same_type { static const bool value = false; }; -template struct is_same_type { static const bool value = true; }; - -template struct is_ref { static const bool value = false; }; -template struct is_ref { static const bool value = true; }; - -#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT -//! std::void_t internal implementation (to avoid GCC < 4.7 "template aliases" absence) -template struct void_t { typedef void type; }; -#endif - -#if __TBB_CPP11_RVALUE_REF_PRESENT && __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT - -//! Allows to store a function parameter pack as a variable and later pass it to another function -template< typename... Types > -struct stored_pack; - -template<> -struct stored_pack<> -{ - typedef stored_pack<> pack_type; - stored_pack() {} - - // Friend front-end functions - template< typename F, typename Pack > friend void call( F&& f, Pack&& p ); - template< typename Ret, typename F, typename Pack > friend Ret call_and_return( F&& f, Pack&& p ); - -protected: - // Ideally, ref-qualified non-static methods would be used, - // but that would greatly reduce the set of compilers where it works. - template< typename Ret, typename F, typename... Preceding > - static Ret call( F&& f, const pack_type& /*pack*/, Preceding&&... params ) { - return std::forward(f)( std::forward(params)... ); - } - template< typename Ret, typename F, typename... Preceding > - static Ret call( F&& f, pack_type&& /*pack*/, Preceding&&... params ) { - return std::forward(f)( std::forward(params)... ); - } -}; - -template< typename T, typename... Types > -struct stored_pack : stored_pack -{ - typedef stored_pack pack_type; - typedef stored_pack pack_remainder; - // Since lifetime of original values is out of control, copies should be made. - // Thus references should be stripped away from the deduced type. - typename strip::type leftmost_value; - - // Here rvalue references act in the same way as forwarding references, - // as long as class template parameters were deduced via forwarding references. - stored_pack( T&& t, Types&&... types ) - : pack_remainder(std::forward(types)...), leftmost_value(std::forward(t)) {} - - // Friend front-end functions - template< typename F, typename Pack > friend void call( F&& f, Pack&& p ); - template< typename Ret, typename F, typename Pack > friend Ret call_and_return( F&& f, Pack&& p ); - -protected: - template< typename Ret, typename F, typename... Preceding > - static Ret call( F&& f, pack_type& pack, Preceding&&... params ) { - return pack_remainder::template call( - std::forward(f), static_cast(pack), - std::forward(params)... , pack.leftmost_value - ); - } - template< typename Ret, typename F, typename... Preceding > - static Ret call( F&& f, const pack_type& pack, Preceding&&... params ) { - return pack_remainder::template call( - std::forward(f), static_cast(pack), - std::forward(params)... , pack.leftmost_value - ); - } - template< typename Ret, typename F, typename... Preceding > - static Ret call( F&& f, pack_type&& pack, Preceding&&... params ) { - return pack_remainder::template call( - std::forward(f), static_cast(pack), - std::forward(params)... , std::move(pack.leftmost_value) - ); - } -}; - -//! Calls the given function with arguments taken from a stored_pack -template< typename F, typename Pack > -void call( F&& f, Pack&& p ) { - strip::type::template call( std::forward(f), std::forward(p) ); -} - -template< typename Ret, typename F, typename Pack > -Ret call_and_return( F&& f, Pack&& p ) { - return strip::type::template call( std::forward(f), std::forward(p) ); -} - -template< typename... Types > -stored_pack save_pack( Types&&... types ) { - return stored_pack( std::forward(types)... ); -} - -#endif /* __TBB_CPP11_RVALUE_REF_PRESENT && __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT */ -} } // namespace internal, namespace tbb - -#endif /* __TBB_template_helpers_H */ diff --git a/src/tbb/include/tbb/internal/_x86_eliding_mutex_impl.h b/src/tbb/include/tbb/internal/_x86_eliding_mutex_impl.h deleted file mode 100644 index ef5f92239..000000000 --- a/src/tbb/include/tbb/internal/_x86_eliding_mutex_impl.h +++ /dev/null @@ -1,148 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB__x86_eliding_mutex_impl_H -#define __TBB__x86_eliding_mutex_impl_H - -#ifndef __TBB_spin_mutex_H -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#if ( __TBB_x86_32 || __TBB_x86_64 ) - -namespace tbb { -namespace interface7 { -namespace internal { - -template -class padded_mutex; - -//! An eliding lock that occupies a single byte. -/** A x86_eliding_mutex is an HLE-enabled spin mutex. It is recommended to - put the mutex on a cache line that is not shared by the data it protects. - It should be used for locking short critical sections where the lock is - contended but the data it protects are not. If zero-initialized, the - mutex is considered unheld. - @ingroup synchronization */ -class x86_eliding_mutex : tbb::internal::mutex_copy_deprecated_and_disabled { - //! 0 if lock is released, 1 if lock is acquired. - __TBB_atomic_flag flag; - - friend class padded_mutex; - -public: - //! Construct unacquired lock. - /** Equivalent to zero-initialization of *this. */ - x86_eliding_mutex() : flag(0) {} - -// bug in gcc 3.x.x causes syntax error in spite of the friend declaration above. -// Make the scoped_lock public in that case. -#if __TBB_USE_X86_ELIDING_MUTEX || __TBB_GCC_VERSION < 40000 -#else - // by default we will not provide the scoped_lock interface. The user - // should use the padded version of the mutex. scoped_lock is used in - // padded_mutex template. -private: -#endif - // scoped_lock in padded_mutex<> is the interface to use. - //! Represents acquisition of a mutex. - class scoped_lock : tbb::internal::no_copy { - private: - //! Points to currently held mutex, or NULL if no lock is held. - x86_eliding_mutex* my_mutex; - - public: - //! Construct without acquiring a mutex. - scoped_lock() : my_mutex(NULL) {} - - //! Construct and acquire lock on a mutex. - scoped_lock( x86_eliding_mutex& m ) : my_mutex(NULL) { acquire(m); } - - //! Acquire lock. - void acquire( x86_eliding_mutex& m ) { - __TBB_ASSERT( !my_mutex, "already holding a lock" ); - - my_mutex=&m; - my_mutex->lock(); - } - - //! Try acquiring lock (non-blocking) - /** Return true if lock acquired; false otherwise. */ - bool try_acquire( x86_eliding_mutex& m ) { - __TBB_ASSERT( !my_mutex, "already holding a lock" ); - - bool result = m.try_lock(); - if( result ) { - my_mutex = &m; - } - return result; - } - - //! Release lock - void release() { - __TBB_ASSERT( my_mutex, "release on scoped_lock that is not holding a lock" ); - - my_mutex->unlock(); - my_mutex = NULL; - } - - //! Destroy lock. If holding a lock, releases the lock first. - ~scoped_lock() { - if( my_mutex ) { - release(); - } - } - }; -#if __TBB_USE_X86_ELIDING_MUTEX || __TBB_GCC_VERSION < 40000 -#else -public: -#endif /* __TBB_USE_X86_ELIDING_MUTEX */ - - // Mutex traits - static const bool is_rw_mutex = false; - static const bool is_recursive_mutex = false; - static const bool is_fair_mutex = false; - - // ISO C++0x compatibility methods - - //! Acquire lock - void lock() { - __TBB_LockByteElided(flag); - } - - //! Try acquiring lock (non-blocking) - /** Return true if lock acquired; false otherwise. */ - bool try_lock() { - return __TBB_TryLockByteElided(flag); - } - - //! Release lock - void unlock() { - __TBB_UnlockByteElided( flag ); - } -}; // end of x86_eliding_mutex - -} // namespace internal -} // namespace interface7 -} // namespace tbb - -#endif /* ( __TBB_x86_32 || __TBB_x86_64 ) */ - -#endif /* __TBB__x86_eliding_mutex_impl_H */ diff --git a/src/tbb/include/tbb/internal/_x86_rtm_rw_mutex_impl.h b/src/tbb/include/tbb/internal/_x86_rtm_rw_mutex_impl.h deleted file mode 100644 index b08c23313..000000000 --- a/src/tbb/include/tbb/internal/_x86_rtm_rw_mutex_impl.h +++ /dev/null @@ -1,225 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB__x86_rtm_rw_mutex_impl_H -#define __TBB__x86_rtm_rw_mutex_impl_H - -#ifndef __TBB_spin_rw_mutex_H -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#if __TBB_TSX_AVAILABLE - -#include "../tbb_stddef.h" -#include "../tbb_machine.h" -#include "../tbb_profiling.h" -#include "../spin_rw_mutex.h" - -namespace tbb { -namespace interface8 { -namespace internal { - -enum RTM_type { - RTM_not_in_mutex, - RTM_transacting_reader, - RTM_transacting_writer, - RTM_real_reader, - RTM_real_writer -}; - -static const unsigned long speculation_granularity = 64; - -//! Fast, unfair, spinning speculation-enabled reader-writer lock with backoff and -// writer-preference -/** @ingroup synchronization */ -class x86_rtm_rw_mutex: private spin_rw_mutex { -#if __TBB_USE_X86_RTM_RW_MUTEX || __TBB_GCC_VERSION < 40000 -// bug in gcc 3.x.x causes syntax error in spite of the friend declaration below. -// Make the scoped_lock public in that case. -public: -#else -private: -#endif - friend class interface7::internal::padded_mutex; - class scoped_lock; // should be private - friend class scoped_lock; -private: - //! @cond INTERNAL - - //! Internal construct unacquired mutex. - void __TBB_EXPORTED_METHOD internal_construct(); - - //! Internal acquire write lock. - // only_speculate == true if we're doing a try_lock, else false. - void __TBB_EXPORTED_METHOD internal_acquire_writer(x86_rtm_rw_mutex::scoped_lock&, bool only_speculate=false); - - //! Internal acquire read lock. - // only_speculate == true if we're doing a try_lock, else false. - void __TBB_EXPORTED_METHOD internal_acquire_reader(x86_rtm_rw_mutex::scoped_lock&, bool only_speculate=false); - - //! Internal upgrade reader to become a writer. - bool __TBB_EXPORTED_METHOD internal_upgrade( x86_rtm_rw_mutex::scoped_lock& ); - - //! Out of line code for downgrading a writer to a reader. - bool __TBB_EXPORTED_METHOD internal_downgrade( x86_rtm_rw_mutex::scoped_lock& ); - - //! Internal try_acquire write lock. - bool __TBB_EXPORTED_METHOD internal_try_acquire_writer( x86_rtm_rw_mutex::scoped_lock& ); - - //! Internal release lock. - void __TBB_EXPORTED_METHOD internal_release( x86_rtm_rw_mutex::scoped_lock& ); - - static x86_rtm_rw_mutex* internal_get_mutex( const spin_rw_mutex::scoped_lock& lock ) - { - return static_cast( lock.internal_get_mutex() ); - } - static void internal_set_mutex( spin_rw_mutex::scoped_lock& lock, spin_rw_mutex* mtx ) - { - lock.internal_set_mutex( mtx ); - } - //! @endcond -public: - //! Construct unacquired mutex. - x86_rtm_rw_mutex() { - w_flag = false; -#if TBB_USE_THREADING_TOOLS - internal_construct(); -#endif - } - -#if TBB_USE_ASSERT - //! Empty destructor. - ~x86_rtm_rw_mutex() {} -#endif /* TBB_USE_ASSERT */ - - // Mutex traits - static const bool is_rw_mutex = true; - static const bool is_recursive_mutex = false; - static const bool is_fair_mutex = false; - -#if __TBB_USE_X86_RTM_RW_MUTEX || __TBB_GCC_VERSION < 40000 -#else - // by default we will not provide the scoped_lock interface. The user - // should use the padded version of the mutex. scoped_lock is used in - // padded_mutex template. -private: -#endif - //! The scoped locking pattern - /** It helps to avoid the common problem of forgetting to release lock. - It also nicely provides the "node" for queuing locks. */ - // Speculation-enabled scoped lock for spin_rw_mutex - // The idea is to be able to reuse the acquire/release methods of spin_rw_mutex - // and its scoped lock wherever possible. The only way to use a speculative lock is to use - // a scoped_lock. (because transaction_state must be local) - - class scoped_lock : tbb::internal::no_copy { - friend class x86_rtm_rw_mutex; - spin_rw_mutex::scoped_lock my_scoped_lock; - - RTM_type transaction_state; - - public: - //! Construct lock that has not acquired a mutex. - /** Equivalent to zero-initialization of *this. */ - scoped_lock() : my_scoped_lock(), transaction_state(RTM_not_in_mutex) { - } - - //! Acquire lock on given mutex. - scoped_lock( x86_rtm_rw_mutex& m, bool write = true ) : my_scoped_lock(), - transaction_state(RTM_not_in_mutex) { - acquire(m, write); - } - - //! Release lock (if lock is held). - ~scoped_lock() { - if(transaction_state != RTM_not_in_mutex) release(); - } - - //! Acquire lock on given mutex. - void acquire( x86_rtm_rw_mutex& m, bool write = true ) { - if( write ) m.internal_acquire_writer(*this); - else m.internal_acquire_reader(*this); - } - - //! Release lock - void release() { - x86_rtm_rw_mutex* mutex = x86_rtm_rw_mutex::internal_get_mutex(my_scoped_lock); - __TBB_ASSERT( mutex, "lock is not acquired" ); - __TBB_ASSERT( transaction_state!=RTM_not_in_mutex, "lock is not acquired" ); - return mutex->internal_release(*this); - } - - //! Upgrade reader to become a writer. - /** Returns whether the upgrade happened without releasing and re-acquiring the lock */ - bool upgrade_to_writer() { - x86_rtm_rw_mutex* mutex = x86_rtm_rw_mutex::internal_get_mutex(my_scoped_lock); - __TBB_ASSERT( mutex, "lock is not acquired" ); - __TBB_ASSERT( transaction_state==RTM_transacting_reader || transaction_state==RTM_real_reader, "Invalid state for upgrade" ); - return mutex->internal_upgrade(*this); - } - - //! Downgrade writer to become a reader. - /** Returns whether the downgrade happened without releasing and re-acquiring the lock */ - bool downgrade_to_reader() { - x86_rtm_rw_mutex* mutex = x86_rtm_rw_mutex::internal_get_mutex(my_scoped_lock); - __TBB_ASSERT( mutex, "lock is not acquired" ); - __TBB_ASSERT( transaction_state==RTM_transacting_writer || transaction_state==RTM_real_writer, "Invalid state for downgrade" ); - return mutex->internal_downgrade(*this); - } - - //! Attempt to acquire mutex. - /** returns true if successful. */ - bool try_acquire( x86_rtm_rw_mutex& m, bool write = true ) { -#if TBB_USE_ASSERT - x86_rtm_rw_mutex* mutex = x86_rtm_rw_mutex::internal_get_mutex(my_scoped_lock); - __TBB_ASSERT( !mutex, "lock is already acquired" ); -#endif - // have to assign m to our mutex. - // cannot set the mutex, because try_acquire in spin_rw_mutex depends on it being NULL. - if(write) return m.internal_try_acquire_writer(*this); - // speculatively acquire the lock. If this fails, do try_acquire on the spin_rw_mutex. - m.internal_acquire_reader(*this, /*only_speculate=*/true); - if(transaction_state == RTM_transacting_reader) return true; - if( my_scoped_lock.try_acquire(m, false)) { - transaction_state = RTM_real_reader; - return true; - } - return false; - } - - }; // class x86_rtm_rw_mutex::scoped_lock - - // ISO C++0x compatibility methods not provided because we cannot maintain - // state about whether a thread is in a transaction. - -private: - char pad[speculation_granularity-sizeof(spin_rw_mutex)]; // padding - - // If true, writer holds the spin_rw_mutex. - tbb::atomic w_flag; // want this on a separate cache line - -}; // x86_rtm_rw_mutex - -} // namespace internal -} // namespace interface8 -} // namespace tbb - -#endif /* __TBB_TSX_AVAILABLE */ -#endif /* __TBB__x86_rtm_rw_mutex_impl_H */ diff --git a/src/tbb/include/tbb/machine/gcc_armv7.h b/src/tbb/include/tbb/machine/gcc_armv7.h deleted file mode 100644 index 642c14fe2..000000000 --- a/src/tbb/include/tbb/machine/gcc_armv7.h +++ /dev/null @@ -1,217 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -/* - Platform isolation layer for the ARMv7-a architecture. -*/ - -#ifndef __TBB_machine_H -#error Do not include this file directly; include tbb_machine.h instead -#endif - -//TODO: is ARMv7 is the only version ever to support? -#if !(__ARM_ARCH_7A__) -#error compilation requires an ARMv7-a architecture. -#endif - -#include -#include - -#define __TBB_WORDSIZE 4 - -// Traditionally ARM is little-endian. -// Note that, since only the layout of aligned 32-bit words is of interest, -// any apparent PDP-endianness of 32-bit words at half-word alignment or -// any little-endian ordering of big-endian 32-bit words in 64-bit quantities -// may be disregarded for this setting. -#if __BIG_ENDIAN__ || (defined(__BYTE_ORDER__) && __BYTE_ORDER__==__ORDER_BIG_ENDIAN__) - #define __TBB_ENDIANNESS __TBB_ENDIAN_BIG -#elif __LITTLE_ENDIAN__ || (defined(__BYTE_ORDER__) && __BYTE_ORDER__==__ORDER_LITTLE_ENDIAN__) - #define __TBB_ENDIANNESS __TBB_ENDIAN_LITTLE -#elif defined(__BYTE_ORDER__) - #define __TBB_ENDIANNESS __TBB_ENDIAN_UNSUPPORTED -#else - #define __TBB_ENDIANNESS __TBB_ENDIAN_DETECT -#endif - - -#define __TBB_compiler_fence() __asm__ __volatile__("": : :"memory") -#define __TBB_full_memory_fence() __asm__ __volatile__("dmb ish": : :"memory") -#define __TBB_control_consistency_helper() __TBB_full_memory_fence() -#define __TBB_acquire_consistency_helper() __TBB_full_memory_fence() -#define __TBB_release_consistency_helper() __TBB_full_memory_fence() - -//-------------------------------------------------- -// Compare and swap -//-------------------------------------------------- - -/** - * Atomic CAS for 32 bit values, if *ptr==comparand, then *ptr=value, returns *ptr - * @param ptr pointer to value in memory to be swapped with value if *ptr==comparand - * @param value value to assign *ptr to if *ptr==comparand - * @param comparand value to compare with *ptr - * @return value originally in memory at ptr, regardless of success -*/ -static inline int32_t __TBB_machine_cmpswp4(volatile void *ptr, int32_t value, int32_t comparand ) -{ - int32_t oldval, res; - - __TBB_full_memory_fence(); - - do { - __asm__ __volatile__( - "ldrex %1, [%3]\n" - "mov %0, #0\n" - "cmp %1, %4\n" - "it eq\n" - "strexeq %0, %5, [%3]\n" - : "=&r" (res), "=&r" (oldval), "+Qo" (*(volatile int32_t*)ptr) - : "r" ((volatile int32_t *)ptr), "Ir" (comparand), "r" (value) - : "cc"); - } while (res); - - __TBB_full_memory_fence(); - - return oldval; -} - -/** - * Atomic CAS for 64 bit values, if *ptr==comparand, then *ptr=value, returns *ptr - * @param ptr pointer to value in memory to be swapped with value if *ptr==comparand - * @param value value to assign *ptr to if *ptr==comparand - * @param comparand value to compare with *ptr - * @return value originally in memory at ptr, regardless of success - */ -static inline int64_t __TBB_machine_cmpswp8(volatile void *ptr, int64_t value, int64_t comparand ) -{ - int64_t oldval; - int32_t res; - - __TBB_full_memory_fence(); - - do { - __asm__ __volatile__( - "mov %0, #0\n" - "ldrexd %1, %H1, [%3]\n" - "cmp %1, %4\n" - "it eq\n" - "cmpeq %H1, %H4\n" - "it eq\n" - "strexdeq %0, %5, %H5, [%3]" - : "=&r" (res), "=&r" (oldval), "+Qo" (*(volatile int64_t*)ptr) - : "r" ((volatile int64_t *)ptr), "r" (comparand), "r" (value) - : "cc"); - } while (res); - - __TBB_full_memory_fence(); - - return oldval; -} - -static inline int32_t __TBB_machine_fetchadd4(volatile void* ptr, int32_t addend) -{ - unsigned long tmp; - int32_t result, tmp2; - - __TBB_full_memory_fence(); - - __asm__ __volatile__( -"1: ldrex %0, [%4]\n" -" add %3, %0, %5\n" -" strex %1, %3, [%4]\n" -" cmp %1, #0\n" -" bne 1b\n" - : "=&r" (result), "=&r" (tmp), "+Qo" (*(volatile int32_t*)ptr), "=&r"(tmp2) - : "r" ((volatile int32_t *)ptr), "Ir" (addend) - : "cc"); - - __TBB_full_memory_fence(); - - return result; -} - -static inline int64_t __TBB_machine_fetchadd8(volatile void *ptr, int64_t addend) -{ - unsigned long tmp; - int64_t result, tmp2; - - __TBB_full_memory_fence(); - - __asm__ __volatile__( -"1: ldrexd %0, %H0, [%4]\n" -" adds %3, %0, %5\n" -" adc %H3, %H0, %H5\n" -" strexd %1, %3, %H3, [%4]\n" -" cmp %1, #0\n" -" bne 1b" - : "=&r" (result), "=&r" (tmp), "+Qo" (*(volatile int64_t*)ptr), "=&r"(tmp2) - : "r" ((volatile int64_t *)ptr), "r" (addend) - : "cc"); - - - __TBB_full_memory_fence(); - - return result; -} - -inline void __TBB_machine_pause (int32_t delay ) -{ - while(delay>0) - { - __TBB_compiler_fence(); - delay--; - } -} - -namespace tbb { -namespace internal { - template - struct machine_load_store_relaxed { - static inline T load ( const volatile T& location ) { - const T value = location; - - /* - * An extra memory barrier is required for errata #761319 - * Please see http://infocenter.arm.com/help/topic/com.arm.doc.uan0004a - */ - __TBB_acquire_consistency_helper(); - return value; - } - - static inline void store ( volatile T& location, T value ) { - location = value; - } - }; -}} // namespaces internal, tbb - -// Machine specific atomic operations - -#define __TBB_CompareAndSwap4(P,V,C) __TBB_machine_cmpswp4(P,V,C) -#define __TBB_CompareAndSwap8(P,V,C) __TBB_machine_cmpswp8(P,V,C) -#define __TBB_Pause(V) __TBB_machine_pause(V) - -// Use generics for some things -#define __TBB_USE_GENERIC_PART_WORD_CAS 1 -#define __TBB_USE_GENERIC_PART_WORD_FETCH_ADD 1 -#define __TBB_USE_GENERIC_PART_WORD_FETCH_STORE 1 -#define __TBB_USE_GENERIC_FETCH_STORE 1 -#define __TBB_USE_GENERIC_HALF_FENCED_LOAD_STORE 1 -#define __TBB_USE_GENERIC_DWORD_LOAD_STORE 1 -#define __TBB_USE_GENERIC_SEQUENTIAL_CONSISTENCY_LOAD_STORE 1 diff --git a/src/tbb/include/tbb/machine/gcc_generic.h b/src/tbb/include/tbb/machine/gcc_generic.h deleted file mode 100644 index 5fc2a9011..000000000 --- a/src/tbb/include/tbb/machine/gcc_generic.h +++ /dev/null @@ -1,184 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#if !defined(__TBB_machine_H) || defined(__TBB_machine_gcc_generic_H) -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#define __TBB_machine_gcc_generic_H - -#include -#include - -#define __TBB_WORDSIZE __SIZEOF_POINTER__ - -#if __TBB_GCC_64BIT_ATOMIC_BUILTINS_BROKEN - #define __TBB_64BIT_ATOMICS 0 -#endif - -/** FPU control setting not available for non-Intel architectures on Android **/ -#if __ANDROID__ && __TBB_generic_arch - #define __TBB_CPU_CTL_ENV_PRESENT 0 -#endif - -// __BYTE_ORDER__ is used in accordance with http://gcc.gnu.org/onlinedocs/cpp/Common-Predefined-Macros.html, -// but __BIG_ENDIAN__ or __LITTLE_ENDIAN__ may be more commonly found instead. -#if __BIG_ENDIAN__ || (defined(__BYTE_ORDER__) && __BYTE_ORDER__==__ORDER_BIG_ENDIAN__) - #define __TBB_ENDIANNESS __TBB_ENDIAN_BIG -#elif __LITTLE_ENDIAN__ || (defined(__BYTE_ORDER__) && __BYTE_ORDER__==__ORDER_LITTLE_ENDIAN__) - #define __TBB_ENDIANNESS __TBB_ENDIAN_LITTLE -#elif defined(__BYTE_ORDER__) - #define __TBB_ENDIANNESS __TBB_ENDIAN_UNSUPPORTED -#else - #define __TBB_ENDIANNESS __TBB_ENDIAN_DETECT -#endif - -#if __TBB_GCC_VERSION < 40700 -// Use __sync_* builtins - -/** As this generic implementation has absolutely no information about underlying - hardware, its performance most likely will be sub-optimal because of full memory - fence usages where a more lightweight synchronization means (or none at all) - could suffice. Thus if you use this header to enable TBB on a new platform, - consider forking it and relaxing below helpers as appropriate. **/ -#define __TBB_acquire_consistency_helper() __sync_synchronize() -#define __TBB_release_consistency_helper() __sync_synchronize() -#define __TBB_full_memory_fence() __sync_synchronize() -#define __TBB_control_consistency_helper() __sync_synchronize() - -#define __TBB_MACHINE_DEFINE_ATOMICS(S,T) \ -inline T __TBB_machine_cmpswp##S( volatile void *ptr, T value, T comparand ) { \ - return __sync_val_compare_and_swap(reinterpret_cast(ptr),comparand,value); \ -} \ -inline T __TBB_machine_fetchadd##S( volatile void *ptr, T value ) { \ - return __sync_fetch_and_add(reinterpret_cast(ptr),value); \ -} - -#define __TBB_USE_GENERIC_FETCH_STORE 1 - -#else -// __TBB_GCC_VERSION >= 40700; use __atomic_* builtins available since gcc 4.7 - -#define __TBB_compiler_fence() __asm__ __volatile__("": : :"memory") -// Acquire and release fence intrinsics in GCC might miss compiler fence. -// Adding it at both sides of an intrinsic, as we do not know what reordering can be made. -#define __TBB_acquire_consistency_helper() __TBB_compiler_fence(); __atomic_thread_fence(__ATOMIC_ACQUIRE); __TBB_compiler_fence() -#define __TBB_release_consistency_helper() __TBB_compiler_fence(); __atomic_thread_fence(__ATOMIC_RELEASE); __TBB_compiler_fence() -#define __TBB_full_memory_fence() __atomic_thread_fence(__ATOMIC_SEQ_CST) -#define __TBB_control_consistency_helper() __TBB_acquire_consistency_helper() - -#define __TBB_MACHINE_DEFINE_ATOMICS(S,T) \ -inline T __TBB_machine_cmpswp##S( volatile void *ptr, T value, T comparand ) { \ - (void)__atomic_compare_exchange_n(reinterpret_cast(ptr), &comparand, value, \ - false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); \ - return comparand; \ -} \ -inline T __TBB_machine_fetchadd##S( volatile void *ptr, T value ) { \ - return __atomic_fetch_add(reinterpret_cast(ptr), value, __ATOMIC_SEQ_CST); \ -} \ -inline T __TBB_machine_fetchstore##S( volatile void *ptr, T value ) { \ - return __atomic_exchange_n(reinterpret_cast(ptr), value, __ATOMIC_SEQ_CST); \ -} - -#endif // __TBB_GCC_VERSION < 40700 - -__TBB_MACHINE_DEFINE_ATOMICS(1,int8_t) -__TBB_MACHINE_DEFINE_ATOMICS(2,int16_t) -__TBB_MACHINE_DEFINE_ATOMICS(4,int32_t) -__TBB_MACHINE_DEFINE_ATOMICS(8,int64_t) - -#undef __TBB_MACHINE_DEFINE_ATOMICS - -namespace tbb{ namespace internal { namespace gcc_builtins { - inline int clz(unsigned int x){ return __builtin_clz(x);}; - inline int clz(unsigned long int x){ return __builtin_clzl(x);}; - inline int clz(unsigned long long int x){ return __builtin_clzll(x);}; -}}} -//gcc __builtin_clz builtin count _number_ of leading zeroes -static inline intptr_t __TBB_machine_lg( uintptr_t x ) { - return sizeof(x)*8 - tbb::internal::gcc_builtins::clz(x) -1 ; -} - - -typedef unsigned char __TBB_Flag; -typedef __TBB_atomic __TBB_Flag __TBB_atomic_flag; - -#if __TBB_GCC_VERSION < 40700 -// Use __sync_* builtins - -static inline void __TBB_machine_or( volatile void *ptr, uintptr_t addend ) { - __sync_fetch_and_or(reinterpret_cast(ptr),addend); -} - -static inline void __TBB_machine_and( volatile void *ptr, uintptr_t addend ) { - __sync_fetch_and_and(reinterpret_cast(ptr),addend); -} - -inline bool __TBB_machine_try_lock_byte( __TBB_atomic_flag &flag ) { - return __sync_lock_test_and_set(&flag,1)==0; -} - -inline void __TBB_machine_unlock_byte( __TBB_atomic_flag &flag ) { - __sync_lock_release(&flag); -} - -#else -// __TBB_GCC_VERSION >= 40700; use __atomic_* builtins available since gcc 4.7 - -static inline void __TBB_machine_or( volatile void *ptr, uintptr_t addend ) { - __atomic_fetch_or(reinterpret_cast(ptr),addend,__ATOMIC_SEQ_CST); -} - -static inline void __TBB_machine_and( volatile void *ptr, uintptr_t addend ) { - __atomic_fetch_and(reinterpret_cast(ptr),addend,__ATOMIC_SEQ_CST); -} - -inline bool __TBB_machine_try_lock_byte( __TBB_atomic_flag &flag ) { - return !__atomic_test_and_set(&flag,__ATOMIC_ACQUIRE); -} - -inline void __TBB_machine_unlock_byte( __TBB_atomic_flag &flag ) { - __atomic_clear(&flag,__ATOMIC_RELEASE); -} - -#endif // __TBB_GCC_VERSION < 40700 - -// Machine specific atomic operations -#define __TBB_AtomicOR(P,V) __TBB_machine_or(P,V) -#define __TBB_AtomicAND(P,V) __TBB_machine_and(P,V) - -#define __TBB_TryLockByte __TBB_machine_try_lock_byte -#define __TBB_UnlockByte __TBB_machine_unlock_byte - -// Definition of other functions -#define __TBB_Log2(V) __TBB_machine_lg(V) - -// TODO: implement with __atomic_* builtins where available -#define __TBB_USE_GENERIC_HALF_FENCED_LOAD_STORE 1 -#define __TBB_USE_GENERIC_RELAXED_LOAD_STORE 1 -#define __TBB_USE_GENERIC_SEQUENTIAL_CONSISTENCY_LOAD_STORE 1 - -#if __TBB_WORDSIZE==4 - #define __TBB_USE_GENERIC_DWORD_LOAD_STORE 1 -#endif - -#if __TBB_x86_32 || __TBB_x86_64 -#include "gcc_itsx.h" -#endif diff --git a/src/tbb/include/tbb/machine/gcc_ia32_common.h b/src/tbb/include/tbb/machine/gcc_ia32_common.h deleted file mode 100644 index f5efc375b..000000000 --- a/src/tbb/include/tbb/machine/gcc_ia32_common.h +++ /dev/null @@ -1,112 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_machine_gcc_ia32_common_H -#define __TBB_machine_gcc_ia32_common_H - -//TODO: Add a higher-level function, e.g. tbb::internal::log2(), into tbb_stddef.h, which -//uses __TBB_Log2 and contains the assert and remove the assert from here and all other -//platform-specific headers. -//TODO: Check if use of gcc intrinsic gives a better chance for cross call optimizations -template -static inline intptr_t __TBB_machine_lg( T x ) { - __TBB_ASSERT(x>0, "The logarithm of a non-positive value is undefined."); - uintptr_t j, i = x; - __asm__("bsr %1,%0" : "=r"(j) : "r"(i)); - return j; -} -#define __TBB_Log2(V) __TBB_machine_lg(V) - -#ifndef __TBB_Pause -//TODO: check if raising a ratio of pause instructions to loop control instructions -//(via e.g. loop unrolling) gives any benefit for HT. E.g, the current implementation -//does about 2 CPU-consuming instructions for every pause instruction. Perhaps for -//high pause counts it should use an unrolled loop to raise the ratio, and thus free -//up more integer cycles for the other hyperthread. On the other hand, if the loop is -//unrolled too far, it won't fit in the core's loop cache, and thus take away -//instruction decode slots from the other hyperthread. - -//TODO: check if use of gcc __builtin_ia32_pause intrinsic gives a "some how" better performing code -static inline void __TBB_machine_pause( int32_t delay ) { - for (int32_t i = 0; i < delay; i++) { - __asm__ __volatile__("pause;"); - } - return; -} -#define __TBB_Pause(V) __TBB_machine_pause(V) -#endif /* !__TBB_Pause */ - -namespace tbb { namespace internal { typedef uint64_t machine_tsc_t; } } -static inline tbb::internal::machine_tsc_t __TBB_machine_time_stamp() { -#if __INTEL_COMPILER - return _rdtsc(); -#else - tbb::internal::uint32_t hi, lo; - __asm__ __volatile__("rdtsc" : "=d"(hi), "=a"(lo)); - return (tbb::internal::machine_tsc_t( hi ) << 32) | lo; -#endif -} -#define __TBB_time_stamp() __TBB_machine_time_stamp() - -// API to retrieve/update FPU control setting -#ifndef __TBB_CPU_CTL_ENV_PRESENT -#define __TBB_CPU_CTL_ENV_PRESENT 1 -namespace tbb { -namespace internal { -class cpu_ctl_env { -private: - int mxcsr; - short x87cw; - static const int MXCSR_CONTROL_MASK = ~0x3f; /* all except last six status bits */ -public: - bool operator!=( const cpu_ctl_env& ctl ) const { return mxcsr != ctl.mxcsr || x87cw != ctl.x87cw; } - void get_env() { - #if __TBB_ICC_12_0_INL_ASM_FSTCW_BROKEN - cpu_ctl_env loc_ctl; - __asm__ __volatile__ ( - "stmxcsr %0\n\t" - "fstcw %1" - : "=m"(loc_ctl.mxcsr), "=m"(loc_ctl.x87cw) - ); - *this = loc_ctl; - #else - __asm__ __volatile__ ( - "stmxcsr %0\n\t" - "fstcw %1" - : "=m"(mxcsr), "=m"(x87cw) - ); - #endif - mxcsr &= MXCSR_CONTROL_MASK; - } - void set_env() const { - __asm__ __volatile__ ( - "ldmxcsr %0\n\t" - "fldcw %1" - : : "m"(mxcsr), "m"(x87cw) - ); - } -}; -} // namespace internal -} // namespace tbb -#endif /* !__TBB_CPU_CTL_ENV_PRESENT */ - -#include "gcc_itsx.h" - -#endif /* __TBB_machine_gcc_ia32_common_H */ diff --git a/src/tbb/include/tbb/machine/gcc_itsx.h b/src/tbb/include/tbb/machine/gcc_itsx.h deleted file mode 100644 index caa35441f..000000000 --- a/src/tbb/include/tbb/machine/gcc_itsx.h +++ /dev/null @@ -1,123 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#if !defined(__TBB_machine_H) || defined(__TBB_machine_gcc_itsx_H) -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#define __TBB_machine_gcc_itsx_H - -#define __TBB_OP_XACQUIRE 0xF2 -#define __TBB_OP_XRELEASE 0xF3 -#define __TBB_OP_LOCK 0xF0 - -#define __TBB_STRINGIZE_INTERNAL(arg) #arg -#define __TBB_STRINGIZE(arg) __TBB_STRINGIZE_INTERNAL(arg) - -#ifdef __TBB_x86_64 -#define __TBB_r_out "=r" -#else -#define __TBB_r_out "=q" -#endif - -inline static uint8_t __TBB_machine_try_lock_elided( volatile uint8_t* lk ) -{ - uint8_t value = 1; - __asm__ volatile (".byte " __TBB_STRINGIZE(__TBB_OP_XACQUIRE)"; lock; xchgb %0, %1;" - : __TBB_r_out(value), "=m"(*lk) : "0"(value), "m"(*lk) : "memory" ); - return uint8_t(value^1); -} - -inline static void __TBB_machine_try_lock_elided_cancel() -{ - // 'pause' instruction aborts HLE/RTM transactions - __asm__ volatile ("pause\n" : : : "memory" ); -} - -inline static void __TBB_machine_unlock_elided( volatile uint8_t* lk ) -{ - __asm__ volatile (".byte " __TBB_STRINGIZE(__TBB_OP_XRELEASE)"; movb $0, %0" - : "=m"(*lk) : "m"(*lk) : "memory" ); -} - -#if __TBB_TSX_INTRINSICS_PRESENT -#include - -#define __TBB_machine_is_in_transaction _xtest -#define __TBB_machine_begin_transaction _xbegin -#define __TBB_machine_end_transaction _xend -#define __TBB_machine_transaction_conflict_abort() _xabort(0xff) - -#else - -/*! - * Check if the instruction is executed in a transaction or not - */ -inline static bool __TBB_machine_is_in_transaction() -{ - int8_t res = 0; -#if __TBB_x86_32 - __asm__ volatile (".byte 0x0F; .byte 0x01; .byte 0xD6;\n" - "setz %0" : "=q"(res) : : "memory" ); -#else - __asm__ volatile (".byte 0x0F; .byte 0x01; .byte 0xD6;\n" - "setz %0" : "=r"(res) : : "memory" ); -#endif - return res==0; -} - -/*! - * Enter speculative execution mode. - * @return -1 on success - * abort cause ( or 0 ) on abort - */ -inline static uint32_t __TBB_machine_begin_transaction() -{ - uint32_t res = ~uint32_t(0); // success value - __asm__ volatile ("1: .byte 0xC7; .byte 0xF8;\n" // XBEGIN - " .long 2f-1b-6\n" // 2f-1b == difference in addresses of start - // of XBEGIN and the MOVL - // 2f - 1b - 6 == that difference minus the size of the - // XBEGIN instruction. This is the abort offset to - // 2: below. - " jmp 3f\n" // success (leave -1 in res) - "2: movl %%eax,%0\n" // store failure code in res - "3:" - :"=r"(res):"0"(res):"memory","%eax"); - return res; -} - -/*! - * Attempt to commit/end transaction - */ -inline static void __TBB_machine_end_transaction() -{ - __asm__ volatile (".byte 0x0F; .byte 0x01; .byte 0xD5" :::"memory"); // XEND -} - -/* - * aborts with code 0xFF (lock already held) - */ -inline static void __TBB_machine_transaction_conflict_abort() -{ - __asm__ volatile (".byte 0xC6; .byte 0xF8; .byte 0xFF" :::"memory"); -} - -#endif /* __TBB_TSX_INTRINSICS_PRESENT */ diff --git a/src/tbb/include/tbb/machine/ibm_aix51.h b/src/tbb/include/tbb/machine/ibm_aix51.h deleted file mode 100644 index a905b4e12..000000000 --- a/src/tbb/include/tbb/machine/ibm_aix51.h +++ /dev/null @@ -1,70 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -// TODO: revise by comparing with mac_ppc.h - -#if !defined(__TBB_machine_H) || defined(__TBB_machine_ibm_aix51_H) -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#define __TBB_machine_ibm_aix51_H - -#define __TBB_WORDSIZE 8 -#define __TBB_ENDIANNESS __TBB_ENDIAN_BIG // assumption based on operating system - -#include -#include -#include - -extern "C" { -int32_t __TBB_machine_cas_32 (volatile void* ptr, int32_t value, int32_t comparand); -int64_t __TBB_machine_cas_64 (volatile void* ptr, int64_t value, int64_t comparand); -void __TBB_machine_flush (); -void __TBB_machine_lwsync (); -void __TBB_machine_isync (); -} - -// Mapping of old entry point names retained for the sake of backward binary compatibility -#define __TBB_machine_cmpswp4 __TBB_machine_cas_32 -#define __TBB_machine_cmpswp8 __TBB_machine_cas_64 - -#define __TBB_Yield() sched_yield() - -#define __TBB_USE_GENERIC_PART_WORD_CAS 1 -#define __TBB_USE_GENERIC_FETCH_ADD 1 -#define __TBB_USE_GENERIC_FETCH_STORE 1 -#define __TBB_USE_GENERIC_HALF_FENCED_LOAD_STORE 1 -#define __TBB_USE_GENERIC_RELAXED_LOAD_STORE 1 -#define __TBB_USE_GENERIC_SEQUENTIAL_CONSISTENCY_LOAD_STORE 1 - -#if __GNUC__ - #define __TBB_control_consistency_helper() __asm__ __volatile__( "isync": : :"memory") - #define __TBB_acquire_consistency_helper() __asm__ __volatile__("lwsync": : :"memory") - #define __TBB_release_consistency_helper() __asm__ __volatile__("lwsync": : :"memory") - #define __TBB_full_memory_fence() __asm__ __volatile__( "sync": : :"memory") -#else - // IBM C++ Compiler does not support inline assembly - // TODO: Since XL 9.0 or earlier GCC syntax is supported. Replace with more - // lightweight implementation (like in mac_ppc.h) - #define __TBB_control_consistency_helper() __TBB_machine_isync () - #define __TBB_acquire_consistency_helper() __TBB_machine_lwsync () - #define __TBB_release_consistency_helper() __TBB_machine_lwsync () - #define __TBB_full_memory_fence() __TBB_machine_flush () -#endif diff --git a/src/tbb/include/tbb/machine/icc_generic.h b/src/tbb/include/tbb/machine/icc_generic.h deleted file mode 100644 index 048630007..000000000 --- a/src/tbb/include/tbb/machine/icc_generic.h +++ /dev/null @@ -1,262 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#if !defined(__TBB_machine_H) || defined(__TBB_machine_icc_generic_H) -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#if ! __TBB_ICC_BUILTIN_ATOMICS_PRESENT - #error "Intel C++ Compiler of at least 12.0 version is needed to use ICC intrinsics port" -#endif - -#define __TBB_machine_icc_generic_H - -//ICC mimics the "native" target compiler -#if _MSC_VER - #include "msvc_ia32_common.h" -#else - #include "gcc_ia32_common.h" -#endif - -//TODO: Make __TBB_WORDSIZE macro optional for ICC intrinsics port. -//As compiler intrinsics are used for all the operations it is possible to do. - -#if __TBB_x86_32 - #define __TBB_WORDSIZE 4 -#else - #define __TBB_WORDSIZE 8 -#endif -#define __TBB_ENDIANNESS __TBB_ENDIAN_LITTLE - -//__TBB_compiler_fence() defined just in case, as it seems not to be used on its own anywhere else -#ifndef __TBB_compiler_fence -#if _MSC_VER - //TODO: any way to use same intrinsics on windows and linux? - #pragma intrinsic(_ReadWriteBarrier) - #define __TBB_compiler_fence() _ReadWriteBarrier() -#else - #define __TBB_compiler_fence() __asm__ __volatile__("": : :"memory") -#endif -#endif - -#ifndef __TBB_full_memory_fence -#if _MSC_VER - //TODO: any way to use same intrinsics on windows and linux? - #pragma intrinsic(_mm_mfence) - #define __TBB_full_memory_fence() _mm_mfence() -#else - #define __TBB_full_memory_fence() __asm__ __volatile__("mfence": : :"memory") -#endif -#endif - -#ifndef __TBB_control_consistency_helper -#define __TBB_control_consistency_helper() __TBB_compiler_fence() -#endif - -namespace tbb { namespace internal { -//TODO: is there any way to reuse definition of memory_order enum from ICC instead of copy paste. -//however it seems unlikely that ICC will silently change exact enum values, as they are defined -//in the ISO exactly like this. -//TODO: add test that exact values of the enum are same as in the ISO C++11 -typedef enum memory_order { - memory_order_relaxed, memory_order_consume, memory_order_acquire, - memory_order_release, memory_order_acq_rel, memory_order_seq_cst -} memory_order; - -namespace icc_intrinsics_port { - template - T convert_argument(T value){ - return value; - } - //The overload below is needed to have explicit conversion of pointer to void* in argument list. - //compiler bug? - //TODO: add according broken macro and recheck with ICC 13.0 if the overload is still needed - template - void* convert_argument(T* value){ - return (void*)value; - } -} -//TODO: code below is a bit repetitive, consider simplifying it -template -struct machine_load_store { - static T load_with_acquire ( const volatile T& location ) { - return __atomic_load_explicit(&location, memory_order_acquire); - } - static void store_with_release ( volatile T &location, T value ) { - __atomic_store_explicit(&location, icc_intrinsics_port::convert_argument(value), memory_order_release); - } -}; - -template -struct machine_load_store_relaxed { - static inline T load ( const T& location ) { - return __atomic_load_explicit(&location, memory_order_relaxed); - } - static inline void store ( T& location, T value ) { - __atomic_store_explicit(&location, icc_intrinsics_port::convert_argument(value), memory_order_relaxed); - } -}; - -template -struct machine_load_store_seq_cst { - static T load ( const volatile T& location ) { - return __atomic_load_explicit(&location, memory_order_seq_cst); - } - - static void store ( volatile T &location, T value ) { - __atomic_store_explicit(&location, value, memory_order_seq_cst); - } -}; - -}} // namespace tbb::internal - -namespace tbb{ namespace internal { namespace icc_intrinsics_port{ - typedef enum memory_order_map { - relaxed = memory_order_relaxed, - acquire = memory_order_acquire, - release = memory_order_release, - full_fence= memory_order_seq_cst - } memory_order_map; -}}}// namespace tbb::internal - -#define __TBB_MACHINE_DEFINE_ATOMICS(S,T,M) \ -inline T __TBB_machine_cmpswp##S##M( volatile void *ptr, T value, T comparand ) { \ - __atomic_compare_exchange_strong_explicit( \ - (T*)ptr \ - ,&comparand \ - ,value \ - , tbb::internal::icc_intrinsics_port::M \ - , tbb::internal::icc_intrinsics_port::M); \ - return comparand; \ -} \ - \ -inline T __TBB_machine_fetchstore##S##M(volatile void *ptr, T value) { \ - return __atomic_exchange_explicit((T*)ptr, value, tbb::internal::icc_intrinsics_port::M); \ -} \ - \ -inline T __TBB_machine_fetchadd##S##M(volatile void *ptr, T value) { \ - return __atomic_fetch_add_explicit((T*)ptr, value, tbb::internal::icc_intrinsics_port::M); \ -} \ - -__TBB_MACHINE_DEFINE_ATOMICS(1,tbb::internal::int8_t, full_fence) -__TBB_MACHINE_DEFINE_ATOMICS(1,tbb::internal::int8_t, acquire) -__TBB_MACHINE_DEFINE_ATOMICS(1,tbb::internal::int8_t, release) -__TBB_MACHINE_DEFINE_ATOMICS(1,tbb::internal::int8_t, relaxed) - -__TBB_MACHINE_DEFINE_ATOMICS(2,tbb::internal::int16_t, full_fence) -__TBB_MACHINE_DEFINE_ATOMICS(2,tbb::internal::int16_t, acquire) -__TBB_MACHINE_DEFINE_ATOMICS(2,tbb::internal::int16_t, release) -__TBB_MACHINE_DEFINE_ATOMICS(2,tbb::internal::int16_t, relaxed) - -__TBB_MACHINE_DEFINE_ATOMICS(4,tbb::internal::int32_t, full_fence) -__TBB_MACHINE_DEFINE_ATOMICS(4,tbb::internal::int32_t, acquire) -__TBB_MACHINE_DEFINE_ATOMICS(4,tbb::internal::int32_t, release) -__TBB_MACHINE_DEFINE_ATOMICS(4,tbb::internal::int32_t, relaxed) - -__TBB_MACHINE_DEFINE_ATOMICS(8,tbb::internal::int64_t, full_fence) -__TBB_MACHINE_DEFINE_ATOMICS(8,tbb::internal::int64_t, acquire) -__TBB_MACHINE_DEFINE_ATOMICS(8,tbb::internal::int64_t, release) -__TBB_MACHINE_DEFINE_ATOMICS(8,tbb::internal::int64_t, relaxed) - - -#undef __TBB_MACHINE_DEFINE_ATOMICS - -#define __TBB_USE_FENCED_ATOMICS 1 - -namespace tbb { namespace internal { -#if __TBB_FORCE_64BIT_ALIGNMENT_BROKEN -__TBB_MACHINE_DEFINE_LOAD8_GENERIC_FENCED(full_fence) -__TBB_MACHINE_DEFINE_STORE8_GENERIC_FENCED(full_fence) - -__TBB_MACHINE_DEFINE_LOAD8_GENERIC_FENCED(acquire) -__TBB_MACHINE_DEFINE_STORE8_GENERIC_FENCED(release) - -__TBB_MACHINE_DEFINE_LOAD8_GENERIC_FENCED(relaxed) -__TBB_MACHINE_DEFINE_STORE8_GENERIC_FENCED(relaxed) - -template -struct machine_load_store { - static T load_with_acquire ( const volatile T& location ) { - if( tbb::internal::is_aligned(&location,8)) { - return __atomic_load_explicit(&location, memory_order_acquire); - } else { - return __TBB_machine_generic_load8acquire(&location); - } - } - static void store_with_release ( volatile T &location, T value ) { - if( tbb::internal::is_aligned(&location,8)) { - __atomic_store_explicit(&location, icc_intrinsics_port::convert_argument(value), memory_order_release); - } else { - return __TBB_machine_generic_store8release(&location,value); - } - } -}; - -template -struct machine_load_store_relaxed { - static T load( const volatile T& location ) { - if( tbb::internal::is_aligned(&location,8)) { - return __atomic_load_explicit(&location, memory_order_relaxed); - } else { - return __TBB_machine_generic_load8relaxed(&location); - } - } - static void store( volatile T &location, T value ) { - if( tbb::internal::is_aligned(&location,8)) { - __atomic_store_explicit(&location, icc_intrinsics_port::convert_argument(value), memory_order_relaxed); - } else { - return __TBB_machine_generic_store8relaxed(&location,value); - } - } -}; - -template -struct machine_load_store_seq_cst { - static T load ( const volatile T& location ) { - if( tbb::internal::is_aligned(&location,8)) { - return __atomic_load_explicit(&location, memory_order_seq_cst); - } else { - return __TBB_machine_generic_load8full_fence(&location); - } - - } - - static void store ( volatile T &location, T value ) { - if( tbb::internal::is_aligned(&location,8)) { - __atomic_store_explicit(&location, value, memory_order_seq_cst); - } else { - return __TBB_machine_generic_store8full_fence(&location,value); - } - - } -}; - -#endif -}} // namespace tbb::internal -template -inline void __TBB_machine_OR( T *operand, T addend ) { - __atomic_fetch_or_explicit(operand, addend, tbb::internal::memory_order_seq_cst); -} - -template -inline void __TBB_machine_AND( T *operand, T addend ) { - __atomic_fetch_and_explicit(operand, addend, tbb::internal::memory_order_seq_cst); -} - diff --git a/src/tbb/include/tbb/machine/linux_common.h b/src/tbb/include/tbb/machine/linux_common.h deleted file mode 100644 index 4d2d355b1..000000000 --- a/src/tbb/include/tbb/machine/linux_common.h +++ /dev/null @@ -1,84 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_machine_H -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#include -#define __TBB_Yield() sched_yield() - -#include -/* Futex definitions */ -#include - -#if defined(SYS_futex) - -#define __TBB_USE_FUTEX 1 -#include -#include -// Unfortunately, some versions of Linux do not have a header that defines FUTEX_WAIT and FUTEX_WAKE. - -#ifdef FUTEX_WAIT -#define __TBB_FUTEX_WAIT FUTEX_WAIT -#else -#define __TBB_FUTEX_WAIT 0 -#endif - -#ifdef FUTEX_WAKE -#define __TBB_FUTEX_WAKE FUTEX_WAKE -#else -#define __TBB_FUTEX_WAKE 1 -#endif - -#ifndef __TBB_ASSERT -#error machine specific headers must be included after tbb_stddef.h -#endif - -namespace tbb { - -namespace internal { - -inline int futex_wait( void *futex, int comparand ) { - int r = syscall( SYS_futex,futex,__TBB_FUTEX_WAIT,comparand,NULL,NULL,0 ); -#if TBB_USE_ASSERT - int e = errno; - __TBB_ASSERT( r==0||r==EWOULDBLOCK||(r==-1&&(e==EAGAIN||e==EINTR)), "futex_wait failed." ); -#endif /* TBB_USE_ASSERT */ - return r; -} - -inline int futex_wakeup_one( void *futex ) { - int r = ::syscall( SYS_futex,futex,__TBB_FUTEX_WAKE,1,NULL,NULL,0 ); - __TBB_ASSERT( r==0||r==1, "futex_wakeup_one: more than one thread woken up?" ); - return r; -} - -inline int futex_wakeup_all( void *futex ) { - int r = ::syscall( SYS_futex,futex,__TBB_FUTEX_WAKE,INT_MAX,NULL,NULL,0 ); - __TBB_ASSERT( r>=0, "futex_wakeup_all: error in waking up threads" ); - return r; -} - -} /* namespace internal */ - -} /* namespace tbb */ - -#endif /* SYS_futex */ diff --git a/src/tbb/include/tbb/machine/linux_ia32.h b/src/tbb/include/tbb/machine/linux_ia32.h deleted file mode 100644 index 932d343d1..000000000 --- a/src/tbb/include/tbb/machine/linux_ia32.h +++ /dev/null @@ -1,232 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#if !defined(__TBB_machine_H) || defined(__TBB_machine_linux_ia32_H) -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#define __TBB_machine_linux_ia32_H - -#include -#include "gcc_ia32_common.h" - -#define __TBB_WORDSIZE 4 -#define __TBB_ENDIANNESS __TBB_ENDIAN_LITTLE - -#define __TBB_compiler_fence() __asm__ __volatile__("": : :"memory") -#define __TBB_control_consistency_helper() __TBB_compiler_fence() -#define __TBB_acquire_consistency_helper() __TBB_compiler_fence() -#define __TBB_release_consistency_helper() __TBB_compiler_fence() -#define __TBB_full_memory_fence() __asm__ __volatile__("mfence": : :"memory") - -#if __TBB_ICC_ASM_VOLATILE_BROKEN -#define __TBB_VOLATILE -#else -#define __TBB_VOLATILE volatile -#endif - -#define __TBB_MACHINE_DEFINE_ATOMICS(S,T,X,R) \ -static inline T __TBB_machine_cmpswp##S (volatile void *ptr, T value, T comparand ) \ -{ \ - T result; \ - \ - __asm__ __volatile__("lock\ncmpxchg" X " %2,%1" \ - : "=a"(result), "=m"(*(__TBB_VOLATILE T*)ptr) \ - : "q"(value), "0"(comparand), "m"(*(__TBB_VOLATILE T*)ptr) \ - : "memory"); \ - return result; \ -} \ - \ -static inline T __TBB_machine_fetchadd##S(volatile void *ptr, T addend) \ -{ \ - T result; \ - __asm__ __volatile__("lock\nxadd" X " %0,%1" \ - : R (result), "=m"(*(__TBB_VOLATILE T*)ptr) \ - : "0"(addend), "m"(*(__TBB_VOLATILE T*)ptr) \ - : "memory"); \ - return result; \ -} \ - \ -static inline T __TBB_machine_fetchstore##S(volatile void *ptr, T value) \ -{ \ - T result; \ - __asm__ __volatile__("lock\nxchg" X " %0,%1" \ - : R (result), "=m"(*(__TBB_VOLATILE T*)ptr) \ - : "0"(value), "m"(*(__TBB_VOLATILE T*)ptr) \ - : "memory"); \ - return result; \ -} \ - -__TBB_MACHINE_DEFINE_ATOMICS(1,int8_t,"","=q") -__TBB_MACHINE_DEFINE_ATOMICS(2,int16_t,"","=r") -__TBB_MACHINE_DEFINE_ATOMICS(4,int32_t,"l","=r") - -#if __INTEL_COMPILER -#pragma warning( push ) -// reference to EBX in a function requiring stack alignment -#pragma warning( disable: 998 ) -#endif - -#if __TBB_GCC_CAS8_BUILTIN_INLINING_BROKEN -#define __TBB_IA32_CAS8_NOINLINE __attribute__ ((noinline)) -#else -#define __TBB_IA32_CAS8_NOINLINE -#endif - -static inline __TBB_IA32_CAS8_NOINLINE int64_t __TBB_machine_cmpswp8 (volatile void *ptr, int64_t value, int64_t comparand ) { -//TODO: remove the extra part of condition once __TBB_GCC_BUILTIN_ATOMICS_PRESENT is lowered to gcc version 4.1.2 -#if (__TBB_GCC_BUILTIN_ATOMICS_PRESENT || (__TBB_GCC_VERSION >= 40102)) && !__TBB_GCC_64BIT_ATOMIC_BUILTINS_BROKEN - return __sync_val_compare_and_swap( reinterpret_cast(ptr), comparand, value ); -#else /* !__TBB_GCC_BUILTIN_ATOMICS_PRESENT */ - //TODO: look like ICC 13.0 has some issues with this code, investigate it more deeply - int64_t result; - union { - int64_t i64; - int32_t i32[2]; - }; - i64 = value; -#if __PIC__ - /* compiling position-independent code */ - // EBX register preserved for compliance with position-independent code rules on IA32 - int32_t tmp; - __asm__ __volatile__ ( - "movl %%ebx,%2\n\t" - "movl %5,%%ebx\n\t" -#if __GNUC__==3 - "lock\n\t cmpxchg8b %1\n\t" -#else - "lock\n\t cmpxchg8b (%3)\n\t" -#endif - "movl %2,%%ebx" - : "=A"(result) - , "=m"(*(__TBB_VOLATILE int64_t *)ptr) - , "=m"(tmp) -#if __GNUC__==3 - : "m"(*(__TBB_VOLATILE int64_t *)ptr) -#else - : "SD"(ptr) -#endif - , "0"(comparand) - , "m"(i32[0]), "c"(i32[1]) - : "memory" -#if __INTEL_COMPILER - ,"ebx" -#endif - ); -#else /* !__PIC__ */ - __asm__ __volatile__ ( - "lock\n\t cmpxchg8b %1\n\t" - : "=A"(result), "=m"(*(__TBB_VOLATILE int64_t *)ptr) - : "m"(*(__TBB_VOLATILE int64_t *)ptr) - , "0"(comparand) - , "b"(i32[0]), "c"(i32[1]) - : "memory" - ); -#endif /* __PIC__ */ - return result; -#endif /* !__TBB_GCC_BUILTIN_ATOMICS_PRESENT */ -} - -#undef __TBB_IA32_CAS8_NOINLINE - -#if __INTEL_COMPILER -#pragma warning( pop ) -#endif // warning 998 is back - -static inline void __TBB_machine_or( volatile void *ptr, uint32_t addend ) { - __asm__ __volatile__("lock\norl %1,%0" : "=m"(*(__TBB_VOLATILE uint32_t *)ptr) : "r"(addend), "m"(*(__TBB_VOLATILE uint32_t *)ptr) : "memory"); -} - -static inline void __TBB_machine_and( volatile void *ptr, uint32_t addend ) { - __asm__ __volatile__("lock\nandl %1,%0" : "=m"(*(__TBB_VOLATILE uint32_t *)ptr) : "r"(addend), "m"(*(__TBB_VOLATILE uint32_t *)ptr) : "memory"); -} - -//TODO: Check if it possible and profitable for IA-32 architecture on (Linux* and Windows*) -//to use of 64-bit load/store via floating point registers together with full fence -//for sequentially consistent load/store, instead of CAS. - -#if __clang__ -#define __TBB_fildq "fildll" -#define __TBB_fistpq "fistpll" -#else -#define __TBB_fildq "fildq" -#define __TBB_fistpq "fistpq" -#endif - -static inline int64_t __TBB_machine_aligned_load8 (const volatile void *ptr) { - __TBB_ASSERT(tbb::internal::is_aligned(ptr,8),"__TBB_machine_aligned_load8 should be used with 8 byte aligned locations only \n"); - int64_t result; - __asm__ __volatile__ ( __TBB_fildq " %1\n\t" - __TBB_fistpq " %0" : "=m"(result) : "m"(*(const __TBB_VOLATILE uint64_t*)ptr) : "memory" ); - return result; -} - -static inline void __TBB_machine_aligned_store8 (volatile void *ptr, int64_t value ) { - __TBB_ASSERT(tbb::internal::is_aligned(ptr,8),"__TBB_machine_aligned_store8 should be used with 8 byte aligned locations only \n"); - // Aligned store - __asm__ __volatile__ ( __TBB_fildq " %1\n\t" - __TBB_fistpq " %0" : "=m"(*(__TBB_VOLATILE int64_t*)ptr) : "m"(value) : "memory" ); -} - -static inline int64_t __TBB_machine_load8 (const volatile void *ptr) { -#if __TBB_FORCE_64BIT_ALIGNMENT_BROKEN - if( tbb::internal::is_aligned(ptr,8)) { -#endif - return __TBB_machine_aligned_load8(ptr); -#if __TBB_FORCE_64BIT_ALIGNMENT_BROKEN - } else { - // Unaligned load - return __TBB_machine_cmpswp8(const_cast(ptr),0,0); - } -#endif -} - -//! Handles misaligned 8-byte store -/** Defined in tbb_misc.cpp */ -extern "C" void __TBB_machine_store8_slow( volatile void *ptr, int64_t value ); -extern "C" void __TBB_machine_store8_slow_perf_warning( volatile void *ptr ); - -static inline void __TBB_machine_store8(volatile void *ptr, int64_t value) { -#if __TBB_FORCE_64BIT_ALIGNMENT_BROKEN - if( tbb::internal::is_aligned(ptr,8)) { -#endif - __TBB_machine_aligned_store8(ptr,value); -#if __TBB_FORCE_64BIT_ALIGNMENT_BROKEN - } else { - // Unaligned store -#if TBB_USE_PERFORMANCE_WARNINGS - __TBB_machine_store8_slow_perf_warning(ptr); -#endif /* TBB_USE_PERFORMANCE_WARNINGS */ - __TBB_machine_store8_slow(ptr,value); - } -#endif -} - -// Machine specific atomic operations -#define __TBB_AtomicOR(P,V) __TBB_machine_or(P,V) -#define __TBB_AtomicAND(P,V) __TBB_machine_and(P,V) - -#define __TBB_USE_GENERIC_DWORD_FETCH_ADD 1 -#define __TBB_USE_GENERIC_DWORD_FETCH_STORE 1 -#define __TBB_USE_FETCHSTORE_AS_FULL_FENCED_STORE 1 -#define __TBB_USE_GENERIC_HALF_FENCED_LOAD_STORE 1 -#define __TBB_USE_GENERIC_RELAXED_LOAD_STORE 1 -#define __TBB_USE_GENERIC_SEQUENTIAL_CONSISTENCY_LOAD_STORE 1 - diff --git a/src/tbb/include/tbb/machine/linux_ia64.h b/src/tbb/include/tbb/machine/linux_ia64.h deleted file mode 100644 index f47722880..000000000 --- a/src/tbb/include/tbb/machine/linux_ia64.h +++ /dev/null @@ -1,181 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#if !defined(__TBB_machine_H) || defined(__TBB_machine_linux_ia64_H) -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#define __TBB_machine_linux_ia64_H - -#include -#include - -#define __TBB_WORDSIZE 8 -#define __TBB_ENDIANNESS __TBB_ENDIAN_LITTLE - -#if __INTEL_COMPILER - #define __TBB_compiler_fence() - #define __TBB_control_consistency_helper() __TBB_compiler_fence() - #define __TBB_acquire_consistency_helper() - #define __TBB_release_consistency_helper() - #define __TBB_full_memory_fence() __mf() -#else - #define __TBB_compiler_fence() __asm__ __volatile__("": : :"memory") - #define __TBB_control_consistency_helper() __TBB_compiler_fence() - // Even though GCC imbues volatile loads with acquire semantics, it sometimes moves - // loads over the acquire fence. The following helpers stop such incorrect code motion. - #define __TBB_acquire_consistency_helper() __TBB_compiler_fence() - #define __TBB_release_consistency_helper() __TBB_compiler_fence() - #define __TBB_full_memory_fence() __asm__ __volatile__("mf": : :"memory") -#endif /* !__INTEL_COMPILER */ - -// Most of the functions will be in a .s file -// TODO: revise dynamic_link, memory pools and etc. if the library dependency is removed. - -extern "C" { - int8_t __TBB_machine_fetchadd1__TBB_full_fence (volatile void *ptr, int8_t addend); - int8_t __TBB_machine_fetchadd1acquire(volatile void *ptr, int8_t addend); - int8_t __TBB_machine_fetchadd1release(volatile void *ptr, int8_t addend); - - int16_t __TBB_machine_fetchadd2__TBB_full_fence (volatile void *ptr, int16_t addend); - int16_t __TBB_machine_fetchadd2acquire(volatile void *ptr, int16_t addend); - int16_t __TBB_machine_fetchadd2release(volatile void *ptr, int16_t addend); - - int32_t __TBB_machine_fetchadd4__TBB_full_fence (volatile void *ptr, int32_t value); - int32_t __TBB_machine_fetchadd4acquire(volatile void *ptr, int32_t addend); - int32_t __TBB_machine_fetchadd4release(volatile void *ptr, int32_t addend); - - int64_t __TBB_machine_fetchadd8__TBB_full_fence (volatile void *ptr, int64_t value); - int64_t __TBB_machine_fetchadd8acquire(volatile void *ptr, int64_t addend); - int64_t __TBB_machine_fetchadd8release(volatile void *ptr, int64_t addend); - - int8_t __TBB_machine_fetchstore1__TBB_full_fence (volatile void *ptr, int8_t value); - int8_t __TBB_machine_fetchstore1acquire(volatile void *ptr, int8_t value); - int8_t __TBB_machine_fetchstore1release(volatile void *ptr, int8_t value); - - int16_t __TBB_machine_fetchstore2__TBB_full_fence (volatile void *ptr, int16_t value); - int16_t __TBB_machine_fetchstore2acquire(volatile void *ptr, int16_t value); - int16_t __TBB_machine_fetchstore2release(volatile void *ptr, int16_t value); - - int32_t __TBB_machine_fetchstore4__TBB_full_fence (volatile void *ptr, int32_t value); - int32_t __TBB_machine_fetchstore4acquire(volatile void *ptr, int32_t value); - int32_t __TBB_machine_fetchstore4release(volatile void *ptr, int32_t value); - - int64_t __TBB_machine_fetchstore8__TBB_full_fence (volatile void *ptr, int64_t value); - int64_t __TBB_machine_fetchstore8acquire(volatile void *ptr, int64_t value); - int64_t __TBB_machine_fetchstore8release(volatile void *ptr, int64_t value); - - int8_t __TBB_machine_cmpswp1__TBB_full_fence (volatile void *ptr, int8_t value, int8_t comparand); - int8_t __TBB_machine_cmpswp1acquire(volatile void *ptr, int8_t value, int8_t comparand); - int8_t __TBB_machine_cmpswp1release(volatile void *ptr, int8_t value, int8_t comparand); - - int16_t __TBB_machine_cmpswp2__TBB_full_fence (volatile void *ptr, int16_t value, int16_t comparand); - int16_t __TBB_machine_cmpswp2acquire(volatile void *ptr, int16_t value, int16_t comparand); - int16_t __TBB_machine_cmpswp2release(volatile void *ptr, int16_t value, int16_t comparand); - - int32_t __TBB_machine_cmpswp4__TBB_full_fence (volatile void *ptr, int32_t value, int32_t comparand); - int32_t __TBB_machine_cmpswp4acquire(volatile void *ptr, int32_t value, int32_t comparand); - int32_t __TBB_machine_cmpswp4release(volatile void *ptr, int32_t value, int32_t comparand); - - int64_t __TBB_machine_cmpswp8__TBB_full_fence (volatile void *ptr, int64_t value, int64_t comparand); - int64_t __TBB_machine_cmpswp8acquire(volatile void *ptr, int64_t value, int64_t comparand); - int64_t __TBB_machine_cmpswp8release(volatile void *ptr, int64_t value, int64_t comparand); - - int64_t __TBB_machine_lg(uint64_t value); - void __TBB_machine_pause(int32_t delay); - bool __TBB_machine_trylockbyte( volatile unsigned char &ptr ); - int64_t __TBB_machine_lockbyte( volatile unsigned char &ptr ); - - //! Retrieves the current RSE backing store pointer. IA64 specific. - void* __TBB_get_bsp(); - - int32_t __TBB_machine_load1_relaxed(const void *ptr); - int32_t __TBB_machine_load2_relaxed(const void *ptr); - int32_t __TBB_machine_load4_relaxed(const void *ptr); - int64_t __TBB_machine_load8_relaxed(const void *ptr); - - void __TBB_machine_store1_relaxed(void *ptr, int32_t value); - void __TBB_machine_store2_relaxed(void *ptr, int32_t value); - void __TBB_machine_store4_relaxed(void *ptr, int32_t value); - void __TBB_machine_store8_relaxed(void *ptr, int64_t value); -} // extern "C" - -// Mapping old entry points to the names corresponding to the new full_fence identifier. -#define __TBB_machine_fetchadd1full_fence __TBB_machine_fetchadd1__TBB_full_fence -#define __TBB_machine_fetchadd2full_fence __TBB_machine_fetchadd2__TBB_full_fence -#define __TBB_machine_fetchadd4full_fence __TBB_machine_fetchadd4__TBB_full_fence -#define __TBB_machine_fetchadd8full_fence __TBB_machine_fetchadd8__TBB_full_fence -#define __TBB_machine_fetchstore1full_fence __TBB_machine_fetchstore1__TBB_full_fence -#define __TBB_machine_fetchstore2full_fence __TBB_machine_fetchstore2__TBB_full_fence -#define __TBB_machine_fetchstore4full_fence __TBB_machine_fetchstore4__TBB_full_fence -#define __TBB_machine_fetchstore8full_fence __TBB_machine_fetchstore8__TBB_full_fence -#define __TBB_machine_cmpswp1full_fence __TBB_machine_cmpswp1__TBB_full_fence -#define __TBB_machine_cmpswp2full_fence __TBB_machine_cmpswp2__TBB_full_fence -#define __TBB_machine_cmpswp4full_fence __TBB_machine_cmpswp4__TBB_full_fence -#define __TBB_machine_cmpswp8full_fence __TBB_machine_cmpswp8__TBB_full_fence - -// Mapping relaxed operations to the entry points implementing them. -/** On IA64 RMW operations implicitly have acquire semantics. Thus one cannot - actually have completely relaxed RMW operation here. **/ -#define __TBB_machine_fetchadd1relaxed __TBB_machine_fetchadd1acquire -#define __TBB_machine_fetchadd2relaxed __TBB_machine_fetchadd2acquire -#define __TBB_machine_fetchadd4relaxed __TBB_machine_fetchadd4acquire -#define __TBB_machine_fetchadd8relaxed __TBB_machine_fetchadd8acquire -#define __TBB_machine_fetchstore1relaxed __TBB_machine_fetchstore1acquire -#define __TBB_machine_fetchstore2relaxed __TBB_machine_fetchstore2acquire -#define __TBB_machine_fetchstore4relaxed __TBB_machine_fetchstore4acquire -#define __TBB_machine_fetchstore8relaxed __TBB_machine_fetchstore8acquire -#define __TBB_machine_cmpswp1relaxed __TBB_machine_cmpswp1acquire -#define __TBB_machine_cmpswp2relaxed __TBB_machine_cmpswp2acquire -#define __TBB_machine_cmpswp4relaxed __TBB_machine_cmpswp4acquire -#define __TBB_machine_cmpswp8relaxed __TBB_machine_cmpswp8acquire - -#define __TBB_MACHINE_DEFINE_ATOMICS(S,V) \ - template \ - struct machine_load_store_relaxed { \ - static inline T load ( const T& location ) { \ - return (T)__TBB_machine_load##S##_relaxed(&location); \ - } \ - static inline void store ( T& location, T value ) { \ - __TBB_machine_store##S##_relaxed(&location, (V)value); \ - } \ - } - -namespace tbb { -namespace internal { - __TBB_MACHINE_DEFINE_ATOMICS(1,int8_t); - __TBB_MACHINE_DEFINE_ATOMICS(2,int16_t); - __TBB_MACHINE_DEFINE_ATOMICS(4,int32_t); - __TBB_MACHINE_DEFINE_ATOMICS(8,int64_t); -}} // namespaces internal, tbb - -#undef __TBB_MACHINE_DEFINE_ATOMICS - -#define __TBB_USE_FENCED_ATOMICS 1 -#define __TBB_USE_GENERIC_HALF_FENCED_LOAD_STORE 1 -#define __TBB_USE_GENERIC_SEQUENTIAL_CONSISTENCY_LOAD_STORE 1 - -// Definition of Lock functions -#define __TBB_TryLockByte(P) __TBB_machine_trylockbyte(P) -#define __TBB_LockByte(P) __TBB_machine_lockbyte(P) - -// Definition of other utility functions -#define __TBB_Pause(V) __TBB_machine_pause(V) -#define __TBB_Log2(V) __TBB_machine_lg(V) diff --git a/src/tbb/include/tbb/machine/linux_intel64.h b/src/tbb/include/tbb/machine/linux_intel64.h deleted file mode 100644 index 02153c2a4..000000000 --- a/src/tbb/include/tbb/machine/linux_intel64.h +++ /dev/null @@ -1,96 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#if !defined(__TBB_machine_H) || defined(__TBB_machine_linux_intel64_H) -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#define __TBB_machine_linux_intel64_H - -#include -#include "gcc_ia32_common.h" - -#define __TBB_WORDSIZE 8 -#define __TBB_ENDIANNESS __TBB_ENDIAN_LITTLE - -#define __TBB_compiler_fence() __asm__ __volatile__("": : :"memory") -#define __TBB_control_consistency_helper() __TBB_compiler_fence() -#define __TBB_acquire_consistency_helper() __TBB_compiler_fence() -#define __TBB_release_consistency_helper() __TBB_compiler_fence() - -#ifndef __TBB_full_memory_fence -#define __TBB_full_memory_fence() __asm__ __volatile__("mfence": : :"memory") -#endif - -#define __TBB_MACHINE_DEFINE_ATOMICS(S,T,X) \ -static inline T __TBB_machine_cmpswp##S (volatile void *ptr, T value, T comparand ) \ -{ \ - T result; \ - \ - __asm__ __volatile__("lock\ncmpxchg" X " %2,%1" \ - : "=a"(result), "=m"(*(volatile T*)ptr) \ - : "q"(value), "0"(comparand), "m"(*(volatile T*)ptr) \ - : "memory"); \ - return result; \ -} \ - \ -static inline T __TBB_machine_fetchadd##S(volatile void *ptr, T addend) \ -{ \ - T result; \ - __asm__ __volatile__("lock\nxadd" X " %0,%1" \ - : "=r"(result),"=m"(*(volatile T*)ptr) \ - : "0"(addend), "m"(*(volatile T*)ptr) \ - : "memory"); \ - return result; \ -} \ - \ -static inline T __TBB_machine_fetchstore##S(volatile void *ptr, T value) \ -{ \ - T result; \ - __asm__ __volatile__("lock\nxchg" X " %0,%1" \ - : "=r"(result),"=m"(*(volatile T*)ptr) \ - : "0"(value), "m"(*(volatile T*)ptr) \ - : "memory"); \ - return result; \ -} \ - -__TBB_MACHINE_DEFINE_ATOMICS(1,int8_t,"") -__TBB_MACHINE_DEFINE_ATOMICS(2,int16_t,"") -__TBB_MACHINE_DEFINE_ATOMICS(4,int32_t,"") -__TBB_MACHINE_DEFINE_ATOMICS(8,int64_t,"q") - -#undef __TBB_MACHINE_DEFINE_ATOMICS - -static inline void __TBB_machine_or( volatile void *ptr, uint64_t value ) { - __asm__ __volatile__("lock\norq %1,%0" : "=m"(*(volatile uint64_t*)ptr) : "r"(value), "m"(*(volatile uint64_t*)ptr) : "memory"); -} - -static inline void __TBB_machine_and( volatile void *ptr, uint64_t value ) { - __asm__ __volatile__("lock\nandq %1,%0" : "=m"(*(volatile uint64_t*)ptr) : "r"(value), "m"(*(volatile uint64_t*)ptr) : "memory"); -} - -#define __TBB_AtomicOR(P,V) __TBB_machine_or(P,V) -#define __TBB_AtomicAND(P,V) __TBB_machine_and(P,V) - -#define __TBB_USE_FETCHSTORE_AS_FULL_FENCED_STORE 1 -#define __TBB_USE_GENERIC_HALF_FENCED_LOAD_STORE 1 -#define __TBB_USE_GENERIC_RELAXED_LOAD_STORE 1 -#define __TBB_USE_GENERIC_SEQUENTIAL_CONSISTENCY_LOAD_STORE 1 - diff --git a/src/tbb/include/tbb/machine/mac_ppc.h b/src/tbb/include/tbb/machine/mac_ppc.h deleted file mode 100644 index 13f387b40..000000000 --- a/src/tbb/include/tbb/machine/mac_ppc.h +++ /dev/null @@ -1,313 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#if !defined(__TBB_machine_H) || defined(__TBB_machine_gcc_power_H) -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#define __TBB_machine_gcc_power_H - -#include -#include - -// TODO: rename to gcc_power.h? -// This file is for Power Architecture with compilers supporting GNU inline-assembler syntax (currently GNU g++ and IBM XL). -// Note that XL V9.0 (sometimes?) has trouble dealing with empty input and/or clobber lists, so they should be avoided. - -#if __powerpc64__ || __ppc64__ - // IBM XL documents __powerpc64__ (and __PPC64__). - // Apple documents __ppc64__ (with __ppc__ only on 32-bit). - #define __TBB_WORDSIZE 8 -#else - #define __TBB_WORDSIZE 4 -#endif - -// Traditionally Power Architecture is big-endian. -// Little-endian could be just an address manipulation (compatibility with TBB not verified), -// or normal little-endian (on more recent systems). Embedded PowerPC systems may support -// page-specific endianness, but then one endianness must be hidden from TBB so that it still sees only one. -#if __BIG_ENDIAN__ || (defined(__BYTE_ORDER__) && __BYTE_ORDER__==__ORDER_BIG_ENDIAN__) - #define __TBB_ENDIANNESS __TBB_ENDIAN_BIG -#elif __LITTLE_ENDIAN__ || (defined(__BYTE_ORDER__) && __BYTE_ORDER__==__ORDER_LITTLE_ENDIAN__) - #define __TBB_ENDIANNESS __TBB_ENDIAN_LITTLE -#elif defined(__BYTE_ORDER__) - #define __TBB_ENDIANNESS __TBB_ENDIAN_UNSUPPORTED -#else - #define __TBB_ENDIANNESS __TBB_ENDIAN_DETECT -#endif - -// On Power Architecture, (lock-free) 64-bit atomics require 64-bit hardware: -#if __TBB_WORDSIZE==8 - // Do not change the following definition, because TBB itself will use 64-bit atomics in 64-bit builds. - #define __TBB_64BIT_ATOMICS 1 -#elif __bgp__ - // Do not change the following definition, because this is known 32-bit hardware. - #define __TBB_64BIT_ATOMICS 0 -#else - // To enable 64-bit atomics in 32-bit builds, set the value below to 1 instead of 0. - // You must make certain that the program will only use them on actual 64-bit hardware - // (which typically means that the entire program is only executed on such hardware), - // because their implementation involves machine instructions that are illegal elsewhere. - // The setting can be chosen independently per compilation unit, - // which also means that TBB itself does not need to be rebuilt. - // Alternatively (but only for the current architecture and TBB version), - // override the default as a predefined macro when invoking the compiler. - #ifndef __TBB_64BIT_ATOMICS - #define __TBB_64BIT_ATOMICS 0 - #endif -#endif - -inline int32_t __TBB_machine_cmpswp4 (volatile void *ptr, int32_t value, int32_t comparand ) -{ - int32_t result; - - __asm__ __volatile__("sync\n" - "0:\n\t" - "lwarx %[res],0,%[ptr]\n\t" /* load w/ reservation */ - "cmpw %[res],%[cmp]\n\t" /* compare against comparand */ - "bne- 1f\n\t" /* exit if not same */ - "stwcx. %[val],0,%[ptr]\n\t" /* store new value */ - "bne- 0b\n" /* retry if reservation lost */ - "1:\n\t" /* the exit */ - "isync" - : [res]"=&r"(result) - , "+m"(* (int32_t*) ptr) /* redundant with "memory" */ - : [ptr]"r"(ptr) - , [val]"r"(value) - , [cmp]"r"(comparand) - : "memory" /* compiler full fence */ - , "cr0" /* clobbered by cmp and/or stwcx. */ - ); - return result; -} - -#if __TBB_WORDSIZE==8 - -inline int64_t __TBB_machine_cmpswp8 (volatile void *ptr, int64_t value, int64_t comparand ) -{ - int64_t result; - __asm__ __volatile__("sync\n" - "0:\n\t" - "ldarx %[res],0,%[ptr]\n\t" /* load w/ reservation */ - "cmpd %[res],%[cmp]\n\t" /* compare against comparand */ - "bne- 1f\n\t" /* exit if not same */ - "stdcx. %[val],0,%[ptr]\n\t" /* store new value */ - "bne- 0b\n" /* retry if reservation lost */ - "1:\n\t" /* the exit */ - "isync" - : [res]"=&r"(result) - , "+m"(* (int64_t*) ptr) /* redundant with "memory" */ - : [ptr]"r"(ptr) - , [val]"r"(value) - , [cmp]"r"(comparand) - : "memory" /* compiler full fence */ - , "cr0" /* clobbered by cmp and/or stdcx. */ - ); - return result; -} - -#elif __TBB_64BIT_ATOMICS /* && __TBB_WORDSIZE==4 */ - -inline int64_t __TBB_machine_cmpswp8 (volatile void *ptr, int64_t value, int64_t comparand ) -{ - int64_t result; - int64_t value_register, comparand_register, result_register; // dummy variables to allocate registers - __asm__ __volatile__("sync\n\t" - "ld %[val],%[valm]\n\t" - "ld %[cmp],%[cmpm]\n" - "0:\n\t" - "ldarx %[res],0,%[ptr]\n\t" /* load w/ reservation */ - "cmpd %[res],%[cmp]\n\t" /* compare against comparand */ - "bne- 1f\n\t" /* exit if not same */ - "stdcx. %[val],0,%[ptr]\n\t" /* store new value */ - "bne- 0b\n" /* retry if reservation lost */ - "1:\n\t" /* the exit */ - "std %[res],%[resm]\n\t" - "isync" - : [resm]"=m"(result) - , [res] "=&r"( result_register) - , [val] "=&r"( value_register) - , [cmp] "=&r"(comparand_register) - , "+m"(* (int64_t*) ptr) /* redundant with "memory" */ - : [ptr] "r"(ptr) - , [valm]"m"(value) - , [cmpm]"m"(comparand) - : "memory" /* compiler full fence */ - , "cr0" /* clobbered by cmpd and/or stdcx. */ - ); - return result; -} - -#endif /* __TBB_WORDSIZE==4 && __TBB_64BIT_ATOMICS */ - -#define __TBB_MACHINE_DEFINE_LOAD_STORE(S,ldx,stx,cmpx) \ - template \ - struct machine_load_store { \ - static inline T load_with_acquire(const volatile T& location) { \ - T result; \ - __asm__ __volatile__(ldx " %[res],0(%[ptr])\n" \ - "0:\n\t" \ - cmpx " %[res],%[res]\n\t" \ - "bne- 0b\n\t" \ - "isync" \ - : [res]"=r"(result) \ - : [ptr]"b"(&location) /* cannot use register 0 here */ \ - , "m"(location) /* redundant with "memory" */ \ - : "memory" /* compiler acquire fence */ \ - , "cr0" /* clobbered by cmpw/cmpd */); \ - return result; \ - } \ - static inline void store_with_release(volatile T &location, T value) { \ - __asm__ __volatile__("lwsync\n\t" \ - stx " %[val],0(%[ptr])" \ - : "=m"(location) /* redundant with "memory" */ \ - : [ptr]"b"(&location) /* cannot use register 0 here */ \ - , [val]"r"(value) \ - : "memory"/*compiler release fence*/ /*(cr0 not affected)*/); \ - } \ - }; \ - \ - template \ - struct machine_load_store_relaxed { \ - static inline T load (const __TBB_atomic T& location) { \ - T result; \ - __asm__ __volatile__(ldx " %[res],0(%[ptr])" \ - : [res]"=r"(result) \ - : [ptr]"b"(&location) /* cannot use register 0 here */ \ - , "m"(location) \ - ); /*(no compiler fence)*/ /*(cr0 not affected)*/ \ - return result; \ - } \ - static inline void store (__TBB_atomic T &location, T value) { \ - __asm__ __volatile__(stx " %[val],0(%[ptr])" \ - : "=m"(location) \ - : [ptr]"b"(&location) /* cannot use register 0 here */ \ - , [val]"r"(value) \ - ); /*(no compiler fence)*/ /*(cr0 not affected)*/ \ - } \ - }; - -namespace tbb { -namespace internal { - __TBB_MACHINE_DEFINE_LOAD_STORE(1,"lbz","stb","cmpw") - __TBB_MACHINE_DEFINE_LOAD_STORE(2,"lhz","sth","cmpw") - __TBB_MACHINE_DEFINE_LOAD_STORE(4,"lwz","stw","cmpw") - -#if __TBB_WORDSIZE==8 - - __TBB_MACHINE_DEFINE_LOAD_STORE(8,"ld" ,"std","cmpd") - -#elif __TBB_64BIT_ATOMICS /* && __TBB_WORDSIZE==4 */ - - template - struct machine_load_store { - static inline T load_with_acquire(const volatile T& location) { - T result; - T result_register; // dummy variable to allocate a register - __asm__ __volatile__("ld %[res],0(%[ptr])\n\t" - "std %[res],%[resm]\n" - "0:\n\t" - "cmpd %[res],%[res]\n\t" - "bne- 0b\n\t" - "isync" - : [resm]"=m"(result) - , [res]"=&r"(result_register) - : [ptr]"b"(&location) /* cannot use register 0 here */ - , "m"(location) /* redundant with "memory" */ - : "memory" /* compiler acquire fence */ - , "cr0" /* clobbered by cmpd */); - return result; - } - - static inline void store_with_release(volatile T &location, T value) { - T value_register; // dummy variable to allocate a register - __asm__ __volatile__("lwsync\n\t" - "ld %[val],%[valm]\n\t" - "std %[val],0(%[ptr])" - : "=m"(location) /* redundant with "memory" */ - , [val]"=&r"(value_register) - : [ptr]"b"(&location) /* cannot use register 0 here */ - , [valm]"m"(value) - : "memory"/*compiler release fence*/ /*(cr0 not affected)*/); - } - }; - - struct machine_load_store_relaxed { - static inline T load (const volatile T& location) { - T result; - T result_register; // dummy variable to allocate a register - __asm__ __volatile__("ld %[res],0(%[ptr])\n\t" - "std %[res],%[resm]" - : [resm]"=m"(result) - , [res]"=&r"(result_register) - : [ptr]"b"(&location) /* cannot use register 0 here */ - , "m"(location) - ); /*(no compiler fence)*/ /*(cr0 not affected)*/ - return result; - } - - static inline void store (volatile T &location, T value) { - T value_register; // dummy variable to allocate a register - __asm__ __volatile__("ld %[val],%[valm]\n\t" - "std %[val],0(%[ptr])" - : "=m"(location) - , [val]"=&r"(value_register) - : [ptr]"b"(&location) /* cannot use register 0 here */ - , [valm]"m"(value) - ); /*(no compiler fence)*/ /*(cr0 not affected)*/ - } - }; - #define __TBB_machine_load_store_relaxed_8 - -#endif /* __TBB_WORDSIZE==4 && __TBB_64BIT_ATOMICS */ - -}} // namespaces internal, tbb - -#undef __TBB_MACHINE_DEFINE_LOAD_STORE - -#define __TBB_USE_GENERIC_PART_WORD_CAS 1 -#define __TBB_USE_GENERIC_FETCH_ADD 1 -#define __TBB_USE_GENERIC_FETCH_STORE 1 -#define __TBB_USE_GENERIC_SEQUENTIAL_CONSISTENCY_LOAD_STORE 1 - -#define __TBB_control_consistency_helper() __asm__ __volatile__("isync": : :"memory") -#define __TBB_full_memory_fence() __asm__ __volatile__( "sync": : :"memory") - -static inline intptr_t __TBB_machine_lg( uintptr_t x ) { - __TBB_ASSERT(x, "__TBB_Log2(0) undefined"); - // cntlzd/cntlzw starts counting at 2^63/2^31 (ignoring any higher-order bits), and does not affect cr0 -#if __TBB_WORDSIZE==8 - __asm__ __volatile__ ("cntlzd %0,%0" : "+r"(x)); - return 63-static_cast(x); -#else - __asm__ __volatile__ ("cntlzw %0,%0" : "+r"(x)); - return 31-static_cast(x); -#endif -} -#define __TBB_Log2(V) __TBB_machine_lg(V) - -// Assumes implicit alignment for any 32-bit value -typedef uint32_t __TBB_Flag; -#define __TBB_Flag __TBB_Flag - -inline bool __TBB_machine_trylockbyte( __TBB_atomic __TBB_Flag &flag ) { - return __TBB_machine_cmpswp4(&flag,1,0)==0; -} -#define __TBB_TryLockByte(P) __TBB_machine_trylockbyte(P) diff --git a/src/tbb/include/tbb/machine/macos_common.h b/src/tbb/include/tbb/machine/macos_common.h deleted file mode 100644 index 119ad979a..000000000 --- a/src/tbb/include/tbb/machine/macos_common.h +++ /dev/null @@ -1,133 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#if !defined(__TBB_machine_H) || defined(__TBB_machine_macos_common_H) -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#define __TBB_machine_macos_common_H - -#include -#define __TBB_Yield() sched_yield() - -// __TBB_HardwareConcurrency - -#include -#include - -static inline int __TBB_macos_available_cpu() { - int name[2] = {CTL_HW, HW_AVAILCPU}; - int ncpu; - size_t size = sizeof(ncpu); - sysctl( name, 2, &ncpu, &size, NULL, 0 ); - return ncpu; -} - -#define __TBB_HardwareConcurrency() __TBB_macos_available_cpu() - -#ifndef __TBB_full_memory_fence - // TBB has not recognized the architecture (none of the architecture abstraction - // headers was included). - #define __TBB_UnknownArchitecture 1 -#endif - -#if __TBB_UnknownArchitecture -// Implementation of atomic operations based on OS provided primitives -#include - -static inline int64_t __TBB_machine_cmpswp8_OsX(volatile void *ptr, int64_t value, int64_t comparand) -{ - __TBB_ASSERT( tbb::internal::is_aligned(ptr,8), "address not properly aligned for macOS* atomics"); - int64_t* address = (int64_t*)ptr; - while( !OSAtomicCompareAndSwap64Barrier(comparand, value, address) ){ -#if __TBB_WORDSIZE==8 - int64_t snapshot = *address; -#else - int64_t snapshot = OSAtomicAdd64( 0, address ); -#endif - if( snapshot!=comparand ) return snapshot; - } - return comparand; -} - -#define __TBB_machine_cmpswp8 __TBB_machine_cmpswp8_OsX - -#endif /* __TBB_UnknownArchitecture */ - -#if __TBB_UnknownArchitecture - -#ifndef __TBB_WORDSIZE -#define __TBB_WORDSIZE __SIZEOF_POINTER__ -#endif - -#ifdef __TBB_ENDIANNESS - // Already determined based on hardware architecture. -#elif __BIG_ENDIAN__ - #define __TBB_ENDIANNESS __TBB_ENDIAN_BIG -#elif __LITTLE_ENDIAN__ - #define __TBB_ENDIANNESS __TBB_ENDIAN_LITTLE -#else - #define __TBB_ENDIANNESS __TBB_ENDIAN_UNSUPPORTED -#endif - -/** As this generic implementation has absolutely no information about underlying - hardware, its performance most likely will be sub-optimal because of full memory - fence usages where a more lightweight synchronization means (or none at all) - could suffice. Thus if you use this header to enable TBB on a new platform, - consider forking it and relaxing below helpers as appropriate. **/ -#define __TBB_control_consistency_helper() OSMemoryBarrier() -#define __TBB_acquire_consistency_helper() OSMemoryBarrier() -#define __TBB_release_consistency_helper() OSMemoryBarrier() -#define __TBB_full_memory_fence() OSMemoryBarrier() - -static inline int32_t __TBB_machine_cmpswp4(volatile void *ptr, int32_t value, int32_t comparand) -{ - __TBB_ASSERT( tbb::internal::is_aligned(ptr,4), "address not properly aligned for macOS atomics"); - int32_t* address = (int32_t*)ptr; - while( !OSAtomicCompareAndSwap32Barrier(comparand, value, address) ){ - int32_t snapshot = *address; - if( snapshot!=comparand ) return snapshot; - } - return comparand; -} - -static inline int32_t __TBB_machine_fetchadd4(volatile void *ptr, int32_t addend) -{ - __TBB_ASSERT( tbb::internal::is_aligned(ptr,4), "address not properly aligned for macOS atomics"); - return OSAtomicAdd32Barrier(addend, (int32_t*)ptr) - addend; -} - -static inline int64_t __TBB_machine_fetchadd8(volatile void *ptr, int64_t addend) -{ - __TBB_ASSERT( tbb::internal::is_aligned(ptr,8), "address not properly aligned for macOS atomics"); - return OSAtomicAdd64Barrier(addend, (int64_t*)ptr) - addend; -} - -#define __TBB_USE_GENERIC_PART_WORD_CAS 1 -#define __TBB_USE_GENERIC_PART_WORD_FETCH_ADD 1 -#define __TBB_USE_GENERIC_FETCH_STORE 1 -#define __TBB_USE_GENERIC_HALF_FENCED_LOAD_STORE 1 -#define __TBB_USE_GENERIC_RELAXED_LOAD_STORE 1 -#if __TBB_WORDSIZE == 4 - #define __TBB_USE_GENERIC_DWORD_LOAD_STORE 1 -#endif -#define __TBB_USE_GENERIC_SEQUENTIAL_CONSISTENCY_LOAD_STORE 1 - -#endif /* __TBB_UnknownArchitecture */ diff --git a/src/tbb/include/tbb/machine/mic_common.h b/src/tbb/include/tbb/machine/mic_common.h deleted file mode 100644 index 8765d39fb..000000000 --- a/src/tbb/include/tbb/machine/mic_common.h +++ /dev/null @@ -1,57 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_mic_common_H -#define __TBB_mic_common_H - -#ifndef __TBB_machine_H -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#if ! __TBB_DEFINE_MIC - #error mic_common.h should be included only when building for Intel(R) Many Integrated Core Architecture -#endif - -#ifndef __TBB_PREFETCHING -#define __TBB_PREFETCHING 1 -#endif -#if __TBB_PREFETCHING -#include -#define __TBB_cl_prefetch(p) _mm_prefetch((const char*)p, _MM_HINT_T1) -#define __TBB_cl_evict(p) _mm_clevict(p, _MM_HINT_T1) -#endif - -/** Intel(R) Many Integrated Core Architecture does not support mfence and pause instructions **/ -#define __TBB_full_memory_fence() __asm__ __volatile__("lock; addl $0,(%%rsp)":::"memory") -#define __TBB_Pause(x) _mm_delay_32(16*(x)) -#define __TBB_STEALING_PAUSE 1500/16 -#include -#define __TBB_Yield() sched_yield() - -/** Specifics **/ -#define __TBB_STEALING_ABORT_ON_CONTENTION 1 -#define __TBB_YIELD2P 1 -#define __TBB_HOARD_NONLOCAL_TASKS 1 - -#if ! ( __FreeBSD__ || __linux__ ) - #error Intel(R) Many Integrated Core Compiler does not define __FreeBSD__ or __linux__ anymore. Check for the __TBB_XXX_BROKEN defined under __FreeBSD__ or __linux__. -#endif /* ! ( __FreeBSD__ || __linux__ ) */ - -#endif /* __TBB_mic_common_H */ diff --git a/src/tbb/include/tbb/machine/msvc_armv7.h b/src/tbb/include/tbb/machine/msvc_armv7.h deleted file mode 100644 index 40d220207..000000000 --- a/src/tbb/include/tbb/machine/msvc_armv7.h +++ /dev/null @@ -1,171 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#if !defined(__TBB_machine_H) || defined(__TBB_msvc_armv7_H) -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#define __TBB_msvc_armv7_H - -#include -#include - -#define __TBB_WORDSIZE 4 - -#define __TBB_ENDIANNESS __TBB_ENDIAN_UNSUPPORTED - -#if defined(TBB_WIN32_USE_CL_BUILTINS) -// We can test this on _M_IX86 -#pragma intrinsic(_ReadWriteBarrier) -#pragma intrinsic(_mm_mfence) -#define __TBB_compiler_fence() _ReadWriteBarrier() -#define __TBB_full_memory_fence() _mm_mfence() -#define __TBB_control_consistency_helper() __TBB_compiler_fence() -#define __TBB_acquire_consistency_helper() __TBB_compiler_fence() -#define __TBB_release_consistency_helper() __TBB_compiler_fence() -#else -//Now __dmb(_ARM_BARRIER_SY) is used for both compiler and memory fences -//This might be changed later after testing -#define __TBB_compiler_fence() __dmb(_ARM_BARRIER_SY) -#define __TBB_full_memory_fence() __dmb(_ARM_BARRIER_SY) -#define __TBB_control_consistency_helper() __TBB_compiler_fence() -#define __TBB_acquire_consistency_helper() __TBB_full_memory_fence() -#define __TBB_release_consistency_helper() __TBB_full_memory_fence() -#endif - -//-------------------------------------------------- -// Compare and swap -//-------------------------------------------------- - -/** - * Atomic CAS for 32 bit values, if *ptr==comparand, then *ptr=value, returns *ptr - * @param ptr pointer to value in memory to be swapped with value if *ptr==comparand - * @param value value to assign *ptr to if *ptr==comparand - * @param comparand value to compare with *ptr - * @return value originally in memory at ptr, regardless of success -*/ - -#define __TBB_MACHINE_DEFINE_ATOMICS_CMPSWP(S,T,F) \ -inline T __TBB_machine_cmpswp##S( volatile void *ptr, T value, T comparand ) { \ - return _InterlockedCompareExchange##F(reinterpret_cast(ptr),value,comparand); \ -} \ - -#define __TBB_MACHINE_DEFINE_ATOMICS_FETCHADD(S,T,F) \ -inline T __TBB_machine_fetchadd##S( volatile void *ptr, T value ) { \ - return _InterlockedExchangeAdd##F(reinterpret_cast(ptr),value); \ -} \ - -__TBB_MACHINE_DEFINE_ATOMICS_CMPSWP(1,char,8) -__TBB_MACHINE_DEFINE_ATOMICS_CMPSWP(2,short,16) -__TBB_MACHINE_DEFINE_ATOMICS_CMPSWP(4,long,) -__TBB_MACHINE_DEFINE_ATOMICS_CMPSWP(8,__int64,64) -__TBB_MACHINE_DEFINE_ATOMICS_FETCHADD(4,long,) -#if defined(TBB_WIN32_USE_CL_BUILTINS) -// No _InterlockedExchangeAdd64 intrinsic on _M_IX86 -#define __TBB_64BIT_ATOMICS 0 -#else -__TBB_MACHINE_DEFINE_ATOMICS_FETCHADD(8,__int64,64) -#endif - -inline void __TBB_machine_pause (int32_t delay ) -{ - while(delay>0) - { - __TBB_compiler_fence(); - delay--; - } -} - -// API to retrieve/update FPU control setting -#define __TBB_CPU_CTL_ENV_PRESENT 1 - -namespace tbb { -namespace internal { - -template -struct machine_load_store_relaxed { - static inline T load ( const volatile T& location ) { - const T value = location; - - /* - * An extra memory barrier is required for errata #761319 - * Please see http://infocenter.arm.com/help/topic/com.arm.doc.uan0004a - */ - __TBB_acquire_consistency_helper(); - return value; - } - - static inline void store ( volatile T& location, T value ) { - location = value; - } -}; - -class cpu_ctl_env { -private: - unsigned int my_ctl; -public: - bool operator!=( const cpu_ctl_env& ctl ) const { return my_ctl != ctl.my_ctl; } - void get_env() { my_ctl = _control87(0, 0); } - void set_env() const { _control87( my_ctl, ~0U ); } -}; - -} // namespace internal -} // namespaces tbb - -// Machine specific atomic operations -#define __TBB_CompareAndSwap4(P,V,C) __TBB_machine_cmpswp4(P,V,C) -#define __TBB_CompareAndSwap8(P,V,C) __TBB_machine_cmpswp8(P,V,C) -#define __TBB_Pause(V) __TBB_machine_pause(V) - -// Use generics for some things -#define __TBB_USE_FETCHSTORE_AS_FULL_FENCED_STORE 1 -#define __TBB_USE_GENERIC_HALF_FENCED_LOAD_STORE 1 -#define __TBB_USE_GENERIC_PART_WORD_FETCH_ADD 1 -#define __TBB_USE_GENERIC_PART_WORD_FETCH_STORE 1 -#define __TBB_USE_GENERIC_FETCH_STORE 1 -#define __TBB_USE_GENERIC_DWORD_LOAD_STORE 1 -#define __TBB_USE_GENERIC_SEQUENTIAL_CONSISTENCY_LOAD_STORE 1 - -#if defined(TBB_WIN32_USE_CL_BUILTINS) -#if !__TBB_WIN8UI_SUPPORT -extern "C" __declspec(dllimport) int __stdcall SwitchToThread( void ); -#define __TBB_Yield() SwitchToThread() -#else -#include -#define __TBB_Yield() std::this_thread::yield() -#endif -#else -#define __TBB_Yield() __yield() -#endif - -// Machine specific atomic operations -#define __TBB_AtomicOR(P,V) __TBB_machine_OR(P,V) -#define __TBB_AtomicAND(P,V) __TBB_machine_AND(P,V) - -template -inline void __TBB_machine_OR( T1 *operand, T2 addend ) { - _InterlockedOr((long volatile *)operand, (long)addend); -} - -template -inline void __TBB_machine_AND( T1 *operand, T2 addend ) { - _InterlockedAnd((long volatile *)operand, (long)addend); -} - diff --git a/src/tbb/include/tbb/machine/msvc_ia32_common.h b/src/tbb/include/tbb/machine/msvc_ia32_common.h deleted file mode 100644 index 8b4814bcc..000000000 --- a/src/tbb/include/tbb/machine/msvc_ia32_common.h +++ /dev/null @@ -1,280 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#if !defined(__TBB_machine_H) || defined(__TBB_machine_msvc_ia32_common_H) -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#define __TBB_machine_msvc_ia32_common_H - -#include - -//TODO: consider moving this macro to tbb_config.h and using where MSVC asm is used -#if !_M_X64 || __INTEL_COMPILER - #define __TBB_X86_MSVC_INLINE_ASM_AVAILABLE 1 -#else - //MSVC in x64 mode does not accept inline assembler - #define __TBB_X86_MSVC_INLINE_ASM_AVAILABLE 0 - #define __TBB_NO_X86_MSVC_INLINE_ASM_MSG "The compiler being used is not supported (outdated?)" -#endif - -#if _M_X64 - #define __TBB_r(reg_name) r##reg_name - #define __TBB_W(name) name##64 - namespace tbb { namespace internal { namespace msvc_intrinsics { - typedef __int64 word; - }}} -#else - #define __TBB_r(reg_name) e##reg_name - #define __TBB_W(name) name - namespace tbb { namespace internal { namespace msvc_intrinsics { - typedef long word; - }}} -#endif - -#if _MSC_VER>=1600 && (!__INTEL_COMPILER || __INTEL_COMPILER>=1310) - // S is the operand size in bytes, B is the suffix for intrinsics for that size - #define __TBB_MACHINE_DEFINE_ATOMICS(S,B,T,U) \ - __pragma(intrinsic( _InterlockedCompareExchange##B )) \ - static inline T __TBB_machine_cmpswp##S ( volatile void * ptr, U value, U comparand ) { \ - return _InterlockedCompareExchange##B ( (T*)ptr, value, comparand ); \ - } \ - __pragma(intrinsic( _InterlockedExchangeAdd##B )) \ - static inline T __TBB_machine_fetchadd##S ( volatile void * ptr, U addend ) { \ - return _InterlockedExchangeAdd##B ( (T*)ptr, addend ); \ - } \ - __pragma(intrinsic( _InterlockedExchange##B )) \ - static inline T __TBB_machine_fetchstore##S ( volatile void * ptr, U value ) { \ - return _InterlockedExchange##B ( (T*)ptr, value ); \ - } - - // Atomic intrinsics for 1, 2, and 4 bytes are available for x86 & x64 - __TBB_MACHINE_DEFINE_ATOMICS(1,8,char,__int8) - __TBB_MACHINE_DEFINE_ATOMICS(2,16,short,__int16) - __TBB_MACHINE_DEFINE_ATOMICS(4,,long,__int32) - - #if __TBB_WORDSIZE==8 - __TBB_MACHINE_DEFINE_ATOMICS(8,64,__int64,__int64) - #endif - - #undef __TBB_MACHINE_DEFINE_ATOMICS - #define __TBB_ATOMIC_PRIMITIVES_DEFINED -#endif /*_MSC_VER>=1600*/ - -#if _MSC_VER>=1300 || __INTEL_COMPILER>=1100 - #pragma intrinsic(_ReadWriteBarrier) - #pragma intrinsic(_mm_mfence) - #define __TBB_compiler_fence() _ReadWriteBarrier() - #define __TBB_full_memory_fence() _mm_mfence() -#elif __TBB_X86_MSVC_INLINE_ASM_AVAILABLE - #define __TBB_compiler_fence() __asm { __asm nop } - #define __TBB_full_memory_fence() __asm { __asm mfence } -#else - #error Unsupported compiler; define __TBB_{control,acquire,release}_consistency_helper to support it -#endif - -#define __TBB_control_consistency_helper() __TBB_compiler_fence() -#define __TBB_acquire_consistency_helper() __TBB_compiler_fence() -#define __TBB_release_consistency_helper() __TBB_compiler_fence() - -#if (_MSC_VER>=1300) || (__INTEL_COMPILER) - #pragma intrinsic(_mm_pause) - namespace tbb { namespace internal { namespace msvc_intrinsics { - static inline void pause (uintptr_t delay ) { - for (;delay>0; --delay ) - _mm_pause(); - } - }}} - #define __TBB_Pause(V) tbb::internal::msvc_intrinsics::pause(V) - #define __TBB_SINGLE_PAUSE _mm_pause() -#else - #if !__TBB_X86_MSVC_INLINE_ASM_AVAILABLE - #error __TBB_NO_X86_MSVC_INLINE_ASM_MSG - #endif - namespace tbb { namespace internal { namespace msvc_inline_asm - static inline void pause (uintptr_t delay ) { - _asm - { - mov __TBB_r(ax), delay - __TBB_L1: - pause - add __TBB_r(ax), -1 - jne __TBB_L1 - } - return; - } - }}} - #define __TBB_Pause(V) tbb::internal::msvc_inline_asm::pause(V) - #define __TBB_SINGLE_PAUSE __asm pause -#endif - -#if (_MSC_VER>=1400 && !__INTEL_COMPILER) || (__INTEL_COMPILER>=1200) -// MSVC did not have this intrinsic prior to VC8. -// ICL 11.1 fails to compile a TBB example if __TBB_Log2 uses the intrinsic. - #pragma intrinsic(__TBB_W(_BitScanReverse)) - namespace tbb { namespace internal { namespace msvc_intrinsics { - static inline uintptr_t lg_bsr( uintptr_t i ){ - unsigned long j; - __TBB_W(_BitScanReverse)( &j, i ); - return j; - } - }}} - #define __TBB_Log2(V) tbb::internal::msvc_intrinsics::lg_bsr(V) -#else - #if !__TBB_X86_MSVC_INLINE_ASM_AVAILABLE - #error __TBB_NO_X86_MSVC_INLINE_ASM_MSG - #endif - namespace tbb { namespace internal { namespace msvc_inline_asm { - static inline uintptr_t lg_bsr( uintptr_t i ){ - uintptr_t j; - __asm - { - bsr __TBB_r(ax), i - mov j, __TBB_r(ax) - } - return j; - } - }}} - #define __TBB_Log2(V) tbb::internal::msvc_inline_asm::lg_bsr(V) -#endif - -#if _MSC_VER>=1400 - #pragma intrinsic(__TBB_W(_InterlockedOr)) - #pragma intrinsic(__TBB_W(_InterlockedAnd)) - namespace tbb { namespace internal { namespace msvc_intrinsics { - static inline void lock_or( volatile void *operand, intptr_t addend ){ - __TBB_W(_InterlockedOr)((volatile word*)operand, addend); - } - static inline void lock_and( volatile void *operand, intptr_t addend ){ - __TBB_W(_InterlockedAnd)((volatile word*)operand, addend); - } - }}} - #define __TBB_AtomicOR(P,V) tbb::internal::msvc_intrinsics::lock_or(P,V) - #define __TBB_AtomicAND(P,V) tbb::internal::msvc_intrinsics::lock_and(P,V) -#else - #if !__TBB_X86_MSVC_INLINE_ASM_AVAILABLE - #error __TBB_NO_X86_MSVC_INLINE_ASM_MSG - #endif - namespace tbb { namespace internal { namespace msvc_inline_asm { - static inline void lock_or( volatile void *operand, __int32 addend ) { - __asm - { - mov eax, addend - mov edx, [operand] - lock or [edx], eax - } - } - static inline void lock_and( volatile void *operand, __int32 addend ) { - __asm - { - mov eax, addend - mov edx, [operand] - lock and [edx], eax - } - } - }}} - #define __TBB_AtomicOR(P,V) tbb::internal::msvc_inline_asm::lock_or(P,V) - #define __TBB_AtomicAND(P,V) tbb::internal::msvc_inline_asm::lock_and(P,V) -#endif - -#pragma intrinsic(__rdtsc) -namespace tbb { namespace internal { typedef uint64_t machine_tsc_t; } } -static inline tbb::internal::machine_tsc_t __TBB_machine_time_stamp() { - return __rdtsc(); -} -#define __TBB_time_stamp() __TBB_machine_time_stamp() - -// API to retrieve/update FPU control setting -#define __TBB_CPU_CTL_ENV_PRESENT 1 - -namespace tbb { namespace internal { class cpu_ctl_env; } } -#if __TBB_X86_MSVC_INLINE_ASM_AVAILABLE - inline void __TBB_get_cpu_ctl_env ( tbb::internal::cpu_ctl_env* ctl ) { - __asm { - __asm mov __TBB_r(ax), ctl - __asm stmxcsr [__TBB_r(ax)] - __asm fstcw [__TBB_r(ax)+4] - } - } - inline void __TBB_set_cpu_ctl_env ( const tbb::internal::cpu_ctl_env* ctl ) { - __asm { - __asm mov __TBB_r(ax), ctl - __asm ldmxcsr [__TBB_r(ax)] - __asm fldcw [__TBB_r(ax)+4] - } - } -#else - extern "C" { - void __TBB_EXPORTED_FUNC __TBB_get_cpu_ctl_env ( tbb::internal::cpu_ctl_env* ); - void __TBB_EXPORTED_FUNC __TBB_set_cpu_ctl_env ( const tbb::internal::cpu_ctl_env* ); - } -#endif - -namespace tbb { -namespace internal { -class cpu_ctl_env { -private: - int mxcsr; - short x87cw; - static const int MXCSR_CONTROL_MASK = ~0x3f; /* all except last six status bits */ -public: - bool operator!=( const cpu_ctl_env& ctl ) const { return mxcsr != ctl.mxcsr || x87cw != ctl.x87cw; } - void get_env() { - __TBB_get_cpu_ctl_env( this ); - mxcsr &= MXCSR_CONTROL_MASK; - } - void set_env() const { __TBB_set_cpu_ctl_env( this ); } -}; -} // namespace internal -} // namespace tbb - -#if !__TBB_WIN8UI_SUPPORT -extern "C" __declspec(dllimport) int __stdcall SwitchToThread( void ); -#define __TBB_Yield() SwitchToThread() -#else -#include -#define __TBB_Yield() std::this_thread::yield() -#endif - -#undef __TBB_r -#undef __TBB_W -#undef __TBB_word - -extern "C" { - __int8 __TBB_EXPORTED_FUNC __TBB_machine_try_lock_elided (volatile void* ptr); - void __TBB_EXPORTED_FUNC __TBB_machine_unlock_elided (volatile void* ptr); - - // 'pause' instruction aborts HLE/RTM transactions - inline static void __TBB_machine_try_lock_elided_cancel() { __TBB_SINGLE_PAUSE; } - -#if __TBB_TSX_INTRINSICS_PRESENT - #define __TBB_machine_is_in_transaction _xtest - #define __TBB_machine_begin_transaction _xbegin - #define __TBB_machine_end_transaction _xend - // The value (0xFF) below comes from the - // Intel(R) 64 and IA-32 Architectures Optimization Reference Manual 12.4.5 lock not free - #define __TBB_machine_transaction_conflict_abort() _xabort(0xFF) -#else - __int8 __TBB_EXPORTED_FUNC __TBB_machine_is_in_transaction(); - unsigned __int32 __TBB_EXPORTED_FUNC __TBB_machine_begin_transaction(); - void __TBB_EXPORTED_FUNC __TBB_machine_end_transaction(); - void __TBB_EXPORTED_FUNC __TBB_machine_transaction_conflict_abort(); -#endif /* __TBB_TSX_INTRINSICS_PRESENT */ -} diff --git a/src/tbb/include/tbb/machine/sunos_sparc.h b/src/tbb/include/tbb/machine/sunos_sparc.h deleted file mode 100644 index 9119f4028..000000000 --- a/src/tbb/include/tbb/machine/sunos_sparc.h +++ /dev/null @@ -1,203 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - - -#if !defined(__TBB_machine_H) || defined(__TBB_machine_sunos_sparc_H) -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#define __TBB_machine_sunos_sparc_H - -#include -#include - -#define __TBB_WORDSIZE 8 -// Big endian is assumed for SPARC. -// While hardware may support page-specific bi-endianness, only big endian pages may be exposed to TBB -#define __TBB_ENDIANNESS __TBB_ENDIAN_BIG - -/** To those working on SPARC hardware. Consider relaxing acquire and release - consistency helpers to no-op (as this port covers TSO mode only). **/ -#define __TBB_compiler_fence() __asm__ __volatile__ ("": : :"memory") -#define __TBB_control_consistency_helper() __TBB_compiler_fence() -#define __TBB_acquire_consistency_helper() __TBB_compiler_fence() -#define __TBB_release_consistency_helper() __TBB_compiler_fence() -#define __TBB_full_memory_fence() __asm__ __volatile__("membar #LoadLoad|#LoadStore|#StoreStore|#StoreLoad": : : "memory") - -//-------------------------------------------------- -// Compare and swap -//-------------------------------------------------- - -/** - * Atomic CAS for 32 bit values, if *ptr==comparand, then *ptr=value, returns *ptr - * @param ptr pointer to value in memory to be swapped with value if *ptr==comparand - * @param value value to assign *ptr to if *ptr==comparand - * @param comparand value to compare with *ptr - ( @return value originally in memory at ptr, regardless of success -*/ -static inline int32_t __TBB_machine_cmpswp4(volatile void *ptr, int32_t value, int32_t comparand ){ - int32_t result; - __asm__ __volatile__( - "cas\t[%5],%4,%1" - : "=m"(*(int32_t *)ptr), "=r"(result) - : "m"(*(int32_t *)ptr), "1"(value), "r"(comparand), "r"(ptr) - : "memory"); - return result; -} - -/** - * Atomic CAS for 64 bit values, if *ptr==comparand, then *ptr=value, returns *ptr - * @param ptr pointer to value in memory to be swapped with value if *ptr==comparand - * @param value value to assign *ptr to if *ptr==comparand - * @param comparand value to compare with *ptr - ( @return value originally in memory at ptr, regardless of success - */ -static inline int64_t __TBB_machine_cmpswp8(volatile void *ptr, int64_t value, int64_t comparand ){ - int64_t result; - __asm__ __volatile__( - "casx\t[%5],%4,%1" - : "=m"(*(int64_t *)ptr), "=r"(result) - : "m"(*(int64_t *)ptr), "1"(value), "r"(comparand), "r"(ptr) - : "memory"); - return result; -} - -//--------------------------------------------------- -// Fetch and add -//--------------------------------------------------- - -/** - * Atomic fetch and add for 32 bit values, in this case implemented by continuously checking success of atomicity - * @param ptr pointer to value to add addend to - * @param addened value to add to *ptr - * @return value at ptr before addened was added - */ -static inline int32_t __TBB_machine_fetchadd4(volatile void *ptr, int32_t addend){ - int32_t result; - __asm__ __volatile__ ( - "0:\t add\t %3, %4, %0\n" // do addition - "\t cas\t [%2], %3, %0\n" // cas to store result in memory - "\t cmp\t %3, %0\n" // check if value from memory is original - "\t bne,a,pn\t %%icc, 0b\n" // if not try again - "\t mov %0, %3\n" // use branch delay slot to move new value in memory to be added - : "=&r"(result), "=m"(*(int32_t *)ptr) - : "r"(ptr), "r"(*(int32_t *)ptr), "r"(addend), "m"(*(int32_t *)ptr) - : "ccr", "memory"); - return result; -} - -/** - * Atomic fetch and add for 64 bit values, in this case implemented by continuously checking success of atomicity - * @param ptr pointer to value to add addend to - * @param addened value to add to *ptr - * @return value at ptr before addened was added - */ -static inline int64_t __TBB_machine_fetchadd8(volatile void *ptr, int64_t addend){ - int64_t result; - __asm__ __volatile__ ( - "0:\t add\t %3, %4, %0\n" // do addition - "\t casx\t [%2], %3, %0\n" // cas to store result in memory - "\t cmp\t %3, %0\n" // check if value from memory is original - "\t bne,a,pn\t %%xcc, 0b\n" // if not try again - "\t mov %0, %3\n" // use branch delay slot to move new value in memory to be added - : "=&r"(result), "=m"(*(int64_t *)ptr) - : "r"(ptr), "r"(*(int64_t *)ptr), "r"(addend), "m"(*(int64_t *)ptr) - : "ccr", "memory"); - return result; -} - -//-------------------------------------------------------- -// Logarithm (base two, integer) -//-------------------------------------------------------- - -static inline int64_t __TBB_machine_lg( uint64_t x ) { - __TBB_ASSERT(x, "__TBB_Log2(0) undefined"); - uint64_t count; - // one hot encode - x |= (x >> 1); - x |= (x >> 2); - x |= (x >> 4); - x |= (x >> 8); - x |= (x >> 16); - x |= (x >> 32); - // count 1's - __asm__ ("popc %1, %0" : "=r"(count) : "r"(x) ); - return count-1; -} - -//-------------------------------------------------------- - -static inline void __TBB_machine_or( volatile void *ptr, uint64_t value ) { - __asm__ __volatile__ ( - "0:\t or\t %2, %3, %%g1\n" // do operation - "\t casx\t [%1], %2, %%g1\n" // cas to store result in memory - "\t cmp\t %2, %%g1\n" // check if value from memory is original - "\t bne,a,pn\t %%xcc, 0b\n" // if not try again - "\t mov %%g1, %2\n" // use branch delay slot to move new value in memory to be added - : "=m"(*(int64_t *)ptr) - : "r"(ptr), "r"(*(int64_t *)ptr), "r"(value), "m"(*(int64_t *)ptr) - : "ccr", "g1", "memory"); -} - -static inline void __TBB_machine_and( volatile void *ptr, uint64_t value ) { - __asm__ __volatile__ ( - "0:\t and\t %2, %3, %%g1\n" // do operation - "\t casx\t [%1], %2, %%g1\n" // cas to store result in memory - "\t cmp\t %2, %%g1\n" // check if value from memory is original - "\t bne,a,pn\t %%xcc, 0b\n" // if not try again - "\t mov %%g1, %2\n" // use branch delay slot to move new value in memory to be added - : "=m"(*(int64_t *)ptr) - : "r"(ptr), "r"(*(int64_t *)ptr), "r"(value), "m"(*(int64_t *)ptr) - : "ccr", "g1", "memory"); -} - - -static inline void __TBB_machine_pause( int32_t delay ) { - // do nothing, inlined, doesn't matter -} - -// put 0xff in memory location, return memory value, -// generic trylockbyte puts 0x01, however this is fine -// because all that matters is that 0 is unlocked -static inline bool __TBB_machine_trylockbyte(unsigned char &flag){ - unsigned char result; - __asm__ __volatile__ ( - "ldstub\t [%2], %0\n" - : "=r"(result), "=m"(flag) - : "r"(&flag), "m"(flag) - : "memory"); - return result == 0; -} - -#define __TBB_USE_GENERIC_PART_WORD_CAS 1 -#define __TBB_USE_GENERIC_PART_WORD_FETCH_ADD 1 -#define __TBB_USE_GENERIC_FETCH_STORE 1 -#define __TBB_USE_GENERIC_HALF_FENCED_LOAD_STORE 1 -#define __TBB_USE_GENERIC_RELAXED_LOAD_STORE 1 -#define __TBB_USE_GENERIC_SEQUENTIAL_CONSISTENCY_LOAD_STORE 1 - -#define __TBB_AtomicOR(P,V) __TBB_machine_or(P,V) -#define __TBB_AtomicAND(P,V) __TBB_machine_and(P,V) - -// Definition of other functions -#define __TBB_Pause(V) __TBB_machine_pause(V) -#define __TBB_Log2(V) __TBB_machine_lg(V) - -#define __TBB_TryLockByte(P) __TBB_machine_trylockbyte(P) diff --git a/src/tbb/include/tbb/machine/windows_api.h b/src/tbb/include/tbb/machine/windows_api.h deleted file mode 100644 index d362abc5e..000000000 --- a/src/tbb/include/tbb/machine/windows_api.h +++ /dev/null @@ -1,69 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_machine_windows_api_H -#define __TBB_machine_windows_api_H - -#if _WIN32 || _WIN64 - -#include - -#if _WIN32_WINNT < 0x0600 -// The following Windows API function is declared explicitly; -// otherwise it fails to compile by VS2005. -#if !defined(WINBASEAPI) || (_WIN32_WINNT < 0x0501 && _MSC_VER == 1400) -#define __TBB_WINBASEAPI extern "C" -#else -#define __TBB_WINBASEAPI WINBASEAPI -#endif -__TBB_WINBASEAPI BOOL WINAPI TryEnterCriticalSection( LPCRITICAL_SECTION ); -__TBB_WINBASEAPI BOOL WINAPI InitializeCriticalSectionAndSpinCount( LPCRITICAL_SECTION, DWORD ); -// Overloading WINBASEAPI macro and using local functions missing in Windows XP/2003 -#define InitializeCriticalSectionEx inlineInitializeCriticalSectionEx -#define CreateSemaphoreEx inlineCreateSemaphoreEx -#define CreateEventEx inlineCreateEventEx -inline BOOL WINAPI inlineInitializeCriticalSectionEx( LPCRITICAL_SECTION lpCriticalSection, DWORD dwSpinCount, DWORD ) -{ - return InitializeCriticalSectionAndSpinCount( lpCriticalSection, dwSpinCount ); -} -inline HANDLE WINAPI inlineCreateSemaphoreEx( LPSECURITY_ATTRIBUTES lpSemaphoreAttributes, LONG lInitialCount, LONG lMaximumCount, LPCTSTR lpName, DWORD, DWORD ) -{ - return CreateSemaphore( lpSemaphoreAttributes, lInitialCount, lMaximumCount, lpName ); -} -inline HANDLE WINAPI inlineCreateEventEx( LPSECURITY_ATTRIBUTES lpEventAttributes, LPCTSTR lpName, DWORD dwFlags, DWORD ) -{ - BOOL manual_reset = dwFlags&0x00000001 ? TRUE : FALSE; // CREATE_EVENT_MANUAL_RESET - BOOL initial_set = dwFlags&0x00000002 ? TRUE : FALSE; // CREATE_EVENT_INITIAL_SET - return CreateEvent( lpEventAttributes, manual_reset, initial_set, lpName ); -} -#endif - -#if defined(RTL_SRWLOCK_INIT) -#ifndef __TBB_USE_SRWLOCK -// TODO: turn it on when bug 1952 will be fixed -#define __TBB_USE_SRWLOCK 0 -#endif -#endif - -#else -#error tbb/machine/windows_api.h should only be used for Windows based platforms -#endif // _WIN32 || _WIN64 - -#endif // __TBB_machine_windows_api_H diff --git a/src/tbb/include/tbb/machine/windows_ia32.h b/src/tbb/include/tbb/machine/windows_ia32.h deleted file mode 100644 index 8db0d2b82..000000000 --- a/src/tbb/include/tbb/machine/windows_ia32.h +++ /dev/null @@ -1,109 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#if !defined(__TBB_machine_H) || defined(__TBB_machine_windows_ia32_H) -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#define __TBB_machine_windows_ia32_H - -#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) - // Workaround for overzealous compiler warnings in /Wp64 mode - #pragma warning (push) - #pragma warning (disable: 4244 4267) -#endif - -#include "msvc_ia32_common.h" - -#define __TBB_WORDSIZE 4 -#define __TBB_ENDIANNESS __TBB_ENDIAN_LITTLE - -extern "C" { - __int64 __TBB_EXPORTED_FUNC __TBB_machine_cmpswp8 (volatile void *ptr, __int64 value, __int64 comparand ); - __int64 __TBB_EXPORTED_FUNC __TBB_machine_fetchadd8 (volatile void *ptr, __int64 addend ); - __int64 __TBB_EXPORTED_FUNC __TBB_machine_fetchstore8 (volatile void *ptr, __int64 value ); - void __TBB_EXPORTED_FUNC __TBB_machine_store8 (volatile void *ptr, __int64 value ); - __int64 __TBB_EXPORTED_FUNC __TBB_machine_load8 (const volatile void *ptr); -} - -#ifndef __TBB_ATOMIC_PRIMITIVES_DEFINED - -#define __TBB_MACHINE_DEFINE_ATOMICS(S,T,U,A,C) \ -static inline T __TBB_machine_cmpswp##S ( volatile void * ptr, U value, U comparand ) { \ - T result; \ - volatile T *p = (T *)ptr; \ - __asm \ - { \ - __asm mov edx, p \ - __asm mov C , value \ - __asm mov A , comparand \ - __asm lock cmpxchg [edx], C \ - __asm mov result, A \ - } \ - return result; \ -} \ -\ -static inline T __TBB_machine_fetchadd##S ( volatile void * ptr, U addend ) { \ - T result; \ - volatile T *p = (T *)ptr; \ - __asm \ - { \ - __asm mov edx, p \ - __asm mov A, addend \ - __asm lock xadd [edx], A \ - __asm mov result, A \ - } \ - return result; \ -}\ -\ -static inline T __TBB_machine_fetchstore##S ( volatile void * ptr, U value ) { \ - T result; \ - volatile T *p = (T *)ptr; \ - __asm \ - { \ - __asm mov edx, p \ - __asm mov A, value \ - __asm lock xchg [edx], A \ - __asm mov result, A \ - } \ - return result; \ -} - - -__TBB_MACHINE_DEFINE_ATOMICS(1, __int8, __int8, al, cl) -__TBB_MACHINE_DEFINE_ATOMICS(2, __int16, __int16, ax, cx) -__TBB_MACHINE_DEFINE_ATOMICS(4, ptrdiff_t, ptrdiff_t, eax, ecx) - -#undef __TBB_MACHINE_DEFINE_ATOMICS - -#endif /*__TBB_ATOMIC_PRIMITIVES_DEFINED*/ - -//TODO: Check if it possible and profitable for IA-32 architecture on (Linux and Windows) -//to use of 64-bit load/store via floating point registers together with full fence -//for sequentially consistent load/store, instead of CAS. -#define __TBB_USE_FETCHSTORE_AS_FULL_FENCED_STORE 1 -#define __TBB_USE_GENERIC_HALF_FENCED_LOAD_STORE 1 -#define __TBB_USE_GENERIC_RELAXED_LOAD_STORE 1 -#define __TBB_USE_GENERIC_SEQUENTIAL_CONSISTENCY_LOAD_STORE 1 - - -#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) - #pragma warning (pop) -#endif // warnings 4244, 4267 are back diff --git a/src/tbb/include/tbb/machine/windows_intel64.h b/src/tbb/include/tbb/machine/windows_intel64.h deleted file mode 100644 index 86abd6ad1..000000000 --- a/src/tbb/include/tbb/machine/windows_intel64.h +++ /dev/null @@ -1,74 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#if !defined(__TBB_machine_H) || defined(__TBB_machine_windows_intel64_H) -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#define __TBB_machine_windows_intel64_H - -#define __TBB_WORDSIZE 8 -#define __TBB_ENDIANNESS __TBB_ENDIAN_LITTLE - -#include "msvc_ia32_common.h" - -#ifndef __TBB_ATOMIC_PRIMITIVES_DEFINED - -#include -#pragma intrinsic(_InterlockedCompareExchange,_InterlockedExchangeAdd,_InterlockedExchange) -#pragma intrinsic(_InterlockedCompareExchange64,_InterlockedExchangeAdd64,_InterlockedExchange64) - -// ATTENTION: if you ever change argument types in machine-specific primitives, -// please take care of atomic_word<> specializations in tbb/atomic.h -extern "C" { - __int8 __TBB_EXPORTED_FUNC __TBB_machine_cmpswp1 (volatile void *ptr, __int8 value, __int8 comparand ); - __int8 __TBB_EXPORTED_FUNC __TBB_machine_fetchadd1 (volatile void *ptr, __int8 addend ); - __int8 __TBB_EXPORTED_FUNC __TBB_machine_fetchstore1 (volatile void *ptr, __int8 value ); - __int16 __TBB_EXPORTED_FUNC __TBB_machine_cmpswp2 (volatile void *ptr, __int16 value, __int16 comparand ); - __int16 __TBB_EXPORTED_FUNC __TBB_machine_fetchadd2 (volatile void *ptr, __int16 addend ); - __int16 __TBB_EXPORTED_FUNC __TBB_machine_fetchstore2 (volatile void *ptr, __int16 value ); -} - -inline long __TBB_machine_cmpswp4 (volatile void *ptr, __int32 value, __int32 comparand ) { - return _InterlockedCompareExchange( (long*)ptr, value, comparand ); -} -inline long __TBB_machine_fetchadd4 (volatile void *ptr, __int32 addend ) { - return _InterlockedExchangeAdd( (long*)ptr, addend ); -} -inline long __TBB_machine_fetchstore4 (volatile void *ptr, __int32 value ) { - return _InterlockedExchange( (long*)ptr, value ); -} - -inline __int64 __TBB_machine_cmpswp8 (volatile void *ptr, __int64 value, __int64 comparand ) { - return _InterlockedCompareExchange64( (__int64*)ptr, value, comparand ); -} -inline __int64 __TBB_machine_fetchadd8 (volatile void *ptr, __int64 addend ) { - return _InterlockedExchangeAdd64( (__int64*)ptr, addend ); -} -inline __int64 __TBB_machine_fetchstore8 (volatile void *ptr, __int64 value ) { - return _InterlockedExchange64( (__int64*)ptr, value ); -} - -#endif /*__TBB_ATOMIC_PRIMITIVES_DEFINED*/ - -#define __TBB_USE_FETCHSTORE_AS_FULL_FENCED_STORE 1 -#define __TBB_USE_GENERIC_HALF_FENCED_LOAD_STORE 1 -#define __TBB_USE_GENERIC_RELAXED_LOAD_STORE 1 -#define __TBB_USE_GENERIC_SEQUENTIAL_CONSISTENCY_LOAD_STORE 1 diff --git a/src/tbb/include/tbb/memory_pool.h b/src/tbb/include/tbb/memory_pool.h index b3bba397a..cefe96e36 100644 --- a/src/tbb/include/tbb/memory_pool.h +++ b/src/tbb/include/tbb/memory_pool.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,268 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ -#ifndef __TBB_memory_pool_H -#define __TBB_memory_pool_H - -#if !TBB_PREVIEW_MEMORY_POOL -#error Set TBB_PREVIEW_MEMORY_POOL to include memory_pool.h -#endif -/** @file */ - -#include "scalable_allocator.h" -#include // std::bad_alloc -#include // std::runtime_error, std::invalid_argument -// required in C++03 to construct std::runtime_error and std::invalid_argument -#include -#if __TBB_ALLOCATOR_CONSTRUCT_VARIADIC -#include // std::forward -#endif - -#if __TBB_EXTRA_DEBUG -#define __TBBMALLOC_ASSERT ASSERT -#else -#define __TBBMALLOC_ASSERT(a,b) ((void)0) -#endif - -namespace tbb { -namespace interface6 { -//! @cond INTERNAL -namespace internal { - -//! Base of thread-safe pool allocator for variable-size requests -class pool_base : tbb::internal::no_copy { - // Pool interface is separate from standard allocator classes because it has - // to maintain internal state, no copy or assignment. Move and swap are possible. -public: - //! Reset pool to reuse its memory (free all objects at once) - void recycle() { rml::pool_reset(my_pool); } - - //! The "malloc" analogue to allocate block of memory of size bytes - void *malloc(size_t size) { return rml::pool_malloc(my_pool, size); } - - //! The "free" analogue to discard a previously allocated piece of memory. - void free(void* ptr) { rml::pool_free(my_pool, ptr); } - - //! The "realloc" analogue complementing pool_malloc. - // Enables some low-level optimization possibilities - void *realloc(void* ptr, size_t size) { - return rml::pool_realloc(my_pool, ptr, size); - } - -protected: - //! destroy pool - must be called in a child class - void destroy() { rml::pool_destroy(my_pool); } - - rml::MemoryPool *my_pool; -}; - -} // namespace internal -//! @endcond - -#if _MSC_VER && !defined(__INTEL_COMPILER) - // Workaround for erroneous "unreferenced parameter" warning in method destroy. - #pragma warning (push) - #pragma warning (disable: 4100) -#endif - -//! Meets "allocator" requirements of ISO C++ Standard, Section 20.1.5 -/** @ingroup memory_allocation */ -template -class memory_pool_allocator { -protected: - typedef P pool_type; - pool_type *my_pool; - template - friend class memory_pool_allocator; - template - friend bool operator==( const memory_pool_allocator& a, const memory_pool_allocator& b); - template - friend bool operator!=( const memory_pool_allocator& a, const memory_pool_allocator& b); -public: - typedef typename tbb::internal::allocator_type::value_type value_type; - typedef value_type* pointer; - typedef const value_type* const_pointer; - typedef value_type& reference; - typedef const value_type& const_reference; - typedef size_t size_type; - typedef ptrdiff_t difference_type; - template struct rebind { - typedef memory_pool_allocator other; - }; - - explicit memory_pool_allocator(pool_type &pool) throw() : my_pool(&pool) {} - memory_pool_allocator(const memory_pool_allocator& src) throw() : my_pool(src.my_pool) {} - template - memory_pool_allocator(const memory_pool_allocator& src) throw() : my_pool(src.my_pool) {} - - pointer address(reference x) const { return &x; } - const_pointer address(const_reference x) const { return &x; } - - //! Allocate space for n objects. - pointer allocate( size_type n, const void* /*hint*/ = 0) { - pointer p = static_cast( my_pool->malloc( n*sizeof(value_type) ) ); - if (!p) - tbb::internal::throw_exception(std::bad_alloc()); - return p; - } - //! Free previously allocated block of memory. - void deallocate( pointer p, size_type ) { - my_pool->free(p); - } - //! Largest value for which method allocate might succeed. - size_type max_size() const throw() { - size_type max = static_cast(-1) / sizeof (value_type); - return (max > 0 ? max : 1); - } - //! Copy-construct value at location pointed to by p. -#if __TBB_ALLOCATOR_CONSTRUCT_VARIADIC - template - void construct(U *p, Args&&... args) - { ::new((void *)p) U(std::forward(args)...); } -#else // __TBB_ALLOCATOR_CONSTRUCT_VARIADIC -#if __TBB_CPP11_RVALUE_REF_PRESENT - void construct( pointer p, value_type&& value ) {::new((void*)(p)) value_type(std::move(value));} -#endif - void construct( pointer p, const value_type& value ) { ::new((void*)(p)) value_type(value); } -#endif // __TBB_ALLOCATOR_CONSTRUCT_VARIADIC - - //! Destroy value at location pointed to by p. - void destroy( pointer p ) { p->~value_type(); } - -}; - -#if _MSC_VER && !defined(__INTEL_COMPILER) - #pragma warning (pop) -#endif // warning 4100 is back - -//! Analogous to std::allocator, as defined in ISO C++ Standard, Section 20.4.1 -/** @ingroup memory_allocation */ -template -class memory_pool_allocator { -public: - typedef P pool_type; - typedef void* pointer; - typedef const void* const_pointer; - typedef void value_type; - template struct rebind { - typedef memory_pool_allocator other; - }; - - explicit memory_pool_allocator( pool_type &pool) throw() : my_pool(&pool) {} - memory_pool_allocator( const memory_pool_allocator& src) throw() : my_pool(src.my_pool) {} - template - memory_pool_allocator(const memory_pool_allocator& src) throw() : my_pool(src.my_pool) {} - -protected: - pool_type *my_pool; - template - friend class memory_pool_allocator; - template - friend bool operator==( const memory_pool_allocator& a, const memory_pool_allocator& b); - template - friend bool operator!=( const memory_pool_allocator& a, const memory_pool_allocator& b); -}; - -template -inline bool operator==( const memory_pool_allocator& a, const memory_pool_allocator& b) {return a.my_pool==b.my_pool;} - -template -inline bool operator!=( const memory_pool_allocator& a, const memory_pool_allocator& b) {return a.my_pool!=b.my_pool;} - - -//! Thread-safe growable pool allocator for variable-size requests -template -class memory_pool : public internal::pool_base { - Alloc my_alloc; // TODO: base-class optimization - static void *allocate_request(intptr_t pool_id, size_t & bytes); - static int deallocate_request(intptr_t pool_id, void*, size_t raw_bytes); - -public: - //! construct pool with underlying allocator - explicit memory_pool(const Alloc &src = Alloc()); - - //! destroy pool - ~memory_pool() { destroy(); } // call the callbacks first and destroy my_alloc latter - -}; - -class fixed_pool : public internal::pool_base { - void *my_buffer; - size_t my_size; - inline static void *allocate_request(intptr_t pool_id, size_t & bytes); - -public: - //! construct pool with underlying allocator - inline fixed_pool(void *buf, size_t size); - //! destroy pool - ~fixed_pool() { destroy(); } -}; - -//////////////// Implementation /////////////// - -template -memory_pool::memory_pool(const Alloc &src) : my_alloc(src) { - rml::MemPoolPolicy args(allocate_request, deallocate_request, - sizeof(typename Alloc::value_type)); - rml::MemPoolError res = rml::pool_create_v1(intptr_t(this), &args, &my_pool); - if (res!=rml::POOL_OK) - tbb::internal::throw_exception(std::runtime_error("Can't create pool")); -} -template -void *memory_pool::allocate_request(intptr_t pool_id, size_t & bytes) { - memory_pool &self = *reinterpret_cast*>(pool_id); - const size_t unit_size = sizeof(typename Alloc::value_type); - __TBBMALLOC_ASSERT( 0 == bytes%unit_size, NULL); - void *ptr; - __TBB_TRY { ptr = self.my_alloc.allocate( bytes/unit_size ); } - __TBB_CATCH(...) { return 0; } - return ptr; -} -#if __TBB_MSVC_UNREACHABLE_CODE_IGNORED - // Workaround for erroneous "unreachable code" warning in the template below. - // Specific for VC++ 17-18 compiler - #pragma warning (push) - #pragma warning (disable: 4702) -#endif -template -int memory_pool::deallocate_request(intptr_t pool_id, void* raw_ptr, size_t raw_bytes) { - memory_pool &self = *reinterpret_cast*>(pool_id); - const size_t unit_size = sizeof(typename Alloc::value_type); - __TBBMALLOC_ASSERT( 0 == raw_bytes%unit_size, NULL); - self.my_alloc.deallocate( static_cast(raw_ptr), raw_bytes/unit_size ); - return 0; -} -#if __TBB_MSVC_UNREACHABLE_CODE_IGNORED - #pragma warning (pop) -#endif -inline fixed_pool::fixed_pool(void *buf, size_t size) : my_buffer(buf), my_size(size) { - if (!buf || !size) - // TODO: improve support for mode with exceptions disabled - tbb::internal::throw_exception(std::invalid_argument("Zero in parameter is invalid")); - rml::MemPoolPolicy args(allocate_request, 0, size, /*fixedPool=*/true); - rml::MemPoolError res = rml::pool_create_v1(intptr_t(this), &args, &my_pool); - if (res!=rml::POOL_OK) - tbb::internal::throw_exception(std::runtime_error("Can't create pool")); -} -inline void *fixed_pool::allocate_request(intptr_t pool_id, size_t & bytes) { - fixed_pool &self = *reinterpret_cast(pool_id); - __TBBMALLOC_ASSERT(0 != self.my_size, "The buffer must not be used twice."); - bytes = self.my_size; - self.my_size = 0; // remember that buffer has been used - return self.my_buffer; -} - -} //namespace interface6 -using interface6::memory_pool_allocator; -using interface6::memory_pool; -using interface6::fixed_pool; -} //namespace tbb - -#undef __TBBMALLOC_ASSERT -#endif// __TBB_memory_pool_H +#include "../oneapi/tbb/memory_pool.h" diff --git a/src/tbb/include/tbb/mutex.h b/src/tbb/include/tbb/mutex.h index e40b4cd02..91dbee0fa 100644 --- a/src/tbb/include/tbb/mutex.h +++ b/src/tbb/include/tbb/mutex.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2023 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,222 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ -#ifndef __TBB_mutex_H -#define __TBB_mutex_H - -#if _WIN32||_WIN64 -#include "machine/windows_api.h" -#else -#include -#endif /* _WIN32||_WIN64 */ - -#include -#include "aligned_space.h" -#include "tbb_stddef.h" -#include "tbb_profiling.h" - -namespace tbb { - -//! Wrapper around the platform's native lock. -/** @ingroup synchronization */ -class mutex : internal::mutex_copy_deprecated_and_disabled { -public: - //! Construct unacquired mutex. - mutex() { -#if TBB_USE_ASSERT || TBB_USE_THREADING_TOOLS - internal_construct(); -#else - #if _WIN32||_WIN64 - InitializeCriticalSectionEx(&impl, 4000, 0); - #else - int error_code = pthread_mutex_init(&impl,NULL); - if( error_code ) - tbb::internal::handle_perror(error_code,"mutex: pthread_mutex_init failed"); - #endif /* _WIN32||_WIN64*/ -#endif /* TBB_USE_ASSERT */ - }; - - ~mutex() { -#if TBB_USE_ASSERT - internal_destroy(); -#else - #if _WIN32||_WIN64 - DeleteCriticalSection(&impl); - #else - pthread_mutex_destroy(&impl); - - #endif /* _WIN32||_WIN64 */ -#endif /* TBB_USE_ASSERT */ - }; - - class scoped_lock; - friend class scoped_lock; - - //! The scoped locking pattern - /** It helps to avoid the common problem of forgetting to release lock. - It also nicely provides the "node" for queuing locks. */ - class scoped_lock : internal::no_copy { - public: - //! Construct lock that has not acquired a mutex. - scoped_lock() : my_mutex(NULL) {}; - - //! Acquire lock on given mutex. - scoped_lock( mutex& mutex ) { - acquire( mutex ); - } - - //! Release lock (if lock is held). - ~scoped_lock() { - if( my_mutex ) - release(); - } - - //! Acquire lock on given mutex. - void acquire( mutex& mutex ) { -#if TBB_USE_ASSERT - internal_acquire(mutex); -#else - mutex.lock(); - my_mutex = &mutex; -#endif /* TBB_USE_ASSERT */ - } - - //! Try acquire lock on given mutex. - bool try_acquire( mutex& mutex ) { -#if TBB_USE_ASSERT - return internal_try_acquire (mutex); -#else - bool result = mutex.try_lock(); - if( result ) - my_mutex = &mutex; - return result; -#endif /* TBB_USE_ASSERT */ - } - - //! Release lock - void release() { -#if TBB_USE_ASSERT - internal_release (); -#else - my_mutex->unlock(); - my_mutex = NULL; -#endif /* TBB_USE_ASSERT */ - } - - private: - //! The pointer to the current mutex to work - mutex* my_mutex; - - //! All checks from acquire using mutex.state were moved here - void __TBB_EXPORTED_METHOD internal_acquire( mutex& m ); - - //! All checks from try_acquire using mutex.state were moved here - bool __TBB_EXPORTED_METHOD internal_try_acquire( mutex& m ); - - //! All checks from release using mutex.state were moved here - void __TBB_EXPORTED_METHOD internal_release(); - - friend class mutex; - }; - - // Mutex traits - static const bool is_rw_mutex = false; - static const bool is_recursive_mutex = false; - static const bool is_fair_mutex = false; - - // ISO C++0x compatibility methods - - //! Acquire lock - void lock() { -#if TBB_USE_ASSERT - aligned_space tmp; - new(tmp.begin()) scoped_lock(*this); -#else - #if _WIN32||_WIN64 - EnterCriticalSection(&impl); - #else - int error_code = pthread_mutex_lock(&impl); - if( error_code ) - tbb::internal::handle_perror(error_code,"mutex: pthread_mutex_lock failed"); - #endif /* _WIN32||_WIN64 */ -#endif /* TBB_USE_ASSERT */ - } - - //! Try acquiring lock (non-blocking) - /** Return true if lock acquired; false otherwise. */ - bool try_lock() { -#if TBB_USE_ASSERT - aligned_space tmp; - scoped_lock& s = *tmp.begin(); - s.my_mutex = NULL; - return s.internal_try_acquire(*this); -#else - #if _WIN32||_WIN64 - return TryEnterCriticalSection(&impl)!=0; - #else - return pthread_mutex_trylock(&impl)==0; - #endif /* _WIN32||_WIN64 */ -#endif /* TBB_USE_ASSERT */ - } - - //! Release lock - void unlock() { -#if TBB_USE_ASSERT - aligned_space tmp; - scoped_lock& s = *tmp.begin(); - s.my_mutex = this; - s.internal_release(); -#else - #if _WIN32||_WIN64 - LeaveCriticalSection(&impl); - #else - pthread_mutex_unlock(&impl); - #endif /* _WIN32||_WIN64 */ -#endif /* TBB_USE_ASSERT */ - } - - //! Return native_handle - #if _WIN32||_WIN64 - typedef LPCRITICAL_SECTION native_handle_type; - #else - typedef pthread_mutex_t* native_handle_type; - #endif - native_handle_type native_handle() { return (native_handle_type) &impl; } - - enum state_t { - INITIALIZED=0x1234, - DESTROYED=0x789A, - HELD=0x56CD - }; -private: -#if _WIN32||_WIN64 - CRITICAL_SECTION impl; - enum state_t state; -#else - pthread_mutex_t impl; -#endif /* _WIN32||_WIN64 */ - - //! All checks from mutex constructor using mutex.state were moved here - void __TBB_EXPORTED_METHOD internal_construct(); - - //! All checks from mutex destructor using mutex.state were moved here - void __TBB_EXPORTED_METHOD internal_destroy(); - -#if _WIN32||_WIN64 -public: - //! Set the internal state - void set_state( state_t to ) { state = to; } -#endif -}; - -__TBB_DEFINE_PROFILING_SET_NAME(mutex) - -} // namespace tbb - -#endif /* __TBB_mutex_H */ +#include "../oneapi/tbb/mutex.h" diff --git a/src/tbb/include/tbb/null_mutex.h b/src/tbb/include/tbb/null_mutex.h index 85c660e31..63218bf06 100644 --- a/src/tbb/include/tbb/null_mutex.h +++ b/src/tbb/include/tbb/null_mutex.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,43 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ -#ifndef __TBB_null_mutex_H -#define __TBB_null_mutex_H - -#include "tbb_stddef.h" - -namespace tbb { - -//! A mutex which does nothing -/** A null_mutex does no operation and simulates success. - @ingroup synchronization */ -class null_mutex : internal::mutex_copy_deprecated_and_disabled { -public: - //! Represents acquisition of a mutex. - class scoped_lock : internal::no_copy { - public: - scoped_lock() {} - scoped_lock( null_mutex& ) {} - ~scoped_lock() {} - void acquire( null_mutex& ) {} - bool try_acquire( null_mutex& ) { return true; } - void release() {} - }; - - null_mutex() {} - - // Mutex traits - static const bool is_rw_mutex = false; - static const bool is_recursive_mutex = true; - static const bool is_fair_mutex = true; -}; - -} - -#endif /* __TBB_null_mutex_H */ +#include "../oneapi/tbb/null_mutex.h" diff --git a/src/tbb/include/tbb/null_rw_mutex.h b/src/tbb/include/tbb/null_rw_mutex.h index fa0c80359..71c42fe26 100644 --- a/src/tbb/include/tbb/null_rw_mutex.h +++ b/src/tbb/include/tbb/null_rw_mutex.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,45 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ -#ifndef __TBB_null_rw_mutex_H -#define __TBB_null_rw_mutex_H - -#include "tbb_stddef.h" - -namespace tbb { - -//! A rw mutex which does nothing -/** A null_rw_mutex is a rw mutex that does nothing and simulates successful operation. - @ingroup synchronization */ -class null_rw_mutex : internal::mutex_copy_deprecated_and_disabled { -public: - //! Represents acquisition of a mutex. - class scoped_lock : internal::no_copy { - public: - scoped_lock() {} - scoped_lock( null_rw_mutex& , bool = true ) {} - ~scoped_lock() {} - void acquire( null_rw_mutex& , bool = true ) {} - bool upgrade_to_writer() { return true; } - bool downgrade_to_reader() { return true; } - bool try_acquire( null_rw_mutex& , bool = true ) { return true; } - void release() {} - }; - - null_rw_mutex() {} - - // Mutex traits - static const bool is_rw_mutex = true; - static const bool is_recursive_mutex = true; - static const bool is_fair_mutex = true; -}; - -} - -#endif /* __TBB_null_rw_mutex_H */ +#include "../oneapi/tbb/null_rw_mutex.h" diff --git a/src/tbb/include/tbb/parallel_do.h b/src/tbb/include/tbb/parallel_do.h deleted file mode 100644 index 152756825..000000000 --- a/src/tbb/include/tbb/parallel_do.h +++ /dev/null @@ -1,551 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_parallel_do_H -#define __TBB_parallel_do_H - -#include "internal/_range_iterator.h" -#include "internal/_template_helpers.h" -#include "task.h" -#include "aligned_space.h" -#include - -namespace tbb { -namespace interface9 { -//! @cond INTERNAL -namespace internal { - template class parallel_do_feeder_impl; -} // namespace internal -//! @endcond - -//! Class the user supplied algorithm body uses to add new tasks -/** \param Item Work item type **/ - template - class parallel_do_feeder: ::tbb::internal::no_copy - { - parallel_do_feeder() {} - virtual ~parallel_do_feeder () {} - virtual void internal_add_copy( const Item& item ) = 0; -#if __TBB_CPP11_RVALUE_REF_PRESENT - virtual void internal_add_move( Item&& item ) = 0; -#endif - template friend class internal::parallel_do_feeder_impl; - public: - //! Add a work item to a running parallel_do. - void add( const Item& item ) {internal_add_copy(item);} -#if __TBB_CPP11_RVALUE_REF_PRESENT - void add( Item&& item ) {internal_add_move(std::move(item));} -#endif - }; - -//! @cond INTERNAL -namespace internal { - template class do_group_task; - - //! For internal use only. - /** Selects one of the two possible forms of function call member operator. - @ingroup algorithms **/ - template - class parallel_do_operator_selector - { - typedef parallel_do_feeder Feeder; - template - static void internal_call( const Body& obj, __TBB_FORWARDING_REF(A1) arg1, A2&, void (Body::*)(CvItem) const ) { - obj(tbb::internal::forward(arg1)); - } - template - static void internal_call( const Body& obj, __TBB_FORWARDING_REF(A1) arg1, A2& arg2, void (Body::*)(CvItem, parallel_do_feeder&) const ) { - obj(tbb::internal::forward(arg1), arg2); - } - template - static void internal_call( const Body& obj, __TBB_FORWARDING_REF(A1) arg1, A2&, void (Body::*)(CvItem&) const ) { - obj(arg1); - } - template - static void internal_call( const Body& obj, __TBB_FORWARDING_REF(A1) arg1, A2& arg2, void (Body::*)(CvItem&, parallel_do_feeder&) const ) { - obj(arg1, arg2); - } - public: - template - static void call( const Body& obj, __TBB_FORWARDING_REF(A1) arg1, A2& arg2 ) - { - internal_call( obj, tbb::internal::forward(arg1), arg2, &Body::operator() ); - } - }; - - //! For internal use only. - /** Executes one iteration of a do. - @ingroup algorithms */ - template - class do_iteration_task: public task - { - typedef parallel_do_feeder_impl feeder_type; - - Item my_value; - feeder_type& my_feeder; - - do_iteration_task( const Item& value, feeder_type& feeder ) : - my_value(value), my_feeder(feeder) - {} - -#if __TBB_CPP11_RVALUE_REF_PRESENT - do_iteration_task( Item&& value, feeder_type& feeder ) : - my_value(std::move(value)), my_feeder(feeder) - {} -#endif - - task* execute() __TBB_override - { - parallel_do_operator_selector::call(*my_feeder.my_body, tbb::internal::move(my_value), my_feeder); - return NULL; - } - - template friend class parallel_do_feeder_impl; - }; // class do_iteration_task - - template - class do_iteration_task_iter: public task - { - typedef parallel_do_feeder_impl feeder_type; - - Iterator my_iter; - feeder_type& my_feeder; - - do_iteration_task_iter( const Iterator& iter, feeder_type& feeder ) : - my_iter(iter), my_feeder(feeder) - {} - - task* execute() __TBB_override - { - parallel_do_operator_selector::call(*my_feeder.my_body, *my_iter, my_feeder); - return NULL; - } - - template friend class do_group_task_forward; - template friend class do_group_task_input; - template friend class do_task_iter; - }; // class do_iteration_task_iter - - //! For internal use only. - /** Implements new task adding procedure. - @ingroup algorithms **/ - template - class parallel_do_feeder_impl : public parallel_do_feeder - { -#if __TBB_CPP11_RVALUE_REF_PRESENT - //Avoiding use of copy constructor in a virtual method if the type does not support it - void internal_add_copy_impl(std::true_type, const Item& item) { - typedef do_iteration_task iteration_type; - iteration_type& t = *new (task::allocate_additional_child_of(*my_barrier)) iteration_type(item, *this); - task::spawn(t); - } - void internal_add_copy_impl(std::false_type, const Item&) { - __TBB_ASSERT(false, "Overloading for r-value reference doesn't work or it's not movable and not copyable object"); - } - void internal_add_copy( const Item& item ) __TBB_override - { -#if __TBB_CPP11_IS_COPY_CONSTRUCTIBLE_PRESENT - internal_add_copy_impl(typename std::is_copy_constructible::type(), item); -#else - internal_add_copy_impl(std::true_type(), item); -#endif - } - void internal_add_move( Item&& item ) __TBB_override - { - typedef do_iteration_task iteration_type; - iteration_type& t = *new (task::allocate_additional_child_of(*my_barrier)) iteration_type(std::move(item), *this); - task::spawn(t); - } -#else /* ! __TBB_CPP11_RVALUE_REF_PRESENT */ - void internal_add_copy(const Item& item) __TBB_override { - typedef do_iteration_task iteration_type; - iteration_type& t = *new (task::allocate_additional_child_of(*my_barrier)) iteration_type(item, *this); - task::spawn(t); - } -#endif /* __TBB_CPP11_RVALUE_REF_PRESENT */ - public: - const Body* my_body; - empty_task* my_barrier; - - parallel_do_feeder_impl() - { - my_barrier = new( task::allocate_root() ) empty_task(); - __TBB_ASSERT(my_barrier, "root task allocation failed"); - } - -#if __TBB_TASK_GROUP_CONTEXT - parallel_do_feeder_impl(tbb::task_group_context &context) - { - my_barrier = new( task::allocate_root(context) ) empty_task(); - __TBB_ASSERT(my_barrier, "root task allocation failed"); - } -#endif - - ~parallel_do_feeder_impl() - { - my_barrier->destroy(*my_barrier); - } - }; // class parallel_do_feeder_impl - - - //! For internal use only - /** Unpacks a block of iterations. - @ingroup algorithms */ - - template - class do_group_task_forward: public task - { - static const size_t max_arg_size = 4; - - typedef parallel_do_feeder_impl feeder_type; - - feeder_type& my_feeder; - Iterator my_first; - size_t my_size; - - do_group_task_forward( Iterator first, size_t size, feeder_type& feeder ) - : my_feeder(feeder), my_first(first), my_size(size) - {} - - task* execute() __TBB_override - { - typedef do_iteration_task_iter iteration_type; - __TBB_ASSERT( my_size>0, NULL ); - task_list list; - task* t; - size_t k=0; - for(;;) { - t = new( allocate_child() ) iteration_type( my_first, my_feeder ); - ++my_first; - if( ++k==my_size ) break; - list.push_back(*t); - } - set_ref_count(int(k+1)); - spawn(list); - spawn_and_wait_for_all(*t); - return NULL; - } - - template friend class do_task_iter; - }; // class do_group_task_forward - - template - class do_group_task_input: public task - { - static const size_t max_arg_size = 4; - - typedef parallel_do_feeder_impl feeder_type; - - feeder_type& my_feeder; - size_t my_size; - aligned_space my_arg; - - do_group_task_input( feeder_type& feeder ) - : my_feeder(feeder), my_size(0) - {} - - task* execute() __TBB_override - { -#if __TBB_CPP11_RVALUE_REF_PRESENT - typedef std::move_iterator Item_iterator; -#else - typedef Item* Item_iterator; -#endif - typedef do_iteration_task_iter iteration_type; - __TBB_ASSERT( my_size>0, NULL ); - task_list list; - task* t; - size_t k=0; - for(;;) { - t = new( allocate_child() ) iteration_type( Item_iterator(my_arg.begin() + k), my_feeder ); - if( ++k==my_size ) break; - list.push_back(*t); - } - set_ref_count(int(k+1)); - spawn(list); - spawn_and_wait_for_all(*t); - return NULL; - } - - ~do_group_task_input(){ - for( size_t k=0; k~Item(); - } - - template friend class do_task_iter; - }; // class do_group_task_input - - //! For internal use only. - /** Gets block of iterations and packages them into a do_group_task. - @ingroup algorithms */ - template - class do_task_iter: public task - { - typedef parallel_do_feeder_impl feeder_type; - - public: - do_task_iter( Iterator first, Iterator last , feeder_type& feeder ) : - my_first(first), my_last(last), my_feeder(feeder) - {} - - private: - Iterator my_first; - Iterator my_last; - feeder_type& my_feeder; - - /* Do not merge run(xxx) and run_xxx() methods. They are separated in order - to make sure that compilers will eliminate unused argument of type xxx - (that is will not put it on stack). The sole purpose of this argument - is overload resolution. - - An alternative could be using template functions, but explicit specialization - of member function templates is not supported for non specialized class - templates. Besides template functions would always fall back to the least - efficient variant (the one for input iterators) in case of iterators having - custom tags derived from basic ones. */ - task* execute() __TBB_override - { - typedef typename std::iterator_traits::iterator_category iterator_tag; - return run( (iterator_tag*)NULL ); - } - - /** This is the most restricted variant that operates on input iterators or - iterators with unknown tags (tags not derived from the standard ones). **/ - inline task* run( void* ) { return run_for_input_iterator(); } - - task* run_for_input_iterator() { - typedef do_group_task_input block_type; - - block_type& t = *new( allocate_additional_child_of(*my_feeder.my_barrier) ) block_type(my_feeder); - size_t k=0; - while( !(my_first == my_last) ) { - // Move semantics are automatically used when supported by the iterator - new (t.my_arg.begin() + k) Item(*my_first); - ++my_first; - if( ++k==block_type::max_arg_size ) { - if ( !(my_first == my_last) ) - recycle_to_reexecute(); - break; - } - } - if( k==0 ) { - destroy(t); - return NULL; - } else { - t.my_size = k; - return &t; - } - } - - inline task* run( std::forward_iterator_tag* ) { return run_for_forward_iterator(); } - - task* run_for_forward_iterator() { - typedef do_group_task_forward block_type; - - Iterator first = my_first; - size_t k=0; - while( !(my_first==my_last) ) { - ++my_first; - if( ++k==block_type::max_arg_size ) { - if ( !(my_first==my_last) ) - recycle_to_reexecute(); - break; - } - } - return k==0 ? NULL : new( allocate_additional_child_of(*my_feeder.my_barrier) ) block_type(first, k, my_feeder); - } - - inline task* run( std::random_access_iterator_tag* ) { return run_for_random_access_iterator(); } - - task* run_for_random_access_iterator() { - typedef do_group_task_forward block_type; - typedef do_iteration_task_iter iteration_type; - - size_t k = static_cast(my_last-my_first); - if( k > block_type::max_arg_size ) { - Iterator middle = my_first + k/2; - - empty_task& c = *new( allocate_continuation() ) empty_task; - do_task_iter& b = *new( c.allocate_child() ) do_task_iter(middle, my_last, my_feeder); - recycle_as_child_of(c); - - my_last = middle; - c.set_ref_count(2); - c.spawn(b); - return this; - }else if( k != 0 ) { - task_list list; - task* t; - size_t k1=0; - for(;;) { - t = new( allocate_child() ) iteration_type(my_first, my_feeder); - ++my_first; - if( ++k1==k ) break; - list.push_back(*t); - } - set_ref_count(int(k+1)); - spawn(list); - spawn_and_wait_for_all(*t); - } - return NULL; - } - }; // class do_task_iter - - //! For internal use only. - /** Implements parallel iteration over a range. - @ingroup algorithms */ - template - void run_parallel_do( Iterator first, Iterator last, const Body& body -#if __TBB_TASK_GROUP_CONTEXT - , task_group_context& context -#endif - ) - { - typedef do_task_iter root_iteration_task; -#if __TBB_TASK_GROUP_CONTEXT - parallel_do_feeder_impl feeder(context); -#else - parallel_do_feeder_impl feeder; -#endif - feeder.my_body = &body; - - root_iteration_task &t = *new( feeder.my_barrier->allocate_child() ) root_iteration_task(first, last, feeder); - - feeder.my_barrier->set_ref_count(2); - feeder.my_barrier->spawn_and_wait_for_all(t); - } - - //! For internal use only. - /** Detects types of Body's operator function arguments. - @ingroup algorithms **/ - template - void select_parallel_do( Iterator first, Iterator last, const Body& body, void (Body::*)(Item) const -#if __TBB_TASK_GROUP_CONTEXT - , task_group_context& context -#endif - ) - { - run_parallel_do::type>( first, last, body -#if __TBB_TASK_GROUP_CONTEXT - , context -#endif - ); - } - - //! For internal use only. - /** Detects types of Body's operator function arguments. - @ingroup algorithms **/ - template - void select_parallel_do( Iterator first, Iterator last, const Body& body, void (Body::*)(Item, parallel_do_feeder<_Item>&) const -#if __TBB_TASK_GROUP_CONTEXT - , task_group_context& context -#endif - ) - { - run_parallel_do::type>( first, last, body -#if __TBB_TASK_GROUP_CONTEXT - , context -#endif - ); - } - -} // namespace internal -} // namespace interface9 -//! @endcond - -/** \page parallel_do_body_req Requirements on parallel_do body - Class \c Body implementing the concept of parallel_do body must define: - - \code - B::operator()( - cv_item_type item, - parallel_do_feeder& feeder - ) const - - OR - - B::operator()( cv_item_type& item ) const - \endcode Process item. - May be invoked concurrently for the same \c this but different \c item. - - - \code item_type( const item_type& ) \endcode - Copy a work item. - - \code ~item_type() \endcode Destroy a work item -**/ - -/** \name parallel_do - See also requirements on \ref parallel_do_body_req "parallel_do Body". **/ -//@{ -//! Parallel iteration over a range, with optional addition of more work. -/** @ingroup algorithms */ -template -void parallel_do( Iterator first, Iterator last, const Body& body ) -{ - if ( first == last ) - return; -#if __TBB_TASK_GROUP_CONTEXT - task_group_context context; -#endif - interface9::internal::select_parallel_do( first, last, body, &Body::operator() -#if __TBB_TASK_GROUP_CONTEXT - , context -#endif - ); -} - -template -void parallel_do(Range& rng, const Body& body) { - parallel_do(tbb::internal::first(rng), tbb::internal::last(rng), body); -} - -template -void parallel_do(const Range& rng, const Body& body) { - parallel_do(tbb::internal::first(rng), tbb::internal::last(rng), body); -} - -#if __TBB_TASK_GROUP_CONTEXT -//! Parallel iteration over a range, with optional addition of more work and user-supplied context -/** @ingroup algorithms */ -template -void parallel_do( Iterator first, Iterator last, const Body& body, task_group_context& context ) -{ - if ( first == last ) - return; - interface9::internal::select_parallel_do( first, last, body, &Body::operator(), context ); -} - -template -void parallel_do(Range& rng, const Body& body, task_group_context& context) { - parallel_do(tbb::internal::first(rng), tbb::internal::last(rng), body, context); -} - -template -void parallel_do(const Range& rng, const Body& body, task_group_context& context) { - parallel_do(tbb::internal::first(rng), tbb::internal::last(rng), body, context); -} - -#endif // __TBB_TASK_GROUP_CONTEXT - -//@} - -using interface9::parallel_do_feeder; - -} // namespace - -#endif /* __TBB_parallel_do_H */ diff --git a/src/tbb/include/tbb/parallel_for.h b/src/tbb/include/tbb/parallel_for.h index 19549e195..fea1d1b9f 100644 --- a/src/tbb/include/tbb/parallel_for.h +++ b/src/tbb/include/tbb/parallel_for.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,412 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ -#ifndef __TBB_parallel_for_H -#define __TBB_parallel_for_H - -#include -#include "task.h" -#include "partitioner.h" -#include "blocked_range.h" -#include "tbb_exception.h" -#include "internal/_tbb_trace_impl.h" - -namespace tbb { - -namespace interface9 { -//! @cond INTERNAL -namespace internal { - - //! allocate right task with new parent - void* allocate_sibling(task* start_for_task, size_t bytes); - - //! Task type used in parallel_for - /** @ingroup algorithms */ - template - class start_for: public task { - Range my_range; - const Body my_body; - typename Partitioner::task_partition_type my_partition; - task* execute() __TBB_override; - - //! Update affinity info, if any. - void note_affinity( affinity_id id ) __TBB_override { - my_partition.note_affinity( id ); - } - - public: - //! Constructor for root task. - start_for( const Range& range, const Body& body, Partitioner& partitioner ) : - my_range(range), - my_body(body), - my_partition(partitioner) - { - tbb::internal::fgt_algorithm(tbb::internal::FGT_PARALLEL_FOR, this, NULL); - } - //! Splitting constructor used to generate children. - /** parent_ becomes left child. Newly constructed object is right child. */ - start_for( start_for& parent_, typename Partitioner::split_type& split_obj) : - my_range(parent_.my_range, split_obj), - my_body(parent_.my_body), - my_partition(parent_.my_partition, split_obj) - { - my_partition.set_affinity(*this); - tbb::internal::fgt_algorithm(tbb::internal::FGT_PARALLEL_FOR, this, (void *)&parent_); - } - //! Construct right child from the given range as response to the demand. - /** parent_ remains left child. Newly constructed object is right child. */ - start_for( start_for& parent_, const Range& r, depth_t d ) : - my_range(r), - my_body(parent_.my_body), - my_partition(parent_.my_partition, split()) - { - my_partition.set_affinity(*this); - my_partition.align_depth( d ); - tbb::internal::fgt_algorithm(tbb::internal::FGT_PARALLEL_FOR, this, (void *)&parent_); - } - static void run( const Range& range, const Body& body, Partitioner& partitioner ) { - if( !range.empty() ) { -#if !__TBB_TASK_GROUP_CONTEXT || TBB_JOIN_OUTER_TASK_GROUP - start_for& a = *new(task::allocate_root()) start_for(range,body,partitioner); -#else - // Bound context prevents exceptions from body to affect nesting or sibling algorithms, - // and allows users to handle exceptions safely by wrapping parallel_for in the try-block. - task_group_context context; - start_for& a = *new(task::allocate_root(context)) start_for(range,body,partitioner); -#endif /* __TBB_TASK_GROUP_CONTEXT && !TBB_JOIN_OUTER_TASK_GROUP */ - // REGION BEGIN - fgt_begin_algorithm( tbb::internal::FGT_PARALLEL_FOR, (void*)&a ); - task::spawn_root_and_wait(a); - fgt_end_algorithm( (void*)&a ); - // REGION END - } - } -#if __TBB_TASK_GROUP_CONTEXT - static void run( const Range& range, const Body& body, Partitioner& partitioner, task_group_context& context ) { - if( !range.empty() ) { - start_for& a = *new(task::allocate_root(context)) start_for(range,body,partitioner); - // REGION BEGIN - fgt_begin_algorithm( tbb::internal::FGT_PARALLEL_FOR, (void*)&a ); - task::spawn_root_and_wait(a); - fgt_end_algorithm( (void*)&a ); - // END REGION - } - } -#endif /* __TBB_TASK_GROUP_CONTEXT */ - //! Run body for range, serves as callback for partitioner - void run_body( Range &r ) { - fgt_alg_begin_body( tbb::internal::FGT_PARALLEL_FOR, (void *)const_cast(&(this->my_body)), (void*)this ); - my_body( r ); - fgt_alg_end_body( (void *)const_cast(&(this->my_body)) ); - } - - //! spawn right task, serves as callback for partitioner - void offer_work(typename Partitioner::split_type& split_obj) { - spawn( *new( allocate_sibling(static_cast(this), sizeof(start_for)) ) start_for(*this, split_obj) ); - } - //! spawn right task, serves as callback for partitioner - void offer_work(const Range& r, depth_t d = 0) { - spawn( *new( allocate_sibling(static_cast(this), sizeof(start_for)) ) start_for(*this, r, d) ); - } - }; - - //! allocate right task with new parent - // TODO: 'inline' here is to avoid multiple definition error but for sake of code size this should not be inlined - inline void* allocate_sibling(task* start_for_task, size_t bytes) { - task* parent_ptr = new( start_for_task->allocate_continuation() ) flag_task(); - start_for_task->set_parent(parent_ptr); - parent_ptr->set_ref_count(2); - return &parent_ptr->allocate_child().allocate(bytes); - } - - //! execute task for parallel_for - template - task* start_for::execute() { - my_partition.check_being_stolen( *this ); - my_partition.execute(*this, my_range); - return NULL; - } -} // namespace internal -//! @endcond -} // namespace interfaceX - -//! @cond INTERNAL -namespace internal { - using interface9::internal::start_for; - - //! Calls the function with values from range [begin, end) with a step provided - template - class parallel_for_body : internal::no_assign { - const Function &my_func; - const Index my_begin; - const Index my_step; - public: - parallel_for_body( const Function& _func, Index& _begin, Index& _step ) - : my_func(_func), my_begin(_begin), my_step(_step) {} - - void operator()( const tbb::blocked_range& r ) const { - // A set of local variables to help the compiler with vectorization of the following loop. - Index b = r.begin(); - Index e = r.end(); - Index ms = my_step; - Index k = my_begin + b*ms; - -#if __INTEL_COMPILER -#pragma ivdep -#if __TBB_ASSERT_ON_VECTORIZATION_FAILURE -#pragma vector always assert -#endif -#endif - for ( Index i = b; i < e; ++i, k += ms ) { - my_func( k ); - } - } - }; -} // namespace internal -//! @endcond - -// Requirements on Range concept are documented in blocked_range.h - -/** \page parallel_for_body_req Requirements on parallel_for body - Class \c Body implementing the concept of parallel_for body must define: - - \code Body::Body( const Body& ); \endcode Copy constructor - - \code Body::~Body(); \endcode Destructor - - \code void Body::operator()( Range& r ) const; \endcode Function call operator applying the body to range \c r. -**/ - -/** \name parallel_for - See also requirements on \ref range_req "Range" and \ref parallel_for_body_req "parallel_for Body". **/ -//@{ - -//! Parallel iteration over range with default partitioner. -/** @ingroup algorithms **/ -template -void parallel_for( const Range& range, const Body& body ) { - internal::start_for::run(range,body,__TBB_DEFAULT_PARTITIONER()); -} - -//! Parallel iteration over range with simple partitioner. -/** @ingroup algorithms **/ -template -void parallel_for( const Range& range, const Body& body, const simple_partitioner& partitioner ) { - internal::start_for::run(range,body,partitioner); -} - -//! Parallel iteration over range with auto_partitioner. -/** @ingroup algorithms **/ -template -void parallel_for( const Range& range, const Body& body, const auto_partitioner& partitioner ) { - internal::start_for::run(range,body,partitioner); -} - -//! Parallel iteration over range with static_partitioner. -/** @ingroup algorithms **/ -template -void parallel_for( const Range& range, const Body& body, const static_partitioner& partitioner ) { - internal::start_for::run(range,body,partitioner); -} - -//! Parallel iteration over range with affinity_partitioner. -/** @ingroup algorithms **/ -template -void parallel_for( const Range& range, const Body& body, affinity_partitioner& partitioner ) { - internal::start_for::run(range,body,partitioner); -} - -#if __TBB_TASK_GROUP_CONTEXT -//! Parallel iteration over range with default partitioner and user-supplied context. -/** @ingroup algorithms **/ -template -void parallel_for( const Range& range, const Body& body, task_group_context& context ) { - internal::start_for::run(range, body, __TBB_DEFAULT_PARTITIONER(), context); -} - -//! Parallel iteration over range with simple partitioner and user-supplied context. -/** @ingroup algorithms **/ -template -void parallel_for( const Range& range, const Body& body, const simple_partitioner& partitioner, task_group_context& context ) { - internal::start_for::run(range, body, partitioner, context); -} - -//! Parallel iteration over range with auto_partitioner and user-supplied context. -/** @ingroup algorithms **/ -template -void parallel_for( const Range& range, const Body& body, const auto_partitioner& partitioner, task_group_context& context ) { - internal::start_for::run(range, body, partitioner, context); -} - -//! Parallel iteration over range with static_partitioner and user-supplied context. -/** @ingroup algorithms **/ -template -void parallel_for( const Range& range, const Body& body, const static_partitioner& partitioner, task_group_context& context ) { - internal::start_for::run(range, body, partitioner, context); -} - -//! Parallel iteration over range with affinity_partitioner and user-supplied context. -/** @ingroup algorithms **/ -template -void parallel_for( const Range& range, const Body& body, affinity_partitioner& partitioner, task_group_context& context ) { - internal::start_for::run(range,body,partitioner, context); -} -#endif /* __TBB_TASK_GROUP_CONTEXT */ -//@} - -namespace strict_ppl { - -//@{ -//! Implementation of parallel iteration over stepped range of integers with explicit step and partitioner -template -void parallel_for_impl(Index first, Index last, Index step, const Function& f, Partitioner& partitioner) { - if (step <= 0 ) - internal::throw_exception(internal::eid_nonpositive_step); // throws std::invalid_argument - else if (last > first) { - // Above "else" avoids "potential divide by zero" warning on some platforms - Index end = (last - first - Index(1)) / step + Index(1); - tbb::blocked_range range(static_cast(0), end); - internal::parallel_for_body body(f, first, step); - tbb::parallel_for(range, body, partitioner); - } -} - -//! Parallel iteration over a range of integers with a step provided and default partitioner -template -void parallel_for(Index first, Index last, Index step, const Function& f) { - parallel_for_impl(first, last, step, f, auto_partitioner()); -} -//! Parallel iteration over a range of integers with a step provided and simple partitioner -template -void parallel_for(Index first, Index last, Index step, const Function& f, const simple_partitioner& partitioner) { - parallel_for_impl(first, last, step, f, partitioner); -} -//! Parallel iteration over a range of integers with a step provided and auto partitioner -template -void parallel_for(Index first, Index last, Index step, const Function& f, const auto_partitioner& partitioner) { - parallel_for_impl(first, last, step, f, partitioner); -} -//! Parallel iteration over a range of integers with a step provided and static partitioner -template -void parallel_for(Index first, Index last, Index step, const Function& f, const static_partitioner& partitioner) { - parallel_for_impl(first, last, step, f, partitioner); -} -//! Parallel iteration over a range of integers with a step provided and affinity partitioner -template -void parallel_for(Index first, Index last, Index step, const Function& f, affinity_partitioner& partitioner) { - parallel_for_impl(first, last, step, f, partitioner); -} - -//! Parallel iteration over a range of integers with a default step value and default partitioner -template -void parallel_for(Index first, Index last, const Function& f) { - parallel_for_impl(first, last, static_cast(1), f, auto_partitioner()); -} -//! Parallel iteration over a range of integers with a default step value and simple partitioner -template -void parallel_for(Index first, Index last, const Function& f, const simple_partitioner& partitioner) { - parallel_for_impl(first, last, static_cast(1), f, partitioner); -} -//! Parallel iteration over a range of integers with a default step value and auto partitioner -template -void parallel_for(Index first, Index last, const Function& f, const auto_partitioner& partitioner) { - parallel_for_impl(first, last, static_cast(1), f, partitioner); -} -//! Parallel iteration over a range of integers with a default step value and static partitioner -template -void parallel_for(Index first, Index last, const Function& f, const static_partitioner& partitioner) { - parallel_for_impl(first, last, static_cast(1), f, partitioner); -} -//! Parallel iteration over a range of integers with a default step value and affinity partitioner -template -void parallel_for(Index first, Index last, const Function& f, affinity_partitioner& partitioner) { - parallel_for_impl(first, last, static_cast(1), f, partitioner); -} - -#if __TBB_TASK_GROUP_CONTEXT -//! Implementation of parallel iteration over stepped range of integers with explicit step, task group context, and partitioner -template -void parallel_for_impl(Index first, Index last, Index step, const Function& f, Partitioner& partitioner, tbb::task_group_context &context) { - if (step <= 0 ) - internal::throw_exception(internal::eid_nonpositive_step); // throws std::invalid_argument - else if (last > first) { - // Above "else" avoids "potential divide by zero" warning on some platforms - Index end = (last - first - Index(1)) / step + Index(1); - tbb::blocked_range range(static_cast(0), end); - internal::parallel_for_body body(f, first, step); - tbb::parallel_for(range, body, partitioner, context); - } -} - -//! Parallel iteration over a range of integers with explicit step, task group context, and default partitioner -template -void parallel_for(Index first, Index last, Index step, const Function& f, tbb::task_group_context &context) { - parallel_for_impl(first, last, step, f, auto_partitioner(), context); -} -//! Parallel iteration over a range of integers with explicit step, task group context, and simple partitioner - template -void parallel_for(Index first, Index last, Index step, const Function& f, const simple_partitioner& partitioner, tbb::task_group_context &context) { - parallel_for_impl(first, last, step, f, partitioner, context); -} -//! Parallel iteration over a range of integers with explicit step, task group context, and auto partitioner - template -void parallel_for(Index first, Index last, Index step, const Function& f, const auto_partitioner& partitioner, tbb::task_group_context &context) { - parallel_for_impl(first, last, step, f, partitioner, context); -} -//! Parallel iteration over a range of integers with explicit step, task group context, and static partitioner -template -void parallel_for(Index first, Index last, Index step, const Function& f, const static_partitioner& partitioner, tbb::task_group_context &context) { - parallel_for_impl(first, last, step, f, partitioner, context); -} -//! Parallel iteration over a range of integers with explicit step, task group context, and affinity partitioner - template -void parallel_for(Index first, Index last, Index step, const Function& f, affinity_partitioner& partitioner, tbb::task_group_context &context) { - parallel_for_impl(first, last, step, f, partitioner, context); -} - - -//! Parallel iteration over a range of integers with a default step value, explicit task group context, and default partitioner -template -void parallel_for(Index first, Index last, const Function& f, tbb::task_group_context &context) { - parallel_for_impl(first, last, static_cast(1), f, auto_partitioner(), context); -} -//! Parallel iteration over a range of integers with a default step value, explicit task group context, and simple partitioner - template -void parallel_for(Index first, Index last, const Function& f, const simple_partitioner& partitioner, tbb::task_group_context &context) { - parallel_for_impl(first, last, static_cast(1), f, partitioner, context); -} -//! Parallel iteration over a range of integers with a default step value, explicit task group context, and auto partitioner - template -void parallel_for(Index first, Index last, const Function& f, const auto_partitioner& partitioner, tbb::task_group_context &context) { - parallel_for_impl(first, last, static_cast(1), f, partitioner, context); -} -//! Parallel iteration over a range of integers with a default step value, explicit task group context, and static partitioner -template -void parallel_for(Index first, Index last, const Function& f, const static_partitioner& partitioner, tbb::task_group_context &context) { - parallel_for_impl(first, last, static_cast(1), f, partitioner, context); -} -//! Parallel iteration over a range of integers with a default step value, explicit task group context, and affinity_partitioner - template -void parallel_for(Index first, Index last, const Function& f, affinity_partitioner& partitioner, tbb::task_group_context &context) { - parallel_for_impl(first, last, static_cast(1), f, partitioner, context); -} - -#endif /* __TBB_TASK_GROUP_CONTEXT */ -//@} - -} // namespace strict_ppl - -using strict_ppl::parallel_for; - -} // namespace tbb - -#if TBB_PREVIEW_SERIAL_SUBSET -#define __TBB_NORMAL_EXECUTION -#include "../serial/tbb/parallel_for.h" -#undef __TBB_NORMAL_EXECUTION -#endif - -#endif /* __TBB_parallel_for_H */ +#include "../oneapi/tbb/parallel_for.h" diff --git a/src/tbb/include/tbb/parallel_for_each.h b/src/tbb/include/tbb/parallel_for_each.h index 6c2ec9f02..27c2ab172 100644 --- a/src/tbb/include/tbb/parallel_for_each.h +++ b/src/tbb/include/tbb/parallel_for_each.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,126 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ -#ifndef __TBB_parallel_for_each_H -#define __TBB_parallel_for_each_H - -#include "parallel_do.h" -#include "parallel_for.h" - -namespace tbb { - -//! @cond INTERNAL -namespace internal { - // The class calls user function in operator() - template - class parallel_for_each_body_do : internal::no_assign { - const Function &my_func; - public: - parallel_for_each_body_do(const Function &_func) : my_func(_func) {} - - void operator()(typename std::iterator_traits::reference value) const { - my_func(value); - } - }; - - // The class calls user function in operator() - template - class parallel_for_each_body_for : internal::no_assign { - const Function &my_func; - public: - parallel_for_each_body_for(const Function &_func) : my_func(_func) {} - - void operator()(tbb::blocked_range range) const { -#if __INTEL_COMPILER -#pragma ivdep -#endif - for(Iterator it = range.begin(), end = range.end(); it != end; ++it) { - my_func(*it); - } - } - }; - - template - struct parallel_for_each_impl { -#if __TBB_TASK_GROUP_CONTEXT - static void doit(Iterator first, Iterator last, const Function& f, task_group_context &context) { - internal::parallel_for_each_body_do body(f); - tbb::parallel_do(first, last, body, context); - } -#endif - static void doit(Iterator first, Iterator last, const Function& f) { - internal::parallel_for_each_body_do body(f); - tbb::parallel_do(first, last, body); - } - }; - template - struct parallel_for_each_impl { -#if __TBB_TASK_GROUP_CONTEXT - static void doit(Iterator first, Iterator last, const Function& f, task_group_context &context) { - internal::parallel_for_each_body_for body(f); - tbb::parallel_for(tbb::blocked_range(first, last), body, context); - } -#endif - static void doit(Iterator first, Iterator last, const Function& f) { - internal::parallel_for_each_body_for body(f); - tbb::parallel_for(tbb::blocked_range(first, last), body); - } - }; -} // namespace internal -//! @endcond - -/** \name parallel_for_each - **/ -//@{ -//! Calls function f for all items from [first, last) interval using user-supplied context -/** @ingroup algorithms */ -#if __TBB_TASK_GROUP_CONTEXT -template -void parallel_for_each(Iterator first, Iterator last, const Function& f, task_group_context &context) { - internal::parallel_for_each_impl::iterator_category>::doit(first, last, f, context); -} - -//! Calls function f for all items from rng using user-supplied context -/** @ingroup algorithms */ -template -void parallel_for_each(Range& rng, const Function& f, task_group_context& context) { - parallel_for_each(tbb::internal::first(rng), tbb::internal::last(rng), f, context); -} - -//! Calls function f for all items from const rng user-supplied context -/** @ingroup algorithms */ -template -void parallel_for_each(const Range& rng, const Function& f, task_group_context& context) { - parallel_for_each(tbb::internal::first(rng), tbb::internal::last(rng), f, context); -} -#endif /* __TBB_TASK_GROUP_CONTEXT */ - -//! Uses default context -template -void parallel_for_each(Iterator first, Iterator last, const Function& f) { - internal::parallel_for_each_impl::iterator_category>::doit(first, last, f); -} - -//! Uses default context -template -void parallel_for_each(Range& rng, const Function& f) { - parallel_for_each(tbb::internal::first(rng), tbb::internal::last(rng), f); -} - -//! Uses default context -template -void parallel_for_each(const Range& rng, const Function& f) { - parallel_for_each(tbb::internal::first(rng), tbb::internal::last(rng), f); -} - -//@} - -} // namespace - -#endif /* __TBB_parallel_for_each_H */ +#include "../oneapi/tbb/parallel_for_each.h" diff --git a/src/tbb/include/tbb/parallel_invoke.h b/src/tbb/include/tbb/parallel_invoke.h index 0dd7590e8..6c21100e7 100644 --- a/src/tbb/include/tbb/parallel_invoke.h +++ b/src/tbb/include/tbb/parallel_invoke.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,444 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ -#ifndef __TBB_parallel_invoke_H -#define __TBB_parallel_invoke_H - -#include "task.h" - -#if __TBB_VARIADIC_PARALLEL_INVOKE - #include // std::forward -#endif - -namespace tbb { - -#if !__TBB_TASK_GROUP_CONTEXT - /** Dummy to avoid cluttering the bulk of the header with enormous amount of ifdefs. **/ - struct task_group_context {}; -#endif /* __TBB_TASK_GROUP_CONTEXT */ - -//! @cond INTERNAL -namespace internal { - // Simple task object, executing user method - template - class function_invoker : public task{ - public: - function_invoker(const function& _function) : my_function(_function) {} - private: - const function &my_function; - task* execute() __TBB_override - { - my_function(); - return NULL; - } - }; - - // The class spawns two or three child tasks - template - class spawner : public task { - private: - const function1& my_func1; - const function2& my_func2; - const function3& my_func3; - bool is_recycled; - - task* execute () __TBB_override { - if(is_recycled){ - return NULL; - }else{ - __TBB_ASSERT(N==2 || N==3, "Number of arguments passed to spawner is wrong"); - set_ref_count(N); - recycle_as_safe_continuation(); - internal::function_invoker* invoker2 = new (allocate_child()) internal::function_invoker(my_func2); - __TBB_ASSERT(invoker2, "Child task allocation failed"); - spawn(*invoker2); - size_t n = N; // To prevent compiler warnings - if (n>2) { - internal::function_invoker* invoker3 = new (allocate_child()) internal::function_invoker(my_func3); - __TBB_ASSERT(invoker3, "Child task allocation failed"); - spawn(*invoker3); - } - my_func1(); - is_recycled = true; - return NULL; - } - } // execute - - public: - spawner(const function1& _func1, const function2& _func2, const function3& _func3) : my_func1(_func1), my_func2(_func2), my_func3(_func3), is_recycled(false) {} - }; - - // Creates and spawns child tasks - class parallel_invoke_helper : public empty_task { - public: - // Dummy functor class - class parallel_invoke_noop { - public: - void operator() () const {} - }; - // Creates a helper object with user-defined number of children expected - parallel_invoke_helper(int number_of_children) - { - set_ref_count(number_of_children + 1); - } - -#if __TBB_VARIADIC_PARALLEL_INVOKE - void add_children() {} - void add_children(tbb::task_group_context&) {} - - template - void add_children(function&& _func) - { - internal::function_invoker* invoker = new (allocate_child()) internal::function_invoker(std::forward(_func)); - __TBB_ASSERT(invoker, "Child task allocation failed"); - spawn(*invoker); - } - - template - void add_children(function&& _func, tbb::task_group_context&) - { - add_children(std::forward(_func)); - } - - // Adds child(ren) task(s) and spawns them - template - void add_children(function1&& _func1, function2&& _func2, function&&... _func) - { - // The third argument is dummy, it is ignored actually. - parallel_invoke_noop noop; - typedef internal::spawner<2, function1, function2, parallel_invoke_noop> spawner_type; - spawner_type & sub_root = *new(allocate_child()) spawner_type(std::forward(_func1), std::forward(_func2), noop); - spawn(sub_root); - add_children(std::forward(_func)...); - } -#else - // Adds child task and spawns it - template - void add_children (const function &_func) - { - internal::function_invoker* invoker = new (allocate_child()) internal::function_invoker(_func); - __TBB_ASSERT(invoker, "Child task allocation failed"); - spawn(*invoker); - } - - // Adds a task with multiple child tasks and spawns it - // two arguments - template - void add_children (const function1& _func1, const function2& _func2) - { - // The third argument is dummy, it is ignored actually. - parallel_invoke_noop noop; - internal::spawner<2, function1, function2, parallel_invoke_noop>& sub_root = *new(allocate_child())internal::spawner<2, function1, function2, parallel_invoke_noop>(_func1, _func2, noop); - spawn(sub_root); - } - // three arguments - template - void add_children (const function1& _func1, const function2& _func2, const function3& _func3) - { - internal::spawner<3, function1, function2, function3>& sub_root = *new(allocate_child())internal::spawner<3, function1, function2, function3>(_func1, _func2, _func3); - spawn(sub_root); - } -#endif // __TBB_VARIADIC_PARALLEL_INVOKE - - // Waits for all child tasks - template - void run_and_finish(const F0& f0) - { - internal::function_invoker* invoker = new (allocate_child()) internal::function_invoker(f0); - __TBB_ASSERT(invoker, "Child task allocation failed"); - spawn_and_wait_for_all(*invoker); - } - }; - // The class destroys root if exception occurred as well as in normal case - class parallel_invoke_cleaner: internal::no_copy { - public: -#if __TBB_TASK_GROUP_CONTEXT - parallel_invoke_cleaner(int number_of_children, tbb::task_group_context& context) - : root(*new(task::allocate_root(context)) internal::parallel_invoke_helper(number_of_children)) -#else - parallel_invoke_cleaner(int number_of_children, tbb::task_group_context&) - : root(*new(task::allocate_root()) internal::parallel_invoke_helper(number_of_children)) -#endif /* !__TBB_TASK_GROUP_CONTEXT */ - {} - - ~parallel_invoke_cleaner(){ - root.destroy(root); - } - internal::parallel_invoke_helper& root; - }; - -#if __TBB_VARIADIC_PARALLEL_INVOKE -// Determine whether the last parameter in a pack is task_group_context - template struct impl_selector; // to workaround a GCC bug - - template struct impl_selector { - typedef typename impl_selector::type type; - }; - - template struct impl_selector { - typedef false_type type; - }; - template<> struct impl_selector { - typedef true_type type; - }; - - // Select task_group_context parameter from the back of a pack - inline task_group_context& get_context( task_group_context& tgc ) { return tgc; } - - template - task_group_context& get_context( T1&& /*ignored*/, T&&... t ) - { return get_context( std::forward(t)... ); } - - // task_group_context is known to be at the back of the parameter pack - template - void parallel_invoke_impl(true_type, F0&& f0, F1&& f1, F&&... f) { - __TBB_STATIC_ASSERT(sizeof...(F)>0, "Variadic parallel_invoke implementation broken?"); - // # of child tasks: f0, f1, and a task for each two elements of the pack except the last - const size_t number_of_children = 2 + sizeof...(F)/2; - parallel_invoke_cleaner cleaner(number_of_children, get_context(std::forward(f)...)); - parallel_invoke_helper& root = cleaner.root; - - root.add_children(std::forward(f)...); - root.add_children(std::forward(f1)); - root.run_and_finish(std::forward(f0)); - } - - // task_group_context is not in the pack, needs to be added - template - void parallel_invoke_impl(false_type, F0&& f0, F1&& f1, F&&... f) { - tbb::task_group_context context; - // Add context to the arguments, and redirect to the other overload - parallel_invoke_impl(true_type(), std::forward(f0), std::forward(f1), std::forward(f)..., context); - } -#endif -} // namespace internal -//! @endcond - -/** \name parallel_invoke - **/ -//@{ -//! Executes a list of tasks in parallel and waits for all tasks to complete. -/** @ingroup algorithms */ - -#if __TBB_VARIADIC_PARALLEL_INVOKE - -// parallel_invoke for two or more arguments via variadic templates -// presence of task_group_context is defined automatically -template -void parallel_invoke(F0&& f0, F1&& f1, F&&... f) { - typedef typename internal::impl_selector::type selector_type; - internal::parallel_invoke_impl(selector_type(), std::forward(f0), std::forward(f1), std::forward(f)...); -} - -#else - -// parallel_invoke with user-defined context -// two arguments -template -void parallel_invoke(const F0& f0, const F1& f1, tbb::task_group_context& context) { - internal::parallel_invoke_cleaner cleaner(2, context); - internal::parallel_invoke_helper& root = cleaner.root; - - root.add_children(f1); - - root.run_and_finish(f0); -} - -// three arguments -template -void parallel_invoke(const F0& f0, const F1& f1, const F2& f2, tbb::task_group_context& context) { - internal::parallel_invoke_cleaner cleaner(3, context); - internal::parallel_invoke_helper& root = cleaner.root; - - root.add_children(f2); - root.add_children(f1); - - root.run_and_finish(f0); -} - -// four arguments -template -void parallel_invoke(const F0& f0, const F1& f1, const F2& f2, const F3& f3, - tbb::task_group_context& context) -{ - internal::parallel_invoke_cleaner cleaner(4, context); - internal::parallel_invoke_helper& root = cleaner.root; - - root.add_children(f3); - root.add_children(f2); - root.add_children(f1); - - root.run_and_finish(f0); -} - -// five arguments -template -void parallel_invoke(const F0& f0, const F1& f1, const F2& f2, const F3& f3, const F4& f4, - tbb::task_group_context& context) -{ - internal::parallel_invoke_cleaner cleaner(3, context); - internal::parallel_invoke_helper& root = cleaner.root; - - root.add_children(f4, f3); - root.add_children(f2, f1); - - root.run_and_finish(f0); -} - -// six arguments -template -void parallel_invoke(const F0& f0, const F1& f1, const F2& f2, const F3& f3, const F4& f4, const F5& f5, - tbb::task_group_context& context) -{ - internal::parallel_invoke_cleaner cleaner(3, context); - internal::parallel_invoke_helper& root = cleaner.root; - - root.add_children(f5, f4, f3); - root.add_children(f2, f1); - - root.run_and_finish(f0); -} - -// seven arguments -template -void parallel_invoke(const F0& f0, const F1& f1, const F2& f2, const F3& f3, const F4& f4, - const F5& f5, const F6& f6, - tbb::task_group_context& context) -{ - internal::parallel_invoke_cleaner cleaner(3, context); - internal::parallel_invoke_helper& root = cleaner.root; - - root.add_children(f6, f5, f4); - root.add_children(f3, f2, f1); - - root.run_and_finish(f0); -} - -// eight arguments -template -void parallel_invoke(const F0& f0, const F1& f1, const F2& f2, const F3& f3, const F4& f4, - const F5& f5, const F6& f6, const F7& f7, - tbb::task_group_context& context) -{ - internal::parallel_invoke_cleaner cleaner(4, context); - internal::parallel_invoke_helper& root = cleaner.root; - - root.add_children(f7, f6, f5); - root.add_children(f4, f3); - root.add_children(f2, f1); - - root.run_and_finish(f0); -} - -// nine arguments -template -void parallel_invoke(const F0& f0, const F1& f1, const F2& f2, const F3& f3, const F4& f4, - const F5& f5, const F6& f6, const F7& f7, const F8& f8, - tbb::task_group_context& context) -{ - internal::parallel_invoke_cleaner cleaner(4, context); - internal::parallel_invoke_helper& root = cleaner.root; - - root.add_children(f8, f7, f6); - root.add_children(f5, f4, f3); - root.add_children(f2, f1); - - root.run_and_finish(f0); -} - -// ten arguments -template -void parallel_invoke(const F0& f0, const F1& f1, const F2& f2, const F3& f3, const F4& f4, - const F5& f5, const F6& f6, const F7& f7, const F8& f8, const F9& f9, - tbb::task_group_context& context) -{ - internal::parallel_invoke_cleaner cleaner(4, context); - internal::parallel_invoke_helper& root = cleaner.root; - - root.add_children(f9, f8, f7); - root.add_children(f6, f5, f4); - root.add_children(f3, f2, f1); - - root.run_and_finish(f0); -} - -// two arguments -template -void parallel_invoke(const F0& f0, const F1& f1) { - task_group_context context; - parallel_invoke(f0, f1, context); -} -// three arguments -template -void parallel_invoke(const F0& f0, const F1& f1, const F2& f2) { - task_group_context context; - parallel_invoke(f0, f1, f2, context); -} -// four arguments -template -void parallel_invoke(const F0& f0, const F1& f1, const F2& f2, const F3& f3) { - task_group_context context; - parallel_invoke(f0, f1, f2, f3, context); -} -// five arguments -template -void parallel_invoke(const F0& f0, const F1& f1, const F2& f2, const F3& f3, const F4& f4) { - task_group_context context; - parallel_invoke(f0, f1, f2, f3, f4, context); -} -// six arguments -template -void parallel_invoke(const F0& f0, const F1& f1, const F2& f2, const F3& f3, const F4& f4, const F5& f5) { - task_group_context context; - parallel_invoke(f0, f1, f2, f3, f4, f5, context); -} -// seven arguments -template -void parallel_invoke(const F0& f0, const F1& f1, const F2& f2, const F3& f3, const F4& f4, - const F5& f5, const F6& f6) -{ - task_group_context context; - parallel_invoke(f0, f1, f2, f3, f4, f5, f6, context); -} -// eight arguments -template -void parallel_invoke(const F0& f0, const F1& f1, const F2& f2, const F3& f3, const F4& f4, - const F5& f5, const F6& f6, const F7& f7) -{ - task_group_context context; - parallel_invoke(f0, f1, f2, f3, f4, f5, f6, f7, context); -} -// nine arguments -template -void parallel_invoke(const F0& f0, const F1& f1, const F2& f2, const F3& f3, const F4& f4, - const F5& f5, const F6& f6, const F7& f7, const F8& f8) -{ - task_group_context context; - parallel_invoke(f0, f1, f2, f3, f4, f5, f6, f7, f8, context); -} -// ten arguments -template -void parallel_invoke(const F0& f0, const F1& f1, const F2& f2, const F3& f3, const F4& f4, - const F5& f5, const F6& f6, const F7& f7, const F8& f8, const F9& f9) -{ - task_group_context context; - parallel_invoke(f0, f1, f2, f3, f4, f5, f6, f7, f8, f9, context); -} -#endif // __TBB_VARIADIC_PARALLEL_INVOKE -//@} - -} // namespace - -#endif /* __TBB_parallel_invoke_H */ +#include "../oneapi/tbb/parallel_invoke.h" diff --git a/src/tbb/include/tbb/parallel_pipeline.h b/src/tbb/include/tbb/parallel_pipeline.h new file mode 100644 index 000000000..aceee49f8 --- /dev/null +++ b/src/tbb/include/tbb/parallel_pipeline.h @@ -0,0 +1,17 @@ +/* + Copyright (c) 2005-2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "../oneapi/tbb/parallel_pipeline.h" diff --git a/src/tbb/include/tbb/parallel_reduce.h b/src/tbb/include/tbb/parallel_reduce.h index ab9c9f737..83658755a 100644 --- a/src/tbb/include/tbb/parallel_reduce.h +++ b/src/tbb/include/tbb/parallel_reduce.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,626 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ -#ifndef __TBB_parallel_reduce_H -#define __TBB_parallel_reduce_H - -#include -#include "task.h" -#include "aligned_space.h" -#include "partitioner.h" -#include "tbb_profiling.h" - -namespace tbb { - -namespace interface9 { -//! @cond INTERNAL -namespace internal { - - using namespace tbb::internal; - - /** Values for reduction_context. */ - enum { - root_task, left_child, right_child - }; - - /** Represented as a char, not enum, for compactness. */ - typedef char reduction_context; - - //! Task type used to combine the partial results of parallel_reduce. - /** @ingroup algorithms */ - template - class finish_reduce: public flag_task { - //! Pointer to body, or NULL if the left child has not yet finished. - bool has_right_zombie; - const reduction_context my_context; - Body* my_body; - aligned_space zombie_space; - finish_reduce( reduction_context context_ ) : - has_right_zombie(false), // TODO: substitute by flag_task::child_stolen? - my_context(context_), - my_body(NULL) - { - } - ~finish_reduce() { - if( has_right_zombie ) - zombie_space.begin()->~Body(); - } - task* execute() __TBB_override { - if( has_right_zombie ) { - // Right child was stolen. - Body* s = zombie_space.begin(); - my_body->join( *s ); - // Body::join() won't be called if canceled. Defer destruction to destructor - } - if( my_context==left_child ) - itt_store_word_with_release( static_cast(parent())->my_body, my_body ); - return NULL; - } - template - friend class start_reduce; - }; - - //! allocate right task with new parent - void allocate_sibling(task* start_reduce_task, task *tasks[], size_t start_bytes, size_t finish_bytes); - - //! Task type used to split the work of parallel_reduce. - /** @ingroup algorithms */ - template - class start_reduce: public task { - typedef finish_reduce finish_type; - Body* my_body; - Range my_range; - typename Partitioner::task_partition_type my_partition; - reduction_context my_context; - task* execute() __TBB_override; - //! Update affinity info, if any - void note_affinity( affinity_id id ) __TBB_override { - my_partition.note_affinity( id ); - } - template - friend class finish_reduce; - -public: - //! Constructor used for root task - start_reduce( const Range& range, Body* body, Partitioner& partitioner ) : - my_body(body), - my_range(range), - my_partition(partitioner), - my_context(root_task) - { - } - //! Splitting constructor used to generate children. - /** parent_ becomes left child. Newly constructed object is right child. */ - start_reduce( start_reduce& parent_, typename Partitioner::split_type& split_obj ) : - my_body(parent_.my_body), - my_range(parent_.my_range, split_obj), - my_partition(parent_.my_partition, split_obj), - my_context(right_child) - { - my_partition.set_affinity(*this); - parent_.my_context = left_child; - } - //! Construct right child from the given range as response to the demand. - /** parent_ remains left child. Newly constructed object is right child. */ - start_reduce( start_reduce& parent_, const Range& r, depth_t d ) : - my_body(parent_.my_body), - my_range(r), - my_partition(parent_.my_partition, split()), - my_context(right_child) - { - my_partition.set_affinity(*this); - my_partition.align_depth( d ); // TODO: move into constructor of partitioner - parent_.my_context = left_child; - } - static void run( const Range& range, Body& body, Partitioner& partitioner ) { - if( !range.empty() ) { -#if !__TBB_TASK_GROUP_CONTEXT || TBB_JOIN_OUTER_TASK_GROUP - task::spawn_root_and_wait( *new(task::allocate_root()) start_reduce(range,&body,partitioner) ); -#else - // Bound context prevents exceptions from body to affect nesting or sibling algorithms, - // and allows users to handle exceptions safely by wrapping parallel_for in the try-block. - task_group_context context; - task::spawn_root_and_wait( *new(task::allocate_root(context)) start_reduce(range,&body,partitioner) ); -#endif /* __TBB_TASK_GROUP_CONTEXT && !TBB_JOIN_OUTER_TASK_GROUP */ - } - } -#if __TBB_TASK_GROUP_CONTEXT - static void run( const Range& range, Body& body, Partitioner& partitioner, task_group_context& context ) { - if( !range.empty() ) - task::spawn_root_and_wait( *new(task::allocate_root(context)) start_reduce(range,&body,partitioner) ); - } -#endif /* __TBB_TASK_GROUP_CONTEXT */ - //! Run body for range - void run_body( Range &r ) { (*my_body)( r ); } - - //! spawn right task, serves as callback for partitioner - // TODO: remove code duplication from 'offer_work' methods - void offer_work(typename Partitioner::split_type& split_obj) { - task *tasks[2]; - allocate_sibling(static_cast(this), tasks, sizeof(start_reduce), sizeof(finish_type)); - new((void*)tasks[0]) finish_type(my_context); - new((void*)tasks[1]) start_reduce(*this, split_obj); - spawn(*tasks[1]); - } - //! spawn right task, serves as callback for partitioner - void offer_work(const Range& r, depth_t d = 0) { - task *tasks[2]; - allocate_sibling(static_cast(this), tasks, sizeof(start_reduce), sizeof(finish_type)); - new((void*)tasks[0]) finish_type(my_context); - new((void*)tasks[1]) start_reduce(*this, r, d); - spawn(*tasks[1]); - } - }; - - //! allocate right task with new parent - // TODO: 'inline' here is to avoid multiple definition error but for sake of code size this should not be inlined - inline void allocate_sibling(task* start_reduce_task, task *tasks[], size_t start_bytes, size_t finish_bytes) { - tasks[0] = &start_reduce_task->allocate_continuation().allocate(finish_bytes); - start_reduce_task->set_parent(tasks[0]); - tasks[0]->set_ref_count(2); - tasks[1] = &tasks[0]->allocate_child().allocate(start_bytes); - } - - template - task* start_reduce::execute() { - my_partition.check_being_stolen( *this ); - if( my_context==right_child ) { - finish_type* parent_ptr = static_cast(parent()); - if( !itt_load_word_with_acquire(parent_ptr->my_body) ) { // TODO: replace by is_stolen_task() or by parent_ptr->ref_count() == 2??? - my_body = new( parent_ptr->zombie_space.begin() ) Body(*my_body,split()); - parent_ptr->has_right_zombie = true; - } - } else __TBB_ASSERT(my_context==root_task,NULL);// because left leaf spawns right leafs without recycling - my_partition.execute(*this, my_range); - if( my_context==left_child ) { - finish_type* parent_ptr = static_cast(parent()); - __TBB_ASSERT(my_body!=parent_ptr->zombie_space.begin(),NULL); - itt_store_word_with_release(parent_ptr->my_body, my_body ); - } - return NULL; - } - - //! Task type used to combine the partial results of parallel_deterministic_reduce. - /** @ingroup algorithms */ - template - class finish_deterministic_reduce: public task { - Body &my_left_body; - Body my_right_body; - - finish_deterministic_reduce( Body &body ) : - my_left_body( body ), - my_right_body( body, split() ) - { - } - task* execute() __TBB_override { - my_left_body.join( my_right_body ); - return NULL; - } - template - friend class start_deterministic_reduce; - }; - - //! Task type used to split the work of parallel_deterministic_reduce. - /** @ingroup algorithms */ - template - class start_deterministic_reduce: public task { - typedef finish_deterministic_reduce finish_type; - Body &my_body; - Range my_range; - typename Partitioner::task_partition_type my_partition; - task* execute() __TBB_override; - - //! Constructor used for root task - start_deterministic_reduce( const Range& range, Body& body, Partitioner& partitioner ) : - my_body( body ), - my_range( range ), - my_partition( partitioner ) - { - } - //! Splitting constructor used to generate children. - /** parent_ becomes left child. Newly constructed object is right child. */ - start_deterministic_reduce( start_deterministic_reduce& parent_, finish_type& c ) : - my_body( c.my_right_body ), - my_range( parent_.my_range, split() ), - my_partition( parent_.my_partition, split() ) - { - } - -public: - static void run( const Range& range, Body& body, Partitioner& partitioner ) { - if( !range.empty() ) { -#if !__TBB_TASK_GROUP_CONTEXT || TBB_JOIN_OUTER_TASK_GROUP - task::spawn_root_and_wait( *new(task::allocate_root()) start_deterministic_reduce(range,&body,partitioner) ); -#else - // Bound context prevents exceptions from body to affect nesting or sibling algorithms, - // and allows users to handle exceptions safely by wrapping parallel_for in the try-block. - task_group_context context; - task::spawn_root_and_wait( *new(task::allocate_root(context)) start_deterministic_reduce(range,body,partitioner) ); -#endif /* __TBB_TASK_GROUP_CONTEXT && !TBB_JOIN_OUTER_TASK_GROUP */ - } - } -#if __TBB_TASK_GROUP_CONTEXT - static void run( const Range& range, Body& body, Partitioner& partitioner, task_group_context& context ) { - if( !range.empty() ) - task::spawn_root_and_wait( *new(task::allocate_root(context)) start_deterministic_reduce(range,body,partitioner) ); - } -#endif /* __TBB_TASK_GROUP_CONTEXT */ - - void offer_work( typename Partitioner::split_type& ) { - task* tasks[2]; - allocate_sibling(static_cast(this), tasks, sizeof(start_deterministic_reduce), sizeof(finish_type)); - new((void*)tasks[0]) finish_type(my_body); - new((void*)tasks[1]) start_deterministic_reduce(*this, *static_cast(tasks[0])); - spawn(*tasks[1]); - } - - void run_body( Range &r ) { my_body(r); } - }; - - template - task* start_deterministic_reduce::execute() { - my_partition.execute(*this, my_range); - return NULL; - } -} // namespace internal -//! @endcond -} //namespace interfaceX - -//! @cond INTERNAL -namespace internal { - using interface9::internal::start_reduce; - using interface9::internal::start_deterministic_reduce; - //! Auxiliary class for parallel_reduce; for internal use only. - /** The adaptor class that implements \ref parallel_reduce_body_req "parallel_reduce Body" - using given \ref parallel_reduce_lambda_req "anonymous function objects". - **/ - /** @ingroup algorithms */ - template - class lambda_reduce_body { - -//FIXME: decide if my_real_body, my_reduction, and identity_element should be copied or referenced -// (might require some performance measurements) - - const Value& identity_element; - const RealBody& my_real_body; - const Reduction& my_reduction; - Value my_value; - lambda_reduce_body& operator= ( const lambda_reduce_body& other ); - public: - lambda_reduce_body( const Value& identity, const RealBody& body, const Reduction& reduction ) - : identity_element(identity) - , my_real_body(body) - , my_reduction(reduction) - , my_value(identity) - { } - lambda_reduce_body( const lambda_reduce_body& other ) - : identity_element(other.identity_element) - , my_real_body(other.my_real_body) - , my_reduction(other.my_reduction) - , my_value(other.my_value) - { } - lambda_reduce_body( lambda_reduce_body& other, tbb::split ) - : identity_element(other.identity_element) - , my_real_body(other.my_real_body) - , my_reduction(other.my_reduction) - , my_value(other.identity_element) - { } - void operator()(Range& range) { - my_value = my_real_body(range, const_cast(my_value)); - } - void join( lambda_reduce_body& rhs ) { - my_value = my_reduction(const_cast(my_value), const_cast(rhs.my_value)); - } - Value result() const { - return my_value; - } - }; - -} // namespace internal -//! @endcond - -// Requirements on Range concept are documented in blocked_range.h - -/** \page parallel_reduce_body_req Requirements on parallel_reduce body - Class \c Body implementing the concept of parallel_reduce body must define: - - \code Body::Body( Body&, split ); \endcode Splitting constructor. - Must be able to run concurrently with operator() and method \c join - - \code Body::~Body(); \endcode Destructor - - \code void Body::operator()( Range& r ); \endcode Function call operator applying body to range \c r - and accumulating the result - - \code void Body::join( Body& b ); \endcode Join results. - The result in \c b should be merged into the result of \c this -**/ - -/** \page parallel_reduce_lambda_req Requirements on parallel_reduce anonymous function objects (lambda functions) - TO BE DOCUMENTED -**/ - -/** \name parallel_reduce - See also requirements on \ref range_req "Range" and \ref parallel_reduce_body_req "parallel_reduce Body". **/ -//@{ - -//! Parallel iteration with reduction and default partitioner. -/** @ingroup algorithms **/ -template -void parallel_reduce( const Range& range, Body& body ) { - internal::start_reduce::run( range, body, __TBB_DEFAULT_PARTITIONER() ); -} - -//! Parallel iteration with reduction and simple_partitioner -/** @ingroup algorithms **/ -template -void parallel_reduce( const Range& range, Body& body, const simple_partitioner& partitioner ) { - internal::start_reduce::run( range, body, partitioner ); -} - -//! Parallel iteration with reduction and auto_partitioner -/** @ingroup algorithms **/ -template -void parallel_reduce( const Range& range, Body& body, const auto_partitioner& partitioner ) { - internal::start_reduce::run( range, body, partitioner ); -} - -//! Parallel iteration with reduction and static_partitioner -/** @ingroup algorithms **/ -template -void parallel_reduce( const Range& range, Body& body, const static_partitioner& partitioner ) { - internal::start_reduce::run( range, body, partitioner ); -} - -//! Parallel iteration with reduction and affinity_partitioner -/** @ingroup algorithms **/ -template -void parallel_reduce( const Range& range, Body& body, affinity_partitioner& partitioner ) { - internal::start_reduce::run( range, body, partitioner ); -} - -#if __TBB_TASK_GROUP_CONTEXT -//! Parallel iteration with reduction, simple partitioner and user-supplied context. -/** @ingroup algorithms **/ -template -void parallel_reduce( const Range& range, Body& body, const simple_partitioner& partitioner, task_group_context& context ) { - internal::start_reduce::run( range, body, partitioner, context ); -} - -//! Parallel iteration with reduction, auto_partitioner and user-supplied context -/** @ingroup algorithms **/ -template -void parallel_reduce( const Range& range, Body& body, const auto_partitioner& partitioner, task_group_context& context ) { - internal::start_reduce::run( range, body, partitioner, context ); -} - -//! Parallel iteration with reduction, static_partitioner and user-supplied context -/** @ingroup algorithms **/ -template -void parallel_reduce( const Range& range, Body& body, const static_partitioner& partitioner, task_group_context& context ) { - internal::start_reduce::run( range, body, partitioner, context ); -} - -//! Parallel iteration with reduction, affinity_partitioner and user-supplied context -/** @ingroup algorithms **/ -template -void parallel_reduce( const Range& range, Body& body, affinity_partitioner& partitioner, task_group_context& context ) { - internal::start_reduce::run( range, body, partitioner, context ); -} -#endif /* __TBB_TASK_GROUP_CONTEXT */ - -/** parallel_reduce overloads that work with anonymous function objects - (see also \ref parallel_reduce_lambda_req "requirements on parallel_reduce anonymous function objects"). **/ - -//! Parallel iteration with reduction and default partitioner. -/** @ingroup algorithms **/ -template -Value parallel_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction ) { - internal::lambda_reduce_body body(identity, real_body, reduction); - internal::start_reduce,const __TBB_DEFAULT_PARTITIONER> - ::run(range, body, __TBB_DEFAULT_PARTITIONER() ); - return body.result(); -} - -//! Parallel iteration with reduction and simple_partitioner. -/** @ingroup algorithms **/ -template -Value parallel_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, - const simple_partitioner& partitioner ) { - internal::lambda_reduce_body body(identity, real_body, reduction); - internal::start_reduce,const simple_partitioner> - ::run(range, body, partitioner ); - return body.result(); -} - -//! Parallel iteration with reduction and auto_partitioner -/** @ingroup algorithms **/ -template -Value parallel_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, - const auto_partitioner& partitioner ) { - internal::lambda_reduce_body body(identity, real_body, reduction); - internal::start_reduce,const auto_partitioner> - ::run( range, body, partitioner ); - return body.result(); -} - -//! Parallel iteration with reduction and static_partitioner -/** @ingroup algorithms **/ -template -Value parallel_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, - const static_partitioner& partitioner ) { - internal::lambda_reduce_body body(identity, real_body, reduction); - internal::start_reduce,const static_partitioner> - ::run( range, body, partitioner ); - return body.result(); -} - -//! Parallel iteration with reduction and affinity_partitioner -/** @ingroup algorithms **/ -template -Value parallel_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, - affinity_partitioner& partitioner ) { - internal::lambda_reduce_body body(identity, real_body, reduction); - internal::start_reduce,affinity_partitioner> - ::run( range, body, partitioner ); - return body.result(); -} - -#if __TBB_TASK_GROUP_CONTEXT -//! Parallel iteration with reduction, simple partitioner and user-supplied context. -/** @ingroup algorithms **/ -template -Value parallel_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, - const simple_partitioner& partitioner, task_group_context& context ) { - internal::lambda_reduce_body body(identity, real_body, reduction); - internal::start_reduce,const simple_partitioner> - ::run( range, body, partitioner, context ); - return body.result(); -} - -//! Parallel iteration with reduction, auto_partitioner and user-supplied context -/** @ingroup algorithms **/ -template -Value parallel_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, - const auto_partitioner& partitioner, task_group_context& context ) { - internal::lambda_reduce_body body(identity, real_body, reduction); - internal::start_reduce,const auto_partitioner> - ::run( range, body, partitioner, context ); - return body.result(); -} - -//! Parallel iteration with reduction, static_partitioner and user-supplied context -/** @ingroup algorithms **/ -template -Value parallel_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, - const static_partitioner& partitioner, task_group_context& context ) { - internal::lambda_reduce_body body(identity, real_body, reduction); - internal::start_reduce,const static_partitioner> - ::run( range, body, partitioner, context ); - return body.result(); -} - -//! Parallel iteration with reduction, affinity_partitioner and user-supplied context -/** @ingroup algorithms **/ -template -Value parallel_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, - affinity_partitioner& partitioner, task_group_context& context ) { - internal::lambda_reduce_body body(identity, real_body, reduction); - internal::start_reduce,affinity_partitioner> - ::run( range, body, partitioner, context ); - return body.result(); -} -#endif /* __TBB_TASK_GROUP_CONTEXT */ - -//! Parallel iteration with deterministic reduction and default simple partitioner. -/** @ingroup algorithms **/ -template -void parallel_deterministic_reduce( const Range& range, Body& body ) { - internal::start_deterministic_reduce::run(range, body, simple_partitioner()); -} - -//! Parallel iteration with deterministic reduction and simple partitioner. -/** @ingroup algorithms **/ -template -void parallel_deterministic_reduce( const Range& range, Body& body, const simple_partitioner& partitioner ) { - internal::start_deterministic_reduce::run(range, body, partitioner); -} - -//! Parallel iteration with deterministic reduction and static partitioner. -/** @ingroup algorithms **/ -template -void parallel_deterministic_reduce( const Range& range, Body& body, const static_partitioner& partitioner ) { - internal::start_deterministic_reduce::run(range, body, partitioner); -} - -#if __TBB_TASK_GROUP_CONTEXT -//! Parallel iteration with deterministic reduction, default simple partitioner and user-supplied context. -/** @ingroup algorithms **/ -template -void parallel_deterministic_reduce( const Range& range, Body& body, task_group_context& context ) { - internal::start_deterministic_reduce::run( range, body, simple_partitioner(), context ); -} - -//! Parallel iteration with deterministic reduction, simple partitioner and user-supplied context. -/** @ingroup algorithms **/ -template -void parallel_deterministic_reduce( const Range& range, Body& body, const simple_partitioner& partitioner, task_group_context& context ) { - internal::start_deterministic_reduce::run(range, body, partitioner, context); -} - -//! Parallel iteration with deterministic reduction, static partitioner and user-supplied context. -/** @ingroup algorithms **/ -template -void parallel_deterministic_reduce( const Range& range, Body& body, const static_partitioner& partitioner, task_group_context& context ) { - internal::start_deterministic_reduce::run(range, body, partitioner, context); -} -#endif /* __TBB_TASK_GROUP_CONTEXT */ - -/** parallel_reduce overloads that work with anonymous function objects - (see also \ref parallel_reduce_lambda_req "requirements on parallel_reduce anonymous function objects"). **/ - -//! Parallel iteration with deterministic reduction and default simple partitioner. -// TODO: consider making static_partitioner the default -/** @ingroup algorithms **/ -template -Value parallel_deterministic_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction ) { - return parallel_deterministic_reduce(range, identity, real_body, reduction, simple_partitioner()); -} - -//! Parallel iteration with deterministic reduction and simple partitioner. -/** @ingroup algorithms **/ -template -Value parallel_deterministic_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, const simple_partitioner& partitioner ) { - internal::lambda_reduce_body body(identity, real_body, reduction); - internal::start_deterministic_reduce, const simple_partitioner> - ::run(range, body, partitioner); - return body.result(); -} - -//! Parallel iteration with deterministic reduction and static partitioner. -/** @ingroup algorithms **/ -template -Value parallel_deterministic_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, const static_partitioner& partitioner ) { - internal::lambda_reduce_body body(identity, real_body, reduction); - internal::start_deterministic_reduce, const static_partitioner> - ::run(range, body, partitioner); - return body.result(); -} -#if __TBB_TASK_GROUP_CONTEXT -//! Parallel iteration with deterministic reduction, default simple partitioner and user-supplied context. -/** @ingroup algorithms **/ -template -Value parallel_deterministic_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, - task_group_context& context ) { - return parallel_deterministic_reduce(range, identity, real_body, reduction, simple_partitioner(), context); -} - -//! Parallel iteration with deterministic reduction, simple partitioner and user-supplied context. -/** @ingroup algorithms **/ -template -Value parallel_deterministic_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, - const simple_partitioner& partitioner, task_group_context& context ) { - internal::lambda_reduce_body body(identity, real_body, reduction); - internal::start_deterministic_reduce, const simple_partitioner> - ::run(range, body, partitioner, context); - return body.result(); -} - -//! Parallel iteration with deterministic reduction, static partitioner and user-supplied context. -/** @ingroup algorithms **/ -template -Value parallel_deterministic_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, - const static_partitioner& partitioner, task_group_context& context ) { - internal::lambda_reduce_body body(identity, real_body, reduction); - internal::start_deterministic_reduce, const static_partitioner> - ::run(range, body, partitioner, context); - return body.result(); -} -#endif /* __TBB_TASK_GROUP_CONTEXT */ -//@} - -} // namespace tbb - -#endif /* __TBB_parallel_reduce_H */ +#include "../oneapi/tbb/parallel_reduce.h" diff --git a/src/tbb/include/tbb/parallel_scan.h b/src/tbb/include/tbb/parallel_scan.h index d36686956..682032a56 100644 --- a/src/tbb/include/tbb/parallel_scan.h +++ b/src/tbb/include/tbb/parallel_scan.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,403 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ -#ifndef __TBB_parallel_scan_H -#define __TBB_parallel_scan_H - -#include "task.h" -#include "aligned_space.h" -#include -#include "partitioner.h" - -namespace tbb { - -//! Used to indicate that the initial scan is being performed. -/** @ingroup algorithms */ -struct pre_scan_tag { - static bool is_final_scan() {return false;} - operator bool() {return is_final_scan();} -}; - -//! Used to indicate that the final scan is being performed. -/** @ingroup algorithms */ -struct final_scan_tag { - static bool is_final_scan() {return true;} - operator bool() {return is_final_scan();} -}; - -//! @cond INTERNAL -namespace internal { - - //! Performs final scan for a leaf - /** @ingroup algorithms */ - template - class final_sum: public task { - public: - Body my_body; - private: - aligned_space my_range; - //! Where to put result of last subrange, or NULL if not last subrange. - Body* my_stuff_last; - public: - final_sum( Body& body_ ) : - my_body(body_,split()) - { - poison_pointer(my_stuff_last); - } - ~final_sum() { - my_range.begin()->~Range(); - } - void finish_construction( const Range& range_, Body* stuff_last_ ) { - new( my_range.begin() ) Range(range_); - my_stuff_last = stuff_last_; - } - private: - task* execute() __TBB_override { - my_body( *my_range.begin(), final_scan_tag() ); - if( my_stuff_last ) - my_stuff_last->assign(my_body); - return NULL; - } - }; - - //! Split work to be done in the scan. - /** @ingroup algorithms */ - template - class sum_node: public task { - typedef final_sum final_sum_type; - public: - final_sum_type *my_incoming; - final_sum_type *my_body; - Body *my_stuff_last; - private: - final_sum_type *my_left_sum; - sum_node *my_left; - sum_node *my_right; - bool my_left_is_final; - Range my_range; - sum_node( const Range range_, bool left_is_final_ ) : - my_stuff_last(NULL), - my_left_sum(NULL), - my_left(NULL), - my_right(NULL), - my_left_is_final(left_is_final_), - my_range(range_) - { - // Poison fields that will be set by second pass. - poison_pointer(my_body); - poison_pointer(my_incoming); - } - task* create_child( const Range& range_, final_sum_type& f, sum_node* n, final_sum_type* incoming_, Body* stuff_last_ ) { - if( !n ) { - f.recycle_as_child_of( *this ); - f.finish_construction( range_, stuff_last_ ); - return &f; - } else { - n->my_body = &f; - n->my_incoming = incoming_; - n->my_stuff_last = stuff_last_; - return n; - } - } - task* execute() __TBB_override { - if( my_body ) { - if( my_incoming ) - my_left_sum->my_body.reverse_join( my_incoming->my_body ); - recycle_as_continuation(); - sum_node& c = *this; - task* b = c.create_child(Range(my_range,split()),*my_left_sum,my_right,my_left_sum,my_stuff_last); - task* a = my_left_is_final ? NULL : c.create_child(my_range,*my_body,my_left,my_incoming,NULL); - set_ref_count( (a!=NULL)+(b!=NULL) ); - my_body = NULL; - if( a ) spawn(*b); - else a = b; - return a; - } else { - return NULL; - } - } - template - friend class start_scan; - - template - friend class finish_scan; - }; - - //! Combine partial results - /** @ingroup algorithms */ - template - class finish_scan: public task { - typedef sum_node sum_node_type; - typedef final_sum final_sum_type; - final_sum_type** const my_sum; - sum_node_type*& my_return_slot; - public: - final_sum_type* my_right_zombie; - sum_node_type& my_result; - - task* execute() __TBB_override { - __TBB_ASSERT( my_result.ref_count()==(my_result.my_left!=NULL)+(my_result.my_right!=NULL), NULL ); - if( my_result.my_left ) - my_result.my_left_is_final = false; - if( my_right_zombie && my_sum ) - ((*my_sum)->my_body).reverse_join(my_result.my_left_sum->my_body); - __TBB_ASSERT( !my_return_slot, NULL ); - if( my_right_zombie || my_result.my_right ) { - my_return_slot = &my_result; - } else { - destroy( my_result ); - } - if( my_right_zombie && !my_sum && !my_result.my_right ) { - destroy(*my_right_zombie); - my_right_zombie = NULL; - } - return NULL; - } - - finish_scan( sum_node_type*& return_slot_, final_sum_type** sum_, sum_node_type& result_ ) : - my_sum(sum_), - my_return_slot(return_slot_), - my_right_zombie(NULL), - my_result(result_) - { - __TBB_ASSERT( !my_return_slot, NULL ); - } - }; - - //! Initial task to split the work - /** @ingroup algorithms */ - template - class start_scan: public task { - typedef sum_node sum_node_type; - typedef final_sum final_sum_type; - final_sum_type* my_body; - /** Non-null if caller is requesting total. */ - final_sum_type** my_sum; - sum_node_type** my_return_slot; - /** Null if computing root. */ - sum_node_type* my_parent_sum; - bool my_is_final; - bool my_is_right_child; - Range my_range; - typename Partitioner::partition_type my_partition; - task* execute() __TBB_override ; - public: - start_scan( sum_node_type*& return_slot_, start_scan& parent_, sum_node_type* parent_sum_ ) : - my_body(parent_.my_body), - my_sum(parent_.my_sum), - my_return_slot(&return_slot_), - my_parent_sum(parent_sum_), - my_is_final(parent_.my_is_final), - my_is_right_child(false), - my_range(parent_.my_range,split()), - my_partition(parent_.my_partition,split()) - { - __TBB_ASSERT( !*my_return_slot, NULL ); - } - - start_scan( sum_node_type*& return_slot_, const Range& range_, final_sum_type& body_, const Partitioner& partitioner_) : - my_body(&body_), - my_sum(NULL), - my_return_slot(&return_slot_), - my_parent_sum(NULL), - my_is_final(true), - my_is_right_child(false), - my_range(range_), - my_partition(partitioner_) - { - __TBB_ASSERT( !*my_return_slot, NULL ); - } - - static void run( const Range& range_, Body& body_, const Partitioner& partitioner_ ) { - if( !range_.empty() ) { - typedef internal::start_scan start_pass1_type; - internal::sum_node* root = NULL; - final_sum_type* temp_body = new(task::allocate_root()) final_sum_type( body_ ); - start_pass1_type& pass1 = *new(task::allocate_root()) start_pass1_type( - /*my_return_slot=*/root, - range_, - *temp_body, - partitioner_ ); - temp_body->my_body.reverse_join(body_); - task::spawn_root_and_wait( pass1 ); - if( root ) { - root->my_body = temp_body; - root->my_incoming = NULL; - root->my_stuff_last = &body_; - task::spawn_root_and_wait( *root ); - } else { - body_.assign(temp_body->my_body); - temp_body->finish_construction( range_, NULL ); - temp_body->destroy(*temp_body); - } - } - } - }; - - template - task* start_scan::execute() { - typedef internal::finish_scan finish_pass1_type; - finish_pass1_type* p = my_parent_sum ? static_cast( parent() ) : NULL; - // Inspecting p->result.left_sum would ordinarily be a race condition. - // But we inspect it only if we are not a stolen task, in which case we - // know that task assigning to p->result.left_sum has completed. - bool treat_as_stolen = my_is_right_child && (is_stolen_task() || my_body!=p->my_result.my_left_sum); - if( treat_as_stolen ) { - // Invocation is for right child that has been really stolen or needs to be virtually stolen - p->my_right_zombie = my_body = new( allocate_root() ) final_sum_type(my_body->my_body); - my_is_final = false; - } - task* next_task = NULL; - if( (my_is_right_child && !treat_as_stolen) || !my_range.is_divisible() || my_partition.should_execute_range(*this) ) { - if( my_is_final ) - (my_body->my_body)( my_range, final_scan_tag() ); - else if( my_sum ) - (my_body->my_body)( my_range, pre_scan_tag() ); - if( my_sum ) - *my_sum = my_body; - __TBB_ASSERT( !*my_return_slot, NULL ); - } else { - sum_node_type* result; - if( my_parent_sum ) - result = new(allocate_additional_child_of(*my_parent_sum)) sum_node_type(my_range,/*my_left_is_final=*/my_is_final); - else - result = new(task::allocate_root()) sum_node_type(my_range,/*my_left_is_final=*/my_is_final); - finish_pass1_type& c = *new( allocate_continuation()) finish_pass1_type(*my_return_slot,my_sum,*result); - // Split off right child - start_scan& b = *new( c.allocate_child() ) start_scan( /*my_return_slot=*/result->my_right, *this, result ); - b.my_is_right_child = true; - // Left child is recycling of *this. Must recycle this before spawning b, - // otherwise b might complete and decrement c.ref_count() to zero, which - // would cause c.execute() to run prematurely. - recycle_as_child_of(c); - c.set_ref_count(2); - c.spawn(b); - my_sum = &result->my_left_sum; - my_return_slot = &result->my_left; - my_is_right_child = false; - next_task = this; - my_parent_sum = result; - __TBB_ASSERT( !*my_return_slot, NULL ); - } - return next_task; - } - - template - class lambda_scan_body : no_assign { - Value my_sum; - const Value& identity_element; - const Scan& my_scan; - const ReverseJoin& my_reverse_join; - public: - lambda_scan_body( const Value& identity, const Scan& scan, const ReverseJoin& rev_join) - : my_sum(identity) - , identity_element(identity) - , my_scan(scan) - , my_reverse_join(rev_join) {} - - lambda_scan_body( lambda_scan_body& b, split ) - : my_sum(b.identity_element) - , identity_element(b.identity_element) - , my_scan(b.my_scan) - , my_reverse_join(b.my_reverse_join) {} - - template - void operator()( const Range& r, Tag tag ) { - my_sum = my_scan(r, my_sum, tag); - } - - void reverse_join( lambda_scan_body& a ) { - my_sum = my_reverse_join(a.my_sum, my_sum); - } - - void assign( lambda_scan_body& b ) { - my_sum = b.my_sum; - } - - Value result() const { - return my_sum; - } - }; -} // namespace internal -//! @endcond - -// Requirements on Range concept are documented in blocked_range.h - -/** \page parallel_scan_body_req Requirements on parallel_scan body - Class \c Body implementing the concept of parallel_scan body must define: - - \code Body::Body( Body&, split ); \endcode Splitting constructor. - Split \c b so that \c this and \c b can accumulate separately - - \code Body::~Body(); \endcode Destructor - - \code void Body::operator()( const Range& r, pre_scan_tag ); \endcode - Preprocess iterations for range \c r - - \code void Body::operator()( const Range& r, final_scan_tag ); \endcode - Do final processing for iterations of range \c r - - \code void Body::reverse_join( Body& a ); \endcode - Merge preprocessing state of \c a into \c this, where \c a was - created earlier from \c b by b's splitting constructor -**/ - -/** \name parallel_scan - See also requirements on \ref range_req "Range" and \ref parallel_scan_body_req "parallel_scan Body". **/ -//@{ - -//! Parallel prefix with default partitioner -/** @ingroup algorithms **/ -template -void parallel_scan( const Range& range, Body& body ) { - internal::start_scan::run(range,body,__TBB_DEFAULT_PARTITIONER()); -} - -//! Parallel prefix with simple_partitioner -/** @ingroup algorithms **/ -template -void parallel_scan( const Range& range, Body& body, const simple_partitioner& partitioner ) { - internal::start_scan::run(range,body,partitioner); -} - -//! Parallel prefix with auto_partitioner -/** @ingroup algorithms **/ -template -void parallel_scan( const Range& range, Body& body, const auto_partitioner& partitioner ) { - internal::start_scan::run(range,body,partitioner); -} - -//! Parallel prefix with default partitioner -/** @ingroup algorithms **/ -template -Value parallel_scan( const Range& range, const Value& identity, const Scan& scan, const ReverseJoin& reverse_join ) { - internal::lambda_scan_body body(identity, scan, reverse_join); - tbb::parallel_scan(range,body,__TBB_DEFAULT_PARTITIONER()); - return body.result(); -} - -//! Parallel prefix with simple_partitioner -/** @ingroup algorithms **/ -template -Value parallel_scan( const Range& range, const Value& identity, const Scan& scan, const ReverseJoin& reverse_join, const simple_partitioner& partitioner ) { - internal::lambda_scan_body body(identity, scan, reverse_join); - tbb::parallel_scan(range,body,partitioner); - return body.result(); -} - -//! Parallel prefix with auto_partitioner -/** @ingroup algorithms **/ -template -Value parallel_scan( const Range& range, const Value& identity, const Scan& scan, const ReverseJoin& reverse_join, const auto_partitioner& partitioner ) { - internal::lambda_scan_body body(identity, scan, reverse_join); - tbb::parallel_scan(range,body,partitioner); - return body.result(); -} - -//@} - -} // namespace tbb - -#endif /* __TBB_parallel_scan_H */ - +#include "../oneapi/tbb/parallel_scan.h" diff --git a/src/tbb/include/tbb/parallel_sort.h b/src/tbb/include/tbb/parallel_sort.h index 07d3907b1..b238e6caa 100644 --- a/src/tbb/include/tbb/parallel_sort.h +++ b/src/tbb/include/tbb/parallel_sort.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,255 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ -#ifndef __TBB_parallel_sort_H -#define __TBB_parallel_sort_H - -#include "parallel_for.h" -#include "blocked_range.h" -#include "internal/_range_iterator.h" -#include -#include -#include - -namespace tbb { - -namespace interface9 { -//! @cond INTERNAL -namespace internal { - -using tbb::internal::no_assign; - -//! Range used in quicksort to split elements into subranges based on a value. -/** The split operation selects a splitter and places all elements less than or equal - to the value in the first range and the remaining elements in the second range. - @ingroup algorithms */ -template -class quick_sort_range: private no_assign { - - inline size_t median_of_three(const RandomAccessIterator &array, size_t l, size_t m, size_t r) const { - return comp(array[l], array[m]) ? ( comp(array[m], array[r]) ? m : ( comp( array[l], array[r]) ? r : l ) ) - : ( comp(array[r], array[m]) ? m : ( comp( array[r], array[l] ) ? r : l ) ); - } - - inline size_t pseudo_median_of_nine( const RandomAccessIterator &array, const quick_sort_range &range ) const { - size_t offset = range.size/8u; - return median_of_three(array, - median_of_three(array, 0, offset, offset*2), - median_of_three(array, offset*3, offset*4, offset*5), - median_of_three(array, offset*6, offset*7, range.size - 1) ); - - } - - size_t split_range( quick_sort_range& range ) { - using std::iter_swap; - RandomAccessIterator array = range.begin; - RandomAccessIterator key0 = range.begin; - size_t m = pseudo_median_of_nine(array, range); - if (m) iter_swap ( array, array+m ); - - size_t i=0; - size_t j=range.size; - // Partition interval [i+1,j-1] with key *key0. - for(;;) { - __TBB_ASSERT( i=grainsize;} - - quick_sort_range( quick_sort_range& range, split ) - : comp(range.comp) - , size(split_range(range)) - // +1 accounts for the pivot element, which is at its correct place - // already and, therefore, is not included into subranges. - , begin(range.begin+range.size+1) {} -}; - -#if __TBB_TASK_GROUP_CONTEXT -//! Body class used to test if elements in a range are presorted -/** @ingroup algorithms */ -template -class quick_sort_pretest_body : no_assign { - const Compare ∁ - -public: - quick_sort_pretest_body(const Compare &_comp) : comp(_comp) {} - - void operator()( const blocked_range& range ) const { - task &my_task = task::self(); - RandomAccessIterator my_end = range.end(); - - int i = 0; - for (RandomAccessIterator k = range.begin(); k != my_end; ++k, ++i) { - if ( i%64 == 0 && my_task.is_cancelled() ) break; - - // The k-1 is never out-of-range because the first chunk starts at begin+serial_cutoff+1 - if ( comp( *(k), *(k-1) ) ) { - my_task.cancel_group_execution(); - break; - } - } - } - -}; -#endif /* __TBB_TASK_GROUP_CONTEXT */ - -//! Body class used to sort elements in a range that is smaller than the grainsize. -/** @ingroup algorithms */ -template -struct quick_sort_body { - void operator()( const quick_sort_range& range ) const { - //SerialQuickSort( range.begin, range.size, range.comp ); - std::sort( range.begin, range.begin + range.size, range.comp ); - } -}; - -//! Wrapper method to initiate the sort by calling parallel_for. -/** @ingroup algorithms */ -template -void parallel_quick_sort( RandomAccessIterator begin, RandomAccessIterator end, const Compare& comp ) { -#if __TBB_TASK_GROUP_CONTEXT - task_group_context my_context; - const int serial_cutoff = 9; - - __TBB_ASSERT( begin + serial_cutoff < end, "min_parallel_size is smaller than serial cutoff?" ); - RandomAccessIterator k = begin; - for ( ; k != begin + serial_cutoff; ++k ) { - if ( comp( *(k+1), *k ) ) { - goto do_parallel_quick_sort; - } - } - - parallel_for( blocked_range(k+1, end), - quick_sort_pretest_body(comp), - auto_partitioner(), - my_context); - - if (my_context.is_group_execution_cancelled()) -do_parallel_quick_sort: -#endif /* __TBB_TASK_GROUP_CONTEXT */ - parallel_for( quick_sort_range(begin, end-begin, comp ), - quick_sort_body(), - auto_partitioner() ); -} - -} // namespace internal -//! @endcond -} // namespace interfaceX - -/** \page parallel_sort_iter_req Requirements on iterators for parallel_sort - Requirements on the iterator type \c It and its value type \c T for \c parallel_sort: - - - \code void iter_swap( It a, It b ) \endcode Swaps the values of the elements the given - iterators \c a and \c b are pointing to. \c It should be a random access iterator. - - - \code bool Compare::operator()( const T& x, const T& y ) \endcode True if x comes before y; -**/ - -/** \name parallel_sort - See also requirements on \ref parallel_sort_iter_req "iterators for parallel_sort". **/ -//@{ - -//! Sorts the data in [begin,end) using the given comparator -/** The compare function object is used for all comparisons between elements during sorting. - The compare object must define a bool operator() function. - @ingroup algorithms **/ -template -void parallel_sort( RandomAccessIterator begin, RandomAccessIterator end, const Compare& comp) { - const int min_parallel_size = 500; - if( end > begin ) { - if (end - begin < min_parallel_size) { - std::sort(begin, end, comp); - } else { - interface9::internal::parallel_quick_sort(begin, end, comp); - } - } -} - -//! Sorts the data in [begin,end) with a default comparator \c std::less -/** @ingroup algorithms **/ -template -inline void parallel_sort( RandomAccessIterator begin, RandomAccessIterator end ) { - parallel_sort( begin, end, std::less< typename std::iterator_traits::value_type >() ); -} - -//! Sorts the data in rng using the given comparator -/** @ingroup algorithms **/ -template -void parallel_sort(Range& rng, const Compare& comp) { - parallel_sort(tbb::internal::first(rng), tbb::internal::last(rng), comp); -} - -//! Sorts the data in const rng using the given comparator -/** @ingroup algorithms **/ -template -void parallel_sort(const Range& rng, const Compare& comp) { - parallel_sort(tbb::internal::first(rng), tbb::internal::last(rng), comp); -} - -//! Sorts the data in rng with a default comparator \c std::less -/** @ingroup algorithms **/ -template -void parallel_sort(Range& rng) { - parallel_sort(tbb::internal::first(rng), tbb::internal::last(rng)); -} - -//! Sorts the data in const rng with a default comparator \c std::less -/** @ingroup algorithms **/ -template -void parallel_sort(const Range& rng) { - parallel_sort(tbb::internal::first(rng), tbb::internal::last(rng)); -} - -//! Sorts the data in the range \c [begin,end) with a default comparator \c std::less -/** @ingroup algorithms **/ -template -inline void parallel_sort( T * begin, T * end ) { - parallel_sort( begin, end, std::less< T >() ); -} -//@} - - -} // namespace tbb - -#endif - +#include "../oneapi/tbb/parallel_sort.h" diff --git a/src/tbb/include/tbb/parallel_while.h b/src/tbb/include/tbb/parallel_while.h deleted file mode 100644 index 2f37a41a4..000000000 --- a/src/tbb/include/tbb/parallel_while.h +++ /dev/null @@ -1,186 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_parallel_while -#define __TBB_parallel_while - -#include "task.h" -#include - -namespace tbb { - -template -class parallel_while; - -//! @cond INTERNAL -namespace internal { - - template class while_task; - - //! For internal use only. - /** Executes one iteration of a while. - @ingroup algorithms */ - template - class while_iteration_task: public task { - const Body& my_body; - typename Body::argument_type my_value; - task* execute() __TBB_override { - my_body(my_value); - return NULL; - } - while_iteration_task( const typename Body::argument_type& value, const Body& body ) : - my_body(body), my_value(value) - {} - template friend class while_group_task; - friend class tbb::parallel_while; - }; - - //! For internal use only - /** Unpacks a block of iterations. - @ingroup algorithms */ - template - class while_group_task: public task { - static const size_t max_arg_size = 4; - const Body& my_body; - size_t size; - typename Body::argument_type my_arg[max_arg_size]; - while_group_task( const Body& body ) : my_body(body), size(0) {} - task* execute() __TBB_override { - typedef while_iteration_task iteration_type; - __TBB_ASSERT( size>0, NULL ); - task_list list; - task* t; - size_t k=0; - for(;;) { - t = new( allocate_child() ) iteration_type(my_arg[k],my_body); - if( ++k==size ) break; - list.push_back(*t); - } - set_ref_count(int(k+1)); - spawn(list); - spawn_and_wait_for_all(*t); - return NULL; - } - template friend class while_task; - }; - - //! For internal use only. - /** Gets block of iterations from a stream and packages them into a while_group_task. - @ingroup algorithms */ - template - class while_task: public task { - Stream& my_stream; - const Body& my_body; - empty_task& my_barrier; - task* execute() __TBB_override { - typedef while_group_task block_type; - block_type& t = *new( allocate_additional_child_of(my_barrier) ) block_type(my_body); - size_t k=0; - while( my_stream.pop_if_present(t.my_arg[k]) ) { - if( ++k==block_type::max_arg_size ) { - // There might be more iterations. - recycle_to_reexecute(); - break; - } - } - if( k==0 ) { - destroy(t); - return NULL; - } else { - t.size = k; - return &t; - } - } - while_task( Stream& stream, const Body& body, empty_task& barrier ) : - my_stream(stream), - my_body(body), - my_barrier(barrier) - {} - friend class tbb::parallel_while; - }; - -} // namespace internal -//! @endcond - -//! Parallel iteration over a stream, with optional addition of more work. -/** The Body b has the requirement: \n - "b(v)" \n - "b.argument_type" \n - where v is an argument_type - @ingroup algorithms */ -template -class parallel_while: internal::no_copy { -public: - //! Construct empty non-running parallel while. - parallel_while() : my_body(NULL), my_barrier(NULL) {} - - //! Destructor cleans up data members before returning. - ~parallel_while() { - if( my_barrier ) { - my_barrier->destroy(*my_barrier); - my_barrier = NULL; - } - } - - //! Type of items - typedef typename Body::argument_type value_type; - - //! Apply body.apply to each item in the stream. - /** A Stream s has the requirements \n - "S::value_type" \n - "s.pop_if_present(value) is convertible to bool */ - template - void run( Stream& stream, const Body& body ); - - //! Add a work item while running. - /** Should be executed only by body.apply or a thread spawned therefrom. */ - void add( const value_type& item ); - -private: - const Body* my_body; - empty_task* my_barrier; -}; - -template -template -void parallel_while::run( Stream& stream, const Body& body ) { - using namespace internal; - empty_task& barrier = *new( task::allocate_root() ) empty_task(); - my_body = &body; - my_barrier = &barrier; - my_barrier->set_ref_count(2); - while_task& w = *new( my_barrier->allocate_child() ) while_task( stream, body, barrier ); - my_barrier->spawn_and_wait_for_all(w); - my_barrier->destroy(*my_barrier); - my_barrier = NULL; - my_body = NULL; -} - -template -void parallel_while::add( const value_type& item ) { - __TBB_ASSERT(my_barrier,"attempt to add to parallel_while that is not running"); - typedef internal::while_iteration_task iteration_type; - iteration_type& i = *new( task::allocate_additional_child_of(*my_barrier) ) iteration_type(item,*my_body); - task::self().spawn( i ); -} - -} // namespace - -#endif /* __TBB_parallel_while */ diff --git a/src/tbb/include/tbb/partitioner.h b/src/tbb/include/tbb/partitioner.h index d7ebdbfed..b959e35a2 100644 --- a/src/tbb/include/tbb/partitioner.h +++ b/src/tbb/include/tbb/partitioner.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,672 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - -*/ - -#ifndef __TBB_partitioner_H -#define __TBB_partitioner_H - -#ifndef __TBB_INITIAL_CHUNKS -// initial task divisions per thread -#define __TBB_INITIAL_CHUNKS 2 -#endif -#ifndef __TBB_RANGE_POOL_CAPACITY -// maximum number of elements in range pool -#define __TBB_RANGE_POOL_CAPACITY 8 -#endif -#ifndef __TBB_INIT_DEPTH -// initial value for depth of range pool -#define __TBB_INIT_DEPTH 5 -#endif -#ifndef __TBB_DEMAND_DEPTH_ADD -// when imbalance is found range splits this value times more -#define __TBB_DEMAND_DEPTH_ADD 1 -#endif -#ifndef __TBB_STATIC_THRESHOLD -// necessary number of clocks for the work to be distributed among all tasks -#define __TBB_STATIC_THRESHOLD 40000 -#endif -#if __TBB_DEFINE_MIC -#define __TBB_NONUNIFORM_TASK_CREATION 1 -#ifdef __TBB_time_stamp -#define __TBB_USE_MACHINE_TIME_STAMPS 1 -#define __TBB_task_duration() __TBB_STATIC_THRESHOLD -#endif // __TBB_machine_time_stamp -#endif // __TBB_DEFINE_MIC - -#include "task.h" -#include "aligned_space.h" -#include "atomic.h" -#include "internal/_template_helpers.h" - -#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) - // Workaround for overzealous compiler warnings - #pragma warning (push) - #pragma warning (disable: 4244) -#endif - -namespace tbb { - -class auto_partitioner; -class simple_partitioner; -class static_partitioner; -class affinity_partitioner; - -namespace interface9 { - namespace internal { - class affinity_partition_type; - } -} - -namespace internal { //< @cond INTERNAL -size_t __TBB_EXPORTED_FUNC get_initial_auto_partitioner_divisor(); - -//! Defines entry point for affinity partitioner into tbb run-time library. -class affinity_partitioner_base_v3: no_copy { - friend class tbb::affinity_partitioner; - friend class tbb::interface9::internal::affinity_partition_type; - //! Array that remembers affinities of tree positions to affinity_id. - /** NULL if my_size==0. */ - affinity_id* my_array; - //! Number of elements in my_array. - size_t my_size; - //! Zeros the fields. - affinity_partitioner_base_v3() : my_array(NULL), my_size(0) {} - //! Deallocates my_array. - ~affinity_partitioner_base_v3() {resize(0);} - //! Resize my_array. - /** Retains values if resulting size is the same. */ - void __TBB_EXPORTED_METHOD resize( unsigned factor ); -}; - -//! Provides backward-compatible methods for partition objects without affinity. -class partition_type_base { -public: - void set_affinity( task & ) {} - void note_affinity( task::affinity_id ) {} - task* continue_after_execute_range() {return NULL;} - bool decide_whether_to_delay() {return false;} - void spawn_or_delay( bool, task& b ) { - task::spawn(b); - } -}; - -template class start_scan; - -} //< namespace internal @endcond - -namespace serial { -namespace interface9 { -template class start_for; -} -} - -namespace interface9 { -//! @cond INTERNAL -namespace internal { -using namespace tbb::internal; -template class start_for; -template class start_reduce; -template class start_deterministic_reduce; - -//! Join task node that contains shared flag for stealing feedback -class flag_task: public task { -public: - tbb::atomic my_child_stolen; - flag_task() { my_child_stolen = false; } - task* execute() __TBB_override { return NULL; } - static void mark_task_stolen(task &t) { - tbb::atomic &flag = static_cast(t.parent())->my_child_stolen; -#if TBB_USE_THREADING_TOOLS - // Threading tools respect lock prefix but report false-positive data-race via plain store - flag.fetch_and_store(true); -#else - flag = true; -#endif //TBB_USE_THREADING_TOOLS - } - static bool is_peer_stolen(task &t) { - return static_cast(t.parent())->my_child_stolen; - } -}; - -//! Depth is a relative depth of recursive division inside a range pool. Relative depth allows -//! infinite absolute depth of the recursion for heavily unbalanced workloads with range represented -//! by a number that cannot fit into machine word. -typedef unsigned char depth_t; - -//! Range pool stores ranges of type T in a circular buffer with MaxCapacity -template -class range_vector { - depth_t my_head; - depth_t my_tail; - depth_t my_size; - depth_t my_depth[MaxCapacity]; // relative depths of stored ranges - tbb::aligned_space my_pool; - -public: - //! initialize via first range in pool - range_vector(const T& elem) : my_head(0), my_tail(0), my_size(1) { - my_depth[0] = 0; - new( static_cast(my_pool.begin()) ) T(elem);//TODO: std::move? - } - ~range_vector() { - while( !empty() ) pop_back(); - } - bool empty() const { return my_size == 0; } - depth_t size() const { return my_size; } - //! Populates range pool via ranges up to max depth or while divisible - //! max_depth starts from 0, e.g. value 2 makes 3 ranges in the pool up to two 1/4 pieces - void split_to_fill(depth_t max_depth) { - while( my_size < MaxCapacity && is_divisible(max_depth) ) { - depth_t prev = my_head; - my_head = (my_head + 1) % MaxCapacity; - new(my_pool.begin()+my_head) T(my_pool.begin()[prev]); // copy TODO: std::move? - my_pool.begin()[prev].~T(); // instead of assignment - new(my_pool.begin()+prev) T(my_pool.begin()[my_head], split()); // do 'inverse' split - my_depth[my_head] = ++my_depth[prev]; - my_size++; - } - } - void pop_back() { - __TBB_ASSERT(my_size > 0, "range_vector::pop_back() with empty size"); - my_pool.begin()[my_head].~T(); - my_size--; - my_head = (my_head + MaxCapacity - 1) % MaxCapacity; - } - void pop_front() { - __TBB_ASSERT(my_size > 0, "range_vector::pop_front() with empty size"); - my_pool.begin()[my_tail].~T(); - my_size--; - my_tail = (my_tail + 1) % MaxCapacity; - } - T& back() { - __TBB_ASSERT(my_size > 0, "range_vector::back() with empty size"); - return my_pool.begin()[my_head]; - } - T& front() { - __TBB_ASSERT(my_size > 0, "range_vector::front() with empty size"); - return my_pool.begin()[my_tail]; - } - //! similarly to front(), returns depth of the first range in the pool - depth_t front_depth() { - __TBB_ASSERT(my_size > 0, "range_vector::front_depth() with empty size"); - return my_depth[my_tail]; - } - depth_t back_depth() { - __TBB_ASSERT(my_size > 0, "range_vector::back_depth() with empty size"); - return my_depth[my_head]; - } - bool is_divisible(depth_t max_depth) { - return back_depth() < max_depth && back().is_divisible(); - } -}; - -//! Provides default methods for partition objects and common algorithm blocks. -template -struct partition_type_base { - typedef split split_type; - // decision makers - void set_affinity( task & ) {} - void note_affinity( task::affinity_id ) {} - bool check_being_stolen(task &) { return false; } // part of old should_execute_range() - bool check_for_demand(task &) { return false; } - bool is_divisible() { return true; } // part of old should_execute_range() - depth_t max_depth() { return 0; } - void align_depth(depth_t) { } - template split_type get_split() { return split(); } - Partition& self() { return *static_cast(this); } // CRTP helper - - template - void work_balance(StartType &start, Range &range) { - start.run_body( range ); // simple partitioner goes always here - } - - template - void execute(StartType &start, Range &range) { - // The algorithm in a few words ([]-denotes calls to decision methods of partitioner): - // [If this task is stolen, adjust depth and divisions if necessary, set flag]. - // If range is divisible { - // Spread the work while [initial divisions left]; - // Create trap task [if necessary]; - // } - // If not divisible or [max depth is reached], execute, else do the range pool part - if ( range.is_divisible() ) { - if ( self().is_divisible() ) { - do { // split until is divisible - typename Partition::split_type split_obj = self().template get_split(); - start.offer_work( split_obj ); - } while ( range.is_divisible() && self().is_divisible() ); - } - } - self().work_balance(start, range); - } -}; - -//! Class determines whether template parameter has static boolean constant -//! 'is_splittable_in_proportion' initialized with value of 'true' or not. -/** If template parameter has such field that has been initialized with non-zero -* value then class field will be set to 'true', otherwise - 'false' */ -template -class is_splittable_in_proportion { -private: - typedef char yes[1]; - typedef char no[2]; - - template static yes& decide(typename enable_if::type *); - template static no& decide(...); -public: - // equals to 'true' if and only if static const variable 'is_splittable_in_proportion' of template parameter - // initialized with the value of 'true' - static const bool value = (sizeof(decide(0)) == sizeof(yes)); -}; - -//! Provides default splitting strategy for partition objects. -template -struct adaptive_mode : partition_type_base { - typedef Partition my_partition; - size_t my_divisor; - // For affinity_partitioner, my_divisor indicates the number of affinity array indices the task reserves. - // A task which has only one index must produce the right split without reserved index in order to avoid - // it to be overwritten in note_affinity() of the created (right) task. - // I.e. a task created deeper than the affinity array can remember must not save its affinity (LIFO order) - static const unsigned factor = 1; - adaptive_mode() : my_divisor(tbb::internal::get_initial_auto_partitioner_divisor() / 4 * my_partition::factor) {} - adaptive_mode(adaptive_mode &src, split) : my_divisor(do_split(src, split())) {} - /*! Override do_split methods in order to specify splitting strategy */ - size_t do_split(adaptive_mode &src, split) { - return src.my_divisor /= 2u; - } -}; - -//! Provides proportional splitting strategy for partition objects -template -struct proportional_mode : adaptive_mode { - typedef Partition my_partition; - using partition_type_base::self; // CRTP helper to get access to derived classes - - proportional_mode() : adaptive_mode() {} - proportional_mode(proportional_mode &src, split) : adaptive_mode(src, split()) {} - proportional_mode(proportional_mode &src, const proportional_split& split_obj) { self().my_divisor = do_split(src, split_obj); } - size_t do_split(proportional_mode &src, const proportional_split& split_obj) { -#if __TBB_ENABLE_RANGE_FEEDBACK - size_t portion = size_t(float(src.my_divisor) * float(split_obj.right()) - / float(split_obj.left() + split_obj.right()) + 0.5f); -#else - size_t portion = split_obj.right() * my_partition::factor; -#endif - portion = (portion + my_partition::factor/2) & (0ul - my_partition::factor); -#if __TBB_ENABLE_RANGE_FEEDBACK - /** Corner case handling */ - if (!portion) - portion = my_partition::factor; - else if (portion == src.my_divisor) - portion = src.my_divisor - my_partition::factor; -#endif - src.my_divisor -= portion; - return portion; - } - bool is_divisible() { // part of old should_execute_range() - return self().my_divisor > my_partition::factor; - } -#if _MSC_VER && !defined(__INTEL_COMPILER) - // Suppress "conditional expression is constant" warning. - #pragma warning( push ) - #pragma warning( disable: 4127 ) -#endif - template - proportional_split get_split() { - if (is_splittable_in_proportion::value) { - size_t size = self().my_divisor / my_partition::factor; -#if __TBB_NONUNIFORM_TASK_CREATION - size_t right = (size + 2) / 3; -#else - size_t right = size / 2; -#endif - size_t left = size - right; - return proportional_split(left, right); - } else { - return proportional_split(1, 1); - } - } -#if _MSC_VER && !defined(__INTEL_COMPILER) - #pragma warning( pop ) -#endif // warning 4127 is back -}; - -//! Provides default linear indexing of partitioner's sequence -template -struct linear_affinity_mode : proportional_mode { - size_t my_head; - using proportional_mode::self; - linear_affinity_mode() : proportional_mode(), my_head(0) {} - linear_affinity_mode(linear_affinity_mode &src, split) : proportional_mode(src, split()) - , my_head(src.my_head + src.my_divisor) {} - linear_affinity_mode(linear_affinity_mode &src, const proportional_split& split_obj) : proportional_mode(src, split_obj) - , my_head(src.my_head + src.my_divisor) {} - void set_affinity( task &t ) { - if( self().my_divisor ) - t.set_affinity( affinity_id(my_head) + 1 ); - } -}; - -/*! Determine work-balance phase implementing splitting & stealing actions */ -template -struct dynamic_grainsize_mode : Mode { - using Mode::self; -#ifdef __TBB_USE_MACHINE_TIME_STAMPS - tbb::internal::machine_tsc_t my_dst_tsc; -#endif - enum { - begin = 0, - run, - pass - } my_delay; - depth_t my_max_depth; - static const unsigned range_pool_size = __TBB_RANGE_POOL_CAPACITY; - dynamic_grainsize_mode(): Mode() -#ifdef __TBB_USE_MACHINE_TIME_STAMPS - , my_dst_tsc(0) -#endif - , my_delay(begin) - , my_max_depth(__TBB_INIT_DEPTH) {} - dynamic_grainsize_mode(dynamic_grainsize_mode& p, split) - : Mode(p, split()) -#ifdef __TBB_USE_MACHINE_TIME_STAMPS - , my_dst_tsc(0) -#endif - , my_delay(pass) - , my_max_depth(p.my_max_depth) {} - dynamic_grainsize_mode(dynamic_grainsize_mode& p, const proportional_split& split_obj) - : Mode(p, split_obj) -#ifdef __TBB_USE_MACHINE_TIME_STAMPS - , my_dst_tsc(0) -#endif - , my_delay(begin) - , my_max_depth(p.my_max_depth) {} - bool check_being_stolen( task &t) { // part of old should_execute_range() - if( !(self().my_divisor / Mode::my_partition::factor) ) { // if not from the top P tasks of binary tree - self().my_divisor = 1; // TODO: replace by on-stack flag (partition_state's member)? - if( t.is_stolen_task() && t.parent()->ref_count() >= 2 ) { // runs concurrently with the left task -#if __TBB_USE_OPTIONAL_RTTI - // RTTI is available, check whether the cast is valid - __TBB_ASSERT(dynamic_cast(t.parent()), 0); - // correctness of the cast relies on avoiding the root task for which: - // - initial value of my_divisor != 0 (protected by separate assertion) - // - is_stolen_task() always returns false for the root task. -#endif - flag_task::mark_task_stolen(t); - if( !my_max_depth ) my_max_depth++; - my_max_depth += __TBB_DEMAND_DEPTH_ADD; - return true; - } - } - return false; - } - depth_t max_depth() { return my_max_depth; } - void align_depth(depth_t base) { - __TBB_ASSERT(base <= my_max_depth, 0); - my_max_depth -= base; - } - template - void work_balance(StartType &start, Range &range) { - if( !range.is_divisible() || !self().max_depth() ) { - start.run_body( range ); // simple partitioner goes always here - } - else { // do range pool - internal::range_vector range_pool(range); - do { - range_pool.split_to_fill(self().max_depth()); // fill range pool - if( self().check_for_demand( start ) ) { - if( range_pool.size() > 1 ) { - start.offer_work( range_pool.front(), range_pool.front_depth() ); - range_pool.pop_front(); - continue; - } - if( range_pool.is_divisible(self().max_depth()) ) // was not enough depth to fork a task - continue; // note: next split_to_fill() should split range at least once - } - start.run_body( range_pool.back() ); - range_pool.pop_back(); - } while( !range_pool.empty() && !start.is_cancelled() ); - } - } - bool check_for_demand( task &t ) { - if( pass == my_delay ) { - if( self().my_divisor > 1 ) // produce affinitized tasks while they have slot in array - return true; // do not do my_max_depth++ here, but be sure range_pool is splittable once more - else if( self().my_divisor && my_max_depth ) { // make balancing task - self().my_divisor = 0; // once for each task; depth will be decreased in align_depth() - return true; - } - else if( flag_task::is_peer_stolen(t) ) { - my_max_depth += __TBB_DEMAND_DEPTH_ADD; - return true; - } - } else if( begin == my_delay ) { -#ifndef __TBB_USE_MACHINE_TIME_STAMPS - my_delay = pass; -#else - my_dst_tsc = __TBB_time_stamp() + __TBB_task_duration(); - my_delay = run; - } else if( run == my_delay ) { - if( __TBB_time_stamp() < my_dst_tsc ) { - __TBB_ASSERT(my_max_depth > 0, NULL); - my_max_depth--; // increase granularity since tasks seem having too small work - return false; - } - my_delay = pass; - return true; -#endif // __TBB_USE_MACHINE_TIME_STAMPS - } - return false; - } -}; - -class auto_partition_type: public dynamic_grainsize_mode > { -public: - auto_partition_type( const auto_partitioner& ) - : dynamic_grainsize_mode >() { - my_divisor *= __TBB_INITIAL_CHUNKS; - } - auto_partition_type( auto_partition_type& src, split) - : dynamic_grainsize_mode >(src, split()) {} - bool is_divisible() { // part of old should_execute_range() - if( my_divisor > 1 ) return true; - if( my_divisor && my_max_depth ) { // can split the task. TODO: on-stack flag instead - // keep same fragmentation while splitting for the local task pool - my_max_depth--; - my_divisor = 0; // decrease max_depth once per task - return true; - } else return false; - } - bool check_for_demand(task &t) { - if( flag_task::is_peer_stolen(t) ) { - my_max_depth += __TBB_DEMAND_DEPTH_ADD; - return true; - } else return false; - } -}; - -class simple_partition_type: public partition_type_base { -public: - simple_partition_type( const simple_partitioner& ) {} - simple_partition_type( const simple_partition_type&, split ) {} - //! simplified algorithm - template - void execute(StartType &start, Range &range) { - split_type split_obj = split(); // start.offer_work accepts split_type as reference - while( range.is_divisible() ) - start.offer_work( split_obj ); - start.run_body( range ); - } -}; - -class static_partition_type : public linear_affinity_mode { -public: - typedef proportional_split split_type; - static_partition_type( const static_partitioner& ) - : linear_affinity_mode() {} - static_partition_type( static_partition_type& p, split ) - : linear_affinity_mode(p, split()) {} - static_partition_type( static_partition_type& p, const proportional_split& split_obj ) - : linear_affinity_mode(p, split_obj) {} -}; - -class affinity_partition_type : public dynamic_grainsize_mode > { - static const unsigned factor_power = 4; // TODO: get a unified formula based on number of computing units - tbb::internal::affinity_id* my_array; -public: - static const unsigned factor = 1 << factor_power; // number of slots in affinity array per task - typedef proportional_split split_type; - affinity_partition_type( tbb::internal::affinity_partitioner_base_v3& ap ) - : dynamic_grainsize_mode >() { - __TBB_ASSERT( (factor&(factor-1))==0, "factor must be power of two" ); - ap.resize(factor); - my_array = ap.my_array; - my_max_depth = factor_power + 1; - __TBB_ASSERT( my_max_depth < __TBB_RANGE_POOL_CAPACITY, 0 ); - } - affinity_partition_type(affinity_partition_type& p, split) - : dynamic_grainsize_mode >(p, split()) - , my_array(p.my_array) {} - affinity_partition_type(affinity_partition_type& p, const proportional_split& split_obj) - : dynamic_grainsize_mode >(p, split_obj) - , my_array(p.my_array) {} - void set_affinity( task &t ) { - if( my_divisor ) { - if( !my_array[my_head] ) - // TODO: consider new ideas with my_array for both affinity and static partitioner's, then code reuse - t.set_affinity( affinity_id(my_head / factor + 1) ); - else - t.set_affinity( my_array[my_head] ); - } - } - void note_affinity( task::affinity_id id ) { - if( my_divisor ) - my_array[my_head] = id; - } -}; - -//! Backward-compatible partition for auto and affinity partition objects. -class old_auto_partition_type: public tbb::internal::partition_type_base { - size_t num_chunks; - static const size_t VICTIM_CHUNKS = 4; -public: - bool should_execute_range(const task &t) { - if( num_chunks friend class serial::interface9::start_for; - template friend class interface9::internal::start_for; - template friend class interface9::internal::start_reduce; - template friend class interface9::internal::start_deterministic_reduce; - template friend class internal::start_scan; - // backward compatibility - class partition_type: public internal::partition_type_base { - public: - bool should_execute_range(const task& ) {return false;} - partition_type( const simple_partitioner& ) {} - partition_type( const partition_type&, split ) {} - }; - // new implementation just extends existing interface - typedef interface9::internal::simple_partition_type task_partition_type; - - // TODO: consider to make split_type public - typedef interface9::internal::simple_partition_type::split_type split_type; -}; - -//! An auto partitioner -/** The range is initial divided into several large chunks. - Chunks are further subdivided into smaller pieces if demand detected and they are divisible. - @ingroup algorithms */ -class auto_partitioner { -public: - auto_partitioner() {} - -private: - template friend class serial::interface9::start_for; - template friend class interface9::internal::start_for; - template friend class interface9::internal::start_reduce; - template friend class internal::start_scan; - // backward compatibility - typedef interface9::internal::old_auto_partition_type partition_type; - // new implementation just extends existing interface - typedef interface9::internal::auto_partition_type task_partition_type; - - // TODO: consider to make split_type public - typedef interface9::internal::auto_partition_type::split_type split_type; -}; - -//! A static partitioner -class static_partitioner { -public: - static_partitioner() {} -private: - template friend class serial::interface9::start_for; - template friend class interface9::internal::start_for; - template friend class interface9::internal::start_reduce; - template friend class interface9::internal::start_deterministic_reduce; - template friend class internal::start_scan; - // backward compatibility - typedef interface9::internal::old_auto_partition_type partition_type; - // new implementation just extends existing interface - typedef interface9::internal::static_partition_type task_partition_type; - - // TODO: consider to make split_type public - typedef interface9::internal::static_partition_type::split_type split_type; -}; - -//! An affinity partitioner -class affinity_partitioner: internal::affinity_partitioner_base_v3 { -public: - affinity_partitioner() {} - -private: - template friend class serial::interface9::start_for; - template friend class interface9::internal::start_for; - template friend class interface9::internal::start_reduce; - template friend class internal::start_scan; - // backward compatibility - for parallel_scan only - typedef interface9::internal::old_auto_partition_type partition_type; - // new implementation just extends existing interface - typedef interface9::internal::affinity_partition_type task_partition_type; - - // TODO: consider to make split_type public - typedef interface9::internal::affinity_partition_type::split_type split_type; -}; - -} // namespace tbb -#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) - #pragma warning (pop) -#endif // warning 4244 is back -#undef __TBB_INITIAL_CHUNKS -#undef __TBB_RANGE_POOL_CAPACITY -#undef __TBB_INIT_DEPTH -#endif /* __TBB_partitioner_H */ +#include "../oneapi/tbb/partitioner.h" diff --git a/src/tbb/include/tbb/pipeline.h b/src/tbb/include/tbb/pipeline.h deleted file mode 100644 index 20a8ec9bb..000000000 --- a/src/tbb/include/tbb/pipeline.h +++ /dev/null @@ -1,665 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_pipeline_H -#define __TBB_pipeline_H - -#include "atomic.h" -#include "task.h" -#include "tbb_allocator.h" -#include - -#if __TBB_CPP11_TYPE_PROPERTIES_PRESENT || __TBB_TR1_TYPE_PROPERTIES_IN_STD_PRESENT -#include -#endif - -namespace tbb { - -class pipeline; -class filter; - -//! @cond INTERNAL -namespace internal { - -// The argument for PIPELINE_VERSION should be an integer between 2 and 9 -#define __TBB_PIPELINE_VERSION(x) ((unsigned char)(x-2)<<1) - -typedef unsigned long Token; -typedef long tokendiff_t; -class stage_task; -class input_buffer; -class pipeline_root_task; -class pipeline_cleaner; - -} // namespace internal - -namespace interface6 { - template class filter_t; - - namespace internal { - class pipeline_proxy; - } -} - -//! @endcond - -//! A stage in a pipeline. -/** @ingroup algorithms */ -class filter: internal::no_copy { -private: - //! Value used to mark "not in pipeline" - static filter* not_in_pipeline() {return reinterpret_cast(intptr_t(-1));} -protected: - //! The lowest bit 0 is for parallel vs. serial - static const unsigned char filter_is_serial = 0x1; - - //! 4th bit distinguishes ordered vs unordered filters. - /** The bit was not set for parallel filters in TBB 2.1 and earlier, - but is_ordered() function always treats parallel filters as out of order. */ - static const unsigned char filter_is_out_of_order = 0x1<<4; - - //! 5th bit distinguishes thread-bound and regular filters. - static const unsigned char filter_is_bound = 0x1<<5; - - //! 6th bit marks input filters emitting small objects - static const unsigned char filter_may_emit_null = 0x1<<6; - - //! 7th bit defines exception propagation mode expected by the application. - static const unsigned char exact_exception_propagation = -#if TBB_USE_CAPTURED_EXCEPTION - 0x0; -#else - 0x1<<7; -#endif /* TBB_USE_CAPTURED_EXCEPTION */ - - static const unsigned char current_version = __TBB_PIPELINE_VERSION(5); - static const unsigned char version_mask = 0x7<<1; // bits 1-3 are for version -public: - enum mode { - //! processes multiple items in parallel and in no particular order - parallel = current_version | filter_is_out_of_order, - //! processes items one at a time; all such filters process items in the same order - serial_in_order = current_version | filter_is_serial, - //! processes items one at a time and in no particular order - serial_out_of_order = current_version | filter_is_serial | filter_is_out_of_order, - //! @deprecated use serial_in_order instead - serial = serial_in_order - }; -protected: - explicit filter( bool is_serial_ ) : - next_filter_in_pipeline(not_in_pipeline()), - my_input_buffer(NULL), - my_filter_mode(static_cast((is_serial_ ? serial : parallel) | exact_exception_propagation)), - prev_filter_in_pipeline(not_in_pipeline()), - my_pipeline(NULL), - next_segment(NULL) - {} - - explicit filter( mode filter_mode ) : - next_filter_in_pipeline(not_in_pipeline()), - my_input_buffer(NULL), - my_filter_mode(static_cast(filter_mode | exact_exception_propagation)), - prev_filter_in_pipeline(not_in_pipeline()), - my_pipeline(NULL), - next_segment(NULL) - {} - - // signal end-of-input for concrete_filters - void __TBB_EXPORTED_METHOD set_end_of_input(); - -public: - //! True if filter is serial. - bool is_serial() const { - return bool( my_filter_mode & filter_is_serial ); - } - - //! True if filter must receive stream in order. - bool is_ordered() const { - return (my_filter_mode & (filter_is_out_of_order|filter_is_serial))==filter_is_serial; - } - - //! True if filter is thread-bound. - bool is_bound() const { - return ( my_filter_mode & filter_is_bound )==filter_is_bound; - } - - //! true if an input filter can emit null - bool object_may_be_null() { - return ( my_filter_mode & filter_may_emit_null ) == filter_may_emit_null; - } - - //! Operate on an item from the input stream, and return item for output stream. - /** Returns NULL if filter is a sink. */ - virtual void* operator()( void* item ) = 0; - - //! Destroy filter. - /** If the filter was added to a pipeline, the pipeline must be destroyed first. */ - virtual __TBB_EXPORTED_METHOD ~filter(); - -#if __TBB_TASK_GROUP_CONTEXT - //! Destroys item if pipeline was cancelled. - /** Required to prevent memory leaks. - Note it can be called concurrently even for serial filters.*/ - virtual void finalize( void* /*item*/ ) {}; -#endif - -private: - //! Pointer to next filter in the pipeline. - filter* next_filter_in_pipeline; - - //! has the filter not yet processed all the tokens it will ever see? - // (pipeline has not yet reached end_of_input or this filter has not yet - // seen the last token produced by input_filter) - bool has_more_work(); - - //! Buffer for incoming tokens, or NULL if not required. - /** The buffer is required if the filter is serial or follows a thread-bound one. */ - internal::input_buffer* my_input_buffer; - - friend class internal::stage_task; - friend class internal::pipeline_root_task; - friend class pipeline; - friend class thread_bound_filter; - - //! Storage for filter mode and dynamically checked implementation version. - const unsigned char my_filter_mode; - - //! Pointer to previous filter in the pipeline. - filter* prev_filter_in_pipeline; - - //! Pointer to the pipeline. - pipeline* my_pipeline; - - //! Pointer to the next "segment" of filters, or NULL if not required. - /** In each segment, the first filter is not thread-bound but follows a thread-bound one. */ - filter* next_segment; -}; - -//! A stage in a pipeline served by a user thread. -/** @ingroup algorithms */ -class thread_bound_filter: public filter { -public: - enum result_type { - // item was processed - success, - // item is currently not available - item_not_available, - // there are no more items to process - end_of_stream - }; -protected: - explicit thread_bound_filter(mode filter_mode): - filter(static_cast(filter_mode | filter::filter_is_bound)) - { - __TBB_ASSERT(filter_mode & filter::filter_is_serial, "thread-bound filters must be serial"); - } -public: - //! If a data item is available, invoke operator() on that item. - /** This interface is non-blocking. - Returns 'success' if an item was processed. - Returns 'item_not_available' if no item can be processed now - but more may arrive in the future, or if token limit is reached. - Returns 'end_of_stream' if there are no more items to process. */ - result_type __TBB_EXPORTED_METHOD try_process_item(); - - //! Wait until a data item becomes available, and invoke operator() on that item. - /** This interface is blocking. - Returns 'success' if an item was processed. - Returns 'end_of_stream' if there are no more items to process. - Never returns 'item_not_available', as it blocks until another return condition applies. */ - result_type __TBB_EXPORTED_METHOD process_item(); - -private: - //! Internal routine for item processing - result_type internal_process_item(bool is_blocking); -}; - -//! A processing pipeline that applies filters to items. -/** @ingroup algorithms */ -class pipeline { -public: - //! Construct empty pipeline. - __TBB_EXPORTED_METHOD pipeline(); - - /** Though the current implementation declares the destructor virtual, do not rely on this - detail. The virtualness is deprecated and may disappear in future versions of TBB. */ - virtual __TBB_EXPORTED_METHOD ~pipeline(); - - //! Add filter to end of pipeline. - void __TBB_EXPORTED_METHOD add_filter( filter& filter_ ); - - //! Run the pipeline to completion. - void __TBB_EXPORTED_METHOD run( size_t max_number_of_live_tokens ); - -#if __TBB_TASK_GROUP_CONTEXT - //! Run the pipeline to completion with user-supplied context. - void __TBB_EXPORTED_METHOD run( size_t max_number_of_live_tokens, tbb::task_group_context& context ); -#endif - - //! Remove all filters from the pipeline. - void __TBB_EXPORTED_METHOD clear(); - -private: - friend class internal::stage_task; - friend class internal::pipeline_root_task; - friend class filter; - friend class thread_bound_filter; - friend class internal::pipeline_cleaner; - friend class tbb::interface6::internal::pipeline_proxy; - - //! Pointer to first filter in the pipeline. - filter* filter_list; - - //! Pointer to location where address of next filter to be added should be stored. - filter* filter_end; - - //! task who's reference count is used to determine when all stages are done. - task* end_counter; - - //! Number of idle tokens waiting for input stage. - atomic input_tokens; - - //! Global counter of tokens - atomic token_counter; - - //! False until fetch_input returns NULL. - bool end_of_input; - - //! True if the pipeline contains a thread-bound filter; false otherwise. - bool has_thread_bound_filters; - - //! Remove filter from pipeline. - void remove_filter( filter& filter_ ); - - //! Not used, but retained to satisfy old export files. - void __TBB_EXPORTED_METHOD inject_token( task& self ); - -#if __TBB_TASK_GROUP_CONTEXT - //! Does clean up if pipeline is cancelled or exception occurred - void clear_filters(); -#endif -}; - -//------------------------------------------------------------------------ -// Support for lambda-friendly parallel_pipeline interface -//------------------------------------------------------------------------ - -namespace interface6 { - -namespace internal { - template class concrete_filter; -} - -//! input_filter control to signal end-of-input for parallel_pipeline -class flow_control { - bool is_pipeline_stopped; - flow_control() { is_pipeline_stopped = false; } - template friend class internal::concrete_filter; -public: - void stop() { is_pipeline_stopped = true; } -}; - -//! @cond INTERNAL -namespace internal { - -template struct tbb_large_object {enum { value = sizeof(T) > sizeof(void *) }; }; - -// Obtain type properties in one or another way -#if __TBB_CPP11_TYPE_PROPERTIES_PRESENT -template struct tbb_trivially_copyable { enum { value = std::is_trivially_copyable::value }; }; -#elif __TBB_TR1_TYPE_PROPERTIES_IN_STD_PRESENT -template struct tbb_trivially_copyable { enum { value = std::has_trivial_copy_constructor::value }; }; -#else -// Explicitly list the types we wish to be placed as-is in the pipeline input_buffers. -template struct tbb_trivially_copyable { enum { value = false }; }; -template struct tbb_trivially_copyable { enum { value = true }; }; -template<> struct tbb_trivially_copyable { enum { value = true }; }; -template<> struct tbb_trivially_copyable { enum { value = true }; }; -template<> struct tbb_trivially_copyable { enum { value = !tbb_large_object::value }; }; -template<> struct tbb_trivially_copyable { enum { value = !tbb_large_object::value }; }; -template<> struct tbb_trivially_copyable { enum { value = !tbb_large_object::value }; }; -template<> struct tbb_trivially_copyable { enum { value = !tbb_large_object::value }; }; -template<> struct tbb_trivially_copyable { enum { value = !tbb_large_object::value }; }; -template<> struct tbb_trivially_copyable { enum { value = !tbb_large_object::value }; }; -#endif // Obtaining type properties - -template struct is_large_object {enum { value = tbb_large_object::value || !tbb_trivially_copyable::value }; }; - -template class token_helper; - -// large object helper (uses tbb_allocator) -template -class token_helper { - public: - typedef typename tbb::tbb_allocator allocator; - typedef T* pointer; - typedef T value_type; - static pointer create_token(const value_type & source) { - pointer output_t = allocator().allocate(1); - return new (output_t) T(source); - } - static value_type & token(pointer & t) { return *t;} - static void * cast_to_void_ptr(pointer ref) { return (void *) ref; } - static pointer cast_from_void_ptr(void * ref) { return (pointer)ref; } - static void destroy_token(pointer token) { - allocator().destroy(token); - allocator().deallocate(token,1); - } -}; - -// pointer specialization -template -class token_helper { - public: - typedef T* pointer; - typedef T* value_type; - static pointer create_token(const value_type & source) { return source; } - static value_type & token(pointer & t) { return t;} - static void * cast_to_void_ptr(pointer ref) { return (void *)ref; } - static pointer cast_from_void_ptr(void * ref) { return (pointer)ref; } - static void destroy_token( pointer /*token*/) {} -}; - -// small object specialization (converts void* to the correct type, passes objects directly.) -template -class token_helper { - typedef union { - T actual_value; - void * void_overlay; - } type_to_void_ptr_map; - public: - typedef T pointer; // not really a pointer in this case. - typedef T value_type; - static pointer create_token(const value_type & source) { - return source; } - static value_type & token(pointer & t) { return t;} - static void * cast_to_void_ptr(pointer ref) { - type_to_void_ptr_map mymap; - mymap.void_overlay = NULL; - mymap.actual_value = ref; - return mymap.void_overlay; - } - static pointer cast_from_void_ptr(void * ref) { - type_to_void_ptr_map mymap; - mymap.void_overlay = ref; - return mymap.actual_value; - } - static void destroy_token( pointer /*token*/) {} -}; - -template -class concrete_filter: public tbb::filter { - const Body& my_body; - typedef token_helper::value > t_helper; - typedef typename t_helper::pointer t_pointer; - typedef token_helper::value > u_helper; - typedef typename u_helper::pointer u_pointer; - - void* operator()(void* input) __TBB_override { - t_pointer temp_input = t_helper::cast_from_void_ptr(input); - u_pointer output_u = u_helper::create_token(my_body(t_helper::token(temp_input))); - t_helper::destroy_token(temp_input); - return u_helper::cast_to_void_ptr(output_u); - } - - void finalize(void * input) __TBB_override { - t_pointer temp_input = t_helper::cast_from_void_ptr(input); - t_helper::destroy_token(temp_input); - } - -public: - concrete_filter(tbb::filter::mode filter_mode, const Body& body) : filter(filter_mode), my_body(body) {} -}; - -// input -template -class concrete_filter: public filter { - const Body& my_body; - typedef token_helper::value > u_helper; - typedef typename u_helper::pointer u_pointer; - - void* operator()(void*) __TBB_override { - flow_control control; - u_pointer output_u = u_helper::create_token(my_body(control)); - if(control.is_pipeline_stopped) { - u_helper::destroy_token(output_u); - set_end_of_input(); - return NULL; - } - return u_helper::cast_to_void_ptr(output_u); - } - -public: - concrete_filter(tbb::filter::mode filter_mode, const Body& body) : - filter(static_cast(filter_mode | filter_may_emit_null)), - my_body(body) - {} -}; - -template -class concrete_filter: public filter { - const Body& my_body; - typedef token_helper::value > t_helper; - typedef typename t_helper::pointer t_pointer; - - void* operator()(void* input) __TBB_override { - t_pointer temp_input = t_helper::cast_from_void_ptr(input); - my_body(t_helper::token(temp_input)); - t_helper::destroy_token(temp_input); - return NULL; - } - void finalize(void* input) __TBB_override { - t_pointer temp_input = t_helper::cast_from_void_ptr(input); - t_helper::destroy_token(temp_input); - } - -public: - concrete_filter(tbb::filter::mode filter_mode, const Body& body) : filter(filter_mode), my_body(body) {} -}; - -template -class concrete_filter: public filter { - const Body& my_body; - - /** Override privately because it is always called virtually */ - void* operator()(void*) __TBB_override { - flow_control control; - my_body(control); - void* output = control.is_pipeline_stopped ? NULL : (void*)(intptr_t)-1; - return output; - } -public: - concrete_filter(filter::mode filter_mode, const Body& body) : filter(filter_mode), my_body(body) {} -}; - -//! The class that represents an object of the pipeline for parallel_pipeline(). -/** It primarily serves as RAII class that deletes heap-allocated filter instances. */ -class pipeline_proxy { - tbb::pipeline my_pipe; -public: - pipeline_proxy( const filter_t& filter_chain ); - ~pipeline_proxy() { - while( filter* f = my_pipe.filter_list ) - delete f; // filter destructor removes it from the pipeline - } - tbb::pipeline* operator->() { return &my_pipe; } -}; - -//! Abstract base class that represents a node in a parse tree underlying a filter_t. -/** These nodes are always heap-allocated and can be shared by filter_t objects. */ -class filter_node: tbb::internal::no_copy { - /** Count must be atomic because it is hidden state for user, but might be shared by threads. */ - tbb::atomic ref_count; -protected: - filter_node() { - ref_count = 0; -#ifdef __TBB_TEST_FILTER_NODE_COUNT - ++(__TBB_TEST_FILTER_NODE_COUNT); -#endif - } -public: - //! Add concrete_filter to pipeline - virtual void add_to( pipeline& ) = 0; - //! Increment reference count - void add_ref() {++ref_count;} - //! Decrement reference count and delete if it becomes zero. - void remove_ref() { - __TBB_ASSERT(ref_count>0,"ref_count underflow"); - if( --ref_count==0 ) - delete this; - } - virtual ~filter_node() { -#ifdef __TBB_TEST_FILTER_NODE_COUNT - --(__TBB_TEST_FILTER_NODE_COUNT); -#endif - } -}; - -//! Node in parse tree representing result of make_filter. -template -class filter_node_leaf: public filter_node { - const tbb::filter::mode mode; - const Body body; - void add_to( pipeline& p ) __TBB_override { - concrete_filter* f = new concrete_filter(mode,body); - p.add_filter( *f ); - } -public: - filter_node_leaf( tbb::filter::mode m, const Body& b ) : mode(m), body(b) {} -}; - -//! Node in parse tree representing join of two filters. -class filter_node_join: public filter_node { - friend class filter_node; // to suppress GCC 3.2 warnings - filter_node& left; - filter_node& right; - ~filter_node_join() { - left.remove_ref(); - right.remove_ref(); - } - void add_to( pipeline& p ) __TBB_override { - left.add_to(p); - right.add_to(p); - } -public: - filter_node_join( filter_node& x, filter_node& y ) : left(x), right(y) { - left.add_ref(); - right.add_ref(); - } -}; - -} // namespace internal -//! @endcond - -//! Create a filter to participate in parallel_pipeline -template -filter_t make_filter(tbb::filter::mode mode, const Body& body) { - return new internal::filter_node_leaf(mode, body); -} - -template -filter_t operator& (const filter_t& left, const filter_t& right) { - __TBB_ASSERT(left.root,"cannot use default-constructed filter_t as left argument of '&'"); - __TBB_ASSERT(right.root,"cannot use default-constructed filter_t as right argument of '&'"); - return new internal::filter_node_join(*left.root,*right.root); -} - -//! Class representing a chain of type-safe pipeline filters -template -class filter_t { - typedef internal::filter_node filter_node; - filter_node* root; - filter_t( filter_node* root_ ) : root(root_) { - root->add_ref(); - } - friend class internal::pipeline_proxy; - template - friend filter_t make_filter(tbb::filter::mode, const Body& ); - template - friend filter_t operator& (const filter_t& , const filter_t& ); -public: - // TODO: add move-constructors, move-assignment, etc. where C++11 is available. - filter_t() : root(NULL) {} - filter_t( const filter_t& rhs ) : root(rhs.root) { - if( root ) root->add_ref(); - } - template - filter_t( tbb::filter::mode mode, const Body& body ) : - root( new internal::filter_node_leaf(mode, body) ) { - root->add_ref(); - } - - void operator=( const filter_t& rhs ) { - // Order of operations below carefully chosen so that reference counts remain correct - // in unlikely event that remove_ref throws exception. - filter_node* old = root; - root = rhs.root; - if( root ) root->add_ref(); - if( old ) old->remove_ref(); - } - ~filter_t() { - if( root ) root->remove_ref(); - } - void clear() { - // Like operator= with filter_t() on right side. - if( root ) { - filter_node* old = root; - root = NULL; - old->remove_ref(); - } - } -}; - -inline internal::pipeline_proxy::pipeline_proxy( const filter_t& filter_chain ) : my_pipe() { - __TBB_ASSERT( filter_chain.root, "cannot apply parallel_pipeline to default-constructed filter_t" ); - filter_chain.root->add_to(my_pipe); -} - -inline void parallel_pipeline(size_t max_number_of_live_tokens, const filter_t& filter_chain -#if __TBB_TASK_GROUP_CONTEXT - , tbb::task_group_context& context -#endif - ) { - internal::pipeline_proxy pipe(filter_chain); - // tbb::pipeline::run() is called via the proxy - pipe->run(max_number_of_live_tokens -#if __TBB_TASK_GROUP_CONTEXT - , context -#endif - ); -} - -#if __TBB_TASK_GROUP_CONTEXT -inline void parallel_pipeline(size_t max_number_of_live_tokens, const filter_t& filter_chain) { - tbb::task_group_context context; - parallel_pipeline(max_number_of_live_tokens, filter_chain, context); -} -#endif // __TBB_TASK_GROUP_CONTEXT - -} // interface6 - -using interface6::flow_control; -using interface6::filter_t; -using interface6::make_filter; -using interface6::parallel_pipeline; - -} // tbb - -#endif /* __TBB_pipeline_H */ diff --git a/src/tbb/include/tbb/profiling.h b/src/tbb/include/tbb/profiling.h new file mode 100644 index 000000000..c7cea9c59 --- /dev/null +++ b/src/tbb/include/tbb/profiling.h @@ -0,0 +1,17 @@ +/* + Copyright (c) 2005-2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "../oneapi/tbb/profiling.h" diff --git a/src/tbb/include/tbb/queuing_mutex.h b/src/tbb/include/tbb/queuing_mutex.h index 1dcc87c82..ad031e4eb 100644 --- a/src/tbb/include/tbb/queuing_mutex.h +++ b/src/tbb/include/tbb/queuing_mutex.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,99 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ -#ifndef __TBB_queuing_mutex_H -#define __TBB_queuing_mutex_H - -#include -#include "atomic.h" -#include "tbb_profiling.h" - -namespace tbb { - -//! Queuing mutex with local-only spinning. -/** @ingroup synchronization */ -class queuing_mutex : internal::mutex_copy_deprecated_and_disabled { -public: - //! Construct unacquired mutex. - queuing_mutex() { - q_tail = NULL; -#if TBB_USE_THREADING_TOOLS - internal_construct(); -#endif - } - - //! The scoped locking pattern - /** It helps to avoid the common problem of forgetting to release lock. - It also nicely provides the "node" for queuing locks. */ - class scoped_lock: internal::no_copy { - //! Initialize fields to mean "no lock held". - void initialize() { - mutex = NULL; -#if TBB_USE_ASSERT - internal::poison_pointer(next); -#endif /* TBB_USE_ASSERT */ - } - - public: - //! Construct lock that has not acquired a mutex. - /** Equivalent to zero-initialization of *this. */ - scoped_lock() {initialize();} - - //! Acquire lock on given mutex. - scoped_lock( queuing_mutex& m ) { - initialize(); - acquire(m); - } - - //! Release lock (if lock is held). - ~scoped_lock() { - if( mutex ) release(); - } - - //! Acquire lock on given mutex. - void __TBB_EXPORTED_METHOD acquire( queuing_mutex& m ); - - //! Acquire lock on given mutex if free (i.e. non-blocking) - bool __TBB_EXPORTED_METHOD try_acquire( queuing_mutex& m ); - - //! Release lock. - void __TBB_EXPORTED_METHOD release(); - - private: - //! The pointer to the mutex owned, or NULL if not holding a mutex. - queuing_mutex* mutex; - - //! The pointer to the next competitor for a mutex - scoped_lock *next; - - //! The local spin-wait variable - /** Inverted (0 - blocked, 1 - acquired the mutex) for the sake of - zero-initialization. Defining it as an entire word instead of - a byte seems to help performance slightly. */ - uintptr_t going; - }; - - void __TBB_EXPORTED_METHOD internal_construct(); - - // Mutex traits - static const bool is_rw_mutex = false; - static const bool is_recursive_mutex = false; - static const bool is_fair_mutex = true; - -private: - //! The last competitor requesting the lock - atomic q_tail; - -}; - -__TBB_DEFINE_PROFILING_SET_NAME(queuing_mutex) - -} // namespace tbb - -#endif /* __TBB_queuing_mutex_H */ +#include "../oneapi/tbb/queuing_mutex.h" diff --git a/src/tbb/include/tbb/queuing_rw_mutex.h b/src/tbb/include/tbb/queuing_rw_mutex.h index 702cbe357..203727ccc 100644 --- a/src/tbb/include/tbb/queuing_rw_mutex.h +++ b/src/tbb/include/tbb/queuing_rw_mutex.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,139 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ -#ifndef __TBB_queuing_rw_mutex_H -#define __TBB_queuing_rw_mutex_H - -#include -#include "atomic.h" -#include "tbb_profiling.h" - -namespace tbb { - -//! Queuing reader-writer mutex with local-only spinning. -/** Adapted from Krieger, Stumm, et al. pseudocode at - http://www.eecg.toronto.edu/parallel/pubs_abs.html#Krieger_etal_ICPP93 - @ingroup synchronization */ -class queuing_rw_mutex : internal::mutex_copy_deprecated_and_disabled { -public: - //! Construct unacquired mutex. - queuing_rw_mutex() { - q_tail = NULL; -#if TBB_USE_THREADING_TOOLS - internal_construct(); -#endif - } - - //! Destructor asserts if the mutex is acquired, i.e. q_tail is non-NULL - ~queuing_rw_mutex() { -#if TBB_USE_ASSERT - __TBB_ASSERT( !q_tail, "destruction of an acquired mutex"); -#endif - } - - //! The scoped locking pattern - /** It helps to avoid the common problem of forgetting to release lock. - It also nicely provides the "node" for queuing locks. */ - class scoped_lock: internal::no_copy { - //! Initialize fields to mean "no lock held". - void initialize() { - my_mutex = NULL; -#if TBB_USE_ASSERT - my_state = 0xFF; // Set to invalid state - internal::poison_pointer(my_next); - internal::poison_pointer(my_prev); -#endif /* TBB_USE_ASSERT */ - } - - public: - //! Construct lock that has not acquired a mutex. - /** Equivalent to zero-initialization of *this. */ - scoped_lock() {initialize();} - - //! Acquire lock on given mutex. - scoped_lock( queuing_rw_mutex& m, bool write=true ) { - initialize(); - acquire(m,write); - } - - //! Release lock (if lock is held). - ~scoped_lock() { - if( my_mutex ) release(); - } - - //! Acquire lock on given mutex. - void acquire( queuing_rw_mutex& m, bool write=true ); - - //! Acquire lock on given mutex if free (i.e. non-blocking) - bool try_acquire( queuing_rw_mutex& m, bool write=true ); - - //! Release lock. - void release(); - - //! Upgrade reader to become a writer. - /** Returns whether the upgrade happened without releasing and re-acquiring the lock */ - bool upgrade_to_writer(); - - //! Downgrade writer to become a reader. - bool downgrade_to_reader(); - - private: - //! The pointer to the mutex owned, or NULL if not holding a mutex. - queuing_rw_mutex* my_mutex; - - //! The pointer to the previous and next competitors for a mutex - scoped_lock *__TBB_atomic my_prev, *__TBB_atomic my_next; - - typedef unsigned char state_t; - - //! State of the request: reader, writer, active reader, other service states - atomic my_state; - - //! The local spin-wait variable - /** Corresponds to "spin" in the pseudocode but inverted for the sake of zero-initialization */ - unsigned char __TBB_atomic my_going; - - //! A tiny internal lock - unsigned char my_internal_lock; - - //! Acquire the internal lock - void acquire_internal_lock(); - - //! Try to acquire the internal lock - /** Returns true if lock was successfully acquired. */ - bool try_acquire_internal_lock(); - - //! Release the internal lock - void release_internal_lock(); - - //! Wait for internal lock to be released - void wait_for_release_of_internal_lock(); - - //! A helper function - void unblock_or_wait_on_internal_lock( uintptr_t ); - }; - - void __TBB_EXPORTED_METHOD internal_construct(); - - // Mutex traits - static const bool is_rw_mutex = true; - static const bool is_recursive_mutex = false; - static const bool is_fair_mutex = true; - -private: - //! The last competitor requesting the lock - atomic q_tail; - -}; - -__TBB_DEFINE_PROFILING_SET_NAME(queuing_rw_mutex) - -} // namespace tbb - -#endif /* __TBB_queuing_rw_mutex_H */ +#include "../oneapi/tbb/queuing_rw_mutex.h" diff --git a/src/tbb/include/tbb/reader_writer_lock.h b/src/tbb/include/tbb/reader_writer_lock.h deleted file mode 100644 index 353beec5d..000000000 --- a/src/tbb/include/tbb/reader_writer_lock.h +++ /dev/null @@ -1,232 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_reader_writer_lock_H -#define __TBB_reader_writer_lock_H - -#include "tbb_thread.h" -#include "tbb_allocator.h" -#include "atomic.h" - -namespace tbb { -namespace interface5 { -//! Writer-preference reader-writer lock with local-only spinning on readers. -/** Loosely adapted from Mellor-Crummey and Scott pseudocode at - http://www.cs.rochester.edu/research/synchronization/pseudocode/rw.html#s_wp - @ingroup synchronization */ - class reader_writer_lock : tbb::internal::no_copy { - public: - friend class scoped_lock; - friend class scoped_lock_read; - //! Status type for nodes associated with lock instances - /** waiting_nonblocking: the wait state for nonblocking lock - instances; for writes, these transition straight to active - states; for reads, these are unused. - - waiting: the start and spin state for all lock instances; these will - transition to active state when appropriate. Non-blocking write locks - transition from this state to waiting_nonblocking immediately. - - active: the active state means that the lock instance holds - the lock; it will transition to invalid state during node deletion - - invalid: the end state for all nodes; this is set in the - destructor so if we encounter this state, we are looking at - memory that has already been freed - - The state diagrams below describe the status transitions. - Single arrows indicate that the thread that owns the node is - responsible for the transition; double arrows indicate that - any thread could make the transition. - - State diagram for scoped_lock status: - - waiting ----------> waiting_nonblocking - | _____________/ | - V V V - active -----------------> invalid - - State diagram for scoped_lock_read status: - - waiting - | - V - active ----------------->invalid - - */ - enum status_t { waiting_nonblocking, waiting, active, invalid }; - - //! Constructs a new reader_writer_lock - reader_writer_lock() { - internal_construct(); - } - - //! Destructs a reader_writer_lock object - ~reader_writer_lock() { - internal_destroy(); - } - - //! The scoped lock pattern for write locks - /** Scoped locks help avoid the common problem of forgetting to release the lock. - This type also serves as the node for queuing locks. */ - class scoped_lock : tbb::internal::no_copy { - public: - friend class reader_writer_lock; - - //! Construct with blocking attempt to acquire write lock on the passed-in lock - scoped_lock(reader_writer_lock& lock) { - internal_construct(lock); - } - - //! Destructor, releases the write lock - ~scoped_lock() { - internal_destroy(); - } - - void* operator new(size_t s) { - return tbb::internal::allocate_via_handler_v3(s); - } - void operator delete(void* p) { - tbb::internal::deallocate_via_handler_v3(p); - } - - private: - //! The pointer to the mutex to lock - reader_writer_lock *mutex; - //! The next queued competitor for the mutex - scoped_lock* next; - //! Status flag of the thread associated with this node - atomic status; - - //! Construct scoped_lock that is not holding lock - scoped_lock(); - - void __TBB_EXPORTED_METHOD internal_construct(reader_writer_lock&); - void __TBB_EXPORTED_METHOD internal_destroy(); - }; - - //! The scoped lock pattern for read locks - class scoped_lock_read : tbb::internal::no_copy { - public: - friend class reader_writer_lock; - - //! Construct with blocking attempt to acquire read lock on the passed-in lock - scoped_lock_read(reader_writer_lock& lock) { - internal_construct(lock); - } - - //! Destructor, releases the read lock - ~scoped_lock_read() { - internal_destroy(); - } - - void* operator new(size_t s) { - return tbb::internal::allocate_via_handler_v3(s); - } - void operator delete(void* p) { - tbb::internal::deallocate_via_handler_v3(p); - } - - private: - //! The pointer to the mutex to lock - reader_writer_lock *mutex; - //! The next queued competitor for the mutex - scoped_lock_read *next; - //! Status flag of the thread associated with this node - atomic status; - - //! Construct scoped_lock_read that is not holding lock - scoped_lock_read(); - - void __TBB_EXPORTED_METHOD internal_construct(reader_writer_lock&); - void __TBB_EXPORTED_METHOD internal_destroy(); - }; - - //! Acquires the reader_writer_lock for write. - /** If the lock is currently held in write mode by another - context, the writer will block by spinning on a local - variable. Exceptions thrown: improper_lock The context tries - to acquire a reader_writer_lock that it already has write - ownership of.*/ - void __TBB_EXPORTED_METHOD lock(); - - //! Tries to acquire the reader_writer_lock for write. - /** This function does not block. Return Value: True or false, - depending on whether the lock is acquired or not. If the lock - is already held by this acquiring context, try_lock() returns - false. */ - bool __TBB_EXPORTED_METHOD try_lock(); - - //! Acquires the reader_writer_lock for read. - /** If the lock is currently held by a writer, this reader will - block and wait until the writers are done. Exceptions thrown: - improper_lock The context tries to acquire a - reader_writer_lock that it already has write ownership of. */ - void __TBB_EXPORTED_METHOD lock_read(); - - //! Tries to acquire the reader_writer_lock for read. - /** This function does not block. Return Value: True or false, - depending on whether the lock is acquired or not. */ - bool __TBB_EXPORTED_METHOD try_lock_read(); - - //! Releases the reader_writer_lock - void __TBB_EXPORTED_METHOD unlock(); - - private: - void __TBB_EXPORTED_METHOD internal_construct(); - void __TBB_EXPORTED_METHOD internal_destroy(); - - //! Attempts to acquire write lock - /** If unavailable, spins in blocking case, returns false in non-blocking case. */ - bool start_write(scoped_lock *); - //! Sets writer_head to w and attempts to unblock - void set_next_writer(scoped_lock *w); - //! Relinquishes write lock to next waiting writer or group of readers - void end_write(scoped_lock *); - //! Checks if current thread holds write lock - bool is_current_writer(); - - //! Attempts to acquire read lock - /** If unavailable, spins in blocking case, returns false in non-blocking case. */ - void start_read(scoped_lock_read *); - //! Unblocks pending readers - void unblock_readers(); - //! Relinquishes read lock by decrementing counter; last reader wakes pending writer - void end_read(); - - //! The list of pending readers - atomic reader_head; - //! The list of pending writers - atomic writer_head; - //! The last node in the list of pending writers - atomic writer_tail; - //! Writer that owns the mutex; tbb_thread::id() otherwise. - tbb_thread::id my_current_writer; - //! Status of mutex - atomic rdr_count_and_flags; // used with __TBB_AtomicOR, which assumes uintptr_t -}; - -} // namespace interface5 - -using interface5::reader_writer_lock; - -} // namespace tbb - -#endif /* __TBB_reader_writer_lock_H */ diff --git a/src/tbb/include/tbb/recursive_mutex.h b/src/tbb/include/tbb/recursive_mutex.h deleted file mode 100644 index 5a23c097c..000000000 --- a/src/tbb/include/tbb/recursive_mutex.h +++ /dev/null @@ -1,234 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_recursive_mutex_H -#define __TBB_recursive_mutex_H - -#if _WIN32||_WIN64 -#include "machine/windows_api.h" -#else -#include -#endif /* _WIN32||_WIN64 */ - -#include -#include "aligned_space.h" -#include "tbb_stddef.h" -#include "tbb_profiling.h" - -namespace tbb { -//! Mutex that allows recursive mutex acquisition. -/** Mutex that allows recursive mutex acquisition. - @ingroup synchronization */ -class recursive_mutex : internal::mutex_copy_deprecated_and_disabled { -public: - //! Construct unacquired recursive_mutex. - recursive_mutex() { -#if TBB_USE_ASSERT || TBB_USE_THREADING_TOOLS - internal_construct(); -#else - #if _WIN32||_WIN64 - InitializeCriticalSectionEx(&impl, 4000, 0); - #else - pthread_mutexattr_t mtx_attr; - int error_code = pthread_mutexattr_init( &mtx_attr ); - if( error_code ) - tbb::internal::handle_perror(error_code,"recursive_mutex: pthread_mutexattr_init failed"); - - pthread_mutexattr_settype( &mtx_attr, PTHREAD_MUTEX_RECURSIVE ); - error_code = pthread_mutex_init( &impl, &mtx_attr ); - if( error_code ) - tbb::internal::handle_perror(error_code,"recursive_mutex: pthread_mutex_init failed"); - - pthread_mutexattr_destroy( &mtx_attr ); - #endif /* _WIN32||_WIN64*/ -#endif /* TBB_USE_ASSERT */ - }; - - ~recursive_mutex() { -#if TBB_USE_ASSERT - internal_destroy(); -#else - #if _WIN32||_WIN64 - DeleteCriticalSection(&impl); - #else - pthread_mutex_destroy(&impl); - - #endif /* _WIN32||_WIN64 */ -#endif /* TBB_USE_ASSERT */ - }; - - class scoped_lock; - friend class scoped_lock; - - //! The scoped locking pattern - /** It helps to avoid the common problem of forgetting to release lock. - It also nicely provides the "node" for queuing locks. */ - class scoped_lock: internal::no_copy { - public: - //! Construct lock that has not acquired a recursive_mutex. - scoped_lock() : my_mutex(NULL) {}; - - //! Acquire lock on given mutex. - scoped_lock( recursive_mutex& mutex ) { -#if TBB_USE_ASSERT - my_mutex = &mutex; -#endif /* TBB_USE_ASSERT */ - acquire( mutex ); - } - - //! Release lock (if lock is held). - ~scoped_lock() { - if( my_mutex ) - release(); - } - - //! Acquire lock on given mutex. - void acquire( recursive_mutex& mutex ) { -#if TBB_USE_ASSERT - internal_acquire( mutex ); -#else - my_mutex = &mutex; - mutex.lock(); -#endif /* TBB_USE_ASSERT */ - } - - //! Try acquire lock on given recursive_mutex. - bool try_acquire( recursive_mutex& mutex ) { -#if TBB_USE_ASSERT - return internal_try_acquire( mutex ); -#else - bool result = mutex.try_lock(); - if( result ) - my_mutex = &mutex; - return result; -#endif /* TBB_USE_ASSERT */ - } - - //! Release lock - void release() { -#if TBB_USE_ASSERT - internal_release(); -#else - my_mutex->unlock(); - my_mutex = NULL; -#endif /* TBB_USE_ASSERT */ - } - - private: - //! The pointer to the current recursive_mutex to work - recursive_mutex* my_mutex; - - //! All checks from acquire using mutex.state were moved here - void __TBB_EXPORTED_METHOD internal_acquire( recursive_mutex& m ); - - //! All checks from try_acquire using mutex.state were moved here - bool __TBB_EXPORTED_METHOD internal_try_acquire( recursive_mutex& m ); - - //! All checks from release using mutex.state were moved here - void __TBB_EXPORTED_METHOD internal_release(); - - friend class recursive_mutex; - }; - - // Mutex traits - static const bool is_rw_mutex = false; - static const bool is_recursive_mutex = true; - static const bool is_fair_mutex = false; - - // C++0x compatibility interface - - //! Acquire lock - void lock() { -#if TBB_USE_ASSERT - aligned_space tmp; - new(tmp.begin()) scoped_lock(*this); -#else - #if _WIN32||_WIN64 - EnterCriticalSection(&impl); - #else - int error_code = pthread_mutex_lock(&impl); - if( error_code ) - tbb::internal::handle_perror(error_code,"recursive_mutex: pthread_mutex_lock failed"); - #endif /* _WIN32||_WIN64 */ -#endif /* TBB_USE_ASSERT */ - } - - //! Try acquiring lock (non-blocking) - /** Return true if lock acquired; false otherwise. */ - bool try_lock() { -#if TBB_USE_ASSERT - aligned_space tmp; - return (new(tmp.begin()) scoped_lock)->internal_try_acquire(*this); -#else - #if _WIN32||_WIN64 - return TryEnterCriticalSection(&impl)!=0; - #else - return pthread_mutex_trylock(&impl)==0; - #endif /* _WIN32||_WIN64 */ -#endif /* TBB_USE_ASSERT */ - } - - //! Release lock - void unlock() { -#if TBB_USE_ASSERT - aligned_space tmp; - scoped_lock& s = *tmp.begin(); - s.my_mutex = this; - s.internal_release(); -#else - #if _WIN32||_WIN64 - LeaveCriticalSection(&impl); - #else - pthread_mutex_unlock(&impl); - #endif /* _WIN32||_WIN64 */ -#endif /* TBB_USE_ASSERT */ - } - - //! Return native_handle - #if _WIN32||_WIN64 - typedef LPCRITICAL_SECTION native_handle_type; - #else - typedef pthread_mutex_t* native_handle_type; - #endif - native_handle_type native_handle() { return (native_handle_type) &impl; } - -private: -#if _WIN32||_WIN64 - CRITICAL_SECTION impl; - enum state_t { - INITIALIZED=0x1234, - DESTROYED=0x789A, - } state; -#else - pthread_mutex_t impl; -#endif /* _WIN32||_WIN64 */ - - //! All checks from mutex constructor using mutex.state were moved here - void __TBB_EXPORTED_METHOD internal_construct(); - - //! All checks from mutex destructor using mutex.state were moved here - void __TBB_EXPORTED_METHOD internal_destroy(); -}; - -__TBB_DEFINE_PROFILING_SET_NAME(recursive_mutex) - -} // namespace tbb - -#endif /* __TBB_recursive_mutex_H */ diff --git a/src/tbb/include/tbb/runtime_loader.h b/src/tbb/include/tbb/runtime_loader.h deleted file mode 100644 index df2846454..000000000 --- a/src/tbb/include/tbb/runtime_loader.h +++ /dev/null @@ -1,180 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_runtime_loader_H -#define __TBB_runtime_loader_H - -#if ! TBB_PREVIEW_RUNTIME_LOADER - #error Set TBB_PREVIEW_RUNTIME_LOADER to include runtime_loader.h -#endif - -#include "tbb_stddef.h" -#include - -#if _MSC_VER - #if ! __TBB_NO_IMPLICIT_LINKAGE - #ifdef _DEBUG - #pragma comment( linker, "/nodefaultlib:tbb_debug.lib" ) - #pragma comment( linker, "/defaultlib:tbbproxy_debug.lib" ) - #else - #pragma comment( linker, "/nodefaultlib:tbb.lib" ) - #pragma comment( linker, "/defaultlib:tbbproxy.lib" ) - #endif - #endif -#endif - -namespace tbb { - -namespace interface6 { - -//! Load TBB at runtime. -/*! - -\b Usage: - -In source code: - -\code -#include "tbb/runtime_loader.h" - -char const * path[] = { "/lib/ia32", NULL }; -tbb::runtime_loader loader( path ); - -// Now use TBB. -\endcode - -Link with \c tbbproxy.lib (or \c libtbbproxy.a) instead of \c tbb.lib (\c libtbb.dylib, -\c libtbb.so). - -TBB library will be loaded at runtime from \c /lib/ia32 directory. - -\b Attention: - -All \c runtime_loader objects (in the same module, i.e. exe or dll) share some global state. -The most noticeable piece of global state is loaded TBB library. -There are some implications: - - - Only one TBB library can be loaded per module. - - - If one object has already loaded TBB library, another object will not load TBB. - If the loaded TBB library is suitable for the second object, both will use TBB - cooperatively, otherwise the second object will report an error. - - - \c runtime_loader objects will not work (correctly) in parallel due to absence of - synchronization. - -*/ - -class runtime_loader : tbb::internal::no_copy { - - public: - - //! Error mode constants. - enum error_mode { - em_status, //!< Save status of operation and continue. - em_throw, //!< Throw an exception of tbb::runtime_loader::error_code type. - em_abort //!< Print message to \c stderr and call \c abort(). - }; // error_mode - - //! Error codes. - enum error_code { - ec_ok, //!< No errors. - ec_bad_call, //!< Invalid function call (e. g. load() called when TBB is already loaded). - ec_bad_arg, //!< Invalid argument passed. - ec_bad_lib, //!< Invalid library found (e. g. \c TBB_runtime_version symbol not found). - ec_bad_ver, //!< TBB found but version is not suitable. - ec_no_lib //!< No suitable TBB library found. - }; // error_code - - //! Initialize object but do not load TBB. - runtime_loader( error_mode mode = em_abort ); - - //! Initialize object and load TBB. - /*! - See load() for details. - - If error mode is \c em_status, call status() to check whether TBB was loaded or not. - */ - runtime_loader( - char const * path[], //!< List of directories to search TBB in. - int min_ver = TBB_INTERFACE_VERSION, //!< Minimal suitable version of TBB. - int max_ver = INT_MAX, //!< Maximal suitable version of TBB. - error_mode mode = em_abort //!< Error mode for this object. - ); - - //! Destroy object. - ~runtime_loader(); - - //! Load TBB. - /*! - The method searches the directories specified in \c path[] array for the TBB library. - When the library is found, it is loaded and its version is checked. If the version is - not suitable, the library is unloaded, and the search continues. - - \b Note: - - For security reasons, avoid using relative directory names. For example, never load - TBB from current (\c "."), parent (\c "..") or any other relative directory (like - \c "lib" ). Use only absolute directory names (e. g. "/usr/local/lib"). - - For the same security reasons, avoid using system default directories (\c "") on - Windows. (See http://www.microsoft.com/technet/security/advisory/2269637.mspx for - details.) - - Neglecting these rules may cause your program to execute 3-rd party malicious code. - - \b Errors: - - \c ec_bad_call - TBB already loaded by this object. - - \c ec_bad_arg - \p min_ver and/or \p max_ver negative or zero, - or \p min_ver > \p max_ver. - - \c ec_bad_ver - TBB of unsuitable version already loaded by another object. - - \c ec_no_lib - No suitable library found. - */ - error_code - load( - char const * path[], //!< List of directories to search TBB in. - int min_ver = TBB_INTERFACE_VERSION, //!< Minimal suitable version of TBB. - int max_ver = INT_MAX //!< Maximal suitable version of TBB. - - ); - - - //! Report status. - /*! - If error mode is \c em_status, the function returns status of the last operation. - */ - error_code status(); - - private: - - error_mode const my_mode; - error_code my_status; - bool my_loaded; - -}; // class runtime_loader - -} // namespace interface6 - -using interface6::runtime_loader; - -} // namespace tbb - -#endif /* __TBB_runtime_loader_H */ - diff --git a/src/tbb/include/tbb/rw_mutex.h b/src/tbb/include/tbb/rw_mutex.h new file mode 100644 index 000000000..f2499ebac --- /dev/null +++ b/src/tbb/include/tbb/rw_mutex.h @@ -0,0 +1,17 @@ +/* + Copyright (c) 2023 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "../oneapi/tbb/rw_mutex.h" diff --git a/src/tbb/include/tbb/scalable_allocator.h b/src/tbb/include/tbb/scalable_allocator.h index 0bebe21e9..5c654ebd6 100644 --- a/src/tbb/include/tbb/scalable_allocator.h +++ b/src/tbb/include/tbb/scalable_allocator.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,331 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - -*/ - -#ifndef __TBB_scalable_allocator_H -#define __TBB_scalable_allocator_H -/** @file */ - -#include /* Need ptrdiff_t and size_t from here. */ -#if !_MSC_VER -#include /* Need intptr_t from here. */ -#endif - -#if !defined(__cplusplus) && __ICC==1100 - #pragma warning (push) - #pragma warning (disable: 991) -#endif - -#ifdef __cplusplus -extern "C" { -#endif /* __cplusplus */ - -#if _MSC_VER >= 1400 -#define __TBB_EXPORTED_FUNC __cdecl -#else -#define __TBB_EXPORTED_FUNC -#endif - -/** The "malloc" analogue to allocate block of memory of size bytes. - * @ingroup memory_allocation */ -void * __TBB_EXPORTED_FUNC scalable_malloc (size_t size); - -/** The "free" analogue to discard a previously allocated piece of memory. - @ingroup memory_allocation */ -void __TBB_EXPORTED_FUNC scalable_free (void* ptr); - -/** The "realloc" analogue complementing scalable_malloc. - @ingroup memory_allocation */ -void * __TBB_EXPORTED_FUNC scalable_realloc (void* ptr, size_t size); - -/** The "calloc" analogue complementing scalable_malloc. - @ingroup memory_allocation */ -void * __TBB_EXPORTED_FUNC scalable_calloc (size_t nobj, size_t size); - -/** The "posix_memalign" analogue. - @ingroup memory_allocation */ -int __TBB_EXPORTED_FUNC scalable_posix_memalign (void** memptr, size_t alignment, size_t size); - -/** The "_aligned_malloc" analogue. - @ingroup memory_allocation */ -void * __TBB_EXPORTED_FUNC scalable_aligned_malloc (size_t size, size_t alignment); - -/** The "_aligned_realloc" analogue. - @ingroup memory_allocation */ -void * __TBB_EXPORTED_FUNC scalable_aligned_realloc (void* ptr, size_t size, size_t alignment); - -/** The "_aligned_free" analogue. - @ingroup memory_allocation */ -void __TBB_EXPORTED_FUNC scalable_aligned_free (void* ptr); - -/** The analogue of _msize/malloc_size/malloc_usable_size. - Returns the usable size of a memory block previously allocated by scalable_*, - or 0 (zero) if ptr does not point to such a block. - @ingroup memory_allocation */ -size_t __TBB_EXPORTED_FUNC scalable_msize (void* ptr); - -/* Results for scalable_allocation_* functions */ -typedef enum { - TBBMALLOC_OK, - TBBMALLOC_INVALID_PARAM, - TBBMALLOC_UNSUPPORTED, - TBBMALLOC_NO_MEMORY, - TBBMALLOC_NO_EFFECT -} ScalableAllocationResult; - -/* Setting TBB_MALLOC_USE_HUGE_PAGES environment variable to 1 enables huge pages. - scalable_allocation_mode call has priority over environment variable. */ -typedef enum { - TBBMALLOC_USE_HUGE_PAGES, /* value turns using huge pages on and off */ - /* deprecated, kept for backward compatibility only */ - USE_HUGE_PAGES = TBBMALLOC_USE_HUGE_PAGES, - /* try to limit memory consumption value Bytes, clean internal buffers - if limit is exceeded, but not prevents from requesting memory from OS */ - TBBMALLOC_SET_SOFT_HEAP_LIMIT -} AllocationModeParam; - -/** Set TBB allocator-specific allocation modes. - @ingroup memory_allocation */ -int __TBB_EXPORTED_FUNC scalable_allocation_mode(int param, intptr_t value); - -typedef enum { - /* Clean internal allocator buffers for all threads. - Returns TBBMALLOC_NO_EFFECT if no buffers cleaned, - TBBMALLOC_OK if some memory released from buffers. */ - TBBMALLOC_CLEAN_ALL_BUFFERS, - /* Clean internal allocator buffer for current thread only. - Return values same as for TBBMALLOC_CLEAN_ALL_BUFFERS. */ - TBBMALLOC_CLEAN_THREAD_BUFFERS -} ScalableAllocationCmd; - -/** Call TBB allocator-specific commands. - @ingroup memory_allocation */ -int __TBB_EXPORTED_FUNC scalable_allocation_command(int cmd, void *param); - -#ifdef __cplusplus -} /* extern "C" */ -#endif /* __cplusplus */ - -#ifdef __cplusplus - -//! The namespace rml contains components of low-level memory pool interface. -namespace rml { -class MemoryPool; - -typedef void *(*rawAllocType)(intptr_t pool_id, size_t &bytes); -// returns non-zero in case of error -typedef int (*rawFreeType)(intptr_t pool_id, void* raw_ptr, size_t raw_bytes); - -/* -MemPoolPolicy extension must be compatible with such structure fields layout - -struct MemPoolPolicy { - rawAllocType pAlloc; - rawFreeType pFree; - size_t granularity; // granularity of pAlloc allocations -}; */ -struct MemPoolPolicy { - enum { - TBBMALLOC_POOL_VERSION = 1 - }; - - rawAllocType pAlloc; - rawFreeType pFree; - // granularity of pAlloc allocations. 0 means default used. - size_t granularity; - int version; - // all memory consumed at 1st pAlloc call and never returned, - // no more pAlloc calls after 1st - unsigned fixedPool : 1, - // memory consumed but returned only at pool termination - keepAllMemory : 1, - reserved : 30; - - MemPoolPolicy(rawAllocType pAlloc_, rawFreeType pFree_, - size_t granularity_ = 0, bool fixedPool_ = false, - bool keepAllMemory_ = false) : - pAlloc(pAlloc_), pFree(pFree_), granularity(granularity_), version(TBBMALLOC_POOL_VERSION), - fixedPool(fixedPool_), keepAllMemory(keepAllMemory_), - reserved(0) {} -}; - -// enums have same values as appropriate enums from ScalableAllocationResult -// TODO: use ScalableAllocationResult in pool_create directly -enum MemPoolError { - // pool created successfully - POOL_OK = TBBMALLOC_OK, - // invalid policy parameters found - INVALID_POLICY = TBBMALLOC_INVALID_PARAM, - // requested pool policy is not supported by allocator library - UNSUPPORTED_POLICY = TBBMALLOC_UNSUPPORTED, - // lack of memory during pool creation - NO_MEMORY = TBBMALLOC_NO_MEMORY, - // action takes no effect - NO_EFFECT = TBBMALLOC_NO_EFFECT -}; - -MemPoolError pool_create_v1(intptr_t pool_id, const MemPoolPolicy *policy, - rml::MemoryPool **pool); - -bool pool_destroy(MemoryPool* memPool); -void *pool_malloc(MemoryPool* memPool, size_t size); -void *pool_realloc(MemoryPool* memPool, void *object, size_t size); -void *pool_aligned_malloc(MemoryPool* mPool, size_t size, size_t alignment); -void *pool_aligned_realloc(MemoryPool* mPool, void *ptr, size_t size, size_t alignment); -bool pool_reset(MemoryPool* memPool); -bool pool_free(MemoryPool *memPool, void *object); -MemoryPool *pool_identify(void *object); - -} // namespace rml - -#include /* To use new with the placement argument */ - -/* Ensure that including this header does not cause implicit linkage with TBB */ -#ifndef __TBB_NO_IMPLICIT_LINKAGE - #define __TBB_NO_IMPLICIT_LINKAGE 1 - #include "tbb_stddef.h" - #undef __TBB_NO_IMPLICIT_LINKAGE -#else - #include "tbb_stddef.h" -#endif - -#if __TBB_ALLOCATOR_CONSTRUCT_VARIADIC - #include // std::forward -#endif - -namespace tbb { - -#if _MSC_VER && !defined(__INTEL_COMPILER) - // Workaround for erroneous "unreferenced parameter" warning in method destroy. - #pragma warning (push) - #pragma warning (disable: 4100) -#endif - -//! @cond INTERNAL -namespace internal { - -#if TBB_USE_EXCEPTIONS -// forward declaration is for inlining prevention -template __TBB_NOINLINE( void throw_exception(const E &e) ); -#endif - -// keep throw in a separate function to prevent code bloat -template -void throw_exception(const E &e) { - __TBB_THROW(e); -} - -} // namespace internal -//! @endcond - -//! Meets "allocator" requirements of ISO C++ Standard, Section 20.1.5 -/** The members are ordered the same way they are in section 20.4.1 - of the ISO C++ standard. - @ingroup memory_allocation */ -template -class scalable_allocator { -public: - typedef typename internal::allocator_type::value_type value_type; - typedef value_type* pointer; - typedef const value_type* const_pointer; - typedef value_type& reference; - typedef const value_type& const_reference; - typedef size_t size_type; - typedef ptrdiff_t difference_type; - template struct rebind { - typedef scalable_allocator other; - }; - - scalable_allocator() throw() {} - scalable_allocator( const scalable_allocator& ) throw() {} - template scalable_allocator(const scalable_allocator&) throw() {} - - pointer address(reference x) const {return &x;} - const_pointer address(const_reference x) const {return &x;} - - //! Allocate space for n objects. - pointer allocate( size_type n, const void* /*hint*/ =0 ) { - pointer p = static_cast( scalable_malloc( n * sizeof(value_type) ) ); - if (!p) - internal::throw_exception(std::bad_alloc()); - return p; - } - - //! Free previously allocated block of memory - void deallocate( pointer p, size_type ) { - scalable_free( p ); - } - - //! Largest value for which method allocate might succeed. - size_type max_size() const throw() { - size_type absolutemax = static_cast(-1) / sizeof (value_type); - return (absolutemax > 0 ? absolutemax : 1); - } -#if __TBB_ALLOCATOR_CONSTRUCT_VARIADIC - template - void construct(U *p, Args&&... args) - { ::new((void *)p) U(std::forward(args)...); } -#else /* __TBB_ALLOCATOR_CONSTRUCT_VARIADIC */ -#if __TBB_CPP11_RVALUE_REF_PRESENT - void construct( pointer p, value_type&& value ) { ::new((void*)(p)) value_type( std::move( value ) ); } -#endif - void construct( pointer p, const value_type& value ) {::new((void*)(p)) value_type(value);} -#endif /* __TBB_ALLOCATOR_CONSTRUCT_VARIADIC */ - void destroy( pointer p ) {p->~value_type();} -}; - -#if _MSC_VER && !defined(__INTEL_COMPILER) - #pragma warning (pop) -#endif /* warning 4100 is back */ - -//! Analogous to std::allocator, as defined in ISO C++ Standard, Section 20.4.1 -/** @ingroup memory_allocation */ -template<> -class scalable_allocator { -public: - typedef void* pointer; - typedef const void* const_pointer; - typedef void value_type; - template struct rebind { - typedef scalable_allocator other; - }; -}; - -template -inline bool operator==( const scalable_allocator&, const scalable_allocator& ) {return true;} - -template -inline bool operator!=( const scalable_allocator&, const scalable_allocator& ) {return false;} - -} // namespace tbb - -#if _MSC_VER - #if (__TBB_BUILD || __TBBMALLOC_BUILD) && !defined(__TBBMALLOC_NO_IMPLICIT_LINKAGE) - #define __TBBMALLOC_NO_IMPLICIT_LINKAGE 1 - #endif - - #if !__TBBMALLOC_NO_IMPLICIT_LINKAGE - #ifdef _DEBUG - #pragma comment(lib, "tbbmalloc_debug.lib") - #else - #pragma comment(lib, "tbbmalloc.lib") - #endif - #endif - - -#endif - -#endif /* __cplusplus */ - -#if !defined(__cplusplus) && __ICC==1100 - #pragma warning (pop) -#endif /* ICC 11.0 warning 991 is back */ - -#endif /* __TBB_scalable_allocator_H */ +#include "../oneapi/tbb/scalable_allocator.h" diff --git a/src/tbb/include/tbb/spin_mutex.h b/src/tbb/include/tbb/spin_mutex.h index 99ef15c22..1a6f7f077 100644 --- a/src/tbb/include/tbb/spin_mutex.h +++ b/src/tbb/include/tbb/spin_mutex.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,201 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ -#ifndef __TBB_spin_mutex_H -#define __TBB_spin_mutex_H - -#include -#include -#include "aligned_space.h" -#include "tbb_stddef.h" -#include "tbb_machine.h" -#include "tbb_profiling.h" -#include "internal/_mutex_padding.h" - -namespace tbb { - -//! A lock that occupies a single byte. -/** A spin_mutex is a spin mutex that fits in a single byte. - It should be used only for locking short critical sections - (typically less than 20 instructions) when fairness is not an issue. - If zero-initialized, the mutex is considered unheld. - @ingroup synchronization */ -class spin_mutex : internal::mutex_copy_deprecated_and_disabled { - //! 0 if lock is released, 1 if lock is acquired. - __TBB_atomic_flag flag; - -public: - //! Construct unacquired lock. - /** Equivalent to zero-initialization of *this. */ - spin_mutex() : flag(0) { -#if TBB_USE_THREADING_TOOLS - internal_construct(); -#endif - } - - //! Represents acquisition of a mutex. - class scoped_lock : internal::no_copy { - private: - //! Points to currently held mutex, or NULL if no lock is held. - spin_mutex* my_mutex; - - //! Value to store into spin_mutex::flag to unlock the mutex. - /** This variable is no longer used. Instead, 0 and 1 are used to - represent that the lock is free and acquired, respectively. - We keep the member variable here to ensure backward compatibility */ - __TBB_Flag my_unlock_value; - - //! Like acquire, but with ITT instrumentation. - void __TBB_EXPORTED_METHOD internal_acquire( spin_mutex& m ); - - //! Like try_acquire, but with ITT instrumentation. - bool __TBB_EXPORTED_METHOD internal_try_acquire( spin_mutex& m ); - - //! Like release, but with ITT instrumentation. - void __TBB_EXPORTED_METHOD internal_release(); - - friend class spin_mutex; - - public: - //! Construct without acquiring a mutex. - scoped_lock() : my_mutex(NULL), my_unlock_value(0) {} - - //! Construct and acquire lock on a mutex. - scoped_lock( spin_mutex& m ) : my_unlock_value(0) { - internal::suppress_unused_warning(my_unlock_value); -#if TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT - my_mutex=NULL; - internal_acquire(m); -#else - my_mutex=&m; - __TBB_LockByte(m.flag); -#endif /* TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT*/ - } - - //! Acquire lock. - void acquire( spin_mutex& m ) { -#if TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT - internal_acquire(m); -#else - my_mutex = &m; - __TBB_LockByte(m.flag); -#endif /* TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT*/ - } - - //! Try acquiring lock (non-blocking) - /** Return true if lock acquired; false otherwise. */ - bool try_acquire( spin_mutex& m ) { -#if TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT - return internal_try_acquire(m); -#else - bool result = __TBB_TryLockByte(m.flag); - if( result ) - my_mutex = &m; - return result; -#endif /* TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT*/ - } - - //! Release lock - void release() { -#if TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT - internal_release(); -#else - __TBB_UnlockByte(my_mutex->flag); - my_mutex = NULL; -#endif /* TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT */ - } - - //! Destroy lock. If holding a lock, releases the lock first. - ~scoped_lock() { - if( my_mutex ) { -#if TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT - internal_release(); -#else - __TBB_UnlockByte(my_mutex->flag); -#endif /* TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT */ - } - } - }; - - //! Internal constructor with ITT instrumentation. - void __TBB_EXPORTED_METHOD internal_construct(); - - // Mutex traits - static const bool is_rw_mutex = false; - static const bool is_recursive_mutex = false; - static const bool is_fair_mutex = false; - - // ISO C++0x compatibility methods - - //! Acquire lock - void lock() { -#if TBB_USE_THREADING_TOOLS - aligned_space tmp; - new(tmp.begin()) scoped_lock(*this); -#else - __TBB_LockByte(flag); -#endif /* TBB_USE_THREADING_TOOLS*/ - } - - //! Try acquiring lock (non-blocking) - /** Return true if lock acquired; false otherwise. */ - bool try_lock() { -#if TBB_USE_THREADING_TOOLS - aligned_space tmp; - return (new(tmp.begin()) scoped_lock)->internal_try_acquire(*this); -#else - return __TBB_TryLockByte(flag); -#endif /* TBB_USE_THREADING_TOOLS*/ - } - - //! Release lock - void unlock() { -#if TBB_USE_THREADING_TOOLS - aligned_space tmp; - scoped_lock& s = *tmp.begin(); - s.my_mutex = this; - s.internal_release(); -#else - __TBB_UnlockByte(flag); -#endif /* TBB_USE_THREADING_TOOLS */ - } - - friend class scoped_lock; -}; // end of spin_mutex - -__TBB_DEFINE_PROFILING_SET_NAME(spin_mutex) - -} // namespace tbb - -#if ( __TBB_x86_32 || __TBB_x86_64 ) -#include "internal/_x86_eliding_mutex_impl.h" -#endif - -namespace tbb { -//! A cross-platform spin mutex with speculative lock acquisition. -/** On platforms with proper HW support, this lock may speculatively execute - its critical sections, using HW mechanisms to detect real data races and - ensure atomicity of the critical sections. In particular, it uses - Intel(R) Transactional Synchronization Extensions (Intel(R) TSX). - Without such HW support, it behaves like a spin_mutex. - It should be used for locking short critical sections where the lock is - contended but the data it protects are not. If zero-initialized, the - mutex is considered unheld. - @ingroup synchronization */ - -#if ( __TBB_x86_32 || __TBB_x86_64 ) -typedef interface7::internal::padded_mutex speculative_spin_mutex; -#else -typedef interface7::internal::padded_mutex speculative_spin_mutex; -#endif -__TBB_DEFINE_PROFILING_SET_NAME(speculative_spin_mutex) - -} // namespace tbb - -#endif /* __TBB_spin_mutex_H */ +#include "../oneapi/tbb/spin_mutex.h" diff --git a/src/tbb/include/tbb/spin_rw_mutex.h b/src/tbb/include/tbb/spin_rw_mutex.h index b20f4ebda..d36282b48 100644 --- a/src/tbb/include/tbb/spin_rw_mutex.h +++ b/src/tbb/include/tbb/spin_rw_mutex.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,248 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ -#ifndef __TBB_spin_rw_mutex_H -#define __TBB_spin_rw_mutex_H - -#include "tbb_stddef.h" -#include "tbb_machine.h" -#include "tbb_profiling.h" -#include "internal/_mutex_padding.h" - -namespace tbb { - -#if __TBB_TSX_AVAILABLE -namespace interface8 { namespace internal { - class x86_rtm_rw_mutex; -}} -#endif - -class spin_rw_mutex_v3; -typedef spin_rw_mutex_v3 spin_rw_mutex; - -//! Fast, unfair, spinning reader-writer lock with backoff and writer-preference -/** @ingroup synchronization */ -class spin_rw_mutex_v3 : internal::mutex_copy_deprecated_and_disabled { - //! @cond INTERNAL - - //! Internal acquire write lock. - bool __TBB_EXPORTED_METHOD internal_acquire_writer(); - - //! Out of line code for releasing a write lock. - /** This code has debug checking and instrumentation for Intel(R) Thread Checker and Intel(R) Thread Profiler. */ - void __TBB_EXPORTED_METHOD internal_release_writer(); - - //! Internal acquire read lock. - void __TBB_EXPORTED_METHOD internal_acquire_reader(); - - //! Internal upgrade reader to become a writer. - bool __TBB_EXPORTED_METHOD internal_upgrade(); - - //! Out of line code for downgrading a writer to a reader. - /** This code has debug checking and instrumentation for Intel(R) Thread Checker and Intel(R) Thread Profiler. */ - void __TBB_EXPORTED_METHOD internal_downgrade(); - - //! Internal release read lock. - void __TBB_EXPORTED_METHOD internal_release_reader(); - - //! Internal try_acquire write lock. - bool __TBB_EXPORTED_METHOD internal_try_acquire_writer(); - - //! Internal try_acquire read lock. - bool __TBB_EXPORTED_METHOD internal_try_acquire_reader(); - - //! @endcond -public: - //! Construct unacquired mutex. - spin_rw_mutex_v3() : state(0) { -#if TBB_USE_THREADING_TOOLS - internal_construct(); -#endif - } - -#if TBB_USE_ASSERT - //! Destructor asserts if the mutex is acquired, i.e. state is zero. - ~spin_rw_mutex_v3() { - __TBB_ASSERT( !state, "destruction of an acquired mutex"); - }; -#endif /* TBB_USE_ASSERT */ - - //! The scoped locking pattern - /** It helps to avoid the common problem of forgetting to release lock. - It also nicely provides the "node" for queuing locks. */ - class scoped_lock : internal::no_copy { -#if __TBB_TSX_AVAILABLE - friend class tbb::interface8::internal::x86_rtm_rw_mutex; - // helper methods for x86_rtm_rw_mutex - spin_rw_mutex *internal_get_mutex() const { return mutex; } - void internal_set_mutex(spin_rw_mutex* m) { mutex = m; } -#endif - public: - //! Construct lock that has not acquired a mutex. - /** Equivalent to zero-initialization of *this. */ - scoped_lock() : mutex(NULL), is_writer(false) {} - - //! Acquire lock on given mutex. - scoped_lock( spin_rw_mutex& m, bool write = true ) : mutex(NULL) { - acquire(m, write); - } - - //! Release lock (if lock is held). - ~scoped_lock() { - if( mutex ) release(); - } - - //! Acquire lock on given mutex. - void acquire( spin_rw_mutex& m, bool write = true ) { - __TBB_ASSERT( !mutex, "holding mutex already" ); - is_writer = write; - mutex = &m; - if( write ) mutex->internal_acquire_writer(); - else mutex->internal_acquire_reader(); - } - - //! Upgrade reader to become a writer. - /** Returns whether the upgrade happened without releasing and re-acquiring the lock */ - bool upgrade_to_writer() { - __TBB_ASSERT( mutex, "lock is not acquired" ); - __TBB_ASSERT( !is_writer, "not a reader" ); - is_writer = true; - return mutex->internal_upgrade(); - } - - //! Release lock. - void release() { - __TBB_ASSERT( mutex, "lock is not acquired" ); - spin_rw_mutex *m = mutex; - mutex = NULL; -#if TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT - if( is_writer ) m->internal_release_writer(); - else m->internal_release_reader(); -#else - if( is_writer ) __TBB_AtomicAND( &m->state, READERS ); - else __TBB_FetchAndAddWrelease( &m->state, -(intptr_t)ONE_READER); -#endif /* TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT */ - } - - //! Downgrade writer to become a reader. - bool downgrade_to_reader() { - __TBB_ASSERT( mutex, "lock is not acquired" ); - __TBB_ASSERT( is_writer, "not a writer" ); -#if TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT - mutex->internal_downgrade(); -#else - __TBB_FetchAndAddW( &mutex->state, ((intptr_t)ONE_READER-WRITER)); -#endif /* TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT */ - is_writer = false; - return true; - } - - //! Try acquire lock on given mutex. - bool try_acquire( spin_rw_mutex& m, bool write = true ) { - __TBB_ASSERT( !mutex, "holding mutex already" ); - bool result; - is_writer = write; - result = write? m.internal_try_acquire_writer() - : m.internal_try_acquire_reader(); - if( result ) - mutex = &m; - return result; - } - - protected: - - //! The pointer to the current mutex that is held, or NULL if no mutex is held. - spin_rw_mutex* mutex; - - //! If mutex!=NULL, then is_writer is true if holding a writer lock, false if holding a reader lock. - /** Not defined if not holding a lock. */ - bool is_writer; - }; - - // Mutex traits - static const bool is_rw_mutex = true; - static const bool is_recursive_mutex = false; - static const bool is_fair_mutex = false; - - // ISO C++0x compatibility methods - - //! Acquire writer lock - void lock() {internal_acquire_writer();} - - //! Try acquiring writer lock (non-blocking) - /** Return true if lock acquired; false otherwise. */ - bool try_lock() {return internal_try_acquire_writer();} - - //! Release lock - void unlock() { -#if TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT - if( state&WRITER ) internal_release_writer(); - else internal_release_reader(); -#else - if( state&WRITER ) __TBB_AtomicAND( &state, READERS ); - else __TBB_FetchAndAddWrelease( &state, -(intptr_t)ONE_READER); -#endif /* TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT */ - } - - // Methods for reader locks that resemble ISO C++0x compatibility methods. - - //! Acquire reader lock - void lock_read() {internal_acquire_reader();} - - //! Try acquiring reader lock (non-blocking) - /** Return true if reader lock acquired; false otherwise. */ - bool try_lock_read() {return internal_try_acquire_reader();} - -protected: - typedef intptr_t state_t; - static const state_t WRITER = 1; - static const state_t WRITER_PENDING = 2; - static const state_t READERS = ~(WRITER | WRITER_PENDING); - static const state_t ONE_READER = 4; - static const state_t BUSY = WRITER | READERS; - //! State of lock - /** Bit 0 = writer is holding lock - Bit 1 = request by a writer to acquire lock (hint to readers to wait) - Bit 2..N = number of readers holding lock */ - state_t state; - -private: - void __TBB_EXPORTED_METHOD internal_construct(); -}; - -__TBB_DEFINE_PROFILING_SET_NAME(spin_rw_mutex) - -} // namespace tbb - -#if __TBB_TSX_AVAILABLE -#include "internal/_x86_rtm_rw_mutex_impl.h" -#endif - -namespace tbb { -namespace interface8 { -//! A cross-platform spin reader/writer mutex with speculative lock acquisition. -/** On platforms with proper HW support, this lock may speculatively execute - its critical sections, using HW mechanisms to detect real data races and - ensure atomicity of the critical sections. In particular, it uses - Intel(R) Transactional Synchronization Extensions (Intel(R) TSX). - Without such HW support, it behaves like a spin_rw_mutex. - It should be used for locking short critical sections where the lock is - contended but the data it protects are not. - @ingroup synchronization */ -#if __TBB_TSX_AVAILABLE -typedef interface7::internal::padded_mutex speculative_spin_rw_mutex; -#else -typedef interface7::internal::padded_mutex speculative_spin_rw_mutex; -#endif -} // namespace interface8 - -using interface8::speculative_spin_rw_mutex; -__TBB_DEFINE_PROFILING_SET_NAME(speculative_spin_rw_mutex) -} // namespace tbb -#endif /* __TBB_spin_rw_mutex_H */ +#include "../oneapi/tbb/spin_rw_mutex.h" diff --git a/src/tbb/include/tbb/task.h b/src/tbb/include/tbb/task.h index 0b04c6b98..9be95b0d6 100644 --- a/src/tbb/include/tbb/task.h +++ b/src/tbb/include/tbb/task.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,1045 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ -#ifndef __TBB_task_H -#define __TBB_task_H - -#include "tbb_stddef.h" -#include "tbb_machine.h" -#include "tbb_profiling.h" -#include - -typedef struct ___itt_caller *__itt_caller; - -namespace tbb { - -class task; -class task_list; -class task_group_context; - -// MSVC does not allow taking the address of a member that was defined -// privately in task_base and made public in class task via a using declaration. -#if _MSC_VER || (__GNUC__==3 && __GNUC_MINOR__<3) -#define __TBB_TASK_BASE_ACCESS public -#else -#define __TBB_TASK_BASE_ACCESS private -#endif - -namespace internal { //< @cond INTERNAL - - class allocate_additional_child_of_proxy: no_assign { - //! No longer used, but retained for binary layout compatibility. Always NULL. - task* self; - task& parent; - public: - explicit allocate_additional_child_of_proxy( task& parent_ ) : self(NULL), parent(parent_) {} - task& __TBB_EXPORTED_METHOD allocate( size_t size ) const; - void __TBB_EXPORTED_METHOD free( task& ) const; - }; - - struct cpu_ctl_env_space { int space[sizeof(internal::uint64_t)/sizeof(int)]; }; -} //< namespace internal @endcond - -namespace interface5 { - namespace internal { - //! Base class for methods that became static in TBB 3.0. - /** TBB's evolution caused the "this" argument for several methods to become obsolete. - However, for backwards binary compatibility, the new methods need distinct names, - otherwise the One Definition Rule would be broken. Hence the new methods are - defined in this private base class, and then exposed in class task via - using declarations. */ - class task_base: tbb::internal::no_copy { - __TBB_TASK_BASE_ACCESS: - friend class tbb::task; - - //! Schedule task for execution when a worker becomes available. - static void spawn( task& t ); - - //! Spawn multiple tasks and clear list. - static void spawn( task_list& list ); - - //! Like allocate_child, except that task's parent becomes "t", not this. - /** Typically used in conjunction with schedule_to_reexecute to implement while loops. - Atomically increments the reference count of t.parent() */ - static tbb::internal::allocate_additional_child_of_proxy allocate_additional_child_of( task& t ) { - return tbb::internal::allocate_additional_child_of_proxy(t); - } - - //! Destroy a task. - /** Usually, calling this method is unnecessary, because a task is - implicitly deleted after its execute() method runs. However, - sometimes a task needs to be explicitly deallocated, such as - when a root task is used as the parent in spawn_and_wait_for_all. */ - static void __TBB_EXPORTED_FUNC destroy( task& victim ); - }; - } // internal -} // interface5 - -//! @cond INTERNAL -namespace internal { - - class scheduler: no_copy { - public: - //! For internal use only - virtual void spawn( task& first, task*& next ) = 0; - - //! For internal use only - virtual void wait_for_all( task& parent, task* child ) = 0; - - //! For internal use only - virtual void spawn_root_and_wait( task& first, task*& next ) = 0; - - //! Pure virtual destructor; - // Have to have it just to shut up overzealous compilation warnings - virtual ~scheduler() = 0; - - //! For internal use only - virtual void enqueue( task& t, void* reserved ) = 0; - }; - - //! A reference count - /** Should always be non-negative. A signed type is used so that underflow can be detected. */ - typedef intptr_t reference_count; - - //! An id as used for specifying affinity. - typedef unsigned short affinity_id; - -#if __TBB_TASK_ISOLATION - //! A tag for task isolation. - typedef intptr_t isolation_tag; - const isolation_tag no_isolation = 0; -#endif /* __TBB_TASK_ISOLATION */ - -#if __TBB_TASK_GROUP_CONTEXT - class generic_scheduler; - - struct context_list_node_t { - context_list_node_t *my_prev, - *my_next; - }; - - class allocate_root_with_context_proxy: no_assign { - task_group_context& my_context; - public: - allocate_root_with_context_proxy ( task_group_context& ctx ) : my_context(ctx) {} - task& __TBB_EXPORTED_METHOD allocate( size_t size ) const; - void __TBB_EXPORTED_METHOD free( task& ) const; - }; -#endif /* __TBB_TASK_GROUP_CONTEXT */ - - class allocate_root_proxy: no_assign { - public: - static task& __TBB_EXPORTED_FUNC allocate( size_t size ); - static void __TBB_EXPORTED_FUNC free( task& ); - }; - - class allocate_continuation_proxy: no_assign { - public: - task& __TBB_EXPORTED_METHOD allocate( size_t size ) const; - void __TBB_EXPORTED_METHOD free( task& ) const; - }; - - class allocate_child_proxy: no_assign { - public: - task& __TBB_EXPORTED_METHOD allocate( size_t size ) const; - void __TBB_EXPORTED_METHOD free( task& ) const; - }; - - //! Memory prefix to a task object. - /** This class is internal to the library. - Do not reference it directly, except within the library itself. - Fields are ordered in way that preserves backwards compatibility and yields good packing on - typical 32-bit and 64-bit platforms. New fields should be added at the beginning for - backward compatibility with accesses to the task prefix inlined into application code. To - prevent ODR violation, the class shall have the same layout in all application translation - units. If some fields are conditional (e.g. enabled by preview macros) and might get - skipped, use reserved fields to adjust the layout. - - In case task prefix size exceeds 32 or 64 bytes on IA32 and Intel64 architectures - correspondingly, consider dynamic setting of task_alignment and task_prefix_reservation_size - based on the maximal operand size supported by the current CPU. - - @ingroup task_scheduling */ - class task_prefix { - private: - friend class tbb::task; - friend class tbb::interface5::internal::task_base; - friend class tbb::task_list; - friend class internal::scheduler; - friend class internal::allocate_root_proxy; - friend class internal::allocate_child_proxy; - friend class internal::allocate_continuation_proxy; - friend class internal::allocate_additional_child_of_proxy; - -#if __TBB_TASK_ISOLATION - //! The tag used for task isolation. - isolation_tag isolation; -#else - intptr_t reserved_space_for_task_isolation_tag; -#endif /* __TBB_TASK_ISOLATION */ - -#if __TBB_TASK_GROUP_CONTEXT - //! Shared context that is used to communicate asynchronous state changes - /** Currently it is used to broadcast cancellation requests generated both - by users and as the result of unhandled exceptions in the task::execute() - methods. */ - task_group_context *context; -#endif /* __TBB_TASK_GROUP_CONTEXT */ - - //! The scheduler that allocated the task, or NULL if the task is big. - /** Small tasks are pooled by the scheduler that allocated the task. - If a scheduler needs to free a small task allocated by another scheduler, - it returns the task to that other scheduler. This policy avoids - memory space blowup issues for memory allocators that allocate from - thread-specific pools. */ - scheduler* origin; - -#if __TBB_TASK_PRIORITY - union { -#endif /* __TBB_TASK_PRIORITY */ - //! Obsolete. The scheduler that owns the task. - /** Retained only for the sake of backward binary compatibility. - Still used by inline methods in the task.h header. **/ - scheduler* owner; - -#if __TBB_TASK_PRIORITY - //! Pointer to the next offloaded lower priority task. - /** Used to maintain a list of offloaded tasks inside the scheduler. **/ - task* next_offloaded; - }; -#endif /* __TBB_TASK_PRIORITY */ - - //! The task whose reference count includes me. - /** In the "blocking style" of programming, this field points to the parent task. - In the "continuation-passing style" of programming, this field points to the - continuation of the parent. */ - tbb::task* parent; - - //! Reference count used for synchronization. - /** In the "continuation-passing style" of programming, this field is - the difference of the number of allocated children minus the - number of children that have completed. - In the "blocking style" of programming, this field is one more than the difference. */ - __TBB_atomic reference_count ref_count; - - //! Obsolete. Used to be scheduling depth before TBB 2.2 - /** Retained only for the sake of backward binary compatibility. - Not used by TBB anymore. **/ - int depth; - - //! A task::state_type, stored as a byte for compactness. - /** This state is exposed to users via method task::state(). */ - unsigned char state; - - //! Miscellaneous state that is not directly visible to users, stored as a byte for compactness. - /** 0x0 -> version 1.0 task - 0x1 -> version >=2.1 task - 0x10 -> task was enqueued - 0x20 -> task_proxy - 0x40 -> task has live ref_count - 0x80 -> a stolen task */ - unsigned char extra_state; - - affinity_id affinity; - - //! "next" field for list of task - tbb::task* next; - - //! The task corresponding to this task_prefix. - tbb::task& task() {return *reinterpret_cast(this+1);} - }; - -} // namespace internal -//! @endcond - -#if __TBB_TASK_GROUP_CONTEXT - -#if __TBB_TASK_PRIORITY -namespace internal { - static const int priority_stride_v4 = INT_MAX / 4; -} - -enum priority_t { - priority_normal = internal::priority_stride_v4 * 2, - priority_low = priority_normal - internal::priority_stride_v4, - priority_high = priority_normal + internal::priority_stride_v4 -}; - -#endif /* __TBB_TASK_PRIORITY */ - -#if TBB_USE_CAPTURED_EXCEPTION - class tbb_exception; -#else - namespace internal { - class tbb_exception_ptr; - } -#endif /* !TBB_USE_CAPTURED_EXCEPTION */ - -class task_scheduler_init; -namespace interface7 { class task_arena; } - -//! Used to form groups of tasks -/** @ingroup task_scheduling - The context services explicit cancellation requests from user code, and unhandled - exceptions intercepted during tasks execution. Intercepting an exception results - in generating internal cancellation requests (which is processed in exactly the - same way as external ones). - - The context is associated with one or more root tasks and defines the cancellation - group that includes all the descendants of the corresponding root task(s). Association - is established when a context object is passed as an argument to the task::allocate_root() - method. See task_group_context::task_group_context for more details. - - The context can be bound to another one, and other contexts can be bound to it, - forming a tree-like structure: parent -> this -> children. Arrows here designate - cancellation propagation direction. If a task in a cancellation group is cancelled - all the other tasks in this group and groups bound to it (as children) get cancelled too. - - IMPLEMENTATION NOTE: - When adding new members to task_group_context or changing types of existing ones, - update the size of both padding buffers (_leading_padding and _trailing_padding) - appropriately. See also VERSIONING NOTE at the constructor definition below. **/ -class task_group_context : internal::no_copy { -private: - friend class internal::generic_scheduler; - friend class task_scheduler_init; - friend class interface7::task_arena; - -#if TBB_USE_CAPTURED_EXCEPTION - typedef tbb_exception exception_container_type; -#else - typedef internal::tbb_exception_ptr exception_container_type; -#endif - - enum version_traits_word_layout { - traits_offset = 16, - version_mask = 0xFFFF, - traits_mask = 0xFFFFul << traits_offset - }; - -public: - enum kind_type { - isolated, - bound - }; - - enum traits_type { - exact_exception = 0x0001ul << traits_offset, -#if __TBB_FP_CONTEXT - fp_settings = 0x0002ul << traits_offset, -#endif - concurrent_wait = 0x0004ul << traits_offset, -#if TBB_USE_CAPTURED_EXCEPTION - default_traits = 0 -#else - default_traits = exact_exception -#endif /* !TBB_USE_CAPTURED_EXCEPTION */ - }; - -private: - enum state { - may_have_children = 1, - // the following enumerations must be the last, new 2^x values must go above - next_state_value, low_unused_state_bit = (next_state_value-1)*2 - }; - - union { - //! Flavor of this context: bound or isolated. - // TODO: describe asynchronous use, and whether any memory semantics are needed - __TBB_atomic kind_type my_kind; - uintptr_t _my_kind_aligner; - }; - - //! Pointer to the context of the parent cancellation group. NULL for isolated contexts. - task_group_context *my_parent; - - //! Used to form the thread specific list of contexts without additional memory allocation. - /** A context is included into the list of the current thread when its binding to - its parent happens. Any context can be present in the list of one thread only. **/ - internal::context_list_node_t my_node; - - //! Used to set and maintain stack stitching point for Intel Performance Tools. - __itt_caller itt_caller; - - //! Leading padding protecting accesses to frequently used members from false sharing. - /** Read accesses to the field my_cancellation_requested are on the hot path inside - the scheduler. This padding ensures that this field never shares the same cache - line with a local variable that is frequently written to. **/ - char _leading_padding[internal::NFS_MaxLineSize - - 2 * sizeof(uintptr_t)- sizeof(void*) - sizeof(internal::context_list_node_t) - - sizeof(__itt_caller) -#if __TBB_FP_CONTEXT - - sizeof(internal::cpu_ctl_env_space) -#endif - ]; - -#if __TBB_FP_CONTEXT - //! Space for platform-specific FPU settings. - /** Must only be accessed inside TBB binaries, and never directly in user - code or inline methods. */ - internal::cpu_ctl_env_space my_cpu_ctl_env; -#endif - - //! Specifies whether cancellation was requested for this task group. - uintptr_t my_cancellation_requested; - - //! Version for run-time checks and behavioral traits of the context. - /** Version occupies low 16 bits, and traits (zero or more ORed enumerators - from the traits_type enumerations) take the next 16 bits. - Original (zeroth) version of the context did not support any traits. **/ - uintptr_t my_version_and_traits; - - //! Pointer to the container storing exception being propagated across this task group. - exception_container_type *my_exception; - - //! Scheduler instance that registered this context in its thread specific list. - internal::generic_scheduler *my_owner; - - //! Internal state (combination of state flags, currently only may_have_children). - uintptr_t my_state; - -#if __TBB_TASK_PRIORITY - //! Priority level of the task group (in normalized representation) - intptr_t my_priority; -#endif /* __TBB_TASK_PRIORITY */ - - //! Trailing padding protecting accesses to frequently used members from false sharing - /** \sa _leading_padding **/ - char _trailing_padding[internal::NFS_MaxLineSize - 2 * sizeof(uintptr_t) - 2 * sizeof(void*) -#if __TBB_TASK_PRIORITY - - sizeof(intptr_t) -#endif /* __TBB_TASK_PRIORITY */ - ]; - -public: - //! Default & binding constructor. - /** By default a bound context is created. That is this context will be bound - (as child) to the context of the task calling task::allocate_root(this_context) - method. Cancellation requests passed to the parent context are propagated - to all the contexts bound to it. Similarly priority change is propagated - from the parent context to its children. - - If task_group_context::isolated is used as the argument, then the tasks associated - with this context will never be affected by events in any other context. - - Creating isolated contexts involve much less overhead, but they have limited - utility. Normally when an exception occurs in an algorithm that has nested - ones running, it is desirably to have all the nested algorithms cancelled - as well. Such a behavior requires nested algorithms to use bound contexts. - - There is one good place where using isolated algorithms is beneficial. It is - a master thread. That is if a particular algorithm is invoked directly from - the master thread (not from a TBB task), supplying it with explicitly - created isolated context will result in a faster algorithm startup. - - VERSIONING NOTE: - Implementation(s) of task_group_context constructor(s) cannot be made - entirely out-of-line because the run-time version must be set by the user - code. This will become critically important for binary compatibility, if - we ever have to change the size of the context object. - - Boosting the runtime version will also be necessary if new data fields are - introduced in the currently unused padding areas and these fields are updated - by inline methods. **/ - task_group_context ( kind_type relation_with_parent = bound, - uintptr_t t = default_traits ) - : my_kind(relation_with_parent) - , my_version_and_traits(2 | t) - { - init(); - } - - // Do not introduce standalone unbind method since it will break state propagation assumptions - __TBB_EXPORTED_METHOD ~task_group_context (); - - //! Forcefully reinitializes the context after the task tree it was associated with is completed. - /** Because the method assumes that all the tasks that used to be associated with - this context have already finished, calling it while the context is still - in use somewhere in the task hierarchy leads to undefined behavior. - - IMPORTANT: This method is not thread safe! - - The method does not change the context's parent if it is set. **/ - void __TBB_EXPORTED_METHOD reset (); - - //! Initiates cancellation of all tasks in this cancellation group and its subordinate groups. - /** \return false if cancellation has already been requested, true otherwise. - - Note that canceling never fails. When false is returned, it just means that - another thread (or this one) has already sent cancellation request to this - context or to one of its ancestors (if this context is bound). It is guaranteed - that when this method is concurrently called on the same not yet cancelled - context, true will be returned by one and only one invocation. **/ - bool __TBB_EXPORTED_METHOD cancel_group_execution (); - - //! Returns true if the context received cancellation request. - bool __TBB_EXPORTED_METHOD is_group_execution_cancelled () const; - - //! Records the pending exception, and cancels the task group. - /** May be called only from inside a catch-block. If the context is already - cancelled, does nothing. - The method brings the task group associated with this context exactly into - the state it would be in, if one of its tasks threw the currently pending - exception during its execution. In other words, it emulates the actions - of the scheduler's dispatch loop exception handler. **/ - void __TBB_EXPORTED_METHOD register_pending_exception (); - -#if __TBB_FP_CONTEXT - //! Captures the current FPU control settings to the context. - /** Because the method assumes that all the tasks that used to be associated with - this context have already finished, calling it while the context is still - in use somewhere in the task hierarchy leads to undefined behavior. - - IMPORTANT: This method is not thread safe! - - The method does not change the FPU control settings of the context's parent. **/ - void __TBB_EXPORTED_METHOD capture_fp_settings (); -#endif - -#if __TBB_TASK_PRIORITY - //! Changes priority of the task group - void set_priority ( priority_t ); - - //! Retrieves current priority of the current task group - priority_t priority () const; -#endif /* __TBB_TASK_PRIORITY */ - - //! Returns the context's trait - uintptr_t traits() const { return my_version_and_traits & traits_mask; } - -protected: - //! Out-of-line part of the constructor. - /** Singled out to ensure backward binary compatibility of the future versions. **/ - void __TBB_EXPORTED_METHOD init (); - -private: - friend class task; - friend class internal::allocate_root_with_context_proxy; - - static const kind_type binding_required = bound; - static const kind_type binding_completed = kind_type(bound+1); - static const kind_type detached = kind_type(binding_completed+1); - static const kind_type dying = kind_type(detached+1); - - //! Propagates any state change detected to *this, and as an optimisation possibly also upward along the heritage line. - template - void propagate_task_group_state ( T task_group_context::*mptr_state, task_group_context& src, T new_state ); - - //! Registers this context with the local scheduler and binds it to its parent context - void bind_to ( internal::generic_scheduler *local_sched ); - - //! Registers this context with the local scheduler - void register_with ( internal::generic_scheduler *local_sched ); - -#if __TBB_FP_CONTEXT - //! Copies FPU control setting from another context - // TODO: Consider adding #else stub in order to omit #if sections in other code - void copy_fp_settings( const task_group_context &src ); -#endif /* __TBB_FP_CONTEXT */ -}; // class task_group_context - -#endif /* __TBB_TASK_GROUP_CONTEXT */ - -//! Base class for user-defined tasks. -/** @ingroup task_scheduling */ -class task: __TBB_TASK_BASE_ACCESS interface5::internal::task_base { - - //! Set reference count - void __TBB_EXPORTED_METHOD internal_set_ref_count( int count ); - - //! Decrement reference count and return its new value. - internal::reference_count __TBB_EXPORTED_METHOD internal_decrement_ref_count(); - -protected: - //! Default constructor. - task() {prefix().extra_state=1;} - -public: - //! Destructor. - virtual ~task() {} - - //! Should be overridden by derived classes. - virtual task* execute() = 0; - - //! Enumeration of task states that the scheduler considers. - enum state_type { - //! task is running, and will be destroyed after method execute() completes. - executing, - //! task to be rescheduled. - reexecute, - //! task is in ready pool, or is going to be put there, or was just taken off. - ready, - //! task object is freshly allocated or recycled. - allocated, - //! task object is on free list, or is going to be put there, or was just taken off. - freed, - //! task to be recycled as continuation - recycle -#if __TBB_RECYCLE_TO_ENQUEUE - //! task to be scheduled for starvation-resistant execution - ,to_enqueue -#endif - }; - - //------------------------------------------------------------------------ - // Allocating tasks - //------------------------------------------------------------------------ - - //! Returns proxy for overloaded new that allocates a root task. - static internal::allocate_root_proxy allocate_root() { - return internal::allocate_root_proxy(); - } - -#if __TBB_TASK_GROUP_CONTEXT - //! Returns proxy for overloaded new that allocates a root task associated with user supplied context. - static internal::allocate_root_with_context_proxy allocate_root( task_group_context& ctx ) { - return internal::allocate_root_with_context_proxy(ctx); - } -#endif /* __TBB_TASK_GROUP_CONTEXT */ - - //! Returns proxy for overloaded new that allocates a continuation task of *this. - /** The continuation's parent becomes the parent of *this. */ - internal::allocate_continuation_proxy& allocate_continuation() { - return *reinterpret_cast(this); - } - - //! Returns proxy for overloaded new that allocates a child task of *this. - internal::allocate_child_proxy& allocate_child() { - return *reinterpret_cast(this); - } - - //! Define recommended static form via import from base class. - using task_base::allocate_additional_child_of; - -#if __TBB_DEPRECATED_TASK_INTERFACE - //! Destroy a task. - /** Usually, calling this method is unnecessary, because a task is - implicitly deleted after its execute() method runs. However, - sometimes a task needs to be explicitly deallocated, such as - when a root task is used as the parent in spawn_and_wait_for_all. */ - void __TBB_EXPORTED_METHOD destroy( task& t ); -#else /* !__TBB_DEPRECATED_TASK_INTERFACE */ - //! Define recommended static form via import from base class. - using task_base::destroy; -#endif /* !__TBB_DEPRECATED_TASK_INTERFACE */ - - //------------------------------------------------------------------------ - // Recycling of tasks - //------------------------------------------------------------------------ - - //! Change this to be a continuation of its former self. - /** The caller must guarantee that the task's refcount does not become zero until - after the method execute() returns. Typically, this is done by having - method execute() return a pointer to a child of the task. If the guarantee - cannot be made, use method recycle_as_safe_continuation instead. - - Because of the hazard, this method may be deprecated in the future. */ - void recycle_as_continuation() { - __TBB_ASSERT( prefix().state==executing, "execute not running?" ); - prefix().state = allocated; - } - - //! Recommended to use, safe variant of recycle_as_continuation - /** For safety, it requires additional increment of ref_count. - With no descendants and ref_count of 1, it has the semantics of recycle_to_reexecute. */ - void recycle_as_safe_continuation() { - __TBB_ASSERT( prefix().state==executing, "execute not running?" ); - prefix().state = recycle; - } - - //! Change this to be a child of new_parent. - void recycle_as_child_of( task& new_parent ) { - internal::task_prefix& p = prefix(); - __TBB_ASSERT( prefix().state==executing||prefix().state==allocated, "execute not running, or already recycled" ); - __TBB_ASSERT( prefix().ref_count==0, "no child tasks allowed when recycled as a child" ); - __TBB_ASSERT( p.parent==NULL, "parent must be null" ); - __TBB_ASSERT( new_parent.prefix().state<=recycle, "corrupt parent's state" ); - __TBB_ASSERT( new_parent.prefix().state!=freed, "parent already freed" ); - p.state = allocated; - p.parent = &new_parent; -#if __TBB_TASK_GROUP_CONTEXT - p.context = new_parent.prefix().context; -#endif /* __TBB_TASK_GROUP_CONTEXT */ - } - - //! Schedule this for reexecution after current execute() returns. - /** Made obsolete by recycle_as_safe_continuation; may become deprecated. */ - void recycle_to_reexecute() { - __TBB_ASSERT( prefix().state==executing, "execute not running, or already recycled" ); - __TBB_ASSERT( prefix().ref_count==0, "no child tasks allowed when recycled for reexecution" ); - prefix().state = reexecute; - } - -#if __TBB_RECYCLE_TO_ENQUEUE - //! Schedule this to enqueue after descendant tasks complete. - /** Save enqueue/spawn difference, it has the semantics of recycle_as_safe_continuation. */ - void recycle_to_enqueue() { - __TBB_ASSERT( prefix().state==executing, "execute not running, or already recycled" ); - prefix().state = to_enqueue; - } -#endif /* __TBB_RECYCLE_TO_ENQUEUE */ - - //------------------------------------------------------------------------ - // Spawning and blocking - //------------------------------------------------------------------------ - - //! Set reference count - void set_ref_count( int count ) { -#if TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT - internal_set_ref_count(count); -#else - prefix().ref_count = count; -#endif /* TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT */ - } - - //! Atomically increment reference count. - /** Has acquire semantics */ - void increment_ref_count() { - __TBB_FetchAndIncrementWacquire( &prefix().ref_count ); - } - - //! Atomically adds to reference count and returns its new value. - /** Has release-acquire semantics */ - int add_ref_count( int count ) { - internal::call_itt_notify( internal::releasing, &prefix().ref_count ); - internal::reference_count k = count+__TBB_FetchAndAddW( &prefix().ref_count, count ); - __TBB_ASSERT( k>=0, "task's reference count underflowed" ); - if( k==0 ) - internal::call_itt_notify( internal::acquired, &prefix().ref_count ); - return int(k); - } - - //! Atomically decrement reference count and returns its new value. - /** Has release semantics. */ - int decrement_ref_count() { -#if TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT - return int(internal_decrement_ref_count()); -#else - return int(__TBB_FetchAndDecrementWrelease( &prefix().ref_count ))-1; -#endif /* TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT */ - } - - //! Define recommended static forms via import from base class. - using task_base::spawn; - - //! Similar to spawn followed by wait_for_all, but more efficient. - void spawn_and_wait_for_all( task& child ) { - prefix().owner->wait_for_all( *this, &child ); - } - - //! Similar to spawn followed by wait_for_all, but more efficient. - void __TBB_EXPORTED_METHOD spawn_and_wait_for_all( task_list& list ); - - //! Spawn task allocated by allocate_root, wait for it to complete, and deallocate it. - static void spawn_root_and_wait( task& root ) { - root.prefix().owner->spawn_root_and_wait( root, root.prefix().next ); - } - - //! Spawn root tasks on list and wait for all of them to finish. - /** If there are more tasks than worker threads, the tasks are spawned in - order of front to back. */ - static void spawn_root_and_wait( task_list& root_list ); - - //! Wait for reference count to become one, and set reference count to zero. - /** Works on tasks while waiting. */ - void wait_for_all() { - prefix().owner->wait_for_all( *this, NULL ); - } - - //! Enqueue task for starvation-resistant execution. -#if __TBB_TASK_PRIORITY - /** The task will be enqueued on the normal priority level disregarding the - priority of its task group. - - The rationale of such semantics is that priority of an enqueued task is - statically fixed at the moment of its enqueuing, while task group priority - is dynamic. Thus automatic priority inheritance would be generally a subject - to the race, which may result in unexpected behavior. - - Use enqueue() overload with explicit priority value and task::group_priority() - method to implement such priority inheritance when it is really necessary. **/ -#endif /* __TBB_TASK_PRIORITY */ - static void enqueue( task& t ) { - t.prefix().owner->enqueue( t, NULL ); - } - -#if __TBB_TASK_PRIORITY - //! Enqueue task for starvation-resistant execution on the specified priority level. - static void enqueue( task& t, priority_t p ) { - __TBB_ASSERT( p == priority_low || p == priority_normal || p == priority_high, "Invalid priority level value" ); - t.prefix().owner->enqueue( t, (void*)p ); - } -#endif /* __TBB_TASK_PRIORITY */ - - //! The innermost task being executed or destroyed by the current thread at the moment. - static task& __TBB_EXPORTED_FUNC self(); - - //! task on whose behalf this task is working, or NULL if this is a root. - task* parent() const {return prefix().parent;} - - //! sets parent task pointer to specified value - void set_parent(task* p) { -#if __TBB_TASK_GROUP_CONTEXT - __TBB_ASSERT(!p || prefix().context == p->prefix().context, "The tasks must be in the same context"); -#endif - prefix().parent = p; - } - -#if __TBB_TASK_GROUP_CONTEXT - //! This method is deprecated and will be removed in the future. - /** Use method group() instead. **/ - task_group_context* context() {return prefix().context;} - - //! Pointer to the task group descriptor. - task_group_context* group () { return prefix().context; } -#endif /* __TBB_TASK_GROUP_CONTEXT */ - - //! True if task was stolen from the task pool of another thread. - bool is_stolen_task() const { - return (prefix().extra_state & 0x80)!=0; - } - - //------------------------------------------------------------------------ - // Debugging - //------------------------------------------------------------------------ - - //! Current execution state - state_type state() const {return state_type(prefix().state);} - - //! The internal reference count. - int ref_count() const { -#if TBB_USE_ASSERT - internal::reference_count ref_count_ = prefix().ref_count; - __TBB_ASSERT( ref_count_==int(ref_count_), "integer overflow error"); -#endif - return int(prefix().ref_count); - } - - //! Obsolete, and only retained for the sake of backward compatibility. Always returns true. - bool __TBB_EXPORTED_METHOD is_owned_by_current_thread() const; - - //------------------------------------------------------------------------ - // Affinity - //------------------------------------------------------------------------ - - //! An id as used for specifying affinity. - /** Guaranteed to be integral type. Value of 0 means no affinity. */ - typedef internal::affinity_id affinity_id; - - //! Set affinity for this task. - void set_affinity( affinity_id id ) {prefix().affinity = id;} - - //! Current affinity of this task - affinity_id affinity() const {return prefix().affinity;} - - //! Invoked by scheduler to notify task that it ran on unexpected thread. - /** Invoked before method execute() runs, if task is stolen, or task has - affinity but will be executed on another thread. - - The default action does nothing. */ - virtual void __TBB_EXPORTED_METHOD note_affinity( affinity_id id ); - -#if __TBB_TASK_GROUP_CONTEXT - //! Moves this task from its current group into another one. - /** Argument ctx specifies the new group. - - The primary purpose of this method is to associate unique task group context - with a task allocated for subsequent enqueuing. In contrast to spawned tasks - enqueued ones normally outlive the scope where they were created. This makes - traditional usage model where task group context are allocated locally on - the stack inapplicable. Dynamic allocation of context objects is performance - inefficient. Method change_group() allows to make task group context object - a member of the task class, and then associate it with its containing task - object in the latter's constructor. **/ - void __TBB_EXPORTED_METHOD change_group ( task_group_context& ctx ); - - //! Initiates cancellation of all tasks in this cancellation group and its subordinate groups. - /** \return false if cancellation has already been requested, true otherwise. **/ - bool cancel_group_execution () { return prefix().context->cancel_group_execution(); } - - //! Returns true if the context has received cancellation request. - bool is_cancelled () const { return prefix().context->is_group_execution_cancelled(); } -#else - bool is_cancelled () const { return false; } -#endif /* __TBB_TASK_GROUP_CONTEXT */ - -#if __TBB_TASK_PRIORITY - //! Changes priority of the task group this task belongs to. - void set_group_priority ( priority_t p ) { prefix().context->set_priority(p); } - - //! Retrieves current priority of the task group this task belongs to. - priority_t group_priority () const { return prefix().context->priority(); } - -#endif /* __TBB_TASK_PRIORITY */ - -private: - friend class interface5::internal::task_base; - friend class task_list; - friend class internal::scheduler; - friend class internal::allocate_root_proxy; -#if __TBB_TASK_GROUP_CONTEXT - friend class internal::allocate_root_with_context_proxy; -#endif /* __TBB_TASK_GROUP_CONTEXT */ - friend class internal::allocate_continuation_proxy; - friend class internal::allocate_child_proxy; - friend class internal::allocate_additional_child_of_proxy; - - //! Get reference to corresponding task_prefix. - /** Version tag prevents loader on Linux from using the wrong symbol in debug builds. **/ - internal::task_prefix& prefix( internal::version_tag* = NULL ) const { - return reinterpret_cast(const_cast(this))[-1]; - } -}; // class task - -//! task that does nothing. Useful for synchronization. -/** @ingroup task_scheduling */ -class empty_task: public task { - task* execute() __TBB_override { - return NULL; - } -}; - -//! @cond INTERNAL -namespace internal { - template - class function_task : public task { -#if __TBB_ALLOW_MUTABLE_FUNCTORS - F my_func; -#else - const F my_func; -#endif - task* execute() __TBB_override { - my_func(); - return NULL; - } - public: - function_task( const F& f ) : my_func(f) {} -#if __TBB_CPP11_RVALUE_REF_PRESENT - function_task( F&& f ) : my_func( std::move(f) ) {} -#endif - }; -} // namespace internal -//! @endcond - -//! A list of children. -/** Used for method task::spawn_children - @ingroup task_scheduling */ -class task_list: internal::no_copy { -private: - task* first; - task** next_ptr; - friend class task; - friend class interface5::internal::task_base; -public: - //! Construct empty list - task_list() : first(NULL), next_ptr(&first) {} - - //! Destroys the list, but does not destroy the task objects. - ~task_list() {} - - //! True if list if empty; false otherwise. - bool empty() const {return !first;} - - //! Push task onto back of list. - void push_back( task& task ) { - task.prefix().next = NULL; - *next_ptr = &task; - next_ptr = &task.prefix().next; - } -#if __TBB_TODO - // TODO: add this method and implement&document the local execution ordering. See more in generic_scheduler::local_spawn - //! Push task onto front of list (FIFO local execution, like individual spawning in the same order). - void push_front( task& task ) { - if( empty() ) { - push_back(task); - } else { - task.prefix().next = first; - first = &task; - } - } -#endif - //! Pop the front task from the list. - task& pop_front() { - __TBB_ASSERT( !empty(), "attempt to pop item from empty task_list" ); - task* result = first; - first = result->prefix().next; - if( !first ) next_ptr = &first; - return *result; - } - - //! Clear the list - void clear() { - first=NULL; - next_ptr=&first; - } -}; - -inline void interface5::internal::task_base::spawn( task& t ) { - t.prefix().owner->spawn( t, t.prefix().next ); -} - -inline void interface5::internal::task_base::spawn( task_list& list ) { - if( task* t = list.first ) { - t->prefix().owner->spawn( *t, *list.next_ptr ); - list.clear(); - } -} - -inline void task::spawn_root_and_wait( task_list& root_list ) { - if( task* t = root_list.first ) { - t->prefix().owner->spawn_root_and_wait( *t, *root_list.next_ptr ); - root_list.clear(); - } -} - -} // namespace tbb - -inline void *operator new( size_t bytes, const tbb::internal::allocate_root_proxy& ) { - return &tbb::internal::allocate_root_proxy::allocate(bytes); -} - -inline void operator delete( void* task, const tbb::internal::allocate_root_proxy& ) { - tbb::internal::allocate_root_proxy::free( *static_cast(task) ); -} - -#if __TBB_TASK_GROUP_CONTEXT -inline void *operator new( size_t bytes, const tbb::internal::allocate_root_with_context_proxy& p ) { - return &p.allocate(bytes); -} - -inline void operator delete( void* task, const tbb::internal::allocate_root_with_context_proxy& p ) { - p.free( *static_cast(task) ); -} -#endif /* __TBB_TASK_GROUP_CONTEXT */ - -inline void *operator new( size_t bytes, const tbb::internal::allocate_continuation_proxy& p ) { - return &p.allocate(bytes); -} - -inline void operator delete( void* task, const tbb::internal::allocate_continuation_proxy& p ) { - p.free( *static_cast(task) ); -} - -inline void *operator new( size_t bytes, const tbb::internal::allocate_child_proxy& p ) { - return &p.allocate(bytes); -} - -inline void operator delete( void* task, const tbb::internal::allocate_child_proxy& p ) { - p.free( *static_cast(task) ); -} - -inline void *operator new( size_t bytes, const tbb::internal::allocate_additional_child_of_proxy& p ) { - return &p.allocate(bytes); -} - -inline void operator delete( void* task, const tbb::internal::allocate_additional_child_of_proxy& p ) { - p.free( *static_cast(task) ); -} - -#endif /* __TBB_task_H */ +#include "../oneapi/tbb/task.h" diff --git a/src/tbb/include/tbb/task_arena.h b/src/tbb/include/tbb/task_arena.h index 94977aeba..f6e34b3e6 100644 --- a/src/tbb/include/tbb/task_arena.h +++ b/src/tbb/include/tbb/task_arena.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,398 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ -#ifndef __TBB_task_arena_H -#define __TBB_task_arena_H - -#include "task.h" -#include "tbb_exception.h" -#include "internal/_template_helpers.h" -#if TBB_USE_THREADING_TOOLS -#include "atomic.h" // for as_atomic -#endif -#include "aligned_space.h" - -namespace tbb { - -namespace this_task_arena { - int max_concurrency(); -} // namespace this_task_arena - -//! @cond INTERNAL -namespace internal { - //! Internal to library. Should not be used by clients. - /** @ingroup task_scheduling */ - class arena; - class task_scheduler_observer_v3; -} // namespace internal -//! @endcond - -namespace interface7 { -class task_arena; - -//! @cond INTERNAL -namespace internal { -using namespace tbb::internal; //e.g. function_task from task.h - -class delegate_base : no_assign { -public: - virtual void operator()() const = 0; - virtual ~delegate_base() {} -}; - -// If decltype is availabe, the helper detects the return type of functor of specified type, -// otherwise it defines the void type. -template -struct return_type_or_void { -#if __TBB_CPP11_DECLTYPE_PRESENT && !__TBB_CPP11_DECLTYPE_OF_FUNCTION_RETURN_TYPE_BROKEN - typedef decltype(declval()()) type; -#else - typedef void type; -#endif -}; - -template -class delegated_function : public delegate_base { - F &my_func; - tbb::aligned_space my_return_storage; - // The function should be called only once. - void operator()() const __TBB_override { - new (my_return_storage.begin()) R(my_func()); - } -public: - delegated_function(F& f) : my_func(f) {} - // The function can be called only after operator() and only once. - R consume_result() const { - return tbb::internal::move(*(my_return_storage.begin())); - } - ~delegated_function() { - my_return_storage.begin()->~R(); - } -}; - -template -class delegated_function : public delegate_base { - F &my_func; - void operator()() const __TBB_override { - my_func(); - } -public: - delegated_function(F& f) : my_func(f) {} - void consume_result() const {} - - friend class task_arena_base; -}; - -class task_arena_base { -protected: - //! NULL if not currently initialized. - internal::arena* my_arena; - -#if __TBB_TASK_GROUP_CONTEXT - //! default context of the arena - task_group_context *my_context; -#endif - - //! Concurrency level for deferred initialization - int my_max_concurrency; - - //! Reserved master slots - unsigned my_master_slots; - - //! Special settings - intptr_t my_version_and_traits; - - enum { - default_flags = 0 -#if __TBB_TASK_GROUP_CONTEXT - | (task_group_context::default_traits & task_group_context::exact_exception) // 0 or 1 << 16 - , exact_exception_flag = task_group_context::exact_exception // used to specify flag for context directly -#endif - }; - - task_arena_base(int max_concurrency, unsigned reserved_for_masters) - : my_arena(0) -#if __TBB_TASK_GROUP_CONTEXT - , my_context(0) -#endif - , my_max_concurrency(max_concurrency) - , my_master_slots(reserved_for_masters) - , my_version_and_traits(default_flags) - {} - - void __TBB_EXPORTED_METHOD internal_initialize(); - void __TBB_EXPORTED_METHOD internal_terminate(); - void __TBB_EXPORTED_METHOD internal_attach(); - void __TBB_EXPORTED_METHOD internal_enqueue( task&, intptr_t ) const; - void __TBB_EXPORTED_METHOD internal_execute( delegate_base& ) const; - void __TBB_EXPORTED_METHOD internal_wait() const; - static int __TBB_EXPORTED_FUNC internal_current_slot(); - static int __TBB_EXPORTED_FUNC internal_max_concurrency( const task_arena * ); -public: - //! Typedef for number of threads that is automatic. - static const int automatic = -1; - static const int not_initialized = -2; - -}; - -#if __TBB_TASK_ISOLATION -void __TBB_EXPORTED_FUNC isolate_within_arena( delegate_base& d, intptr_t reserved = 0 ); - -template -R isolate_impl(F& f) { - delegated_function d(f); - isolate_within_arena(d); - return d.consume_result(); -} -#endif /* __TBB_TASK_ISOLATION */ -} // namespace internal -//! @endcond - -/** 1-to-1 proxy representation class of scheduler's arena - * Constructors set up settings only, real construction is deferred till the first method invocation - * Destructor only removes one of the references to the inner arena representation. - * Final destruction happens when all the references (and the work) are gone. - */ -class task_arena : public internal::task_arena_base { - friend class tbb::internal::task_scheduler_observer_v3; - friend int tbb::this_task_arena::max_concurrency(); - bool my_initialized; - void mark_initialized() { - __TBB_ASSERT( my_arena, "task_arena initialization is incomplete" ); -#if __TBB_TASK_GROUP_CONTEXT - __TBB_ASSERT( my_context, "task_arena initialization is incomplete" ); -#endif -#if TBB_USE_THREADING_TOOLS - // Actual synchronization happens in internal_initialize & internal_attach. - // The race on setting my_initialized is benign, but should be hidden from Intel(R) Inspector - internal::as_atomic(my_initialized).fetch_and_store(true); -#else - my_initialized = true; -#endif - } - - template - void enqueue_impl( __TBB_FORWARDING_REF(F) f -#if __TBB_TASK_PRIORITY - , priority_t p = priority_t(0) -#endif - ) { -#if !__TBB_TASK_PRIORITY - intptr_t p = 0; -#endif - initialize(); -#if __TBB_TASK_GROUP_CONTEXT - internal_enqueue(*new(task::allocate_root(*my_context)) internal::function_task< typename internal::strip::type >(internal::forward(f)), p); -#else - internal_enqueue(*new(task::allocate_root()) internal::function_task< typename internal::strip::type >(internal::forward(f)), p); -#endif /* __TBB_TASK_GROUP_CONTEXT */ - } - - template - R execute_impl(F& f) { - initialize(); - internal::delegated_function d(f); - internal_execute(d); - return d.consume_result(); - } - -public: - //! Creates task_arena with certain concurrency limits - /** Sets up settings only, real construction is deferred till the first method invocation - * @arg max_concurrency specifies total number of slots in arena where threads work - * @arg reserved_for_masters specifies number of slots to be used by master threads only. - * Value of 1 is default and reflects behavior of implicit arenas. - **/ - task_arena(int max_concurrency_ = automatic, unsigned reserved_for_masters = 1) - : task_arena_base(max_concurrency_, reserved_for_masters) - , my_initialized(false) - {} - - //! Copies settings from another task_arena - task_arena(const task_arena &s) // copy settings but not the reference or instance - : task_arena_base(s.my_max_concurrency, s.my_master_slots) - , my_initialized(false) - {} - - //! Tag class used to indicate the "attaching" constructor - struct attach {}; - - //! Creates an instance of task_arena attached to the current arena of the thread - explicit task_arena( attach ) - : task_arena_base(automatic, 1) // use default settings if attach fails - , my_initialized(false) - { - internal_attach(); - if( my_arena ) my_initialized = true; - } - - //! Forces allocation of the resources for the task_arena as specified in constructor arguments - inline void initialize() { - if( !my_initialized ) { - internal_initialize(); - mark_initialized(); - } - } - - //! Overrides concurrency level and forces initialization of internal representation - inline void initialize(int max_concurrency_, unsigned reserved_for_masters = 1) { - // TODO: decide if this call must be thread-safe - __TBB_ASSERT(!my_arena, "Impossible to modify settings of an already initialized task_arena"); - if( !my_initialized ) { - my_max_concurrency = max_concurrency_; - my_master_slots = reserved_for_masters; - initialize(); - } - } - - //! Attaches this instance to the current arena of the thread - inline void initialize(attach) { - // TODO: decide if this call must be thread-safe - __TBB_ASSERT(!my_arena, "Impossible to modify settings of an already initialized task_arena"); - if( !my_initialized ) { - internal_attach(); - if ( !my_arena ) internal_initialize(); - mark_initialized(); - } - } - - //! Removes the reference to the internal arena representation. - //! Not thread safe wrt concurrent invocations of other methods. - inline void terminate() { - if( my_initialized ) { - internal_terminate(); - my_initialized = false; - } - } - - //! Removes the reference to the internal arena representation, and destroys the external object. - //! Not thread safe wrt concurrent invocations of other methods. - ~task_arena() { - terminate(); - } - - //! Returns true if the arena is active (initialized); false otherwise. - //! The name was chosen to match a task_scheduler_init method with the same semantics. - bool is_active() const { return my_initialized; } - - //! Enqueues a task into the arena to process a functor, and immediately returns. - //! Does not require the calling thread to join the arena - -#if __TBB_CPP11_RVALUE_REF_PRESENT - template - void enqueue( F&& f ) { - enqueue_impl(std::forward(f)); - } -#else - template - void enqueue( const F& f ) { - enqueue_impl(f); - } -#endif - -#if __TBB_TASK_PRIORITY - //! Enqueues a task with priority p into the arena to process a functor f, and immediately returns. - //! Does not require the calling thread to join the arena - template -#if __TBB_CPP11_RVALUE_REF_PRESENT - void enqueue( F&& f, priority_t p ) { - __TBB_ASSERT(p == priority_low || p == priority_normal || p == priority_high, "Invalid priority level value"); - enqueue_impl(std::forward(f), p); - } -#else - void enqueue( const F& f, priority_t p ) { - __TBB_ASSERT(p == priority_low || p == priority_normal || p == priority_high, "Invalid priority level value"); - enqueue_impl(f,p); - } -#endif -#endif// __TBB_TASK_PRIORITY - - //! Joins the arena and executes a mutable functor, then returns - //! If not possible to join, wraps the functor into a task, enqueues it and waits for task completion - //! Can decrement the arena demand for workers, causing a worker to leave and free a slot to the calling thread - //! Since C++11, the method returns the value returned by functor (prior to C++11 it returns void). - template - typename internal::return_type_or_void::type execute(F& f) { - return execute_impl::type>(f); - } - - //! Joins the arena and executes a constant functor, then returns - //! If not possible to join, wraps the functor into a task, enqueues it and waits for task completion - //! Can decrement the arena demand for workers, causing a worker to leave and free a slot to the calling thread - //! Since C++11, the method returns the value returned by functor (prior to C++11 it returns void). - template - typename internal::return_type_or_void::type execute(const F& f) { - return execute_impl::type>(f); - } - -#if __TBB_EXTRA_DEBUG - //! Wait for all work in the arena to be completed - //! Even submitted by other application threads - //! Joins arena if/when possible (in the same way as execute()) - void debug_wait_until_empty() { - initialize(); - internal_wait(); - } -#endif //__TBB_EXTRA_DEBUG - - //! Returns the index, aka slot number, of the calling thread in its current arena - //! This method is deprecated and replaced with this_task_arena::current_thread_index() - inline static int current_thread_index() { - return internal_current_slot(); - } - - //! Returns the maximal number of threads that can work inside the arena - inline int max_concurrency() const { - // Handle special cases inside the library - return (my_max_concurrency>1) ? my_max_concurrency : internal_max_concurrency(this); - } -}; - -#if __TBB_TASK_ISOLATION -namespace this_task_arena { - //! Executes a mutable functor in isolation within the current task arena. - //! Since C++11, the method returns the value returned by functor (prior to C++11 it returns void). - template - typename internal::return_type_or_void::type isolate(F& f) { - return internal::isolate_impl::type>(f); - } - - //! Executes a constant functor in isolation within the current task arena. - //! Since C++11, the method returns the value returned by functor (prior to C++11 it returns void). - template - typename internal::return_type_or_void::type isolate(const F& f) { - return internal::isolate_impl::type>(f); - } -} -#endif /* __TBB_TASK_ISOLATION */ -} // namespace interfaceX - -using interface7::task_arena; -#if __TBB_TASK_ISOLATION -namespace this_task_arena { - using namespace interface7::this_task_arena; -} -#endif /* __TBB_TASK_ISOLATION */ - -namespace this_task_arena { - //! Returns the index, aka slot number, of the calling thread in its current arena - inline int current_thread_index() { - int idx = tbb::task_arena::current_thread_index(); - return idx == -1 ? tbb::task_arena::not_initialized : idx; - } - - //! Returns the maximal number of threads that can work inside the arena - inline int max_concurrency() { - return tbb::task_arena::internal_max_concurrency(NULL); - } -} // namespace this_task_arena - -} // namespace tbb - -#endif /* __TBB_task_arena_H */ +#include "../oneapi/tbb/task_arena.h" diff --git a/src/tbb/include/tbb/task_group.h b/src/tbb/include/tbb/task_group.h index b30addc82..2f0250397 100644 --- a/src/tbb/include/tbb/task_group.h +++ b/src/tbb/include/tbb/task_group.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,232 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ -#ifndef __TBB_task_group_H -#define __TBB_task_group_H - -#include "task.h" -#include "tbb_exception.h" -#include "internal/_template_helpers.h" - -#if __TBB_TASK_GROUP_CONTEXT - -namespace tbb { - -namespace internal { - template class task_handle_task; -} - -class task_group; -class structured_task_group; - -template -class task_handle : internal::no_assign { - template friend class internal::task_handle_task; - friend class task_group; - friend class structured_task_group; - - static const intptr_t scheduled = 0x1; - - F my_func; - intptr_t my_state; - - void mark_scheduled () { - // The check here is intentionally lax to avoid the impact of interlocked operation - if ( my_state & scheduled ) - internal::throw_exception( internal::eid_invalid_multiple_scheduling ); - my_state |= scheduled; - } -public: - task_handle( const F& f ) : my_func(f), my_state(0) {} -#if __TBB_CPP11_RVALUE_REF_PRESENT - task_handle( F&& f ) : my_func( std::move(f)), my_state(0) {} -#endif - - void operator() () const { my_func(); } -}; - -enum task_group_status { - not_complete, - complete, - canceled -}; - -namespace internal { - -template -class task_handle_task : public task { - task_handle& my_handle; - task* execute() __TBB_override { - my_handle(); - return NULL; - } -public: - task_handle_task( task_handle& h ) : my_handle(h) { h.mark_scheduled(); } -}; - -class task_group_base : internal::no_copy { -protected: - empty_task* my_root; - task_group_context my_context; - - task& owner () { return *my_root; } - - template - task_group_status internal_run_and_wait( F& f ) { - __TBB_TRY { - if ( !my_context.is_group_execution_cancelled() ) - f(); - } __TBB_CATCH( ... ) { - my_context.register_pending_exception(); - } - return wait(); - } - - template - void internal_run( __TBB_FORWARDING_REF(F) f ) { - owner().spawn( *new( owner().allocate_additional_child_of(*my_root) ) Task( internal::forward(f) )); - } - -public: - task_group_base( uintptr_t traits = 0 ) - : my_context(task_group_context::bound, task_group_context::default_traits | traits) - { - my_root = new( task::allocate_root(my_context) ) empty_task; - my_root->set_ref_count(1); - } - - ~task_group_base() __TBB_NOEXCEPT(false) { - if( my_root->ref_count() > 1 ) { - bool stack_unwinding_in_progress = std::uncaught_exception(); - // Always attempt to do proper cleanup to avoid inevitable memory corruption - // in case of missing wait (for the sake of better testability & debuggability) - if ( !is_canceling() ) - cancel(); - __TBB_TRY { - my_root->wait_for_all(); - } __TBB_CATCH (...) { - task::destroy(*my_root); - __TBB_RETHROW(); - } - task::destroy(*my_root); - if ( !stack_unwinding_in_progress ) - internal::throw_exception( internal::eid_missing_wait ); - } - else { - task::destroy(*my_root); - } - } - - template - void run( task_handle& h ) { - internal_run< internal::task_handle_task >( h ); - } - - task_group_status wait() { - __TBB_TRY { - my_root->wait_for_all(); - } __TBB_CATCH( ... ) { - my_context.reset(); - __TBB_RETHROW(); - } - if ( my_context.is_group_execution_cancelled() ) { - // TODO: the reset method is not thread-safe. Ensure the correct behavior. - my_context.reset(); - return canceled; - } - return complete; - } - - bool is_canceling() { - return my_context.is_group_execution_cancelled(); - } - - void cancel() { - my_context.cancel_group_execution(); - } -}; // class task_group_base - -} // namespace internal - -class task_group : public internal::task_group_base { -public: - task_group () : task_group_base( task_group_context::concurrent_wait ) {} - -#if __SUNPRO_CC - template - void run( task_handle& h ) { - internal_run< internal::task_handle_task >( h ); - } -#else - using task_group_base::run; -#endif - -#if __TBB_CPP11_RVALUE_REF_PRESENT - template - void run( F&& f ) { - internal_run< internal::function_task< typename internal::strip::type > >( std::forward< F >(f) ); - } -#else - template - void run(const F& f) { - internal_run >(f); - } -#endif - - template - task_group_status run_and_wait( const F& f ) { - return internal_run_and_wait( f ); - } - - // TODO: add task_handle rvalues support - template - task_group_status run_and_wait( task_handle& h ) { - h.mark_scheduled(); - return internal_run_and_wait< task_handle >( h ); - } -}; // class task_group - -class structured_task_group : public internal::task_group_base { -public: - // TODO: add task_handle rvalues support - template - task_group_status run_and_wait ( task_handle& h ) { - h.mark_scheduled(); - return internal_run_and_wait< task_handle >( h ); - } - - task_group_status wait() { - task_group_status res = task_group_base::wait(); - my_root->set_ref_count(1); - return res; - } -}; // class structured_task_group - -inline -bool is_current_task_group_canceling() { - return task::self().is_cancelled(); -} - -#if __TBB_CPP11_RVALUE_REF_PRESENT -template -task_handle< typename internal::strip::type > make_task( F&& f ) { - return task_handle< typename internal::strip::type >( std::forward(f) ); -} -#else -template -task_handle make_task( const F& f ) { - return task_handle( f ); -} -#endif /* __TBB_CPP11_RVALUE_REF_PRESENT */ - -} // namespace tbb - -#endif /* __TBB_TASK_GROUP_CONTEXT */ - -#endif /* __TBB_task_group_H */ +#include "../oneapi/tbb/task_group.h" diff --git a/src/tbb/include/tbb/task_scheduler_init.h b/src/tbb/include/tbb/task_scheduler_init.h deleted file mode 100644 index 928e7a4e5..000000000 --- a/src/tbb/include/tbb/task_scheduler_init.h +++ /dev/null @@ -1,161 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_task_scheduler_init_H -#define __TBB_task_scheduler_init_H - -#include "tbb_stddef.h" -#include "limits.h" -#if __TBB_SUPPORTS_WORKERS_WAITING_IN_TERMINATE -#include // nothrow_t -#endif - -namespace tbb { - -typedef std::size_t stack_size_type; - -//! @cond INTERNAL -namespace internal { - //! Internal to library. Should not be used by clients. - /** @ingroup task_scheduling */ - class scheduler; -} // namespace internal -//! @endcond - -//! Class delimiting the scope of task scheduler activity. -/** A thread can construct a task_scheduler_init object and keep it alive - while it uses TBB's tasking subsystem (including parallel algorithms). - - This class allows to customize properties of the TBB task pool to some extent. - For example it can limit concurrency level of parallel work initiated by the - given thread. It also can be used to specify stack size of the TBB worker threads, - though this setting is not effective if the thread pool has already been created. - - If a parallel construct is used without task_scheduler_init object previously - created, the scheduler will be initialized automatically with default settings, - and will persist until this thread exits. Default concurrency level is defined - as described in task_scheduler_init::initialize(). - @ingroup task_scheduling */ -class task_scheduler_init: internal::no_copy { - enum ExceptionPropagationMode { - propagation_mode_exact = 1u, - propagation_mode_captured = 2u, - propagation_mode_mask = propagation_mode_exact | propagation_mode_captured - }; - - /** NULL if not currently initialized. */ - internal::scheduler* my_scheduler; - - bool internal_terminate( bool blocking ); -#if __TBB_SUPPORTS_WORKERS_WAITING_IN_TERMINATE - bool __TBB_EXPORTED_METHOD internal_blocking_terminate( bool throwing ); -#endif -public: - - //! Typedef for number of threads that is automatic. - static const int automatic = -1; - - //! Argument to initialize() or constructor that causes initialization to be deferred. - static const int deferred = -2; - - //! Ensure that scheduler exists for this thread - /** A value of -1 lets TBB decide on the number of threads, which is usually - maximal hardware concurrency for this process, that is the number of logical - CPUs on the machine (possibly limited by the processor affinity mask of this - process (Windows) or of this thread (Linux, FreeBSD). It is preferable option - for production code because it helps to avoid nasty surprises when several - TBB based components run side-by-side or in a nested fashion inside the same - process. - - The number_of_threads is ignored if any other task_scheduler_inits - currently exist. A thread may construct multiple task_scheduler_inits. - Doing so does no harm because the underlying scheduler is reference counted. */ - void __TBB_EXPORTED_METHOD initialize( int number_of_threads=automatic ); - - //! The overloaded method with stack size parameter - /** Overloading is necessary to preserve ABI compatibility */ - void __TBB_EXPORTED_METHOD initialize( int number_of_threads, stack_size_type thread_stack_size ); - - //! Inverse of method initialize. - void __TBB_EXPORTED_METHOD terminate(); - -#if __TBB_SUPPORTS_WORKERS_WAITING_IN_TERMINATE -#if TBB_USE_EXCEPTIONS - //! terminate() that waits for worker threads termination. Throws exception on error. - void blocking_terminate() { - internal_blocking_terminate( /*throwing=*/true ); - } -#endif - //! terminate() that waits for worker threads termination. Returns false on error. - bool blocking_terminate(const std::nothrow_t&) __TBB_NOEXCEPT(true) { - return internal_blocking_terminate( /*throwing=*/false ); - } -#endif // __TBB_SUPPORTS_WORKERS_WAITING_IN_TERMINATE - - //! Shorthand for default constructor followed by call to initialize(number_of_threads). - task_scheduler_init( int number_of_threads=automatic, stack_size_type thread_stack_size=0 ) : my_scheduler(NULL) - { - // Two lowest order bits of the stack size argument may be taken to communicate - // default exception propagation mode of the client to be used when the - // client manually creates tasks in the master thread and does not use - // explicit task group context object. This is necessary because newer - // TBB binaries with exact propagation enabled by default may be used - // by older clients that expect tbb::captured_exception wrapper. - // All zeros mean old client - no preference. - __TBB_ASSERT( !(thread_stack_size & propagation_mode_mask), "Requested stack size is not aligned" ); -#if TBB_USE_EXCEPTIONS - thread_stack_size |= TBB_USE_CAPTURED_EXCEPTION ? propagation_mode_captured : propagation_mode_exact; -#endif /* TBB_USE_EXCEPTIONS */ - initialize( number_of_threads, thread_stack_size ); - } - - //! Destroy scheduler for this thread if thread has no other live task_scheduler_inits. - ~task_scheduler_init() { - if( my_scheduler ) - terminate(); - internal::poison_pointer( my_scheduler ); - } - //! Returns the number of threads TBB scheduler would create if initialized by default. - /** Result returned by this method does not depend on whether the scheduler - has already been initialized. - - Because tbb 2.0 does not support blocking tasks yet, you may use this method - to boost the number of threads in the tbb's internal pool, if your tasks are - doing I/O operations. The optimal number of additional threads depends on how - much time your tasks spend in the blocked state. - - Before TBB 3.0 U4 this method returned the number of logical CPU in the - system. Currently on Windows, Linux and FreeBSD it returns the number of - logical CPUs available to the current process in accordance with its affinity - mask. - - NOTE: The return value of this method never changes after its first invocation. - This means that changes in the process affinity mask that took place after - this method was first invoked will not affect the number of worker threads - in the TBB worker threads pool. */ - static int __TBB_EXPORTED_FUNC default_num_threads (); - - //! Returns true if scheduler is active (initialized); false otherwise - bool is_active() const { return my_scheduler != NULL; } -}; - -} // namespace tbb - -#endif /* __TBB_task_scheduler_init_H */ diff --git a/src/tbb/include/tbb/task_scheduler_observer.h b/src/tbb/include/tbb/task_scheduler_observer.h index 5586ad4fb..9236f4cdf 100644 --- a/src/tbb/include/tbb/task_scheduler_observer.h +++ b/src/tbb/include/tbb/task_scheduler_observer.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,163 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ -#ifndef __TBB_task_scheduler_observer_H -#define __TBB_task_scheduler_observer_H - -#include "atomic.h" -#if __TBB_ARENA_OBSERVER || __TBB_SLEEP_PERMISSION -#include "task_arena.h" -#endif - -#if __TBB_SCHEDULER_OBSERVER - -namespace tbb { -namespace interface6 { -class task_scheduler_observer; -} -namespace internal { - -class observer_proxy; -class observer_list; - -class task_scheduler_observer_v3 { - friend class observer_proxy; - friend class observer_list; - friend class interface6::task_scheduler_observer; - - //! Pointer to the proxy holding this observer. - /** Observers are proxied by the scheduler to maintain persistent lists of them. **/ - observer_proxy* my_proxy; - - //! Counter preventing the observer from being destroyed while in use by the scheduler. - /** Valid only when observation is on. **/ - atomic my_busy_count; - -public: - //! Enable or disable observation - /** For local observers the method can be used only when the current thread - has the task scheduler initialized or is attached to an arena. - - Repeated calls with the same state are no-ops. **/ - void __TBB_EXPORTED_METHOD observe( bool state=true ); - - //! Returns true if observation is enabled, false otherwise. - bool is_observing() const {return my_proxy!=NULL;} - - //! Construct observer with observation disabled. - task_scheduler_observer_v3() : my_proxy(NULL) { my_busy_count.store(0); } - - //! Entry notification - /** Invoked from inside observe(true) call and whenever a worker enters the arena - this observer is associated with. If a thread is already in the arena when - the observer is activated, the entry notification is called before it - executes the first stolen task. - - Obsolete semantics. For global observers it is called by a thread before - the first steal since observation became enabled. **/ - virtual void on_scheduler_entry( bool /*is_worker*/ ) {} - - //! Exit notification - /** Invoked from inside observe(false) call and whenever a worker leaves the - arena this observer is associated with. - - Obsolete semantics. For global observers it is called by a thread before - the first steal since observation became enabled. **/ - virtual void on_scheduler_exit( bool /*is_worker*/ ) {} - - //! Destructor automatically switches observation off if it is enabled. - virtual ~task_scheduler_observer_v3() { if(my_proxy) observe(false);} -}; - -} // namespace internal - -#if __TBB_ARENA_OBSERVER || __TBB_SLEEP_PERMISSION -namespace interface6 { -class task_scheduler_observer : public internal::task_scheduler_observer_v3 { - friend class internal::task_scheduler_observer_v3; - friend class internal::observer_proxy; - friend class internal::observer_list; - - /** Negative numbers with the largest absolute value to minimize probability - of coincidence in case of a bug in busy count usage. **/ - // TODO: take more high bits for version number - static const intptr_t v6_trait = (intptr_t)((~(uintptr_t)0 >> 1) + 1); - - //! contains task_arena pointer or tag indicating local or global semantics of the observer - intptr_t my_context_tag; - enum { global_tag = 0, implicit_tag = 1 }; - -public: - //! Construct local or global observer in inactive state (observation disabled). - /** For a local observer entry/exit notifications are invoked whenever a worker - thread joins/leaves the arena of the observer's owner thread. If a thread is - already in the arena when the observer is activated, the entry notification is - called before it executes the first stolen task. **/ - /** TODO: Obsolete. - Global observer semantics is obsolete as it violates master thread isolation - guarantees and is not composable. Thus the current default behavior of the - constructor is obsolete too and will be changed in one of the future versions - of the library. **/ - explicit task_scheduler_observer( bool local = false ) { -#if __TBB_ARENA_OBSERVER - my_context_tag = local? implicit_tag : global_tag; -#else - __TBB_ASSERT_EX( !local, NULL ); - my_context_tag = global_tag; -#endif - } - -#if __TBB_ARENA_OBSERVER - //! Construct local observer for a given arena in inactive state (observation disabled). - /** entry/exit notifications are invoked whenever a thread joins/leaves arena. - If a thread is already in the arena when the observer is activated, the entry notification - is called before it executes the first stolen task. **/ - explicit task_scheduler_observer( task_arena & a) { - my_context_tag = (intptr_t)&a; - } -#endif /* __TBB_ARENA_OBSERVER */ - - /** Destructor protects instance of the observer from concurrent notification. - It is recommended to disable observation before destructor of a derived class starts, - otherwise it can lead to concurrent notification callback on partly destroyed object **/ - virtual ~task_scheduler_observer() { if(my_proxy) observe(false); } - - //! Enable or disable observation - /** Warning: concurrent invocations of this method are not safe. - Repeated calls with the same state are no-ops. **/ - void observe( bool state=true ) { - if( state && !my_proxy ) { - __TBB_ASSERT( !my_busy_count, "Inconsistent state of task_scheduler_observer instance"); - my_busy_count.store(v6_trait); - } - internal::task_scheduler_observer_v3::observe(state); - } - -#if __TBB_SLEEP_PERMISSION - //! Return commands for may_sleep() - enum { keep_awake = false, allow_sleep = true }; - - //! The callback can be invoked by a worker thread before it goes to sleep. - /** If it returns false ('keep_awake'), the thread will keep spinning and looking for work. - It will not be called for master threads. **/ - virtual bool may_sleep() { return allow_sleep; } -#endif /*__TBB_SLEEP_PERMISSION*/ -}; - -} //namespace interface6 -using interface6::task_scheduler_observer; -#else /*__TBB_ARENA_OBSERVER || __TBB_SLEEP_PERMISSION*/ -typedef tbb::internal::task_scheduler_observer_v3 task_scheduler_observer; -#endif /*__TBB_ARENA_OBSERVER || __TBB_SLEEP_PERMISSION*/ - -} // namespace tbb - -#endif /* __TBB_SCHEDULER_OBSERVER */ - -#endif /* __TBB_task_scheduler_observer_H */ +#include "../oneapi/tbb/task_scheduler_observer.h" diff --git a/src/tbb/include/tbb/tbb.h b/src/tbb/include/tbb/tbb.h index 5e385ea2f..e443b8f1c 100644 --- a/src/tbb/include/tbb/tbb.h +++ b/src/tbb/include/tbb/tbb.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,74 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ -#ifndef __TBB_tbb_H -#define __TBB_tbb_H - -/** - This header bulk-includes declarations or definitions of all the functionality - provided by TBB (save for malloc dependent headers). - - If you use only a few TBB constructs, consider including specific headers only. - Any header listed below can be included independently of others. -**/ - -#if TBB_PREVIEW_AGGREGATOR -#include "aggregator.h" -#endif -#include "aligned_space.h" -#include "atomic.h" -#include "blocked_range.h" -#include "blocked_range2d.h" -#include "blocked_range3d.h" -#include "cache_aligned_allocator.h" -#include "combinable.h" -#include "concurrent_hash_map.h" -#if TBB_PREVIEW_CONCURRENT_LRU_CACHE -#include "concurrent_lru_cache.h" -#endif -#include "concurrent_priority_queue.h" -#include "concurrent_queue.h" -#include "concurrent_unordered_map.h" -#include "concurrent_unordered_set.h" -#include "concurrent_vector.h" -#include "critical_section.h" -#include "enumerable_thread_specific.h" -#include "flow_graph.h" -#if TBB_PREVIEW_GLOBAL_CONTROL -#include "global_control.h" -#endif -#include "mutex.h" -#include "null_mutex.h" -#include "null_rw_mutex.h" -#include "parallel_do.h" -#include "parallel_for.h" -#include "parallel_for_each.h" -#include "parallel_invoke.h" -#include "parallel_reduce.h" -#include "parallel_scan.h" -#include "parallel_sort.h" -#include "partitioner.h" -#include "pipeline.h" -#include "queuing_mutex.h" -#include "queuing_rw_mutex.h" -#include "reader_writer_lock.h" -#include "recursive_mutex.h" -#include "spin_mutex.h" -#include "spin_rw_mutex.h" -#include "task.h" -#include "task_arena.h" -#include "task_group.h" -#include "task_scheduler_init.h" -#include "task_scheduler_observer.h" -#include "tbb_allocator.h" -#include "tbb_exception.h" -#include "tbb_thread.h" -#include "tick_count.h" - -#endif /* __TBB_tbb_H */ +#include "../oneapi/tbb.h" diff --git a/src/tbb/include/tbb/tbb_allocator.h b/src/tbb/include/tbb/tbb_allocator.h index 047344e7b..81ab9d33b 100644 --- a/src/tbb/include/tbb/tbb_allocator.h +++ b/src/tbb/include/tbb/tbb_allocator.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,196 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ -#ifndef __TBB_tbb_allocator_H -#define __TBB_tbb_allocator_H - -#include "tbb_stddef.h" -#include -#if __TBB_ALLOCATOR_CONSTRUCT_VARIADIC - #include // std::forward -#endif -#include - -namespace tbb { - -//! @cond INTERNAL -namespace internal { - - //! Deallocates memory using FreeHandler - /** The function uses scalable_free if scalable allocator is available and free if not*/ - void __TBB_EXPORTED_FUNC deallocate_via_handler_v3( void *p ); - - //! Allocates memory using MallocHandler - /** The function uses scalable_malloc if scalable allocator is available and malloc if not*/ - void* __TBB_EXPORTED_FUNC allocate_via_handler_v3( size_t n ); - - //! Returns true if standard malloc/free are used to work with memory. - bool __TBB_EXPORTED_FUNC is_malloc_used_v3(); -} -//! @endcond - -#if _MSC_VER && !defined(__INTEL_COMPILER) - // Workaround for erroneous "unreferenced parameter" warning in method destroy. - #pragma warning (push) - #pragma warning (disable: 4100) -#endif - -//! Meets "allocator" requirements of ISO C++ Standard, Section 20.1.5 -/** The class selects the best memory allocation mechanism available - from scalable_malloc and standard malloc. - The members are ordered the same way they are in section 20.4.1 - of the ISO C++ standard. - @ingroup memory_allocation */ -template -class tbb_allocator { -public: - typedef typename internal::allocator_type::value_type value_type; - typedef value_type* pointer; - typedef const value_type* const_pointer; - typedef value_type& reference; - typedef const value_type& const_reference; - typedef size_t size_type; - typedef ptrdiff_t difference_type; - template struct rebind { - typedef tbb_allocator other; - }; - - //! Specifies current allocator - enum malloc_type { - scalable, - standard - }; - - tbb_allocator() throw() {} - tbb_allocator( const tbb_allocator& ) throw() {} - template tbb_allocator(const tbb_allocator&) throw() {} - - pointer address(reference x) const {return &x;} - const_pointer address(const_reference x) const {return &x;} - - //! Allocate space for n objects. - pointer allocate( size_type n, const void* /*hint*/ = 0) { - return pointer(internal::allocate_via_handler_v3( n * sizeof(value_type) )); - } - - //! Free previously allocated block of memory. - void deallocate( pointer p, size_type ) { - internal::deallocate_via_handler_v3(p); - } - - //! Largest value for which method allocate might succeed. - size_type max_size() const throw() { - size_type max = static_cast(-1) / sizeof (value_type); - return (max > 0 ? max : 1); - } - - //! Copy-construct value at location pointed to by p. -#if __TBB_ALLOCATOR_CONSTRUCT_VARIADIC - template - void construct(U *p, Args&&... args) - { ::new((void *)p) U(std::forward(args)...); } -#else // __TBB_ALLOCATOR_CONSTRUCT_VARIADIC -#if __TBB_CPP11_RVALUE_REF_PRESENT - void construct( pointer p, value_type&& value ) {::new((void*)(p)) value_type(std::move(value));} -#endif - void construct( pointer p, const value_type& value ) {::new((void*)(p)) value_type(value);} -#endif // __TBB_ALLOCATOR_CONSTRUCT_VARIADIC - - //! Destroy value at location pointed to by p. - void destroy( pointer p ) {p->~value_type();} - - //! Returns current allocator - static malloc_type allocator_type() { - return internal::is_malloc_used_v3() ? standard : scalable; - } -}; - -#if _MSC_VER && !defined(__INTEL_COMPILER) - #pragma warning (pop) -#endif // warning 4100 is back - -//! Analogous to std::allocator, as defined in ISO C++ Standard, Section 20.4.1 -/** @ingroup memory_allocation */ -template<> -class tbb_allocator { -public: - typedef void* pointer; - typedef const void* const_pointer; - typedef void value_type; - template struct rebind { - typedef tbb_allocator other; - }; -}; - -template -inline bool operator==( const tbb_allocator&, const tbb_allocator& ) {return true;} - -template -inline bool operator!=( const tbb_allocator&, const tbb_allocator& ) {return false;} - -//! Meets "allocator" requirements of ISO C++ Standard, Section 20.1.5 -/** The class is an adapter over an actual allocator that fills the allocation - using memset function with template argument C as the value. - The members are ordered the same way they are in section 20.4.1 - of the ISO C++ standard. - @ingroup memory_allocation */ -template class Allocator = tbb_allocator> -class zero_allocator : public Allocator -{ -public: - typedef Allocator base_allocator_type; - typedef typename base_allocator_type::value_type value_type; - typedef typename base_allocator_type::pointer pointer; - typedef typename base_allocator_type::const_pointer const_pointer; - typedef typename base_allocator_type::reference reference; - typedef typename base_allocator_type::const_reference const_reference; - typedef typename base_allocator_type::size_type size_type; - typedef typename base_allocator_type::difference_type difference_type; - template struct rebind { - typedef zero_allocator other; - }; - - zero_allocator() throw() { } - zero_allocator(const zero_allocator &a) throw() : base_allocator_type( a ) { } - template - zero_allocator(const zero_allocator &a) throw() : base_allocator_type( Allocator( a ) ) { } - - pointer allocate(const size_type n, const void *hint = 0 ) { - pointer ptr = base_allocator_type::allocate( n, hint ); - std::memset( (void*) ptr, 0, n * sizeof(value_type) ); - return ptr; - } -}; - -//! Analogous to std::allocator, as defined in ISO C++ Standard, Section 20.4.1 -/** @ingroup memory_allocation */ -template class Allocator> -class zero_allocator : public Allocator { -public: - typedef Allocator base_allocator_type; - typedef typename base_allocator_type::value_type value_type; - typedef typename base_allocator_type::pointer pointer; - typedef typename base_allocator_type::const_pointer const_pointer; - template struct rebind { - typedef zero_allocator other; - }; -}; - -template class B1, typename T2, template class B2> -inline bool operator==( const zero_allocator &a, const zero_allocator &b) { - return static_cast< B1 >(a) == static_cast< B2 >(b); -} -template class B1, typename T2, template class B2> -inline bool operator!=( const zero_allocator &a, const zero_allocator &b) { - return static_cast< B1 >(a) != static_cast< B2 >(b); -} - -} // namespace tbb - -#endif /* __TBB_tbb_allocator_H */ +#include "../oneapi/tbb/tbb_allocator.h" diff --git a/src/tbb/include/tbb/tbb_config.h b/src/tbb/include/tbb/tbb_config.h deleted file mode 100644 index 3ac2b3e03..000000000 --- a/src/tbb/include/tbb/tbb_config.h +++ /dev/null @@ -1,774 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_tbb_config_H -#define __TBB_tbb_config_H - -/** This header is supposed to contain macro definitions and C style comments only. - The macros defined here are intended to control such aspects of TBB build as - - presence of compiler features - - compilation modes - - feature sets - - known compiler/platform issues -**/ - -/* This macro marks incomplete code or comments describing ideas which are considered for the future. - * See also for plain comment with TODO and FIXME marks for small improvement opportunities. - */ -#define __TBB_TODO 0 - -/* Check which standard library we use. */ -/* __TBB_SYMBOL is defined only while processing exported symbols list where C++ is not allowed. */ -#if !defined(__TBB_SYMBOL) && !__TBB_CONFIG_PREPROC_ONLY - #include -#endif - -// note that when ICC or Clang is in use, __TBB_GCC_VERSION might not fully match -// the actual GCC version on the system. -#define __TBB_GCC_VERSION (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) - -// Since GNU libstdc++ does not have a convenient macro for its version, -// we rely on the version of GCC or the user-specified macro below. -// The format of TBB_USE_GLIBCXX_VERSION should match the __TBB_GCC_VERSION above, -// e.g. it should be set to 40902 for libstdc++ coming with GCC 4.9.2. -#ifdef TBB_USE_GLIBCXX_VERSION -#define __TBB_GLIBCXX_VERSION TBB_USE_GLIBCXX_VERSION -#elif __GLIBCPP__ || __GLIBCXX__ -#define __TBB_GLIBCXX_VERSION __TBB_GCC_VERSION -//TODO: analyze __GLIBCXX__ instead of __TBB_GCC_VERSION ? -#endif - -#if __clang__ - /** according to clang documentation, version can be vendor specific **/ - #define __TBB_CLANG_VERSION (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) -#endif - -/** Target OS is either iOS* or iOS* simulator **/ -#if __ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__ - #define __TBB_IOS 1 -#endif - -/** Preprocessor symbols to determine HW architecture **/ - -#if _WIN32||_WIN64 -# if defined(_M_X64)||defined(__x86_64__) // the latter for MinGW support -# define __TBB_x86_64 1 -# elif defined(_M_IA64) -# define __TBB_ipf 1 -# elif defined(_M_IX86)||defined(__i386__) // the latter for MinGW support -# define __TBB_x86_32 1 -# else -# define __TBB_generic_arch 1 -# endif -#else /* Assume generic Unix */ -# if !__linux__ && !__APPLE__ -# define __TBB_generic_os 1 -# endif -# if __TBB_IOS -# define __TBB_generic_arch 1 -# elif __x86_64__ -# define __TBB_x86_64 1 -# elif __ia64__ -# define __TBB_ipf 1 -# elif __i386__||__i386 // __i386 is for Sun OS -# define __TBB_x86_32 1 -# else -# define __TBB_generic_arch 1 -# endif -#endif - -#if __MIC__ || __MIC2__ -#define __TBB_DEFINE_MIC 1 -#endif - -#define __TBB_TSX_AVAILABLE ((__TBB_x86_32 || __TBB_x86_64) && !__TBB_DEFINE_MIC) - -/** Presence of compiler features **/ - -#if __INTEL_COMPILER == 9999 && __INTEL_COMPILER_BUILD_DATE == 20110811 -/* Intel(R) Composer XE 2011 Update 6 incorrectly sets __INTEL_COMPILER. Fix it. */ - #undef __INTEL_COMPILER - #define __INTEL_COMPILER 1210 -#endif - -#if __clang__ && !__INTEL_COMPILER -#define __TBB_USE_OPTIONAL_RTTI __has_feature(cxx_rtti) -#elif defined(_CPPRTTI) -#define __TBB_USE_OPTIONAL_RTTI 1 -#else -#define __TBB_USE_OPTIONAL_RTTI (__GXX_RTTI || __RTTI || __INTEL_RTTI__) -#endif - -#if __TBB_GCC_VERSION >= 40400 && !defined(__INTEL_COMPILER) - /** warning suppression pragmas available in GCC since 4.4 **/ - #define __TBB_GCC_WARNING_SUPPRESSION_PRESENT 1 -#endif - -/* Select particular features of C++11 based on compiler version. - ICC 12.1 (Linux*), GCC 4.3 and higher, clang 2.9 and higher - set __GXX_EXPERIMENTAL_CXX0X__ in c++11 mode. - - Compilers that mimics other compilers (ICC, clang) must be processed before - compilers they mimic (GCC, MSVC). - - TODO: The following conditions should be extended when new compilers/runtimes - support added. - */ - -/** C++11 mode detection macros for Intel(R) C++ Compiler (enabled by -std=c++XY option): - __INTEL_CXX11_MODE__ for version >=13.0 (not available for ICC 15.0 if -std=c++14 is used), - __STDC_HOSTED__ for version >=12.0 (useful only on Windows), - __GXX_EXPERIMENTAL_CXX0X__ for version >=12.0 on Linux and macOS. **/ -#if __INTEL_COMPILER && !__INTEL_CXX11_MODE__ - // __INTEL_CXX11_MODE__ is not set, try to deduce it - #define __INTEL_CXX11_MODE__ (__GXX_EXPERIMENTAL_CXX0X__ || (_MSC_VER && __STDC_HOSTED__)) -#endif - -// Intel(R) C++ Compiler offloading API to the Intel(R) Graphics Technology presence macro -// TODO: add support for ICC 15.00 _GFX_enqueue API and then decrease Intel C++ Compiler supported version -// TODO: add linux support and restict it with (__linux__ && __TBB_x86_64 && !__ANDROID__) macro -#if __INTEL_COMPILER >= 1600 && _WIN32 -#define __TBB_GFX_PRESENT 1 -#endif - -#if __INTEL_COMPILER && (!_MSC_VER || __INTEL_CXX11_MODE__) - // On Windows, C++11 features supported by Visual Studio 2010 and higher are enabled by default, - // so in absence of /Qstd= use MSVC branch for __TBB_CPP11_* detection. - // On other platforms, no -std= means C++03. - - #define __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT (__INTEL_CXX11_MODE__ && __VARIADIC_TEMPLATES) - // Both r-value reference support in compiler and std::move/std::forward - // presence in C++ standard library is checked. - #define __TBB_CPP11_RVALUE_REF_PRESENT ((_MSC_VER >= 1700 || __GXX_EXPERIMENTAL_CXX0X__ && (__TBB_GLIBCXX_VERSION >= 40500 || _LIBCPP_VERSION)) && __INTEL_COMPILER >= 1400) - #define __TBB_IMPLICIT_MOVE_PRESENT (__INTEL_CXX11_MODE__ && __INTEL_COMPILER >= 1400 && (_MSC_VER >= 1900 || __TBB_GCC_VERSION >= 40600 || __clang__)) - #if _MSC_VER >= 1600 - #define __TBB_EXCEPTION_PTR_PRESENT ( __INTEL_COMPILER > 1300 \ - /*ICC 12.1 Upd 10 and 13 beta Upd 2 fixed exception_ptr linking issue*/ \ - || (__INTEL_COMPILER == 1300 && __INTEL_COMPILER_BUILD_DATE >= 20120530) \ - || (__INTEL_COMPILER == 1210 && __INTEL_COMPILER_BUILD_DATE >= 20120410) ) - /** libstdc++ that comes with GCC 4.6 use C++11 features not supported by ICC 12.1. - * Because of that ICC 12.1 does not support C++11 mode with gcc 4.6 (or higher), - * and therefore does not define __GXX_EXPERIMENTAL_CXX0X__ macro **/ - #elif __TBB_GLIBCXX_VERSION >= 40404 && __TBB_GLIBCXX_VERSION < 40600 - #define __TBB_EXCEPTION_PTR_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __INTEL_COMPILER >= 1200) - #elif __TBB_GLIBCXX_VERSION >= 40600 - #define __TBB_EXCEPTION_PTR_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __INTEL_COMPILER >= 1300) - #elif _LIBCPP_VERSION - #define __TBB_EXCEPTION_PTR_PRESENT __GXX_EXPERIMENTAL_CXX0X__ - #else - #define __TBB_EXCEPTION_PTR_PRESENT 0 - #endif - #define __TBB_STATIC_ASSERT_PRESENT (__INTEL_CXX11_MODE__ || _MSC_VER >= 1600) - #define __TBB_CPP11_TUPLE_PRESENT (_MSC_VER >= 1600 || __GXX_EXPERIMENTAL_CXX0X__ && (__TBB_GLIBCXX_VERSION >= 40300 || _LIBCPP_VERSION)) - #if (__clang__ && __INTEL_COMPILER > 1400) - /* Older versions of Intel C++ Compiler do not have __has_include */ - #if (__has_feature(__cxx_generalized_initializers__) && __has_include()) - #define __TBB_INITIALIZER_LISTS_PRESENT 1 - #endif - #else - #define __TBB_INITIALIZER_LISTS_PRESENT (__INTEL_CXX11_MODE__ && __INTEL_COMPILER >= 1400 && (_MSC_VER >= 1800 || __TBB_GLIBCXX_VERSION >= 40400 || _LIBCPP_VERSION)) - #endif - #define __TBB_CONSTEXPR_PRESENT (__INTEL_CXX11_MODE__ && __INTEL_COMPILER >= 1400) - #define __TBB_DEFAULTED_AND_DELETED_FUNC_PRESENT (__INTEL_CXX11_MODE__ && __INTEL_COMPILER >= 1200) - /** ICC seems to disable support of noexcept event in c++11 when compiling in compatibility mode for gcc <4.6 **/ - #define __TBB_NOEXCEPT_PRESENT (__INTEL_CXX11_MODE__ && __INTEL_COMPILER >= 1300 && (__TBB_GLIBCXX_VERSION >= 40600 || _LIBCPP_VERSION || _MSC_VER)) - #define __TBB_CPP11_STD_BEGIN_END_PRESENT (_MSC_VER >= 1700 || __GXX_EXPERIMENTAL_CXX0X__ && __INTEL_COMPILER >= 1310 && (__TBB_GLIBCXX_VERSION >= 40600 || _LIBCPP_VERSION)) - #define __TBB_CPP11_AUTO_PRESENT (_MSC_VER >= 1600 || __GXX_EXPERIMENTAL_CXX0X__ && __INTEL_COMPILER >= 1210) - #define __TBB_CPP11_DECLTYPE_PRESENT (_MSC_VER >= 1600 || __GXX_EXPERIMENTAL_CXX0X__ && __INTEL_COMPILER >= 1210) - #define __TBB_CPP11_LAMBDAS_PRESENT (__INTEL_CXX11_MODE__ && __INTEL_COMPILER >= 1200) - #define __TBB_CPP11_DEFAULT_FUNC_TEMPLATE_ARGS_PRESENT (_MSC_VER >= 1800 || __GXX_EXPERIMENTAL_CXX0X__ && __INTEL_COMPILER >= 1210) - #define __TBB_OVERRIDE_PRESENT (__INTEL_CXX11_MODE__ && __INTEL_COMPILER >= 1400) - #define __TBB_ALIGNAS_PRESENT (__INTEL_CXX11_MODE__ && __INTEL_COMPILER >= 1500) - #define __TBB_CPP11_TEMPLATE_ALIASES_PRESENT (__INTEL_CXX11_MODE__ && __INTEL_COMPILER >= 1210) -#elif __clang__ -/** TODO: these options need to be rechecked **/ -/** on macOS the only way to get C++11 is to use clang. For library features (e.g. exception_ptr) libc++ is also - * required. So there is no need to check GCC version for clang**/ - #define __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT __has_feature(__cxx_variadic_templates__) - #define __TBB_CPP11_RVALUE_REF_PRESENT (__has_feature(__cxx_rvalue_references__) && (_LIBCPP_VERSION || __TBB_GLIBCXX_VERSION >= 40500)) - #define __TBB_IMPLICIT_MOVE_PRESENT __has_feature(cxx_implicit_moves) -/** TODO: extend exception_ptr related conditions to cover libstdc++ **/ - #define __TBB_EXCEPTION_PTR_PRESENT (__cplusplus >= 201103L && (_LIBCPP_VERSION || __TBB_GLIBCXX_VERSION >= 40600)) - #define __TBB_STATIC_ASSERT_PRESENT __has_feature(__cxx_static_assert__) - /**Clang (preprocessor) has problems with dealing with expression having __has_include in #ifs - * used inside C++ code. (At least version that comes with OS X 10.8 : Apple LLVM version 4.2 (clang-425.0.28) (based on LLVM 3.2svn)) **/ - #if (__GXX_EXPERIMENTAL_CXX0X__ && __has_include()) - #define __TBB_CPP11_TUPLE_PRESENT 1 - #endif - #if (__has_feature(__cxx_generalized_initializers__) && __has_include()) - #define __TBB_INITIALIZER_LISTS_PRESENT 1 - #endif - #define __TBB_CONSTEXPR_PRESENT __has_feature(__cxx_constexpr__) - #define __TBB_DEFAULTED_AND_DELETED_FUNC_PRESENT (__has_feature(__cxx_defaulted_functions__) && __has_feature(__cxx_deleted_functions__)) - /**For some unknown reason __has_feature(__cxx_noexcept) does not yield true for all cases. Compiler bug ? **/ - #define __TBB_NOEXCEPT_PRESENT (__cplusplus >= 201103L) - #define __TBB_CPP11_STD_BEGIN_END_PRESENT (__has_feature(__cxx_range_for__) && (_LIBCPP_VERSION || __TBB_GLIBCXX_VERSION >= 40600)) - #define __TBB_CPP11_AUTO_PRESENT __has_feature(__cxx_auto_type__) - #define __TBB_CPP11_DECLTYPE_PRESENT __has_feature(__cxx_decltype__) - #define __TBB_CPP11_LAMBDAS_PRESENT __has_feature(cxx_lambdas) - #define __TBB_CPP11_DEFAULT_FUNC_TEMPLATE_ARGS_PRESENT __has_feature(cxx_default_function_template_args) - #define __TBB_OVERRIDE_PRESENT __has_feature(cxx_override_control) - #define __TBB_ALIGNAS_PRESENT __has_feature(cxx_alignas) - #define __TBB_CPP11_TEMPLATE_ALIASES_PRESENT __has_feature(cxx_alias_templates) -#elif __GNUC__ - #define __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT __GXX_EXPERIMENTAL_CXX0X__ - #define __TBB_CPP11_VARIADIC_FIXED_LENGTH_EXP_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GCC_VERSION >= 40700) - #define __TBB_CPP11_RVALUE_REF_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GCC_VERSION >= 40500) - #define __TBB_IMPLICIT_MOVE_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GCC_VERSION >= 40600) - /** __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4 here is a substitution for _GLIBCXX_ATOMIC_BUILTINS_4, which is a prerequisite - for exception_ptr but cannot be used in this file because it is defined in a header, not by the compiler. - If the compiler has no atomic intrinsics, the C++ library should not expect those as well. **/ - #define __TBB_EXCEPTION_PTR_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GCC_VERSION >= 40404 && __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4) - #define __TBB_STATIC_ASSERT_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GCC_VERSION >= 40300) - #define __TBB_CPP11_TUPLE_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GCC_VERSION >= 40300) - #define __TBB_INITIALIZER_LISTS_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GCC_VERSION >= 40400) - /** gcc seems have to support constexpr from 4.4 but tests in (test_atomic) seeming reasonable fail to compile prior 4.6**/ - #define __TBB_CONSTEXPR_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GCC_VERSION >= 40400) - #define __TBB_DEFAULTED_AND_DELETED_FUNC_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GCC_VERSION >= 40400) - #define __TBB_NOEXCEPT_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GCC_VERSION >= 40600) - #define __TBB_CPP11_STD_BEGIN_END_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GCC_VERSION >= 40600) - #define __TBB_CPP11_AUTO_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GCC_VERSION >= 40400) - #define __TBB_CPP11_DECLTYPE_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GCC_VERSION >= 40400) - #define __TBB_CPP11_LAMBDAS_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GCC_VERSION >= 40500) - #define __TBB_CPP11_DEFAULT_FUNC_TEMPLATE_ARGS_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GCC_VERSION >= 40300) - #define __TBB_OVERRIDE_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GCC_VERSION >= 40700) - #define __TBB_ALIGNAS_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GCC_VERSION >= 40800) - #define __TBB_CPP11_TEMPLATE_ALIASES_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GCC_VERSION >= 40700) -#elif _MSC_VER - // These definitions are also used with Intel C++ Compiler in "default" mode; see a comment above. - - #define __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT (_MSC_VER >= 1800) - // Contains a workaround for ICC 13 - #define __TBB_CPP11_RVALUE_REF_PRESENT (_MSC_VER >= 1700 && (!__INTEL_COMPILER || __INTEL_COMPILER >= 1400)) - #define __TBB_IMPLICIT_MOVE_PRESENT (_MSC_VER >= 1900) - #define __TBB_EXCEPTION_PTR_PRESENT (_MSC_VER >= 1600) - #define __TBB_STATIC_ASSERT_PRESENT (_MSC_VER >= 1600) - #define __TBB_CPP11_TUPLE_PRESENT (_MSC_VER >= 1600) - #define __TBB_INITIALIZER_LISTS_PRESENT (_MSC_VER >= 1800) - #define __TBB_CONSTEXPR_PRESENT (_MSC_VER >= 1900) - #define __TBB_DEFAULTED_AND_DELETED_FUNC_PRESENT (_MSC_VER >= 1800) - #define __TBB_NOEXCEPT_PRESENT (_MSC_VER >= 1900) - #define __TBB_CPP11_STD_BEGIN_END_PRESENT (_MSC_VER >= 1700) - #define __TBB_CPP11_AUTO_PRESENT (_MSC_VER >= 1600) - #define __TBB_CPP11_DECLTYPE_PRESENT (_MSC_VER >= 1600) - #define __TBB_CPP11_LAMBDAS_PRESENT (_MSC_VER >= 1600) - #define __TBB_CPP11_DEFAULT_FUNC_TEMPLATE_ARGS_PRESENT (_MSC_VER >= 1800) - #define __TBB_OVERRIDE_PRESENT (_MSC_VER >= 1700) - #define __TBB_ALIGNAS_PRESENT (_MSC_VER >= 1900) - #define __TBB_CPP11_TEMPLATE_ALIASES_PRESENT (_MSC_VER >= 1800) -#else - #define __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT 0 - #define __TBB_CPP11_RVALUE_REF_PRESENT 0 - #define __TBB_IMPLICIT_MOVE_PRESENT 0 - #define __TBB_EXCEPTION_PTR_PRESENT 0 - #define __TBB_STATIC_ASSERT_PRESENT 0 - #define __TBB_CPP11_TUPLE_PRESENT 0 - #define __TBB_INITIALIZER_LISTS_PRESENT 0 - #define __TBB_CONSTEXPR_PRESENT 0 - #define __TBB_DEFAULTED_AND_DELETED_FUNC_PRESENT 0 - #define __TBB_NOEXCEPT_PRESENT 0 - #define __TBB_CPP11_STD_BEGIN_END_PRESENT 0 - #define __TBB_CPP11_AUTO_PRESENT 0 - #define __TBB_CPP11_DECLTYPE_PRESENT 0 - #define __TBB_CPP11_LAMBDAS_PRESENT 0 - #define __TBB_CPP11_DEFAULT_FUNC_TEMPLATE_ARGS_PRESENT 0 - #define __TBB_OVERRIDE_PRESENT 0 - #define __TBB_ALIGNAS_PRESENT 0 - #define __TBB_CPP11_TEMPLATE_ALIASES_PRESENT 0 -#endif - -// C++11 standard library features - -#ifndef __TBB_CPP11_VARIADIC_FIXED_LENGTH_EXP_PRESENT -#define __TBB_CPP11_VARIADIC_FIXED_LENGTH_EXP_PRESENT __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT -#endif -#define __TBB_CPP11_VARIADIC_TUPLE_PRESENT (!_MSC_VER || _MSC_VER >=1800) - -#define __TBB_CPP11_TYPE_PROPERTIES_PRESENT (_LIBCPP_VERSION || _MSC_VER >= 1700 || (__TBB_GLIBCXX_VERSION >= 50000 && __GXX_EXPERIMENTAL_CXX0X__)) -#define __TBB_TR1_TYPE_PROPERTIES_IN_STD_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GLIBCXX_VERSION >= 40300 || _MSC_VER >= 1600) -// GCC supported some of type properties since 4.7 -#define __TBB_CPP11_IS_COPY_CONSTRUCTIBLE_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GLIBCXX_VERSION >= 40700 || __TBB_CPP11_TYPE_PROPERTIES_PRESENT) - -// In GCC, std::move_if_noexcept appeared later than noexcept -#define __TBB_MOVE_IF_NOEXCEPT_PRESENT (__TBB_NOEXCEPT_PRESENT && (__TBB_GLIBCXX_VERSION >= 40700 || _MSC_VER >= 1900 || _LIBCPP_VERSION)) -#define __TBB_ALLOCATOR_TRAITS_PRESENT (__cplusplus >= 201103L && _LIBCPP_VERSION || _MSC_VER >= 1700 || \ - __GXX_EXPERIMENTAL_CXX0X__ && __TBB_GLIBCXX_VERSION >= 40700 && !(__TBB_GLIBCXX_VERSION == 40700 && __TBB_DEFINE_MIC)) -#define __TBB_MAKE_EXCEPTION_PTR_PRESENT (__TBB_EXCEPTION_PTR_PRESENT && (_MSC_VER >= 1700 || __TBB_GLIBCXX_VERSION >= 40600 || _LIBCPP_VERSION)) - -// Due to libc++ limitations in C++03 mode, do not pass rvalues to std::make_shared() -#define __TBB_CPP11_SMART_POINTERS_PRESENT ( _MSC_VER >= 1600 || _LIBCPP_VERSION || ((__cplusplus >= 201103L || __GXX_EXPERIMENTAL_CXX0X__) && (__TBB_GLIBCXX_VERSION>=40500 || __TBB_GLIBCXX_VERSION>=40400 && __TBB_USE_OPTIONAL_RTTI)) ) - -#define __TBB_CPP11_FUTURE_PRESENT (_MSC_VER >= 1700 || __TBB_GLIBCXX_VERSION >= 40600 && _GXX_EXPERIMENTAL_CXX0X__ || _LIBCPP_VERSION) - -// std::swap is in only since C++11, though MSVC had it at least since VS2005 -#if _MSC_VER>=1400 || _LIBCPP_VERSION || __GXX_EXPERIMENTAL_CXX0X__ -#define __TBB_STD_SWAP_HEADER -#else -#define __TBB_STD_SWAP_HEADER -#endif - -//TODO: not clear how exactly this macro affects exception_ptr - investigate -// On linux ICC fails to find existing std::exception_ptr in libstdc++ without this define -#if __INTEL_COMPILER && __GNUC__ && __TBB_EXCEPTION_PTR_PRESENT && !defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_4) - #define __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4 1 -#endif - -// Work around a bug in MinGW32 -#if __MINGW32__ && __TBB_EXCEPTION_PTR_PRESENT && !defined(_GLIBCXX_ATOMIC_BUILTINS_4) - #define _GLIBCXX_ATOMIC_BUILTINS_4 -#endif - -#if __GNUC__ || __SUNPRO_CC || __IBMCPP__ - /* ICC defines __GNUC__ and so is covered */ - #define __TBB_ATTRIBUTE_ALIGNED_PRESENT 1 -#elif _MSC_VER && (_MSC_VER >= 1300 || __INTEL_COMPILER) - #define __TBB_DECLSPEC_ALIGN_PRESENT 1 -#endif - -/* Actually ICC supports gcc __sync_* intrinsics starting 11.1, - * but 64 bit support for 32 bit target comes in later ones*/ -/* TODO: change the version back to 4.1.2 once macro __TBB_WORD_SIZE become optional */ -/* Assumed that all clang versions have these gcc compatible intrinsics. */ -#if __TBB_GCC_VERSION >= 40306 || __INTEL_COMPILER >= 1200 || __clang__ - /** built-in atomics available in GCC since 4.1.2 **/ - #define __TBB_GCC_BUILTIN_ATOMICS_PRESENT 1 -#endif - -#if __INTEL_COMPILER >= 1200 - /** built-in C++11 style atomics available in ICC since 12.0 **/ - #define __TBB_ICC_BUILTIN_ATOMICS_PRESENT 1 -#endif - -#define __TBB_TSX_INTRINSICS_PRESENT ((__RTM__ || _MSC_VER>=1700 || __INTEL_COMPILER>=1300) && !__TBB_DEFINE_MIC && !__ANDROID__) - -/** Macro helpers **/ -#define __TBB_CONCAT_AUX(A,B) A##B -// The additional level of indirection is needed to expand macros A and B (not to get the AB macro). -// See [cpp.subst] and [cpp.concat] for more details. -#define __TBB_CONCAT(A,B) __TBB_CONCAT_AUX(A,B) -// The IGNORED argument and comma are needed to always have 2 arguments (even when A is empty). -#define __TBB_IS_MACRO_EMPTY(A,IGNORED) __TBB_CONCAT_AUX(__TBB_MACRO_EMPTY,A) -#define __TBB_MACRO_EMPTY 1 - -/** User controlled TBB features & modes **/ -#ifndef TBB_USE_DEBUG -/* -There are four cases that are supported: - 1. "_DEBUG is undefined" means "no debug"; - 2. "_DEBUG defined to something that is evaluated to 0" (including "garbage", as per [cpp.cond]) means "no debug"; - 3. "_DEBUG defined to something that is evaluated to a non-zero value" means "debug"; - 4. "_DEBUG defined to nothing (empty)" means "debug". -*/ -#ifdef _DEBUG -// Check if _DEBUG is empty. -#define __TBB_IS__DEBUG_EMPTY (__TBB_IS_MACRO_EMPTY(_DEBUG,IGNORED)==__TBB_MACRO_EMPTY) -#if __TBB_IS__DEBUG_EMPTY -#define TBB_USE_DEBUG 1 -#else -#define TBB_USE_DEBUG _DEBUG -#endif /* __TBB_IS__DEBUG_EMPTY */ -#else -#define TBB_USE_DEBUG 0 -#endif -#endif /* TBB_USE_DEBUG */ - -#ifndef TBB_USE_ASSERT -#define TBB_USE_ASSERT TBB_USE_DEBUG -#endif /* TBB_USE_ASSERT */ - -#ifndef TBB_USE_THREADING_TOOLS -#define TBB_USE_THREADING_TOOLS TBB_USE_DEBUG -#endif /* TBB_USE_THREADING_TOOLS */ - -#ifndef TBB_USE_PERFORMANCE_WARNINGS -#ifdef TBB_PERFORMANCE_WARNINGS -#define TBB_USE_PERFORMANCE_WARNINGS TBB_PERFORMANCE_WARNINGS -#else -#define TBB_USE_PERFORMANCE_WARNINGS TBB_USE_DEBUG -#endif /* TBB_PERFORMANCE_WARNINGS */ -#endif /* TBB_USE_PERFORMANCE_WARNINGS */ - -#if __TBB_DEFINE_MIC - #if TBB_USE_EXCEPTIONS - #error The platform does not properly support exception handling. Please do not set TBB_USE_EXCEPTIONS macro or set it to 0. - #elif !defined(TBB_USE_EXCEPTIONS) - #define TBB_USE_EXCEPTIONS 0 - #endif -#elif !(__EXCEPTIONS || defined(_CPPUNWIND) || __SUNPRO_CC) - #if TBB_USE_EXCEPTIONS - #error Compilation settings do not support exception handling. Please do not set TBB_USE_EXCEPTIONS macro or set it to 0. - #elif !defined(TBB_USE_EXCEPTIONS) - #define TBB_USE_EXCEPTIONS 0 - #endif -#elif !defined(TBB_USE_EXCEPTIONS) - #define TBB_USE_EXCEPTIONS 1 -#endif - -#ifndef TBB_IMPLEMENT_CPP0X -/** By default, use C++11 classes if available **/ - #if __clang__ - /* Old versions of Intel C++ Compiler do not have __has_include or cannot use it in #define */ - #if (__INTEL_COMPILER && (__INTEL_COMPILER < 1500 || __INTEL_COMPILER == 1500 && __INTEL_COMPILER_UPDATE <= 1)) - #define TBB_IMPLEMENT_CPP0X (__cplusplus < 201103L || !_LIBCPP_VERSION) - #else - #define TBB_IMPLEMENT_CPP0X (__cplusplus < 201103L || (!__has_include() && !__has_include())) - #endif - #elif __GNUC__ - #define TBB_IMPLEMENT_CPP0X (__TBB_GCC_VERSION < 40400 || !__GXX_EXPERIMENTAL_CXX0X__) - #elif _MSC_VER - #define TBB_IMPLEMENT_CPP0X (_MSC_VER < 1700) - #else - // TODO: Reconsider general approach to be more reliable, e.g. (!(__cplusplus >= 201103L && __ STDC_HOSTED__)) - #define TBB_IMPLEMENT_CPP0X (!__STDCPP_THREADS__) - #endif -#endif /* TBB_IMPLEMENT_CPP0X */ - -/* TBB_USE_CAPTURED_EXCEPTION should be explicitly set to either 0 or 1, as it is used as C++ const */ -#ifndef TBB_USE_CAPTURED_EXCEPTION - /** IA-64 architecture pre-built TBB binaries do not support exception_ptr. **/ - #if __TBB_EXCEPTION_PTR_PRESENT && !defined(__ia64__) - #define TBB_USE_CAPTURED_EXCEPTION 0 - #else - #define TBB_USE_CAPTURED_EXCEPTION 1 - #endif -#else /* defined TBB_USE_CAPTURED_EXCEPTION */ - #if !TBB_USE_CAPTURED_EXCEPTION && !__TBB_EXCEPTION_PTR_PRESENT - #error Current runtime does not support std::exception_ptr. Set TBB_USE_CAPTURED_EXCEPTION and make sure that your code is ready to catch tbb::captured_exception. - #endif -#endif /* defined TBB_USE_CAPTURED_EXCEPTION */ - -/** Check whether the request to use GCC atomics can be satisfied **/ -#if TBB_USE_GCC_BUILTINS && !__TBB_GCC_BUILTIN_ATOMICS_PRESENT - #error "GCC atomic built-ins are not supported." -#endif - -/** Internal TBB features & modes **/ - -/** __TBB_WEAK_SYMBOLS_PRESENT denotes that the system supports the weak symbol mechanism **/ -#ifndef __TBB_WEAK_SYMBOLS_PRESENT -#define __TBB_WEAK_SYMBOLS_PRESENT ( !_WIN32 && !__APPLE__ && !__sun && (__TBB_GCC_VERSION >= 40000 || __INTEL_COMPILER ) ) -#endif - -/** __TBB_DYNAMIC_LOAD_ENABLED describes the system possibility to load shared libraries at run time **/ -#ifndef __TBB_DYNAMIC_LOAD_ENABLED - #define __TBB_DYNAMIC_LOAD_ENABLED 1 -#endif - -/** __TBB_SOURCE_DIRECTLY_INCLUDED is a mode used in whitebox testing when - it's necessary to test internal functions not exported from TBB DLLs -**/ -#if (_WIN32||_WIN64) && (__TBB_SOURCE_DIRECTLY_INCLUDED || TBB_USE_PREVIEW_BINARY) - #define __TBB_NO_IMPLICIT_LINKAGE 1 - #define __TBBMALLOC_NO_IMPLICIT_LINKAGE 1 -#endif - -#ifndef __TBB_COUNT_TASK_NODES - #define __TBB_COUNT_TASK_NODES TBB_USE_ASSERT -#endif - -#ifndef __TBB_TASK_GROUP_CONTEXT - #define __TBB_TASK_GROUP_CONTEXT 1 -#endif /* __TBB_TASK_GROUP_CONTEXT */ - -#ifndef __TBB_SCHEDULER_OBSERVER - #define __TBB_SCHEDULER_OBSERVER 1 -#endif /* __TBB_SCHEDULER_OBSERVER */ - -#ifndef __TBB_FP_CONTEXT - #define __TBB_FP_CONTEXT __TBB_TASK_GROUP_CONTEXT -#endif /* __TBB_FP_CONTEXT */ - -#if __TBB_FP_CONTEXT && !__TBB_TASK_GROUP_CONTEXT - #error __TBB_FP_CONTEXT requires __TBB_TASK_GROUP_CONTEXT to be enabled -#endif - -#define __TBB_RECYCLE_TO_ENQUEUE __TBB_BUILD // keep non-official - -#ifndef __TBB_ARENA_OBSERVER - #define __TBB_ARENA_OBSERVER ((__TBB_BUILD||TBB_PREVIEW_LOCAL_OBSERVER)&& __TBB_SCHEDULER_OBSERVER) -#endif /* __TBB_ARENA_OBSERVER */ - -#ifndef __TBB_SLEEP_PERMISSION - #define __TBB_SLEEP_PERMISSION ((__TBB_CPF_BUILD||TBB_PREVIEW_LOCAL_OBSERVER)&& __TBB_SCHEDULER_OBSERVER) -#endif /* __TBB_SLEEP_PERMISSION */ - -#ifndef __TBB_TASK_ISOLATION - #define __TBB_TASK_ISOLATION 1 -#endif /* __TBB_TASK_ISOLATION */ - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE || TBB_PREVIEW_ALGORITHM_TRACE -// Users of flow-graph and algorithm trace need to explicitly link against the preview -// library. This prevents the linker from implicitly linking an application with a preview -// version of TBB and unexpectedly bringing in other community preview features, which -// might change the behavior of the application. -#define __TBB_NO_IMPLICIT_LINKAGE 1 -#endif /* TBB_PREVIEW_FLOW_GRAPH_TRACE */ - -#ifndef __TBB_ITT_STRUCTURE_API -#define __TBB_ITT_STRUCTURE_API ( !__TBB_DEFINE_MIC && (__TBB_CPF_BUILD || TBB_PREVIEW_FLOW_GRAPH_TRACE || TBB_PREVIEW_ALGORITHM_TRACE) ) -#endif - -#if TBB_USE_EXCEPTIONS && !__TBB_TASK_GROUP_CONTEXT - #error TBB_USE_EXCEPTIONS requires __TBB_TASK_GROUP_CONTEXT to be enabled -#endif - -#ifndef __TBB_TASK_PRIORITY - #define __TBB_TASK_PRIORITY (__TBB_TASK_GROUP_CONTEXT) -#endif /* __TBB_TASK_PRIORITY */ - -#if __TBB_TASK_PRIORITY && !__TBB_TASK_GROUP_CONTEXT - #error __TBB_TASK_PRIORITY requires __TBB_TASK_GROUP_CONTEXT to be enabled -#endif - -#if TBB_PREVIEW_WAITING_FOR_WORKERS || __TBB_BUILD - #define __TBB_SUPPORTS_WORKERS_WAITING_IN_TERMINATE 1 -#endif - -#ifndef __TBB_ENQUEUE_ENFORCED_CONCURRENCY - #define __TBB_ENQUEUE_ENFORCED_CONCURRENCY 1 -#endif - -#if !defined(__TBB_SURVIVE_THREAD_SWITCH) && \ - (_WIN32 || _WIN64 || __APPLE__ || (__linux__ && !__ANDROID__)) - #define __TBB_SURVIVE_THREAD_SWITCH 1 -#endif /* __TBB_SURVIVE_THREAD_SWITCH */ - -#ifndef __TBB_DEFAULT_PARTITIONER -#define __TBB_DEFAULT_PARTITIONER tbb::auto_partitioner -#endif - -#ifndef __TBB_USE_PROPORTIONAL_SPLIT_IN_BLOCKED_RANGES -#define __TBB_USE_PROPORTIONAL_SPLIT_IN_BLOCKED_RANGES 1 -#endif - -#ifndef __TBB_ENABLE_RANGE_FEEDBACK -#define __TBB_ENABLE_RANGE_FEEDBACK 0 -#endif - -#ifdef _VARIADIC_MAX - #define __TBB_VARIADIC_MAX _VARIADIC_MAX -#else - #if _MSC_VER == 1700 - #define __TBB_VARIADIC_MAX 5 // VS11 setting, issue resolved in VS12 - #elif _MSC_VER == 1600 - #define __TBB_VARIADIC_MAX 10 // VS10 setting - #else - #define __TBB_VARIADIC_MAX 15 - #endif -#endif - -/** __TBB_WIN8UI_SUPPORT enables support of Windows* Store Apps and limit a possibility to load - shared libraries at run time only from application container **/ -#if defined(WINAPI_FAMILY) && WINAPI_FAMILY == WINAPI_FAMILY_APP - #define __TBB_WIN8UI_SUPPORT 1 -#else - #define __TBB_WIN8UI_SUPPORT 0 -#endif - -/** Macros of the form __TBB_XXX_BROKEN denote known issues that are caused by - the bugs in compilers, standard or OS specific libraries. They should be - removed as soon as the corresponding bugs are fixed or the buggy OS/compiler - versions go out of the support list. -**/ - -#if __SIZEOF_POINTER__ < 8 && __ANDROID__ && __TBB_GCC_VERSION <= 40403 && !__GCC_HAVE_SYNC_COMPARE_AND_SWAP_8 - /** Necessary because on Android 8-byte CAS and F&A are not available for some processor architectures, - but no mandatory warning message appears from GCC 4.4.3. Instead, only a linkage error occurs when - these atomic operations are used (such as in unit test test_atomic.exe). **/ - #define __TBB_GCC_64BIT_ATOMIC_BUILTINS_BROKEN 1 -#elif __TBB_x86_32 && __TBB_GCC_VERSION == 40102 && ! __GNUC_RH_RELEASE__ - /** GCC 4.1.2 erroneously emit call to external function for 64 bit sync_ intrinsics. - However these functions are not defined anywhere. It seems that this problem was fixed later on - and RHEL got an updated version of gcc 4.1.2. **/ - #define __TBB_GCC_64BIT_ATOMIC_BUILTINS_BROKEN 1 -#endif - -#if __GNUC__ && __TBB_x86_64 && __INTEL_COMPILER == 1200 - #define __TBB_ICC_12_0_INL_ASM_FSTCW_BROKEN 1 -#endif - -#if _MSC_VER && __INTEL_COMPILER && (__INTEL_COMPILER<1110 || __INTEL_COMPILER==1110 && __INTEL_COMPILER_BUILD_DATE < 20091012) - /** Necessary to avoid ICL error (or warning in non-strict mode): - "exception specification for implicitly declared virtual destructor is - incompatible with that of overridden one". **/ - #define __TBB_DEFAULT_DTOR_THROW_SPEC_BROKEN 1 -#endif - -#if !__INTEL_COMPILER && (_MSC_VER && _MSC_VER < 1500 || __GNUC__ && __TBB_GCC_VERSION < 40102) - /** gcc 3.4.6 (and earlier) and VS2005 (and earlier) do not allow declaring template class as a friend - of classes defined in other namespaces. **/ - #define __TBB_TEMPLATE_FRIENDS_BROKEN 1 -#endif - -#if __GLIBC__==2 && __GLIBC_MINOR__==3 || (__APPLE__ && ( __INTEL_COMPILER==1200 && !TBB_USE_DEBUG)) - /** Macro controlling EH usages in TBB tests. - Some older versions of glibc crash when exception handling happens concurrently. **/ - #define __TBB_THROW_ACROSS_MODULE_BOUNDARY_BROKEN 1 -#endif - -#if (_WIN32||_WIN64) && __INTEL_COMPILER == 1110 - /** That's a bug in Intel C++ Compiler 11.1.044/IA-32 architecture/Windows* OS, that leads to a worker thread crash on the thread's startup. **/ - #define __TBB_ICL_11_1_CODE_GEN_BROKEN 1 -#endif - -#if __clang__ || (__GNUC__==3 && __GNUC_MINOR__==3 && !defined(__INTEL_COMPILER)) - /** Bugs with access to nested classes declared in protected area */ - #define __TBB_PROTECTED_NESTED_CLASS_BROKEN 1 -#endif - -#if __MINGW32__ && __TBB_GCC_VERSION < 40200 - /** MinGW has a bug with stack alignment for routines invoked from MS RTLs. - Since GCC 4.2, the bug can be worked around via a special attribute. **/ - #define __TBB_SSE_STACK_ALIGNMENT_BROKEN 1 -#endif - -#if __TBB_GCC_VERSION==40300 && !__INTEL_COMPILER && !__clang__ - /* GCC of this version may rashly ignore control dependencies */ - #define __TBB_GCC_OPTIMIZER_ORDERING_BROKEN 1 -#endif - -#if __FreeBSD__ - /** A bug in FreeBSD 8.0 results in kernel panic when there is contention - on a mutex created with this attribute. **/ - #define __TBB_PRIO_INHERIT_BROKEN 1 - - /** A bug in FreeBSD 8.0 results in test hanging when an exception occurs - during (concurrent?) object construction by means of placement new operator. **/ - #define __TBB_PLACEMENT_NEW_EXCEPTION_SAFETY_BROKEN 1 -#endif /* __FreeBSD__ */ - -#if (__linux__ || __APPLE__) && __i386__ && defined(__INTEL_COMPILER) - /** The Intel C++ Compiler for IA-32 architecture (Linux* OS|macOS) crashes or generates - incorrect code when __asm__ arguments have a cast to volatile. **/ - #define __TBB_ICC_ASM_VOLATILE_BROKEN 1 -#endif - -#if !__INTEL_COMPILER && (_MSC_VER || __GNUC__==3 && __GNUC_MINOR__<=2) - /** Bug in GCC 3.2 and MSVC compilers that sometimes return 0 for __alignof(T) - when T has not yet been instantiated. **/ - #define __TBB_ALIGNOF_NOT_INSTANTIATED_TYPES_BROKEN 1 -#endif - -#if __TBB_DEFINE_MIC - /** Main thread and user's thread have different default thread affinity masks. **/ - #define __TBB_MAIN_THREAD_AFFINITY_BROKEN 1 -#endif - -#if __GXX_EXPERIMENTAL_CXX0X__ && !defined(__EXCEPTIONS) && \ - ((!__INTEL_COMPILER && !__clang__ && (__TBB_GCC_VERSION>=40400 && __TBB_GCC_VERSION<40600)) || \ - (__INTEL_COMPILER<=1400 && (__TBB_GLIBCXX_VERSION>=40400 && __TBB_GLIBCXX_VERSION<=40801))) -/* There is an issue for specific GCC toolchain when C++11 is enabled - and exceptions are disabled: - exceprion_ptr.h/nested_exception.h use throw unconditionally. - GCC can ignore 'throw' since 4.6; but with ICC the issue still exists. - */ - #define __TBB_LIBSTDCPP_EXCEPTION_HEADERS_BROKEN 1 -#endif - -#if __INTEL_COMPILER==1300 && __TBB_GLIBCXX_VERSION>=40700 && defined(__GXX_EXPERIMENTAL_CXX0X__) -/* Some C++11 features used inside libstdc++ are not supported by Intel C++ Compiler. */ - #define __TBB_ICC_13_0_CPP11_STDLIB_SUPPORT_BROKEN 1 -#endif - -#if (__GNUC__==4 && __GNUC_MINOR__==4 ) && !defined(__INTEL_COMPILER) && !defined(__clang__) - /** excessive warnings related to strict aliasing rules in GCC 4.4 **/ - #define __TBB_GCC_STRICT_ALIASING_BROKEN 1 - /* topical remedy: #pragma GCC diagnostic ignored "-Wstrict-aliasing" */ - #if !__TBB_GCC_WARNING_SUPPRESSION_PRESENT - #error Warning suppression is not supported, while should. - #endif -#endif - -/* In a PIC mode some versions of GCC 4.1.2 generate incorrect inlined code for 8 byte __sync_val_compare_and_swap intrinsic */ -#if __TBB_GCC_VERSION == 40102 && __PIC__ && !defined(__INTEL_COMPILER) && !defined(__clang__) - #define __TBB_GCC_CAS8_BUILTIN_INLINING_BROKEN 1 -#endif - -#if __TBB_x86_32 && ( __INTEL_COMPILER || (__GNUC__==5 && __GNUC_MINOR__>=2 && __GXX_EXPERIMENTAL_CXX0X__) \ - || (__GNUC__==3 && __GNUC_MINOR__==3) || (__MINGW32__ && __GNUC__==4 && __GNUC_MINOR__==5) || __SUNPRO_CC ) - // Some compilers for IA-32 architecture fail to provide 8-byte alignment of objects on the stack, - // even if the object specifies 8-byte alignment. On such platforms, the implementation - // of 64 bit atomics for IA-32 architecture (e.g. atomic) use different tactics - // depending upon whether the object is properly aligned or not. - #define __TBB_FORCE_64BIT_ALIGNMENT_BROKEN 1 -#else - // Define to 0 explicitly because the macro is used in a compiled code of test_atomic - #define __TBB_FORCE_64BIT_ALIGNMENT_BROKEN 0 -#endif - -#if __GNUC__ && !__INTEL_COMPILER && !__clang__ && __TBB_DEFAULTED_AND_DELETED_FUNC_PRESENT && __TBB_GCC_VERSION < 40700 - #define __TBB_ZERO_INIT_WITH_DEFAULTED_CTOR_BROKEN 1 -#endif - -#if _MSC_VER && _MSC_VER <= 1800 && !__INTEL_COMPILER - // With MSVC, when an array is passed by const reference to a template function, - // constness from the function parameter may get propagated to the template parameter. - #define __TBB_CONST_REF_TO_ARRAY_TEMPLATE_PARAM_BROKEN 1 -#endif - -// A compiler bug: a disabled copy constructor prevents use of the moving constructor -#define __TBB_IF_NO_COPY_CTOR_MOVE_SEMANTICS_BROKEN (_MSC_VER && (__INTEL_COMPILER >= 1300 && __INTEL_COMPILER <= 1310) && !__INTEL_CXX11_MODE__) - -#define __TBB_CPP11_DECLVAL_BROKEN (_MSC_VER == 1600 || (__GNUC__ && __TBB_GCC_VERSION < 40500) ) -// Intel C++ Compiler has difficulties with copying std::pair with VC11 std::reference_wrapper being a const member -#define __TBB_COPY_FROM_NON_CONST_REF_BROKEN (_MSC_VER == 1700 && __INTEL_COMPILER && __INTEL_COMPILER < 1600) - -// The implicit upcasting of the tuple of a reference of a derived class to a base class fails on icc 13.X if the system's gcc environment is 4.8 -// Also in gcc 4.4 standard library the implementation of the tuple<&> conversion (tuple a = tuple, B is inherited from A) is broken. -#if __GXX_EXPERIMENTAL_CXX0X__ && __GLIBCXX__ && ((__INTEL_COMPILER >=1300 && __INTEL_COMPILER <=1310 && __TBB_GLIBCXX_VERSION>=40700) || (__TBB_GLIBCXX_VERSION < 40500)) -#define __TBB_UPCAST_OF_TUPLE_OF_REF_BROKEN 1 -#endif - -// In some cases decltype of a function adds a reference to a return type. -#define __TBB_CPP11_DECLTYPE_OF_FUNCTION_RETURN_TYPE_BROKEN (_MSC_VER == 1600 && !__INTEL_COMPILER) - -/** End of __TBB_XXX_BROKEN macro section **/ - -#if defined(_MSC_VER) && _MSC_VER>=1500 && !defined(__INTEL_COMPILER) - // A macro to suppress erroneous or benign "unreachable code" MSVC warning (4702) - #define __TBB_MSVC_UNREACHABLE_CODE_IGNORED 1 -#endif - -#define __TBB_ATOMIC_CTORS (__TBB_CONSTEXPR_PRESENT && __TBB_DEFAULTED_AND_DELETED_FUNC_PRESENT && (!__TBB_ZERO_INIT_WITH_DEFAULTED_CTOR_BROKEN)) - -// Many OS versions (Android 4.0.[0-3] for example) need workaround for dlopen to avoid non-recursive loader lock hang -// Setting the workaround for all compile targets ($APP_PLATFORM) below Android 4.4 (android-19) -#if __ANDROID__ -#include -#define __TBB_USE_DLOPEN_REENTRANCY_WORKAROUND (__ANDROID_API__ < 19) -#endif - -#define __TBB_ALLOCATOR_CONSTRUCT_VARIADIC (__TBB_CPP11_VARIADIC_TEMPLATES_PRESENT && __TBB_CPP11_RVALUE_REF_PRESENT) - -#define __TBB_VARIADIC_PARALLEL_INVOKE (TBB_PREVIEW_VARIADIC_PARALLEL_INVOKE && __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT && __TBB_CPP11_RVALUE_REF_PRESENT) -#define __TBB_FLOW_GRAPH_CPP11_FEATURES (__TBB_CPP11_VARIADIC_TEMPLATES_PRESENT \ - && __TBB_CPP11_SMART_POINTERS_PRESENT && __TBB_CPP11_RVALUE_REF_PRESENT && __TBB_CPP11_AUTO_PRESENT) \ - && __TBB_CPP11_VARIADIC_TUPLE_PRESENT && __TBB_CPP11_DEFAULT_FUNC_TEMPLATE_ARGS_PRESENT \ - && !__TBB_UPCAST_OF_TUPLE_OF_REF_BROKEN -#define __TBB_PREVIEW_STREAMING_NODE (__TBB_CPP11_VARIADIC_FIXED_LENGTH_EXP_PRESENT && __TBB_FLOW_GRAPH_CPP11_FEATURES \ - && TBB_PREVIEW_FLOW_GRAPH_NODES && !TBB_IMPLEMENT_CPP0X && !__TBB_UPCAST_OF_TUPLE_OF_REF_BROKEN) -#define __TBB_PREVIEW_OPENCL_NODE (__TBB_PREVIEW_STREAMING_NODE && __TBB_CPP11_TEMPLATE_ALIASES_PRESENT) -#define __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING (TBB_PREVIEW_FLOW_GRAPH_FEATURES || __TBB_PREVIEW_OPENCL_NODE) -#define __TBB_PREVIEW_ASYNC_MSG (TBB_PREVIEW_FLOW_GRAPH_FEATURES && __TBB_FLOW_GRAPH_CPP11_FEATURES) - -#define __TBB_PREVIEW_GFX_FACTORY (__TBB_GFX_PRESENT && TBB_PREVIEW_FLOW_GRAPH_FEATURES && !__TBB_MIC_OFFLOAD \ - && __TBB_FLOW_GRAPH_CPP11_FEATURES && __TBB_CPP11_TEMPLATE_ALIASES_PRESENT \ - && __TBB_CPP11_FUTURE_PRESENT) -#endif /* __TBB_tbb_config_H */ diff --git a/src/tbb/include/tbb/tbb_disable_exceptions.h b/src/tbb/include/tbb/tbb_disable_exceptions.h deleted file mode 100644 index a1d28a60b..000000000 --- a/src/tbb/include/tbb/tbb_disable_exceptions.h +++ /dev/null @@ -1,35 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -//! To disable use of exceptions, include this header before any other header file from the library. - -//! The macro that prevents use of exceptions in the library files -#undef TBB_USE_EXCEPTIONS -#define TBB_USE_EXCEPTIONS 0 - -//! Prevent compilers from issuing exception related warnings. -/** Note that the warnings are suppressed for all the code after this header is included. */ -#if _MSC_VER -#if __INTEL_COMPILER - #pragma warning (disable: 583) -#else - #pragma warning (disable: 4530 4577) -#endif -#endif diff --git a/src/tbb/include/tbb/tbb_exception.h b/src/tbb/include/tbb/tbb_exception.h deleted file mode 100644 index a2f3e1b6b..000000000 --- a/src/tbb/include/tbb/tbb_exception.h +++ /dev/null @@ -1,360 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_exception_H -#define __TBB_exception_H - -#include "tbb_stddef.h" -#include -#include // required for bad_alloc definition, operators new -#include // required to construct std exception classes - -namespace tbb { - -//! Exception for concurrent containers -class bad_last_alloc : public std::bad_alloc { -public: - const char* what() const throw() __TBB_override; -#if __TBB_DEFAULT_DTOR_THROW_SPEC_BROKEN - ~bad_last_alloc() throw() __TBB_override {} -#endif -}; - -//! Exception for PPL locks -class improper_lock : public std::exception { -public: - const char* what() const throw() __TBB_override; -}; - -//! Exception for user-initiated abort -class user_abort : public std::exception { -public: - const char* what() const throw() __TBB_override; -}; - -//! Exception for missing wait on structured_task_group -class missing_wait : public std::exception { -public: - const char* what() const throw() __TBB_override; -}; - -//! Exception for repeated scheduling of the same task_handle -class invalid_multiple_scheduling : public std::exception { -public: - const char* what() const throw() __TBB_override; -}; - -namespace internal { -//! Obsolete -void __TBB_EXPORTED_FUNC throw_bad_last_alloc_exception_v4(); - -enum exception_id { - eid_bad_alloc = 1, - eid_bad_last_alloc, - eid_nonpositive_step, - eid_out_of_range, - eid_segment_range_error, - eid_index_range_error, - eid_missing_wait, - eid_invalid_multiple_scheduling, - eid_improper_lock, - eid_possible_deadlock, - eid_operation_not_permitted, - eid_condvar_wait_failed, - eid_invalid_load_factor, - eid_reserved, // free slot for backward compatibility, can be reused. - eid_invalid_swap, - eid_reservation_length_error, - eid_invalid_key, - eid_user_abort, - eid_reserved1, -#if __TBB_SUPPORTS_WORKERS_WAITING_IN_TERMINATE - // This id is used only from inside the library and only for support of CPF functionality. - // So, if we drop the functionality, eid_reserved1 can be safely renamed and reused. - eid_blocking_thread_join_impossible = eid_reserved1, -#endif - eid_bad_tagged_msg_cast, - //! The last enumerator tracks the number of defined IDs. It must remain the last one. - /** When adding new IDs, place them immediately _before_ this comment (that is - _after_ all the existing IDs. NEVER insert new IDs between the existing ones. **/ - eid_max -}; - -//! Gathers all throw operators in one place. -/** Its purpose is to minimize code bloat that can be caused by throw operators - scattered in multiple places, especially in templates. **/ -void __TBB_EXPORTED_FUNC throw_exception_v4 ( exception_id ); - -//! Versionless convenience wrapper for throw_exception_v4() -inline void throw_exception ( exception_id eid ) { throw_exception_v4(eid); } - -} // namespace internal -} // namespace tbb - -#if __TBB_TASK_GROUP_CONTEXT -#include "tbb_allocator.h" -#include //for typeid - -namespace tbb { - -//! Interface to be implemented by all exceptions TBB recognizes and propagates across the threads. -/** If an unhandled exception of the type derived from tbb::tbb_exception is intercepted - by the TBB scheduler in one of the worker threads, it is delivered to and re-thrown in - the root thread. The root thread is the thread that has started the outermost algorithm - or root task sharing the same task_group_context with the guilty algorithm/task (the one - that threw the exception first). - - Note: when documentation mentions workers with respect to exception handling, - masters are implied as well, because they are completely equivalent in this context. - Consequently a root thread can be master or worker thread. - - NOTE: In case of nested algorithms or complex task hierarchies when the nested - levels share (explicitly or by means of implicit inheritance) the task group - context of the outermost level, the exception may be (re-)thrown multiple times - (ultimately - in each worker on each nesting level) before reaching the root - thread at the outermost level. IMPORTANT: if you intercept an exception derived - from this class on a nested level, you must re-throw it in the catch block by means - of the "throw;" operator. - - TBB provides two implementations of this interface: tbb::captured_exception and - template class tbb::movable_exception. See their declarations for more info. **/ -class tbb_exception : public std::exception -{ - /** No operator new is provided because the TBB usage model assumes dynamic - creation of the TBB exception objects only by means of applying move() - operation on an exception thrown out of TBB scheduler. **/ - void* operator new ( size_t ); - -public: -#if __clang__ - // At -O3 or even -O2 optimization level, Clang may fully throw away an empty destructor - // of tbb_exception from destructors of derived classes. As a result, it does not create - // vtable for tbb_exception, which is a required part of TBB binary interface. - // Making the destructor non-empty (with just a semicolon) prevents that optimization. - ~tbb_exception() throw() { /* keep the semicolon! */ ; } -#endif - - //! Creates and returns pointer to the deep copy of this exception object. - /** Move semantics is allowed. **/ - virtual tbb_exception* move() throw() = 0; - - //! Destroys objects created by the move() method. - /** Frees memory and calls destructor for this exception object. - Can and must be used only on objects created by the move method. **/ - virtual void destroy() throw() = 0; - - //! Throws this exception object. - /** Make sure that if you have several levels of derivation from this interface - you implement or override this method on the most derived level. The implementation - is as simple as "throw *this;". Failure to do this will result in exception - of a base class type being thrown. **/ - virtual void throw_self() = 0; - - //! Returns RTTI name of the originally intercepted exception - virtual const char* name() const throw() = 0; - - //! Returns the result of originally intercepted exception's what() method. - virtual const char* what() const throw() __TBB_override = 0; - - /** Operator delete is provided only to allow using existing smart pointers - with TBB exception objects obtained as the result of applying move() - operation on an exception thrown out of TBB scheduler. - - When overriding method move() make sure to override operator delete as well - if memory is allocated not by TBB's scalable allocator. **/ - void operator delete ( void* p ) { - internal::deallocate_via_handler_v3(p); - } -}; - -//! This class is used by TBB to propagate information about unhandled exceptions into the root thread. -/** Exception of this type is thrown by TBB in the root thread (thread that started a parallel - algorithm ) if an unhandled exception was intercepted during the algorithm execution in one - of the workers. - \sa tbb::tbb_exception **/ -class captured_exception : public tbb_exception -{ -public: - captured_exception( const captured_exception& src ) - : tbb_exception(src), my_dynamic(false) - { - set(src.my_exception_name, src.my_exception_info); - } - - captured_exception( const char* name_, const char* info ) - : my_dynamic(false) - { - set(name_, info); - } - - __TBB_EXPORTED_METHOD ~captured_exception() throw(); - - captured_exception& operator= ( const captured_exception& src ) { - if ( this != &src ) { - clear(); - set(src.my_exception_name, src.my_exception_info); - } - return *this; - } - - captured_exception* __TBB_EXPORTED_METHOD move() throw() __TBB_override; - - void __TBB_EXPORTED_METHOD destroy() throw() __TBB_override; - - void throw_self() __TBB_override { __TBB_THROW(*this); } - - const char* __TBB_EXPORTED_METHOD name() const throw() __TBB_override; - - const char* __TBB_EXPORTED_METHOD what() const throw() __TBB_override; - - void __TBB_EXPORTED_METHOD set( const char* name, const char* info ) throw(); - void __TBB_EXPORTED_METHOD clear() throw(); - -private: - //! Used only by method move(). - captured_exception() {} - - //! Functionally equivalent to {captured_exception e(name,info); return e.move();} - static captured_exception* allocate( const char* name, const char* info ); - - bool my_dynamic; - const char* my_exception_name; - const char* my_exception_info; -}; - -//! Template that can be used to implement exception that transfers arbitrary ExceptionData to the root thread -/** Code using TBB can instantiate this template with an arbitrary ExceptionData type - and throw this exception object. Such exceptions are intercepted by the TBB scheduler - and delivered to the root thread (). - \sa tbb::tbb_exception **/ -template -class movable_exception : public tbb_exception -{ - typedef movable_exception self_type; - -public: - movable_exception( const ExceptionData& data_ ) - : my_exception_data(data_) - , my_dynamic(false) - , my_exception_name( -#if TBB_USE_EXCEPTIONS - typeid(self_type).name() -#else /* !TBB_USE_EXCEPTIONS */ - "movable_exception" -#endif /* !TBB_USE_EXCEPTIONS */ - ) - {} - - movable_exception( const movable_exception& src ) throw () - : tbb_exception(src) - , my_exception_data(src.my_exception_data) - , my_dynamic(false) - , my_exception_name(src.my_exception_name) - {} - - ~movable_exception() throw() {} - - const movable_exception& operator= ( const movable_exception& src ) { - if ( this != &src ) { - my_exception_data = src.my_exception_data; - my_exception_name = src.my_exception_name; - } - return *this; - } - - ExceptionData& data() throw() { return my_exception_data; } - - const ExceptionData& data() const throw() { return my_exception_data; } - - const char* name() const throw() __TBB_override { return my_exception_name; } - - const char* what() const throw() __TBB_override { return "tbb::movable_exception"; } - - movable_exception* move() throw() __TBB_override { - void* e = internal::allocate_via_handler_v3(sizeof(movable_exception)); - if ( e ) { - ::new (e) movable_exception(*this); - ((movable_exception*)e)->my_dynamic = true; - } - return (movable_exception*)e; - } - void destroy() throw() __TBB_override { - __TBB_ASSERT ( my_dynamic, "Method destroy can be called only on dynamically allocated movable_exceptions" ); - if ( my_dynamic ) { - this->~movable_exception(); - internal::deallocate_via_handler_v3(this); - } - } - void throw_self() __TBB_override { __TBB_THROW( *this ); } - -protected: - //! User data - ExceptionData my_exception_data; - -private: - //! Flag specifying whether this object has been dynamically allocated (by the move method) - bool my_dynamic; - - //! RTTI name of this class - /** We rely on the fact that RTTI names are static string constants. **/ - const char* my_exception_name; -}; - -#if !TBB_USE_CAPTURED_EXCEPTION -namespace internal { - -//! Exception container that preserves the exact copy of the original exception -/** This class can be used only when the appropriate runtime support (mandated - by C++11) is present **/ -class tbb_exception_ptr { - std::exception_ptr my_ptr; - -public: - static tbb_exception_ptr* allocate(); - static tbb_exception_ptr* allocate( const tbb_exception& tag ); - //! This overload uses move semantics (i.e. it empties src) - static tbb_exception_ptr* allocate( captured_exception& src ); - - //! Destroys this objects - /** Note that objects of this type can be created only by the allocate() method. **/ - void destroy() throw(); - - //! Throws the contained exception . - void throw_self() { std::rethrow_exception(my_ptr); } - -private: - tbb_exception_ptr( const std::exception_ptr& src ) : my_ptr(src) {} - tbb_exception_ptr( const captured_exception& src ) : - #if __TBB_MAKE_EXCEPTION_PTR_PRESENT - my_ptr(std::make_exception_ptr(src)) // the final function name in C++11 - #else - my_ptr(std::copy_exception(src)) // early C++0x drafts name - #endif - {} -}; // class tbb::internal::tbb_exception_ptr - -} // namespace internal -#endif /* !TBB_USE_CAPTURED_EXCEPTION */ - -} // namespace tbb - -#endif /* __TBB_TASK_GROUP_CONTEXT */ - -#endif /* __TBB_exception_H */ diff --git a/src/tbb/include/tbb/tbb_machine.h b/src/tbb/include/tbb/tbb_machine.h deleted file mode 100644 index cf8ea04bc..000000000 --- a/src/tbb/include/tbb/tbb_machine.h +++ /dev/null @@ -1,982 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_machine_H -#define __TBB_machine_H - -/** This header provides basic platform abstraction layer by hooking up appropriate - architecture/OS/compiler specific headers from the /include/tbb/machine directory. - If a plug-in header does not implement all the required APIs, it must specify - the missing ones by setting one or more of the following macros: - - __TBB_USE_GENERIC_PART_WORD_CAS - __TBB_USE_GENERIC_PART_WORD_FETCH_ADD - __TBB_USE_GENERIC_PART_WORD_FETCH_STORE - __TBB_USE_GENERIC_FETCH_ADD - __TBB_USE_GENERIC_FETCH_STORE - __TBB_USE_GENERIC_DWORD_FETCH_ADD - __TBB_USE_GENERIC_DWORD_FETCH_STORE - __TBB_USE_GENERIC_HALF_FENCED_LOAD_STORE - __TBB_USE_GENERIC_SEQUENTIAL_CONSISTENCY_LOAD_STORE - __TBB_USE_GENERIC_RELAXED_LOAD_STORE - __TBB_USE_FETCHSTORE_AS_FULL_FENCED_STORE - - In this case tbb_machine.h will add missing functionality based on a minimal set - of APIs that are required to be implemented by all plug-n headers as described - further. - Note that these generic implementations may be sub-optimal for a particular - architecture, and thus should be relied upon only after careful evaluation - or as the last resort. - - Additionally __TBB_64BIT_ATOMICS can be set to 0 on a 32-bit architecture to - indicate that the port is not going to support double word atomics. It may also - be set to 1 explicitly, though normally this is not necessary as tbb_machine.h - will set it automatically. - - __TBB_ENDIANNESS macro can be defined by the implementation as well. - It is used only if __TBB_USE_GENERIC_PART_WORD_CAS is set (or for testing), - and must specify the layout of aligned 16-bit and 32-bit data anywhere within a process - (while the details of unaligned 16-bit or 32-bit data or of 64-bit data are irrelevant). - The layout must be the same at all relevant memory locations within the current process; - in case of page-specific endianness, one endianness must be kept "out of sight". - Possible settings, reflecting hardware and possibly O.S. convention, are: - - __TBB_ENDIAN_BIG for big-endian data, - - __TBB_ENDIAN_LITTLE for little-endian data, - - __TBB_ENDIAN_DETECT for run-time detection iff exactly one of the above, - - __TBB_ENDIAN_UNSUPPORTED to prevent undefined behavior if none of the above. - - Prerequisites for each architecture port - ---------------------------------------- - The following functions and macros have no generic implementation. Therefore they must be - implemented in each machine architecture specific header either as a conventional - function or as a functional macro. - - __TBB_WORDSIZE - This is the size of machine word in bytes, i.e. for 32 bit systems it - should be defined to 4. - - __TBB_Yield() - Signals OS that the current thread is willing to relinquish the remainder - of its time quantum. - - __TBB_full_memory_fence() - Must prevent all memory operations from being reordered across it (both - by hardware and compiler). All such fences must be totally ordered (or - sequentially consistent). - - __TBB_machine_cmpswp4( volatile void *ptr, int32_t value, int32_t comparand ) - Must be provided if __TBB_USE_FENCED_ATOMICS is not set. - - __TBB_machine_cmpswp8( volatile void *ptr, int32_t value, int64_t comparand ) - Must be provided for 64-bit architectures if __TBB_USE_FENCED_ATOMICS is not set, - and for 32-bit architectures if __TBB_64BIT_ATOMICS is set - - __TBB_machine_(...), where - = {cmpswp, fetchadd, fetchstore} - = {1, 2, 4, 8} - = {full_fence, acquire, release, relaxed} - Must be provided if __TBB_USE_FENCED_ATOMICS is set. - - __TBB_control_consistency_helper() - Bridges the memory-semantics gap between architectures providing only - implicit C++0x "consume" semantics (like Power Architecture) and those - also implicitly obeying control dependencies (like IA-64 architecture). - It must be used only in conditional code where the condition is itself - data-dependent, and will then make subsequent code behave as if the - original data dependency were acquired. - It needs only a compiler fence where implied by the architecture - either specifically (like IA-64 architecture) or because generally stronger - "acquire" semantics are enforced (like x86). - It is always valid, though potentially suboptimal, to replace - control with acquire on the load and then remove the helper. - - __TBB_acquire_consistency_helper(), __TBB_release_consistency_helper() - Must be provided if __TBB_USE_GENERIC_HALF_FENCED_LOAD_STORE is set. - Enforce acquire and release semantics in generic implementations of fenced - store and load operations. Depending on the particular architecture/compiler - combination they may be a hardware fence, a compiler fence, both or nothing. - **/ - -#include "tbb_stddef.h" - -namespace tbb { -namespace internal { //< @cond INTERNAL - -//////////////////////////////////////////////////////////////////////////////// -// Overridable helpers declarations -// -// A machine/*.h file may choose to define these templates, otherwise it must -// request default implementation by setting appropriate __TBB_USE_GENERIC_XXX macro(s). -// -template -struct machine_load_store; - -template -struct machine_load_store_relaxed; - -template -struct machine_load_store_seq_cst; -// -// End of overridable helpers declarations -//////////////////////////////////////////////////////////////////////////////// - -template struct atomic_selector; - -template<> struct atomic_selector<1> { - typedef int8_t word; - inline static word fetch_store ( volatile void* location, word value ); -}; - -template<> struct atomic_selector<2> { - typedef int16_t word; - inline static word fetch_store ( volatile void* location, word value ); -}; - -template<> struct atomic_selector<4> { -#if _MSC_VER && !_WIN64 - // Work-around that avoids spurious /Wp64 warnings - typedef intptr_t word; -#else - typedef int32_t word; -#endif - inline static word fetch_store ( volatile void* location, word value ); -}; - -template<> struct atomic_selector<8> { - typedef int64_t word; - inline static word fetch_store ( volatile void* location, word value ); -}; - -}} //< namespaces internal @endcond, tbb - -#define __TBB_MACHINE_DEFINE_STORE8_GENERIC_FENCED(M) \ - inline void __TBB_machine_generic_store8##M(volatile void *ptr, int64_t value) { \ - for(;;) { \ - int64_t result = *(volatile int64_t *)ptr; \ - if( __TBB_machine_cmpswp8##M(ptr,value,result)==result ) break; \ - } \ - } \ - -#define __TBB_MACHINE_DEFINE_LOAD8_GENERIC_FENCED(M) \ - inline int64_t __TBB_machine_generic_load8##M(const volatile void *ptr) { \ - /* Comparand and new value may be anything, they only must be equal, and */ \ - /* the value should have a low probability to be actually found in 'location'.*/ \ - const int64_t anyvalue = 2305843009213693951LL; \ - return __TBB_machine_cmpswp8##M(const_cast(ptr),anyvalue,anyvalue); \ - } \ - -// The set of allowed values for __TBB_ENDIANNESS (see above for details) -#define __TBB_ENDIAN_UNSUPPORTED -1 -#define __TBB_ENDIAN_LITTLE 0 -#define __TBB_ENDIAN_BIG 1 -#define __TBB_ENDIAN_DETECT 2 - -#if _WIN32||_WIN64 - -#ifdef _MANAGED -#pragma managed(push, off) -#endif - - #if __MINGW64__ || __MINGW32__ - extern "C" __declspec(dllimport) int __stdcall SwitchToThread( void ); - #define __TBB_Yield() SwitchToThread() - #if (TBB_USE_GCC_BUILTINS && __TBB_GCC_BUILTIN_ATOMICS_PRESENT) - #include "machine/gcc_generic.h" - #elif __MINGW64__ - #include "machine/linux_intel64.h" - #elif __MINGW32__ - #include "machine/linux_ia32.h" - #endif - #elif (TBB_USE_ICC_BUILTINS && __TBB_ICC_BUILTIN_ATOMICS_PRESENT) - #include "machine/icc_generic.h" - #elif defined(_M_IX86) && !defined(__TBB_WIN32_USE_CL_BUILTINS) - #include "machine/windows_ia32.h" - #elif defined(_M_X64) - #include "machine/windows_intel64.h" - #elif defined(_M_ARM) || defined(__TBB_WIN32_USE_CL_BUILTINS) - #include "machine/msvc_armv7.h" - #endif - -#ifdef _MANAGED -#pragma managed(pop) -#endif - -#elif __TBB_DEFINE_MIC - - #include "machine/mic_common.h" - #if (TBB_USE_ICC_BUILTINS && __TBB_ICC_BUILTIN_ATOMICS_PRESENT) - #include "machine/icc_generic.h" - #else - #include "machine/linux_intel64.h" - #endif - -#elif __linux__ || __FreeBSD__ || __NetBSD__ - - #if (TBB_USE_GCC_BUILTINS && __TBB_GCC_BUILTIN_ATOMICS_PRESENT) - #include "machine/gcc_generic.h" - #elif (TBB_USE_ICC_BUILTINS && __TBB_ICC_BUILTIN_ATOMICS_PRESENT) - #include "machine/icc_generic.h" - #elif __i386__ - #include "machine/linux_ia32.h" - #elif __x86_64__ - #include "machine/linux_intel64.h" - #elif __ia64__ - #include "machine/linux_ia64.h" - #elif __powerpc__ - #include "machine/mac_ppc.h" - #elif __ARM_ARCH_7A__ - #include "machine/gcc_armv7.h" - #elif __TBB_GCC_BUILTIN_ATOMICS_PRESENT - #include "machine/gcc_generic.h" - #endif - #include "machine/linux_common.h" - -#elif __APPLE__ - //TODO: TBB_USE_GCC_BUILTINS is not used for Mac, Sun, Aix - #if (TBB_USE_ICC_BUILTINS && __TBB_ICC_BUILTIN_ATOMICS_PRESENT) - #include "machine/icc_generic.h" - #elif __TBB_x86_32 - #include "machine/linux_ia32.h" - #elif __TBB_x86_64 - #include "machine/linux_intel64.h" - #elif __POWERPC__ - #include "machine/mac_ppc.h" - #endif - #include "machine/macos_common.h" - -#elif _AIX - - #include "machine/ibm_aix51.h" - -#elif __sun || __SUNPRO_CC - - #define __asm__ asm - #define __volatile__ volatile - - #if __i386 || __i386__ - #include "machine/linux_ia32.h" - #elif __x86_64__ - #include "machine/linux_intel64.h" - #elif __sparc - #include "machine/sunos_sparc.h" - #endif - #include - - #define __TBB_Yield() sched_yield() - -#endif /* OS selection */ - -#ifndef __TBB_64BIT_ATOMICS - #define __TBB_64BIT_ATOMICS 1 -#endif - -//TODO: replace usage of these functions with usage of tbb::atomic, and then remove them -//TODO: map functions with W suffix to use cast to tbb::atomic and according op, i.e. as_atomic().op() -// Special atomic functions -#if __TBB_USE_FENCED_ATOMICS - #define __TBB_machine_cmpswp1 __TBB_machine_cmpswp1full_fence - #define __TBB_machine_cmpswp2 __TBB_machine_cmpswp2full_fence - #define __TBB_machine_cmpswp4 __TBB_machine_cmpswp4full_fence - #define __TBB_machine_cmpswp8 __TBB_machine_cmpswp8full_fence - - #if __TBB_WORDSIZE==8 - #define __TBB_machine_fetchadd8 __TBB_machine_fetchadd8full_fence - #define __TBB_machine_fetchstore8 __TBB_machine_fetchstore8full_fence - #define __TBB_FetchAndAddWrelease(P,V) __TBB_machine_fetchadd8release(P,V) - #define __TBB_FetchAndIncrementWacquire(P) __TBB_machine_fetchadd8acquire(P,1) - #define __TBB_FetchAndDecrementWrelease(P) __TBB_machine_fetchadd8release(P,(-1)) - #else - #define __TBB_machine_fetchadd4 __TBB_machine_fetchadd4full_fence - #define __TBB_machine_fetchstore4 __TBB_machine_fetchstore4full_fence - #define __TBB_FetchAndAddWrelease(P,V) __TBB_machine_fetchadd4release(P,V) - #define __TBB_FetchAndIncrementWacquire(P) __TBB_machine_fetchadd4acquire(P,1) - #define __TBB_FetchAndDecrementWrelease(P) __TBB_machine_fetchadd4release(P,(-1)) - #endif /* __TBB_WORDSIZE==4 */ -#else /* !__TBB_USE_FENCED_ATOMICS */ - #define __TBB_FetchAndAddWrelease(P,V) __TBB_FetchAndAddW(P,V) - #define __TBB_FetchAndIncrementWacquire(P) __TBB_FetchAndAddW(P,1) - #define __TBB_FetchAndDecrementWrelease(P) __TBB_FetchAndAddW(P,(-1)) -#endif /* !__TBB_USE_FENCED_ATOMICS */ - -#if __TBB_WORDSIZE==4 - #define __TBB_CompareAndSwapW(P,V,C) __TBB_machine_cmpswp4(P,V,C) - #define __TBB_FetchAndAddW(P,V) __TBB_machine_fetchadd4(P,V) - #define __TBB_FetchAndStoreW(P,V) __TBB_machine_fetchstore4(P,V) -#elif __TBB_WORDSIZE==8 - #if __TBB_USE_GENERIC_DWORD_LOAD_STORE || __TBB_USE_GENERIC_DWORD_FETCH_ADD || __TBB_USE_GENERIC_DWORD_FETCH_STORE - #error These macros should only be used on 32-bit platforms. - #endif - - #define __TBB_CompareAndSwapW(P,V,C) __TBB_machine_cmpswp8(P,V,C) - #define __TBB_FetchAndAddW(P,V) __TBB_machine_fetchadd8(P,V) - #define __TBB_FetchAndStoreW(P,V) __TBB_machine_fetchstore8(P,V) -#else /* __TBB_WORDSIZE != 8 */ - #error Unsupported machine word size. -#endif /* __TBB_WORDSIZE */ - -#ifndef __TBB_Pause - inline void __TBB_Pause(int32_t) { - __TBB_Yield(); - } -#endif - -namespace tbb { - -//! Sequentially consistent full memory fence. -inline void atomic_fence () { __TBB_full_memory_fence(); } - -namespace internal { //< @cond INTERNAL - -//! Class that implements exponential backoff. -/** See implementation of spin_wait_while_eq for an example. */ -class atomic_backoff : no_copy { - //! Time delay, in units of "pause" instructions. - /** Should be equal to approximately the number of "pause" instructions - that take the same time as an context switch. Must be a power of two.*/ - static const int32_t LOOPS_BEFORE_YIELD = 16; - int32_t count; -public: - // In many cases, an object of this type is initialized eagerly on hot path, - // as in for(atomic_backoff b; ; b.pause()) { /*loop body*/ } - // For this reason, the construction cost must be very small! - atomic_backoff() : count(1) {} - // This constructor pauses immediately; do not use on hot paths! - atomic_backoff( bool ) : count(1) { pause(); } - - //! Pause for a while. - void pause() { - if( count<=LOOPS_BEFORE_YIELD ) { - __TBB_Pause(count); - // Pause twice as long the next time. - count*=2; - } else { - // Pause is so long that we might as well yield CPU to scheduler. - __TBB_Yield(); - } - } - - //! Pause for a few times and return false if saturated. - bool bounded_pause() { - __TBB_Pause(count); - if( count -void spin_wait_while_eq( const volatile T& location, U value ) { - atomic_backoff backoff; - while( location==value ) backoff.pause(); -} - -//! Spin UNTIL the value of the variable is equal to a given value -/** T and U should be comparable types. */ -template -void spin_wait_until_eq( const volatile T& location, const U value ) { - atomic_backoff backoff; - while( location!=value ) backoff.pause(); -} - -template -void spin_wait_while(predicate_type condition){ - atomic_backoff backoff; - while( condition() ) backoff.pause(); -} - -//////////////////////////////////////////////////////////////////////////////// -// Generic compare-and-swap applied to only a part of a machine word. -// -#ifndef __TBB_ENDIANNESS -#define __TBB_ENDIANNESS __TBB_ENDIAN_DETECT -#endif - -#if __TBB_USE_GENERIC_PART_WORD_CAS && __TBB_ENDIANNESS==__TBB_ENDIAN_UNSUPPORTED -#error Generic implementation of part-word CAS may not be used with __TBB_ENDIAN_UNSUPPORTED -#endif - -#if __TBB_ENDIANNESS!=__TBB_ENDIAN_UNSUPPORTED -// -// This function is the only use of __TBB_ENDIANNESS. -// The following restrictions/limitations apply for this operation: -// - T must be an integer type of at most 4 bytes for the casts and calculations to work -// - T must also be less than 4 bytes to avoid compiler warnings when computing mask -// (and for the operation to be useful at all, so no workaround is applied) -// - the architecture must consistently use either little-endian or big-endian (same for all locations) -// -// TODO: static_assert for the type requirements stated above -template -inline T __TBB_MaskedCompareAndSwap (volatile T * const ptr, const T value, const T comparand ) { - struct endianness{ static bool is_big_endian(){ - #if __TBB_ENDIANNESS==__TBB_ENDIAN_DETECT - const uint32_t probe = 0x03020100; - return (((const char*)(&probe))[0]==0x03); - #elif __TBB_ENDIANNESS==__TBB_ENDIAN_BIG || __TBB_ENDIANNESS==__TBB_ENDIAN_LITTLE - return __TBB_ENDIANNESS==__TBB_ENDIAN_BIG; - #else - #error Unexpected value of __TBB_ENDIANNESS - #endif - }}; - - const uint32_t byte_offset = (uint32_t) ((uintptr_t)ptr & 0x3); - volatile uint32_t * const aligned_ptr = (uint32_t*)((uintptr_t)ptr - byte_offset ); - - // location of T within uint32_t for a C++ shift operation - const uint32_t bits_to_shift = 8*(endianness::is_big_endian() ? (4 - sizeof(T) - (byte_offset)) : byte_offset); - const uint32_t mask = (((uint32_t)1<<(sizeof(T)*8)) - 1 )<> bits_to_shift); - } - else continue; // CAS failed but the bits of interest were not changed - } -} -#endif // __TBB_ENDIANNESS!=__TBB_ENDIAN_UNSUPPORTED -//////////////////////////////////////////////////////////////////////////////// - -template -inline T __TBB_CompareAndSwapGeneric (volatile void *ptr, T value, T comparand ); - -template<> -inline int8_t __TBB_CompareAndSwapGeneric <1,int8_t> (volatile void *ptr, int8_t value, int8_t comparand ) { -#if __TBB_USE_GENERIC_PART_WORD_CAS - return __TBB_MaskedCompareAndSwap((volatile int8_t *)ptr,value,comparand); -#else - return __TBB_machine_cmpswp1(ptr,value,comparand); -#endif -} - -template<> -inline int16_t __TBB_CompareAndSwapGeneric <2,int16_t> (volatile void *ptr, int16_t value, int16_t comparand ) { -#if __TBB_USE_GENERIC_PART_WORD_CAS - return __TBB_MaskedCompareAndSwap((volatile int16_t *)ptr,value,comparand); -#else - return __TBB_machine_cmpswp2(ptr,value,comparand); -#endif -} - -template<> -inline int32_t __TBB_CompareAndSwapGeneric <4,int32_t> (volatile void *ptr, int32_t value, int32_t comparand ) { - // Cast shuts up /Wp64 warning - return (int32_t)__TBB_machine_cmpswp4(ptr,value,comparand); -} - -#if __TBB_64BIT_ATOMICS -template<> -inline int64_t __TBB_CompareAndSwapGeneric <8,int64_t> (volatile void *ptr, int64_t value, int64_t comparand ) { - return __TBB_machine_cmpswp8(ptr,value,comparand); -} -#endif - -template -inline T __TBB_FetchAndAddGeneric (volatile void *ptr, T addend) { - T result; - for( atomic_backoff b;;b.pause() ) { - result = *reinterpret_cast(ptr); - // __TBB_CompareAndSwapGeneric presumed to have full fence. - if( __TBB_CompareAndSwapGeneric ( ptr, result+addend, result )==result ) - break; - } - return result; -} - -template -inline T __TBB_FetchAndStoreGeneric (volatile void *ptr, T value) { - T result; - for( atomic_backoff b;;b.pause() ) { - result = *reinterpret_cast(ptr); - // __TBB_CompareAndSwapGeneric presumed to have full fence. - if( __TBB_CompareAndSwapGeneric ( ptr, value, result )==result ) - break; - } - return result; -} - -#if __TBB_USE_GENERIC_PART_WORD_CAS -#define __TBB_machine_cmpswp1 tbb::internal::__TBB_CompareAndSwapGeneric<1,int8_t> -#define __TBB_machine_cmpswp2 tbb::internal::__TBB_CompareAndSwapGeneric<2,int16_t> -#endif - -#if __TBB_USE_GENERIC_FETCH_ADD || __TBB_USE_GENERIC_PART_WORD_FETCH_ADD -#define __TBB_machine_fetchadd1 tbb::internal::__TBB_FetchAndAddGeneric<1,int8_t> -#define __TBB_machine_fetchadd2 tbb::internal::__TBB_FetchAndAddGeneric<2,int16_t> -#endif - -#if __TBB_USE_GENERIC_FETCH_ADD -#define __TBB_machine_fetchadd4 tbb::internal::__TBB_FetchAndAddGeneric<4,int32_t> -#endif - -#if __TBB_USE_GENERIC_FETCH_ADD || __TBB_USE_GENERIC_DWORD_FETCH_ADD -#define __TBB_machine_fetchadd8 tbb::internal::__TBB_FetchAndAddGeneric<8,int64_t> -#endif - -#if __TBB_USE_GENERIC_FETCH_STORE || __TBB_USE_GENERIC_PART_WORD_FETCH_STORE -#define __TBB_machine_fetchstore1 tbb::internal::__TBB_FetchAndStoreGeneric<1,int8_t> -#define __TBB_machine_fetchstore2 tbb::internal::__TBB_FetchAndStoreGeneric<2,int16_t> -#endif - -#if __TBB_USE_GENERIC_FETCH_STORE -#define __TBB_machine_fetchstore4 tbb::internal::__TBB_FetchAndStoreGeneric<4,int32_t> -#endif - -#if __TBB_USE_GENERIC_FETCH_STORE || __TBB_USE_GENERIC_DWORD_FETCH_STORE -#define __TBB_machine_fetchstore8 tbb::internal::__TBB_FetchAndStoreGeneric<8,int64_t> -#endif - -#if __TBB_USE_FETCHSTORE_AS_FULL_FENCED_STORE -#define __TBB_MACHINE_DEFINE_ATOMIC_SELECTOR_FETCH_STORE(S) \ - atomic_selector::word atomic_selector::fetch_store ( volatile void* location, word value ) { \ - return __TBB_machine_fetchstore##S( location, value ); \ - } - -__TBB_MACHINE_DEFINE_ATOMIC_SELECTOR_FETCH_STORE(1) -__TBB_MACHINE_DEFINE_ATOMIC_SELECTOR_FETCH_STORE(2) -__TBB_MACHINE_DEFINE_ATOMIC_SELECTOR_FETCH_STORE(4) -__TBB_MACHINE_DEFINE_ATOMIC_SELECTOR_FETCH_STORE(8) - -#undef __TBB_MACHINE_DEFINE_ATOMIC_SELECTOR_FETCH_STORE -#endif /* __TBB_USE_FETCHSTORE_AS_FULL_FENCED_STORE */ - -#if __TBB_USE_GENERIC_DWORD_LOAD_STORE -/*TODO: find a more elegant way to handle function names difference*/ -#if ! __TBB_USE_FENCED_ATOMICS - /* This name forwarding is needed for generic implementation of - * load8/store8 defined below (via macro) to pick the right CAS function*/ - #define __TBB_machine_cmpswp8full_fence __TBB_machine_cmpswp8 -#endif -__TBB_MACHINE_DEFINE_LOAD8_GENERIC_FENCED(full_fence) -__TBB_MACHINE_DEFINE_STORE8_GENERIC_FENCED(full_fence) - -#if ! __TBB_USE_FENCED_ATOMICS - #undef __TBB_machine_cmpswp8full_fence -#endif - -#define __TBB_machine_store8 tbb::internal::__TBB_machine_generic_store8full_fence -#define __TBB_machine_load8 tbb::internal::__TBB_machine_generic_load8full_fence -#endif /* __TBB_USE_GENERIC_DWORD_LOAD_STORE */ - -#if __TBB_USE_GENERIC_HALF_FENCED_LOAD_STORE -/** Fenced operations use volatile qualifier to prevent compiler from optimizing - them out, and on architectures with weak memory ordering to induce compiler - to generate code with appropriate acquire/release semantics. - On architectures like IA32, Intel64 (and likely Sparc TSO) volatile has - no effect on code gen, and consistency helpers serve as a compiler fence (the - latter being true for IA64/gcc as well to fix a bug in some gcc versions). - This code assumes that the generated instructions will operate atomically, - which typically requires a type that can be moved in a single instruction, - cooperation from the compiler for effective use of such an instruction, - and appropriate alignment of the data. **/ -template -struct machine_load_store { - static T load_with_acquire ( const volatile T& location ) { - T to_return = location; - __TBB_acquire_consistency_helper(); - return to_return; - } - static void store_with_release ( volatile T &location, T value ) { - __TBB_release_consistency_helper(); - location = value; - } -}; - -//in general, plain load and store of 32bit compiler is not atomic for 64bit types -#if __TBB_WORDSIZE==4 && __TBB_64BIT_ATOMICS -template -struct machine_load_store { - static T load_with_acquire ( const volatile T& location ) { - return (T)__TBB_machine_load8( (const volatile void*)&location ); - } - static void store_with_release ( volatile T& location, T value ) { - __TBB_machine_store8( (volatile void*)&location, (int64_t)value ); - } -}; -#endif /* __TBB_WORDSIZE==4 && __TBB_64BIT_ATOMICS */ -#endif /* __TBB_USE_GENERIC_HALF_FENCED_LOAD_STORE */ - -#if __TBB_USE_GENERIC_SEQUENTIAL_CONSISTENCY_LOAD_STORE -template -struct machine_load_store_seq_cst { - static T load ( const volatile T& location ) { - __TBB_full_memory_fence(); - return machine_load_store::load_with_acquire( location ); - } -#if __TBB_USE_FETCHSTORE_AS_FULL_FENCED_STORE - static void store ( volatile T &location, T value ) { - atomic_selector::fetch_store( (volatile void*)&location, (typename atomic_selector::word)value ); - } -#else /* !__TBB_USE_FETCHSTORE_AS_FULL_FENCED_STORE */ - static void store ( volatile T &location, T value ) { - machine_load_store::store_with_release( location, value ); - __TBB_full_memory_fence(); - } -#endif /* !__TBB_USE_FETCHSTORE_AS_FULL_FENCED_STORE */ -}; - -#if __TBB_WORDSIZE==4 && __TBB_64BIT_ATOMICS -/** The implementation does not use functions __TBB_machine_load8/store8 as they - are not required to be sequentially consistent. **/ -template -struct machine_load_store_seq_cst { - static T load ( const volatile T& location ) { - // Comparand and new value may be anything, they only must be equal, and - // the value should have a low probability to be actually found in 'location'. - const int64_t anyvalue = 2305843009213693951LL; - return __TBB_machine_cmpswp8( (volatile void*)const_cast(&location), anyvalue, anyvalue ); - } - static void store ( volatile T &location, T value ) { -#if __TBB_GCC_VERSION >= 40702 -#pragma GCC diagnostic push -// #pragma GCC diagnostic ignored "-Wmaybe-uninitialized" -#endif - // An atomic initialization leads to reading of uninitialized memory - int64_t result = (volatile int64_t&)location; -#if __TBB_GCC_VERSION >= 40702 -#pragma GCC diagnostic pop -#endif - while ( __TBB_machine_cmpswp8((volatile void*)&location, (int64_t)value, result) != result ) - result = (volatile int64_t&)location; - } -}; -#endif /* __TBB_WORDSIZE==4 && __TBB_64BIT_ATOMICS */ -#endif /*__TBB_USE_GENERIC_SEQUENTIAL_CONSISTENCY_LOAD_STORE */ - -#if __TBB_USE_GENERIC_RELAXED_LOAD_STORE -// Relaxed operations add volatile qualifier to prevent compiler from optimizing them out. -/** Volatile should not incur any additional cost on IA32, Intel64, and Sparc TSO - architectures. However on architectures with weak memory ordering compiler may - generate code with acquire/release semantics for operations on volatile data. **/ -template -struct machine_load_store_relaxed { - static inline T load ( const volatile T& location ) { - return location; - } - static inline void store ( volatile T& location, T value ) { - location = value; - } -}; - -#if __TBB_WORDSIZE==4 && __TBB_64BIT_ATOMICS -template -struct machine_load_store_relaxed { - static inline T load ( const volatile T& location ) { - return (T)__TBB_machine_load8( (const volatile void*)&location ); - } - static inline void store ( volatile T& location, T value ) { - __TBB_machine_store8( (volatile void*)&location, (int64_t)value ); - } -}; -#endif /* __TBB_WORDSIZE==4 && __TBB_64BIT_ATOMICS */ -#endif /* __TBB_USE_GENERIC_RELAXED_LOAD_STORE */ - -#undef __TBB_WORDSIZE //this macro is forbidden to use outside of atomic machinery - -template -inline T __TBB_load_with_acquire(const volatile T &location) { - return machine_load_store::load_with_acquire( location ); -} -template -inline void __TBB_store_with_release(volatile T& location, V value) { - machine_load_store::store_with_release( location, T(value) ); -} -//! Overload that exists solely to avoid /Wp64 warnings. -inline void __TBB_store_with_release(volatile size_t& location, size_t value) { - machine_load_store::store_with_release( location, value ); -} - -template -inline T __TBB_load_full_fence(const volatile T &location) { - return machine_load_store_seq_cst::load( location ); -} -template -inline void __TBB_store_full_fence(volatile T& location, V value) { - machine_load_store_seq_cst::store( location, T(value) ); -} -//! Overload that exists solely to avoid /Wp64 warnings. -inline void __TBB_store_full_fence(volatile size_t& location, size_t value) { - machine_load_store_seq_cst::store( location, value ); -} - -template -inline T __TBB_load_relaxed (const volatile T& location) { - return machine_load_store_relaxed::load( const_cast(location) ); -} -template -inline void __TBB_store_relaxed ( volatile T& location, V value ) { - machine_load_store_relaxed::store( const_cast(location), T(value) ); -} -//! Overload that exists solely to avoid /Wp64 warnings. -inline void __TBB_store_relaxed ( volatile size_t& location, size_t value ) { - machine_load_store_relaxed::store( const_cast(location), value ); -} - -// Macro __TBB_TypeWithAlignmentAtLeastAsStrict(T) should be a type with alignment at least as -// strict as type T. The type should have a trivial default constructor and destructor, so that -// arrays of that type can be declared without initializers. -// It is correct (but perhaps a waste of space) if __TBB_TypeWithAlignmentAtLeastAsStrict(T) expands -// to a type bigger than T. -// The default definition here works on machines where integers are naturally aligned and the -// strictest alignment is 64. -#ifndef __TBB_TypeWithAlignmentAtLeastAsStrict - -#if __TBB_ALIGNAS_PRESENT - -// Use C++11 keywords alignas and alignof -#define __TBB_DefineTypeWithAlignment(PowerOf2) \ -struct alignas(PowerOf2) __TBB_machine_type_with_alignment_##PowerOf2 { \ - uint32_t member[PowerOf2/sizeof(uint32_t)]; \ -}; -#define __TBB_alignof(T) alignof(T) - -#elif __TBB_ATTRIBUTE_ALIGNED_PRESENT - -#define __TBB_DefineTypeWithAlignment(PowerOf2) \ -struct __TBB_machine_type_with_alignment_##PowerOf2 { \ - uint32_t member[PowerOf2/sizeof(uint32_t)]; \ -} __attribute__((aligned(PowerOf2))); -#define __TBB_alignof(T) __alignof__(T) - -#elif __TBB_DECLSPEC_ALIGN_PRESENT - -#define __TBB_DefineTypeWithAlignment(PowerOf2) \ -__declspec(align(PowerOf2)) \ -struct __TBB_machine_type_with_alignment_##PowerOf2 { \ - uint32_t member[PowerOf2/sizeof(uint32_t)]; \ -}; -#define __TBB_alignof(T) __alignof(T) - -#else /* A compiler with unknown syntax for data alignment */ -#error Must define __TBB_TypeWithAlignmentAtLeastAsStrict(T) -#endif - -/* Now declare types aligned to useful powers of two */ -// TODO: Is __TBB_DefineTypeWithAlignment(8) needed on 32 bit platforms? -__TBB_DefineTypeWithAlignment(16) -__TBB_DefineTypeWithAlignment(32) -__TBB_DefineTypeWithAlignment(64) - -typedef __TBB_machine_type_with_alignment_64 __TBB_machine_type_with_strictest_alignment; - -// Primary template is a declaration of incomplete type so that it fails with unknown alignments -template struct type_with_alignment; - -// Specializations for allowed alignments -template<> struct type_with_alignment<1> { char member; }; -template<> struct type_with_alignment<2> { uint16_t member; }; -template<> struct type_with_alignment<4> { uint32_t member; }; -template<> struct type_with_alignment<8> { uint64_t member; }; -template<> struct type_with_alignment<16> {__TBB_machine_type_with_alignment_16 member; }; -template<> struct type_with_alignment<32> {__TBB_machine_type_with_alignment_32 member; }; -template<> struct type_with_alignment<64> {__TBB_machine_type_with_alignment_64 member; }; - -#if __TBB_ALIGNOF_NOT_INSTANTIATED_TYPES_BROKEN -//! Work around for bug in GNU 3.2 and MSVC compilers. -/** Bug is that compiler sometimes returns 0 for __alignof(T) when T has not yet been instantiated. - The work-around forces instantiation by forcing computation of sizeof(T) before __alignof(T). */ -template -struct work_around_alignment_bug { - static const size_t alignment = __TBB_alignof(T); -}; -#define __TBB_TypeWithAlignmentAtLeastAsStrict(T) tbb::internal::type_with_alignment::alignment> -#else -#define __TBB_TypeWithAlignmentAtLeastAsStrict(T) tbb::internal::type_with_alignment<__TBB_alignof(T)> -#endif /* __TBB_ALIGNOF_NOT_INSTANTIATED_TYPES_BROKEN */ - -#endif /* __TBB_TypeWithAlignmentAtLeastAsStrict */ - -// Template class here is to avoid instantiation of the static data for modules that don't use it -template -struct reverse { - static const T byte_table[256]; -}; -// An efficient implementation of the reverse function utilizes a 2^8 lookup table holding the bit-reversed -// values of [0..2^8 - 1]. Those values can also be computed on the fly at a slightly higher cost. -template -const T reverse::byte_table[256] = { - 0x00, 0x80, 0x40, 0xC0, 0x20, 0xA0, 0x60, 0xE0, 0x10, 0x90, 0x50, 0xD0, 0x30, 0xB0, 0x70, 0xF0, - 0x08, 0x88, 0x48, 0xC8, 0x28, 0xA8, 0x68, 0xE8, 0x18, 0x98, 0x58, 0xD8, 0x38, 0xB8, 0x78, 0xF8, - 0x04, 0x84, 0x44, 0xC4, 0x24, 0xA4, 0x64, 0xE4, 0x14, 0x94, 0x54, 0xD4, 0x34, 0xB4, 0x74, 0xF4, - 0x0C, 0x8C, 0x4C, 0xCC, 0x2C, 0xAC, 0x6C, 0xEC, 0x1C, 0x9C, 0x5C, 0xDC, 0x3C, 0xBC, 0x7C, 0xFC, - 0x02, 0x82, 0x42, 0xC2, 0x22, 0xA2, 0x62, 0xE2, 0x12, 0x92, 0x52, 0xD2, 0x32, 0xB2, 0x72, 0xF2, - 0x0A, 0x8A, 0x4A, 0xCA, 0x2A, 0xAA, 0x6A, 0xEA, 0x1A, 0x9A, 0x5A, 0xDA, 0x3A, 0xBA, 0x7A, 0xFA, - 0x06, 0x86, 0x46, 0xC6, 0x26, 0xA6, 0x66, 0xE6, 0x16, 0x96, 0x56, 0xD6, 0x36, 0xB6, 0x76, 0xF6, - 0x0E, 0x8E, 0x4E, 0xCE, 0x2E, 0xAE, 0x6E, 0xEE, 0x1E, 0x9E, 0x5E, 0xDE, 0x3E, 0xBE, 0x7E, 0xFE, - 0x01, 0x81, 0x41, 0xC1, 0x21, 0xA1, 0x61, 0xE1, 0x11, 0x91, 0x51, 0xD1, 0x31, 0xB1, 0x71, 0xF1, - 0x09, 0x89, 0x49, 0xC9, 0x29, 0xA9, 0x69, 0xE9, 0x19, 0x99, 0x59, 0xD9, 0x39, 0xB9, 0x79, 0xF9, - 0x05, 0x85, 0x45, 0xC5, 0x25, 0xA5, 0x65, 0xE5, 0x15, 0x95, 0x55, 0xD5, 0x35, 0xB5, 0x75, 0xF5, - 0x0D, 0x8D, 0x4D, 0xCD, 0x2D, 0xAD, 0x6D, 0xED, 0x1D, 0x9D, 0x5D, 0xDD, 0x3D, 0xBD, 0x7D, 0xFD, - 0x03, 0x83, 0x43, 0xC3, 0x23, 0xA3, 0x63, 0xE3, 0x13, 0x93, 0x53, 0xD3, 0x33, 0xB3, 0x73, 0xF3, - 0x0B, 0x8B, 0x4B, 0xCB, 0x2B, 0xAB, 0x6B, 0xEB, 0x1B, 0x9B, 0x5B, 0xDB, 0x3B, 0xBB, 0x7B, 0xFB, - 0x07, 0x87, 0x47, 0xC7, 0x27, 0xA7, 0x67, 0xE7, 0x17, 0x97, 0x57, 0xD7, 0x37, 0xB7, 0x77, 0xF7, - 0x0F, 0x8F, 0x4F, 0xCF, 0x2F, 0xAF, 0x6F, 0xEF, 0x1F, 0x9F, 0x5F, 0xDF, 0x3F, 0xBF, 0x7F, 0xFF -}; - -} // namespace internal @endcond -} // namespace tbb - -// Preserving access to legacy APIs -using tbb::internal::__TBB_load_with_acquire; -using tbb::internal::__TBB_store_with_release; - -// Mapping historically used names to the ones expected by atomic_load_store_traits -#define __TBB_load_acquire __TBB_load_with_acquire -#define __TBB_store_release __TBB_store_with_release - -#ifndef __TBB_Log2 -inline intptr_t __TBB_Log2( uintptr_t x ) { - if( x==0 ) return -1; - intptr_t result = 0; - -#if !defined(_M_ARM) - uintptr_t tmp_; - if( sizeof(x)>4 && (tmp_ = ((uint64_t)x)>>32) ) { x=tmp_; result += 32; } -#endif - if( uintptr_t tmp = x>>16 ) { x=tmp; result += 16; } - if( uintptr_t tmp = x>>8 ) { x=tmp; result += 8; } - if( uintptr_t tmp = x>>4 ) { x=tmp; result += 4; } - if( uintptr_t tmp = x>>2 ) { x=tmp; result += 2; } - - return (x&2)? result+1: result; -} -#endif - -#ifndef __TBB_AtomicOR -inline void __TBB_AtomicOR( volatile void *operand, uintptr_t addend ) { - for( tbb::internal::atomic_backoff b;;b.pause() ) { - uintptr_t tmp = *(volatile uintptr_t *)operand; - uintptr_t result = __TBB_CompareAndSwapW(operand, tmp|addend, tmp); - if( result==tmp ) break; - } -} -#endif - -#ifndef __TBB_AtomicAND -inline void __TBB_AtomicAND( volatile void *operand, uintptr_t addend ) { - for( tbb::internal::atomic_backoff b;;b.pause() ) { - uintptr_t tmp = *(volatile uintptr_t *)operand; - uintptr_t result = __TBB_CompareAndSwapW(operand, tmp&addend, tmp); - if( result==tmp ) break; - } -} -#endif - -#if __TBB_PREFETCHING -#ifndef __TBB_cl_prefetch -#error This platform does not define cache management primitives required for __TBB_PREFETCHING -#endif - -#ifndef __TBB_cl_evict -#define __TBB_cl_evict(p) -#endif -#endif - -#ifndef __TBB_Flag -typedef unsigned char __TBB_Flag; -#endif -typedef __TBB_atomic __TBB_Flag __TBB_atomic_flag; - -#ifndef __TBB_TryLockByte -inline bool __TBB_TryLockByte( __TBB_atomic_flag &flag ) { - return __TBB_machine_cmpswp1(&flag,1,0)==0; -} -#endif - -#ifndef __TBB_LockByte -inline __TBB_Flag __TBB_LockByte( __TBB_atomic_flag& flag ) { - tbb::internal::atomic_backoff backoff; - while( !__TBB_TryLockByte(flag) ) backoff.pause(); - return 0; -} -#endif - -#ifndef __TBB_UnlockByte -#define __TBB_UnlockByte(addr) __TBB_store_with_release((addr),0) -#endif - -// lock primitives with Intel(R) Transactional Synchronization Extensions (Intel(R) TSX) -#if ( __TBB_x86_32 || __TBB_x86_64 ) /* only on ia32/intel64 */ -inline void __TBB_TryLockByteElidedCancel() { __TBB_machine_try_lock_elided_cancel(); } - -inline bool __TBB_TryLockByteElided( __TBB_atomic_flag& flag ) { - bool res = __TBB_machine_try_lock_elided( &flag )!=0; - // to avoid the "lemming" effect, we need to abort the transaction - // if __TBB_machine_try_lock_elided returns false (i.e., someone else - // has acquired the mutex non-speculatively). - if( !res ) __TBB_TryLockByteElidedCancel(); - return res; -} - -inline void __TBB_LockByteElided( __TBB_atomic_flag& flag ) -{ - for(;;) { - tbb::internal::spin_wait_while_eq( flag, 1 ); - if( __TBB_machine_try_lock_elided( &flag ) ) - return; - // Another thread acquired the lock "for real". - // To avoid the "lemming" effect, we abort the transaction. - __TBB_TryLockByteElidedCancel(); - } -} - -inline void __TBB_UnlockByteElided( __TBB_atomic_flag& flag ) { - __TBB_machine_unlock_elided( &flag ); -} -#endif - -#ifndef __TBB_ReverseByte -inline unsigned char __TBB_ReverseByte(unsigned char src) { - return tbb::internal::reverse::byte_table[src]; -} -#endif - -template -T __TBB_ReverseBits(T src) { - T dst; - unsigned char *original = (unsigned char *) &src; - unsigned char *reversed = (unsigned char *) &dst; - - for( int i = sizeof(T)-1; i >= 0; i-- ) - reversed[i] = __TBB_ReverseByte( original[sizeof(T)-i-1] ); - - return dst; -} - -#endif /* __TBB_machine_H */ diff --git a/src/tbb/include/tbb/tbb_profiling.h b/src/tbb/include/tbb/tbb_profiling.h deleted file mode 100644 index ffaf98b91..000000000 --- a/src/tbb/include/tbb/tbb_profiling.h +++ /dev/null @@ -1,284 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_profiling_H -#define __TBB_profiling_H - -namespace tbb { - namespace internal { - - // - // This is not under __TBB_ITT_STRUCTURE_API because these values are used directly in flow_graph.h. - // - - // include list of index names - #define TBB_STRING_RESOURCE(index_name,str) index_name, - enum string_index { - #include "internal/_tbb_strings.h" - NUM_STRINGS - }; - #undef TBB_STRING_RESOURCE - - enum itt_relation - { - __itt_relation_is_unknown = 0, - __itt_relation_is_dependent_on, /**< "A is dependent on B" means that A cannot start until B completes */ - __itt_relation_is_sibling_of, /**< "A is sibling of B" means that A and B were created as a group */ - __itt_relation_is_parent_of, /**< "A is parent of B" means that A created B */ - __itt_relation_is_continuation_of, /**< "A is continuation of B" means that A assumes the dependencies of B */ - __itt_relation_is_child_of, /**< "A is child of B" means that A was created by B (inverse of is_parent_of) */ - __itt_relation_is_continued_by, /**< "A is continued by B" means that B assumes the dependencies of A (inverse of is_continuation_of) */ - __itt_relation_is_predecessor_to /**< "A is predecessor to B" means that B cannot start until A completes (inverse of is_dependent_on) */ - }; - - } -} - -// Check if the tools support is enabled -#if (_WIN32||_WIN64||__linux__) && !__MINGW32__ && TBB_USE_THREADING_TOOLS - -#if _WIN32||_WIN64 -#include /* mbstowcs_s */ -#endif -#include "tbb_stddef.h" - -namespace tbb { - namespace internal { - -#if _WIN32||_WIN64 - void __TBB_EXPORTED_FUNC itt_set_sync_name_v3( void *obj, const wchar_t* name ); - inline size_t multibyte_to_widechar( wchar_t* wcs, const char* mbs, size_t bufsize) { -#if _MSC_VER>=1400 - size_t len; - mbstowcs_s( &len, wcs, bufsize, mbs, _TRUNCATE ); - return len; // mbstowcs_s counts null terminator -#else - size_t len = mbstowcs( wcs, mbs, bufsize ); - if(wcs && len!=size_t(-1) ) - wcs[len - inline void itt_store_word_with_release(tbb::atomic& dst, U src) { -#if TBB_USE_THREADING_TOOLS - // This assertion should be replaced with static_assert - __TBB_ASSERT(sizeof(T) == sizeof(void *), "Type must be word-sized."); - itt_store_pointer_with_release_v3(&dst, (void *)uintptr_t(src)); -#else - dst = src; -#endif // TBB_USE_THREADING_TOOLS - } - - template - inline T itt_load_word_with_acquire(const tbb::atomic& src) { -#if TBB_USE_THREADING_TOOLS - // This assertion should be replaced with static_assert - __TBB_ASSERT(sizeof(T) == sizeof(void *), "Type must be word-sized."); -#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) - // Workaround for overzealous compiler warnings - #pragma warning (push) - #pragma warning (disable: 4311) -#endif - T result = (T)itt_load_pointer_with_acquire_v3(&src); -#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) - #pragma warning (pop) -#endif - return result; -#else - return src; -#endif // TBB_USE_THREADING_TOOLS - } - - template - inline void itt_store_word_with_release(T& dst, T src) { -#if TBB_USE_THREADING_TOOLS - // This assertion should be replaced with static_assert - __TBB_ASSERT(sizeof(T) == sizeof(void *), "Type must be word-sized."); - itt_store_pointer_with_release_v3(&dst, (void *)src); -#else - __TBB_store_with_release(dst, src); -#endif // TBB_USE_THREADING_TOOLS - } - - template - inline T itt_load_word_with_acquire(const T& src) { -#if TBB_USE_THREADING_TOOLS - // This assertion should be replaced with static_assert - __TBB_ASSERT(sizeof(T) == sizeof(void *), "Type must be word-sized"); - return (T)itt_load_pointer_with_acquire_v3(&src); -#else - return __TBB_load_with_acquire(src); -#endif // TBB_USE_THREADING_TOOLS - } - - template - inline void itt_hide_store_word(T& dst, T src) { -#if TBB_USE_THREADING_TOOLS - //TODO: This assertion should be replaced with static_assert - __TBB_ASSERT(sizeof(T) == sizeof(void *), "Type must be word-sized"); - itt_store_pointer_with_release_v3(&dst, (void *)src); -#else - dst = src; -#endif - } - - //TODO: rename to itt_hide_load_word_relaxed - template - inline T itt_hide_load_word(const T& src) { -#if TBB_USE_THREADING_TOOLS - //TODO: This assertion should be replaced with static_assert - __TBB_ASSERT(sizeof(T) == sizeof(void *), "Type must be word-sized."); - return (T)itt_load_pointer_v3(&src); -#else - return src; -#endif - } - -#if TBB_USE_THREADING_TOOLS - inline void call_itt_notify(notify_type t, void *ptr) { - call_itt_notify_v5((int)t, ptr); - } - -#else - inline void call_itt_notify(notify_type /*t*/, void * /*ptr*/) {} - -#endif // TBB_USE_THREADING_TOOLS - -#if __TBB_ITT_STRUCTURE_API - inline void itt_make_task_group( itt_domain_enum domain, void *group, unsigned long long group_extra, - void *parent, unsigned long long parent_extra, string_index name_index ) { - itt_make_task_group_v7( domain, group, group_extra, parent, parent_extra, name_index ); - } - - inline void itt_metadata_str_add( itt_domain_enum domain, void *addr, unsigned long long addr_extra, - string_index key, const char *value ) { - itt_metadata_str_add_v7( domain, addr, addr_extra, key, value ); - } - - inline void itt_relation_add( itt_domain_enum domain, void *addr0, unsigned long long addr0_extra, - itt_relation relation, void *addr1, unsigned long long addr1_extra ) { - itt_relation_add_v7( domain, addr0, addr0_extra, relation, addr1, addr1_extra ); - } - - inline void itt_task_begin( itt_domain_enum domain, void *task, unsigned long long task_extra, - void *parent, unsigned long long parent_extra, string_index name_index ) { - itt_task_begin_v7( domain, task, task_extra, parent, parent_extra, name_index ); - } - - inline void itt_task_end( itt_domain_enum domain ) { - itt_task_end_v7( domain ); - } - - inline void itt_region_begin( itt_domain_enum domain, void *region, unsigned long long region_extra, - void *parent, unsigned long long parent_extra, string_index name_index ) { - itt_region_begin_v9( domain, region, region_extra, parent, parent_extra, name_index ); - } - - inline void itt_region_end( itt_domain_enum domain, void *region, unsigned long long region_extra ) { - itt_region_end_v9( domain, region, region_extra ); - } -#endif // __TBB_ITT_STRUCTURE_API - - } // namespace internal -} // namespace tbb - -#endif /* __TBB_profiling_H */ diff --git a/src/tbb/include/tbb/tbb_stddef.h b/src/tbb/include/tbb/tbb_stddef.h deleted file mode 100644 index b4ee78101..000000000 --- a/src/tbb/include/tbb/tbb_stddef.h +++ /dev/null @@ -1,549 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_tbb_stddef_H -#define __TBB_tbb_stddef_H - -// Marketing-driven product version -#define TBB_VERSION_MAJOR 2018 -#define TBB_VERSION_MINOR 0 - -// Engineering-focused interface version -#define TBB_INTERFACE_VERSION 10001 -#define TBB_INTERFACE_VERSION_MAJOR TBB_INTERFACE_VERSION/1000 - -// The oldest major interface version still supported -// To be used in SONAME, manifests, etc. -#define TBB_COMPATIBLE_INTERFACE_VERSION 2 - -#define __TBB_STRING_AUX(x) #x -#define __TBB_STRING(x) __TBB_STRING_AUX(x) - -// We do not need defines below for resource processing on windows -#if !defined RC_INVOKED - -// Define groups for Doxygen documentation -/** - * @defgroup algorithms Algorithms - * @defgroup containers Containers - * @defgroup memory_allocation Memory Allocation - * @defgroup synchronization Synchronization - * @defgroup timing Timing - * @defgroup task_scheduling Task Scheduling - */ - -// Simple text that is displayed on the main page of Doxygen documentation. -/** - * \mainpage Main Page - * - * Click the tabs above for information about the - * - Modules (groups of functionality) implemented by the library - * - Classes provided by the library - * - Files constituting the library. - * . - * Please note that significant part of TBB functionality is implemented in the form of - * template functions, descriptions of which are not accessible on the Classes - * tab. Use Modules or Namespace/Namespace Members - * tabs to find them. - * - * Additional pieces of information can be found here - * - \subpage concepts - * . - */ - -/** \page concepts TBB concepts - - A concept is a set of requirements to a type, which are necessary and sufficient - for the type to model a particular behavior or a set of behaviors. Some concepts - are specific to a particular algorithm (e.g. algorithm body), while other ones - are common to several algorithms (e.g. range concept). - - All TBB algorithms make use of different classes implementing various concepts. - Implementation classes are supplied by the user as type arguments of template - parameters and/or as objects passed as function call arguments. The library - provides predefined implementations of some concepts (e.g. several kinds of - \ref range_req "ranges"), while other ones must always be implemented by the user. - - TBB defines a set of minimal requirements each concept must conform to. Here is - the list of different concepts hyperlinked to the corresponding requirements specifications: - - \subpage range_req - - \subpage parallel_do_body_req - - \subpage parallel_for_body_req - - \subpage parallel_reduce_body_req - - \subpage parallel_scan_body_req - - \subpage parallel_sort_iter_req -**/ - -// tbb_config.h should be included the first since it contains macro definitions used in other headers -#include "tbb_config.h" - -#if _MSC_VER >=1400 - #define __TBB_EXPORTED_FUNC __cdecl - #define __TBB_EXPORTED_METHOD __thiscall -#else - #define __TBB_EXPORTED_FUNC - #define __TBB_EXPORTED_METHOD -#endif - -#if __INTEL_COMPILER || _MSC_VER -#define __TBB_NOINLINE(decl) __declspec(noinline) decl -#elif __GNUC__ -#define __TBB_NOINLINE(decl) decl __attribute__ ((noinline)) -#else -#define __TBB_NOINLINE(decl) decl -#endif - -#if __TBB_NOEXCEPT_PRESENT -#define __TBB_NOEXCEPT(expression) noexcept(expression) -#else -#define __TBB_NOEXCEPT(expression) -#endif - -#include /* Need size_t and ptrdiff_t */ - -#if _MSC_VER - #define __TBB_tbb_windef_H - #include "internal/_tbb_windef.h" - #undef __TBB_tbb_windef_H -#endif -#if !defined(_MSC_VER) || _MSC_VER>=1600 - #include -#endif - -//! Type for an assertion handler -typedef void(*assertion_handler_type)( const char* filename, int line, const char* expression, const char * comment ); - -#if __TBBMALLOC_BUILD -namespace rml { namespace internal { - #define __TBB_ASSERT_RELEASE(predicate,message) ((predicate)?((void)0) : rml::internal::assertion_failure(__FILE__,__LINE__,#predicate,message)) -#else -namespace tbb { - #define __TBB_ASSERT_RELEASE(predicate,message) ((predicate)?((void)0) : tbb::assertion_failure(__FILE__,__LINE__,#predicate,message)) -#endif - - //! Set assertion handler and return previous value of it. - assertion_handler_type __TBB_EXPORTED_FUNC set_assertion_handler( assertion_handler_type new_handler ); - - //! Process an assertion failure. - /** Normally called from __TBB_ASSERT macro. - If assertion handler is null, print message for assertion failure and abort. - Otherwise call the assertion handler. */ - void __TBB_EXPORTED_FUNC assertion_failure( const char* filename, int line, const char* expression, const char* comment ); - -#if __TBBMALLOC_BUILD -}} // namespace rml::internal -#else -} // namespace tbb -#endif - -#if TBB_USE_ASSERT - - //! Assert that x is true. - /** If x is false, print assertion failure message. - If the comment argument is not NULL, it is printed as part of the failure message. - The comment argument has no other effect. */ - #define __TBB_ASSERT(predicate,message) __TBB_ASSERT_RELEASE(predicate,message) - - #define __TBB_ASSERT_EX __TBB_ASSERT - -#else /* !TBB_USE_ASSERT */ - - //! No-op version of __TBB_ASSERT. - #define __TBB_ASSERT(predicate,comment) ((void)0) - //! "Extended" version is useful to suppress warnings if a variable is only used with an assert - #define __TBB_ASSERT_EX(predicate,comment) ((void)(1 && (predicate))) - -#endif /* !TBB_USE_ASSERT */ - -//! The namespace tbb contains all components of the library. -namespace tbb { - - namespace internal { -#if _MSC_VER && _MSC_VER<1600 - typedef __int8 int8_t; - typedef __int16 int16_t; - typedef __int32 int32_t; - typedef __int64 int64_t; - typedef unsigned __int8 uint8_t; - typedef unsigned __int16 uint16_t; - typedef unsigned __int32 uint32_t; - typedef unsigned __int64 uint64_t; -#else /* Posix */ - using ::int8_t; - using ::int16_t; - using ::int32_t; - using ::int64_t; - using ::uint8_t; - using ::uint16_t; - using ::uint32_t; - using ::uint64_t; -#endif /* Posix */ - } // namespace internal - - using std::size_t; - using std::ptrdiff_t; - -//! The function returns the interface version of the TBB shared library being used. -/** - * The version it returns is determined at runtime, not at compile/link time. - * So it can be different than the value of TBB_INTERFACE_VERSION obtained at compile time. - */ -extern "C" int __TBB_EXPORTED_FUNC TBB_runtime_interface_version(); - -/** - * @cond INTERNAL - * @brief Identifiers declared inside namespace internal should never be used directly by client code. - */ -namespace internal { - -//! Compile-time constant that is upper bound on cache line/sector size. -/** It should be used only in situations where having a compile-time upper - bound is more useful than a run-time exact answer. - @ingroup memory_allocation */ -const size_t NFS_MaxLineSize = 128; - -/** Label for data that may be accessed from different threads, and that may eventually become wrapped - in a formal atomic type. - - Note that no problems have yet been observed relating to the definition currently being empty, - even if at least "volatile" would seem to be in order to avoid data sometimes temporarily hiding - in a register (although "volatile" as a "poor man's atomic" lacks several other features of a proper - atomic, some of which are now provided instead through specialized functions). - - Note that usage is intentionally compatible with a definition as qualifier "volatile", - both as a way to have the compiler help enforce use of the label and to quickly rule out - one potential issue. - - Note however that, with some architecture/compiler combinations, e.g. on IA-64 architecture, "volatile" - also has non-portable memory semantics that are needlessly expensive for "relaxed" operations. - - Note that this must only be applied to data that will not change bit patterns when cast to/from - an integral type of the same length; tbb::atomic must be used instead for, e.g., floating-point types. - - TODO: apply wherever relevant **/ -#define __TBB_atomic // intentionally empty, see above - -#if __TBB_OVERRIDE_PRESENT -#define __TBB_override override -#else -#define __TBB_override // formal comment only -#endif - -template -struct padded_base : T { - char pad[S - R]; -}; -template struct padded_base : T {}; - -//! Pads type T to fill out to a multiple of cache line size. -template -struct padded : padded_base {}; - -//! Extended variant of the standard offsetof macro -/** The standard offsetof macro is not sufficient for TBB as it can be used for - POD-types only. The constant 0x1000 (not NULL) is necessary to appease GCC. **/ -#define __TBB_offsetof(class_name, member_name) \ - ((ptrdiff_t)&(reinterpret_cast(0x1000)->member_name) - 0x1000) - -//! Returns address of the object containing a member with the given name and address -#define __TBB_get_object_ref(class_name, member_name, member_addr) \ - (*reinterpret_cast((char*)member_addr - __TBB_offsetof(class_name, member_name))) - -//! Throws std::runtime_error with what() returning error_code description prefixed with aux_info -void __TBB_EXPORTED_FUNC handle_perror( int error_code, const char* aux_info ); - -#if TBB_USE_EXCEPTIONS - #define __TBB_TRY try - #define __TBB_CATCH(e) catch(e) - #define __TBB_THROW(e) throw e - #define __TBB_RETHROW() throw -#else /* !TBB_USE_EXCEPTIONS */ - inline bool __TBB_false() { return false; } - #define __TBB_TRY - #define __TBB_CATCH(e) if ( tbb::internal::__TBB_false() ) - #define __TBB_THROW(e) tbb::internal::suppress_unused_warning(e) - #define __TBB_RETHROW() ((void)0) -#endif /* !TBB_USE_EXCEPTIONS */ - -//! Report a runtime warning. -void __TBB_EXPORTED_FUNC runtime_warning( const char* format, ... ); - -#if TBB_USE_ASSERT -static void* const poisoned_ptr = reinterpret_cast(-1); - -//! Set p to invalid pointer value. -// Also works for regular (non-__TBB_atomic) pointers. -template -inline void poison_pointer( T* __TBB_atomic & p ) { p = reinterpret_cast(poisoned_ptr); } - -/** Expected to be used in assertions only, thus no empty form is defined. **/ -template -inline bool is_poisoned( T* p ) { return p == reinterpret_cast(poisoned_ptr); } -#else -template -inline void poison_pointer( T* __TBB_atomic & ) {/*do nothing*/} -#endif /* !TBB_USE_ASSERT */ - -//! Cast between unrelated pointer types. -/** This method should be used sparingly as a last resort for dealing with - situations that inherently break strict ISO C++ aliasing rules. */ -// T is a pointer type because it will be explicitly provided by the programmer as a template argument; -// U is a referent type to enable the compiler to check that "ptr" is a pointer, deducing U in the process. -template -inline T punned_cast( U* ptr ) { - uintptr_t x = reinterpret_cast(ptr); - return reinterpret_cast(x); -} - -//! Base class for types that should not be assigned. -class no_assign { - // Deny assignment - void operator=( const no_assign& ); -public: -#if __GNUC__ - //! Explicitly define default construction, because otherwise gcc issues gratuitous warning. - no_assign() {} -#endif /* __GNUC__ */ -}; - -//! Base class for types that should not be copied or assigned. -class no_copy: no_assign { - //! Deny copy construction - no_copy( const no_copy& ); -public: - //! Allow default construction - no_copy() {} -}; - -#if TBB_DEPRECATED_MUTEX_COPYING -class mutex_copy_deprecated_and_disabled {}; -#else -// By default various implementations of mutexes are not copy constructible -// and not copy assignable. -class mutex_copy_deprecated_and_disabled : no_copy {}; -#endif - -//! A function to check if passed in pointer is aligned on a specific border -template -inline bool is_aligned(T* pointer, uintptr_t alignment) { - return 0==((uintptr_t)pointer & (alignment-1)); -} - -//! A function to check if passed integer is a power of 2 -template -inline bool is_power_of_two(integer_type arg) { - return arg && (0 == (arg & (arg - 1))); -} - -//! A function to compute arg modulo divisor where divisor is a power of 2. -template -inline argument_integer_type modulo_power_of_two(argument_integer_type arg, divisor_integer_type divisor) { - __TBB_ASSERT( is_power_of_two(divisor), "Divisor should be a power of two" ); - return (arg & (divisor - 1)); -} - - -//! A function to determine if arg is a power of 2 at least as big as another power of 2. -// i.e. for strictly positive i and j, with j being a power of 2, -// determines whether i==j< -inline bool is_power_of_two_at_least(argument_integer_type arg, power2_integer_type power2) { - __TBB_ASSERT( is_power_of_two(power2), "Divisor should be a power of two" ); - return 0 == (arg & (arg - power2)); -} - -//! Utility template function to prevent "unused" warnings by various compilers. -template void suppress_unused_warning( const T1& ) {} -template void suppress_unused_warning( const T1&, const T2& ) {} -template void suppress_unused_warning( const T1&, const T2&, const T3& ) {} - -// Struct to be used as a version tag for inline functions. -/** Version tag can be necessary to prevent loader on Linux from using the wrong - symbol in debug builds (when inline functions are compiled as out-of-line). **/ -struct version_tag_v3 {}; - -typedef version_tag_v3 version_tag; - -} // internal - -//! Dummy type that distinguishes splitting constructor from copy constructor. -/** - * See description of parallel_for and parallel_reduce for example usages. - * @ingroup algorithms - */ -class split { -}; - -//! Type enables transmission of splitting proportion from partitioners to range objects -/** - * In order to make use of such facility Range objects must implement - * splitting constructor with this type passed and initialize static - * constant boolean field 'is_splittable_in_proportion' with the value - * of 'true' - */ -class proportional_split: internal::no_assign { -public: - proportional_split(size_t _left = 1, size_t _right = 1) : my_left(_left), my_right(_right) { } - - size_t left() const { return my_left; } - size_t right() const { return my_right; } - - // used when range does not support proportional split - operator split() const { return split(); } - -#if __TBB_ENABLE_RANGE_FEEDBACK - void set_proportion(size_t _left, size_t _right) { - my_left = _left; - my_right = _right; - } -#endif -private: - size_t my_left, my_right; -}; - -} // tbb - -// Following is a set of classes and functions typically used in compile-time "metaprogramming". -// TODO: move all that to a separate header - -#if __TBB_ALLOCATOR_TRAITS_PRESENT || __TBB_CPP11_SMART_POINTERS_PRESENT -#include // for allocator_traits, unique_ptr -#endif - -#if __TBB_CPP11_RVALUE_REF_PRESENT || __TBB_CPP11_DECLTYPE_PRESENT || _LIBCPP_VERSION -#include // for std::move, std::forward, std::declval -#endif - -namespace tbb { -namespace internal { - -#if __TBB_CPP11_SMART_POINTERS_PRESENT && __TBB_CPP11_RVALUE_REF_PRESENT && __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT - template - std::unique_ptr make_unique(Args&&... args) { - return std::unique_ptr(new T(std::forward(args)...)); - } -#endif - -//! Class for determining type of std::allocator::value_type. -template -struct allocator_type { - typedef T value_type; -}; - -#if _MSC_VER -//! Microsoft std::allocator has non-standard extension that strips const from a type. -template -struct allocator_type { - typedef T value_type; -}; -#endif - -// Ad-hoc implementation of true_type & false_type -// Intended strictly for internal use! For public APIs (traits etc), use C++11 analogues. -template -struct bool_constant { - static /*constexpr*/ const bool value = v; -}; -typedef bool_constant true_type; -typedef bool_constant false_type; - -#if __TBB_ALLOCATOR_TRAITS_PRESENT -using std::allocator_traits; -#else -template -struct allocator_traits{ - typedef tbb::internal::false_type propagate_on_container_move_assignment; -}; -#endif - -//! A template to select either 32-bit or 64-bit constant as compile time, depending on machine word size. -template -struct select_size_t_constant { - //Explicit cast is needed to avoid compiler warnings about possible truncation. - //The value of the right size, which is selected by ?:, is anyway not truncated or promoted. - static const size_t value = (size_t)((sizeof(size_t)==sizeof(u)) ? u : ull); -}; - -#if __TBB_CPP11_RVALUE_REF_PRESENT -using std::move; -using std::forward; -#elif defined(_LIBCPP_NAMESPACE) -// libc++ defines "pre-C++11 move and forward" similarly to ours; use it to avoid name conflicts in some cases. -using std::_LIBCPP_NAMESPACE::move; -using std::_LIBCPP_NAMESPACE::forward; -#else -// It is assumed that cv qualifiers, if any, are part of the deduced type. -template -T& move( T& x ) { return x; } -template -T& forward( T& x ) { return x; } -#endif /* __TBB_CPP11_RVALUE_REF_PRESENT */ - -// Helper macros to simplify writing templates working with both C++03 and C++11. -#if __TBB_CPP11_RVALUE_REF_PRESENT -#define __TBB_FORWARDING_REF(A) A&& -#else -// It is assumed that cv qualifiers, if any, are part of a deduced type. -// Thus this macro should not be used in public interfaces. -#define __TBB_FORWARDING_REF(A) A& -#endif -#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT -#define __TBB_PARAMETER_PACK ... -#define __TBB_PACK_EXPANSION(A) A... -#else -#define __TBB_PARAMETER_PACK -#define __TBB_PACK_EXPANSION(A) A -#endif /* __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT */ - -#if __TBB_CPP11_DECLTYPE_PRESENT -#if __TBB_CPP11_DECLVAL_BROKEN -// Ad-hoc implementation of std::declval -template __TBB_FORWARDING_REF(T) declval() /*noexcept*/; -#else -using std::declval; -#endif -#endif - -template -struct STATIC_ASSERTION_FAILED; - -template <> -struct STATIC_ASSERTION_FAILED { enum {value=1};}; - -template<> -struct STATIC_ASSERTION_FAILED; //intentionally left undefined to cause compile time error - -//! @endcond -}} // namespace tbb::internal - -#if __TBB_STATIC_ASSERT_PRESENT -#define __TBB_STATIC_ASSERT(condition,msg) static_assert(condition,msg) -#else -//please note condition is intentionally inverted to get a bit more understandable error msg -#define __TBB_STATIC_ASSERT_IMPL1(condition,msg,line) \ - enum {static_assert_on_line_##line = tbb::internal::STATIC_ASSERTION_FAILED::value} - -#define __TBB_STATIC_ASSERT_IMPL(condition,msg,line) __TBB_STATIC_ASSERT_IMPL1(condition,msg,line) -//! Verify condition, at compile time -#define __TBB_STATIC_ASSERT(condition,msg) __TBB_STATIC_ASSERT_IMPL(condition,msg,__LINE__) -#endif - -#endif /* RC_INVOKED */ -#endif /* __TBB_tbb_stddef_H */ diff --git a/src/tbb/include/tbb/tbb_thread.h b/src/tbb/include/tbb/tbb_thread.h deleted file mode 100644 index c2846b9e5..000000000 --- a/src/tbb/include/tbb/tbb_thread.h +++ /dev/null @@ -1,332 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_tbb_thread_H -#define __TBB_tbb_thread_H - -#include "tbb_stddef.h" - -#if _WIN32||_WIN64 -#include "machine/windows_api.h" -#define __TBB_NATIVE_THREAD_ROUTINE unsigned WINAPI -#define __TBB_NATIVE_THREAD_ROUTINE_PTR(r) unsigned (WINAPI* r)( void* ) -namespace tbb { namespace internal { -#if __TBB_WIN8UI_SUPPORT - typedef size_t thread_id_type; -#else // __TBB_WIN8UI_SUPPORT - typedef DWORD thread_id_type; -#endif // __TBB_WIN8UI_SUPPORT -}} //namespace tbb::internal -#else -#define __TBB_NATIVE_THREAD_ROUTINE void* -#define __TBB_NATIVE_THREAD_ROUTINE_PTR(r) void* (*r)( void* ) -#include -namespace tbb { namespace internal { - typedef pthread_t thread_id_type; -}} //namespace tbb::internal -#endif // _WIN32||_WIN64 - -#include "atomic.h" -#include "internal/_tbb_hash_compare_impl.h" -#include "tick_count.h" - -#include __TBB_STD_SWAP_HEADER -#include - -namespace tbb { - -namespace internal { - class tbb_thread_v3; -} - -inline void swap( internal::tbb_thread_v3& t1, internal::tbb_thread_v3& t2 ) __TBB_NOEXCEPT(true); - -namespace internal { - - //! Allocate a closure - void* __TBB_EXPORTED_FUNC allocate_closure_v3( size_t size ); - //! Free a closure allocated by allocate_closure_v3 - void __TBB_EXPORTED_FUNC free_closure_v3( void* ); - - struct thread_closure_base { - void* operator new( size_t size ) {return allocate_closure_v3(size);} - void operator delete( void* ptr ) {free_closure_v3(ptr);} - }; - - template struct thread_closure_0: thread_closure_base { - F function; - - static __TBB_NATIVE_THREAD_ROUTINE start_routine( void* c ) { - thread_closure_0 *self = static_cast(c); - self->function(); - delete self; - return 0; - } - thread_closure_0( const F& f ) : function(f) {} - }; - //! Structure used to pass user function with 1 argument to thread. - template struct thread_closure_1: thread_closure_base { - F function; - X arg1; - //! Routine passed to Windows's _beginthreadex by thread::internal_start() inside tbb.dll - static __TBB_NATIVE_THREAD_ROUTINE start_routine( void* c ) { - thread_closure_1 *self = static_cast(c); - self->function(self->arg1); - delete self; - return 0; - } - thread_closure_1( const F& f, const X& x ) : function(f), arg1(x) {} - }; - template struct thread_closure_2: thread_closure_base { - F function; - X arg1; - Y arg2; - //! Routine passed to Windows's _beginthreadex by thread::internal_start() inside tbb.dll - static __TBB_NATIVE_THREAD_ROUTINE start_routine( void* c ) { - thread_closure_2 *self = static_cast(c); - self->function(self->arg1, self->arg2); - delete self; - return 0; - } - thread_closure_2( const F& f, const X& x, const Y& y ) : function(f), arg1(x), arg2(y) {} - }; - - //! Versioned thread class. - class tbb_thread_v3 { -#if __TBB_IF_NO_COPY_CTOR_MOVE_SEMANTICS_BROKEN - // Workaround for a compiler bug: declaring the copy constructor as public - // enables use of the moving constructor. - // The definition is not provided in order to prohibit copying. - public: -#endif - tbb_thread_v3(const tbb_thread_v3&); // = delete; // Deny access - public: -#if _WIN32||_WIN64 - typedef HANDLE native_handle_type; -#else - typedef pthread_t native_handle_type; -#endif // _WIN32||_WIN64 - - class id; - //! Constructs a thread object that does not represent a thread of execution. - tbb_thread_v3() __TBB_NOEXCEPT(true) : my_handle(0) -#if _WIN32||_WIN64 - , my_thread_id(0) -#endif // _WIN32||_WIN64 - {} - - //! Constructs an object and executes f() in a new thread - template explicit tbb_thread_v3(F f) { - typedef internal::thread_closure_0 closure_type; - internal_start(closure_type::start_routine, new closure_type(f)); - } - //! Constructs an object and executes f(x) in a new thread - template tbb_thread_v3(F f, X x) { - typedef internal::thread_closure_1 closure_type; - internal_start(closure_type::start_routine, new closure_type(f,x)); - } - //! Constructs an object and executes f(x,y) in a new thread - template tbb_thread_v3(F f, X x, Y y) { - typedef internal::thread_closure_2 closure_type; - internal_start(closure_type::start_routine, new closure_type(f,x,y)); - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - tbb_thread_v3(tbb_thread_v3&& x) __TBB_NOEXCEPT(true) - : my_handle(x.my_handle) -#if _WIN32||_WIN64 - , my_thread_id(x.my_thread_id) -#endif - { - x.internal_wipe(); - } - tbb_thread_v3& operator=(tbb_thread_v3&& x) __TBB_NOEXCEPT(true) { - internal_move(x); - return *this; - } - private: - tbb_thread_v3& operator=(const tbb_thread_v3& x); // = delete; - public: -#else // __TBB_CPP11_RVALUE_REF_PRESENT - tbb_thread_v3& operator=(tbb_thread_v3& x) { - internal_move(x); - return *this; - } -#endif // __TBB_CPP11_RVALUE_REF_PRESENT - - void swap( tbb_thread_v3& t ) __TBB_NOEXCEPT(true) {tbb::swap( *this, t );} - bool joinable() const __TBB_NOEXCEPT(true) {return my_handle!=0; } - //! The completion of the thread represented by *this happens before join() returns. - void __TBB_EXPORTED_METHOD join(); - //! When detach() returns, *this no longer represents the possibly continuing thread of execution. - void __TBB_EXPORTED_METHOD detach(); - ~tbb_thread_v3() {if( joinable() ) detach();} - inline id get_id() const __TBB_NOEXCEPT(true); - native_handle_type native_handle() { return my_handle; } - - //! The number of hardware thread contexts. - /** Before TBB 3.0 U4 this methods returned the number of logical CPU in - the system. Currently on Windows, Linux and FreeBSD it returns the - number of logical CPUs available to the current process in accordance - with its affinity mask. - - NOTE: The return value of this method never changes after its first - invocation. This means that changes in the process affinity mask that - took place after this method was first invoked will not affect the - number of worker threads in the TBB worker threads pool. **/ - static unsigned __TBB_EXPORTED_FUNC hardware_concurrency() __TBB_NOEXCEPT(true); - private: - native_handle_type my_handle; -#if _WIN32||_WIN64 - thread_id_type my_thread_id; -#endif // _WIN32||_WIN64 - - void internal_wipe() __TBB_NOEXCEPT(true) { - my_handle = 0; -#if _WIN32||_WIN64 - my_thread_id = 0; -#endif - } - void internal_move(tbb_thread_v3& x) __TBB_NOEXCEPT(true) { - if (joinable()) detach(); - my_handle = x.my_handle; -#if _WIN32||_WIN64 - my_thread_id = x.my_thread_id; -#endif // _WIN32||_WIN64 - x.internal_wipe(); - } - - /** Runs start_routine(closure) on another thread and sets my_handle to the handle of the created thread. */ - void __TBB_EXPORTED_METHOD internal_start( __TBB_NATIVE_THREAD_ROUTINE_PTR(start_routine), - void* closure ); - friend void __TBB_EXPORTED_FUNC move_v3( tbb_thread_v3& t1, tbb_thread_v3& t2 ); - friend void tbb::swap( tbb_thread_v3& t1, tbb_thread_v3& t2 ) __TBB_NOEXCEPT(true); - }; - - class tbb_thread_v3::id { - thread_id_type my_id; - id( thread_id_type id_ ) : my_id(id_) {} - - friend class tbb_thread_v3; - public: - id() __TBB_NOEXCEPT(true) : my_id(0) {} - - friend bool operator==( tbb_thread_v3::id x, tbb_thread_v3::id y ) __TBB_NOEXCEPT(true); - friend bool operator!=( tbb_thread_v3::id x, tbb_thread_v3::id y ) __TBB_NOEXCEPT(true); - friend bool operator<( tbb_thread_v3::id x, tbb_thread_v3::id y ) __TBB_NOEXCEPT(true); - friend bool operator<=( tbb_thread_v3::id x, tbb_thread_v3::id y ) __TBB_NOEXCEPT(true); - friend bool operator>( tbb_thread_v3::id x, tbb_thread_v3::id y ) __TBB_NOEXCEPT(true); - friend bool operator>=( tbb_thread_v3::id x, tbb_thread_v3::id y ) __TBB_NOEXCEPT(true); - - template - friend std::basic_ostream& - operator<< (std::basic_ostream &out, - tbb_thread_v3::id id) - { - out << id.my_id; - return out; - } - friend tbb_thread_v3::id __TBB_EXPORTED_FUNC thread_get_id_v3(); - - friend inline size_t tbb_hasher( const tbb_thread_v3::id& id ) { - __TBB_STATIC_ASSERT(sizeof(id.my_id) <= sizeof(size_t), "Implementaion assumes that thread_id_type fits into machine word"); - return tbb::tbb_hasher(id.my_id); - } - - // A workaround for lack of tbb::atomic (which would require id to be POD in C++03). - friend id atomic_compare_and_swap(id& location, const id& value, const id& comparand){ - return as_atomic(location.my_id).compare_and_swap(value.my_id, comparand.my_id); - } - }; // tbb_thread_v3::id - - tbb_thread_v3::id tbb_thread_v3::get_id() const __TBB_NOEXCEPT(true) { -#if _WIN32||_WIN64 - return id(my_thread_id); -#else - return id(my_handle); -#endif // _WIN32||_WIN64 - } - - void __TBB_EXPORTED_FUNC move_v3( tbb_thread_v3& t1, tbb_thread_v3& t2 ); - tbb_thread_v3::id __TBB_EXPORTED_FUNC thread_get_id_v3(); - void __TBB_EXPORTED_FUNC thread_yield_v3(); - void __TBB_EXPORTED_FUNC thread_sleep_v3(const tick_count::interval_t &i); - - inline bool operator==(tbb_thread_v3::id x, tbb_thread_v3::id y) __TBB_NOEXCEPT(true) - { - return x.my_id == y.my_id; - } - inline bool operator!=(tbb_thread_v3::id x, tbb_thread_v3::id y) __TBB_NOEXCEPT(true) - { - return x.my_id != y.my_id; - } - inline bool operator<(tbb_thread_v3::id x, tbb_thread_v3::id y) __TBB_NOEXCEPT(true) - { - return x.my_id < y.my_id; - } - inline bool operator<=(tbb_thread_v3::id x, tbb_thread_v3::id y) __TBB_NOEXCEPT(true) - { - return x.my_id <= y.my_id; - } - inline bool operator>(tbb_thread_v3::id x, tbb_thread_v3::id y) __TBB_NOEXCEPT(true) - { - return x.my_id > y.my_id; - } - inline bool operator>=(tbb_thread_v3::id x, tbb_thread_v3::id y) __TBB_NOEXCEPT(true) - { - return x.my_id >= y.my_id; - } - -} // namespace internal; - -//! Users reference thread class by name tbb_thread -typedef internal::tbb_thread_v3 tbb_thread; - -using internal::operator==; -using internal::operator!=; -using internal::operator<; -using internal::operator>; -using internal::operator<=; -using internal::operator>=; - -inline void move( tbb_thread& t1, tbb_thread& t2 ) { - internal::move_v3(t1, t2); -} - -inline void swap( internal::tbb_thread_v3& t1, internal::tbb_thread_v3& t2 ) __TBB_NOEXCEPT(true) { - std::swap(t1.my_handle, t2.my_handle); -#if _WIN32||_WIN64 - std::swap(t1.my_thread_id, t2.my_thread_id); -#endif /* _WIN32||_WIN64 */ -} - -namespace this_tbb_thread { - inline tbb_thread::id get_id() { return internal::thread_get_id_v3(); } - //! Offers the operating system the opportunity to schedule another thread. - inline void yield() { internal::thread_yield_v3(); } - //! The current thread blocks at least until the time specified. - inline void sleep(const tick_count::interval_t &i) { - internal::thread_sleep_v3(i); - } -} // namespace this_tbb_thread - -} // namespace tbb - -#endif /* __TBB_tbb_thread_H */ diff --git a/src/tbb/include/tbb/tbbmalloc_proxy.h b/src/tbb/include/tbb/tbbmalloc_proxy.h index 76cbd6d72..93eaa18e8 100644 --- a/src/tbb/include/tbb/tbbmalloc_proxy.h +++ b/src/tbb/include/tbb/tbbmalloc_proxy.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,55 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - -*/ - -/* -Replacing the standard memory allocation routines in Microsoft* C/C++ RTL -(malloc/free, global new/delete, etc.) with the TBB memory allocator. - -Include the following header to a source of any binary which is loaded during -application startup - -#include "tbb/tbbmalloc_proxy.h" - -or add following parameters to the linker options for the binary which is -loaded during application startup. It can be either exe-file or dll. - -For win32 -tbbmalloc_proxy.lib /INCLUDE:"___TBB_malloc_proxy" -win64 -tbbmalloc_proxy.lib /INCLUDE:"__TBB_malloc_proxy" */ -#ifndef __TBB_tbbmalloc_proxy_H -#define __TBB_tbbmalloc_proxy_H - -#if _MSC_VER - -#ifdef _DEBUG - #pragma comment(lib, "tbbmalloc_proxy_debug.lib") -#else - #pragma comment(lib, "tbbmalloc_proxy.lib") -#endif - -#if defined(_WIN64) - #pragma comment(linker, "/include:__TBB_malloc_proxy") -#else - #pragma comment(linker, "/include:___TBB_malloc_proxy") -#endif - -#else -/* Primarily to support MinGW */ - -extern "C" void __TBB_malloc_proxy(); -struct __TBB_malloc_proxy_caller { - __TBB_malloc_proxy_caller() { __TBB_malloc_proxy(); } -} volatile __TBB_malloc_proxy_helper_object; - -#endif // _MSC_VER - -#endif //__TBB_tbbmalloc_proxy_H +#include "../oneapi/tbb/tbbmalloc_proxy.h" diff --git a/src/tbb/include/tbb/tick_count.h b/src/tbb/include/tbb/tick_count.h index a7f4e0f37..170074aeb 100644 --- a/src/tbb/include/tbb/tick_count.h +++ b/src/tbb/include/tbb/tick_count.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,129 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ -#ifndef __TBB_tick_count_H -#define __TBB_tick_count_H - -#include "tbb_stddef.h" - -#if _WIN32||_WIN64 -#include "machine/windows_api.h" -#elif __linux__ -#include -#else /* generic Unix */ -#include -#endif /* (choice of OS) */ - -namespace tbb { - -//! Absolute timestamp -/** @ingroup timing */ -class tick_count { -public: - //! Relative time interval. - class interval_t { - long long value; - explicit interval_t( long long value_ ) : value(value_) {} - public: - //! Construct a time interval representing zero time duration - interval_t() : value(0) {}; - - //! Construct a time interval representing sec seconds time duration - explicit interval_t( double sec ); - - //! Return the length of a time interval in seconds - double seconds() const; - - friend class tbb::tick_count; - - //! Extract the intervals from the tick_counts and subtract them. - friend interval_t operator-( const tick_count& t1, const tick_count& t0 ); - - //! Add two intervals. - friend interval_t operator+( const interval_t& i, const interval_t& j ) { - return interval_t(i.value+j.value); - } - - //! Subtract two intervals. - friend interval_t operator-( const interval_t& i, const interval_t& j ) { - return interval_t(i.value-j.value); - } - - //! Accumulation operator - interval_t& operator+=( const interval_t& i ) {value += i.value; return *this;} - - //! Subtraction operator - interval_t& operator-=( const interval_t& i ) {value -= i.value; return *this;} - private: - static long long ticks_per_second(){ -#if _WIN32||_WIN64 - LARGE_INTEGER qpfreq; - int rval = QueryPerformanceFrequency(&qpfreq); - __TBB_ASSERT_EX(rval, "QueryPerformanceFrequency returned zero"); - return static_cast(qpfreq.QuadPart); -#elif __linux__ - return static_cast(1E9); -#else /* generic Unix */ - return static_cast(1E6); -#endif /* (choice of OS) */ - } - }; - - //! Construct an absolute timestamp initialized to zero. - tick_count() : my_count(0) {}; - - //! Return current time. - static tick_count now(); - - //! Subtract two timestamps to get the time interval between - friend interval_t operator-( const tick_count& t1, const tick_count& t0 ); - - //! Return the resolution of the clock in seconds per tick. - static double resolution() { return 1.0 / interval_t::ticks_per_second(); } - -private: - long long my_count; -}; - -inline tick_count tick_count::now() { - tick_count result; -#if _WIN32||_WIN64 - LARGE_INTEGER qpcnt; - int rval = QueryPerformanceCounter(&qpcnt); - __TBB_ASSERT_EX(rval, "QueryPerformanceCounter failed"); - result.my_count = qpcnt.QuadPart; -#elif __linux__ - struct timespec ts; - int status = clock_gettime( CLOCK_REALTIME, &ts ); - __TBB_ASSERT_EX( status==0, "CLOCK_REALTIME not supported" ); - result.my_count = static_cast(1000000000UL)*static_cast(ts.tv_sec) + static_cast(ts.tv_nsec); -#else /* generic Unix */ - struct timeval tv; - int status = gettimeofday(&tv, NULL); - __TBB_ASSERT_EX( status==0, "gettimeofday failed" ); - result.my_count = static_cast(1000000)*static_cast(tv.tv_sec) + static_cast(tv.tv_usec); -#endif /*(choice of OS) */ - return result; -} - -inline tick_count::interval_t::interval_t( double sec ) { - value = static_cast(sec*interval_t::ticks_per_second()); -} - -inline tick_count::interval_t operator-( const tick_count& t1, const tick_count& t0 ) { - return tick_count::interval_t( t1.my_count-t0.my_count ); -} - -inline double tick_count::interval_t::seconds() const { - return value*tick_count::resolution(); -} - -} // namespace tbb - -#endif /* __TBB_tick_count_H */ +#include "../oneapi/tbb/tick_count.h" diff --git a/src/tbb/include/tbb/version.h b/src/tbb/include/tbb/version.h new file mode 100644 index 000000000..cd13a83a1 --- /dev/null +++ b/src/tbb/include/tbb/version.h @@ -0,0 +1,17 @@ +/* + Copyright (c) 2005-2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "../oneapi/tbb/version.h" diff --git a/src/tbb/integration/cmake/generate_vars.cmake b/src/tbb/integration/cmake/generate_vars.cmake new file mode 100644 index 000000000..90690610c --- /dev/null +++ b/src/tbb/integration/cmake/generate_vars.cmake @@ -0,0 +1,51 @@ +# Copyright (c) 2020-2021 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Reuired parameters: +# SOURCE_DIR - incoming path to oneTBB source directory. +# BINARY_DIR - incoming path to oneTBB build directory. +# BIN_PATH - incoming path to oneTBB binaries directory. +# TBB_INSTALL_VARS - install vars generation trigger +# TBB_CMAKE_INSTALL_LIBDIR - subdir for shared object files installation path (used only in TBB_INSTALL_VARS mode) +# VARS_TEMPLATE - path to the vars template file +# VARS_NAME - name of the output vars script + +set(INPUT_FILE "${SOURCE_DIR}/integration/${VARS_TEMPLATE}") +set(OUTPUT_FILE "${BIN_PATH}/${VARS_NAME}") + +file(TO_NATIVE_PATH "${SOURCE_DIR}" TBBROOT_REPLACEMENT) +file(TO_NATIVE_PATH "${BIN_PATH}" LIBRARY_PATH_REPLACEMENT) +if (WIN32) + file(TO_NATIVE_PATH "${BIN_PATH}" BINARY_PATH_REPLACEMENT) +endif() + +if (NOT EXISTS ${OUTPUT_FILE}) + configure_file(${INPUT_FILE} ${OUTPUT_FILE} @ONLY) +endif() + +if (TBB_INSTALL_VARS) + set(OUTPUT_FILE "${BINARY_DIR}/internal_install_vars") + if (UNIX) + set(TBBROOT_REPLACEMENT "$(cd $(dirname \${BASH_SOURCE}) && pwd -P)/..") + set(LIBRARY_PATH_REPLACEMENT "$TBBROOT/${TBB_CMAKE_INSTALL_LIBDIR}/") + set(CMAKE_ENVIRONMENT_SOURCING_STRING "CMAKE_PREFIX_PATH=\"\${TBBROOT}/${TBB_CMAKE_INSTALL_LIBDIR}/cmake/TBB:${CMAKE_PREFIX_PATH}\"; export CMAKE_PREFIX_PATH") + else() + set(TBBROOT_REPLACEMENT "%~d0%~p0..") + set(LIBRARY_PATH_REPLACEMENT "%TBBROOT%\\${TBB_CMAKE_INSTALL_LIBDIR}") + set(BINARY_PATH_REPLACEMENT "%TBBROOT%\\bin") + set(CMAKE_ENVIRONMENT_SOURCING_STRING "set \"CMAKE_PREFIX_PATH=%TBBROOT%\\${TBB_CMAKE_INSTALL_LIBDIR}\\cmake\\TBB;%CMAKE_PREFIX_PATH%\"") + endif() + + configure_file( ${INPUT_FILE} ${OUTPUT_FILE} @ONLY ) +endif() diff --git a/src/tbb/integration/linux/env/vars.sh b/src/tbb/integration/linux/env/vars.sh new file mode 100644 index 000000000..5d913bb61 --- /dev/null +++ b/src/tbb/integration/linux/env/vars.sh @@ -0,0 +1,183 @@ +#!/bin/sh +# shellcheck shell=sh +# +# Copyright (c) 2005-2023 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The script is setting up environment for oneTBB. +# Supported arguments: +# intel64|ia32 - architecture, intel64 is default. + +# Get absolute path to script. Gets a relative path as argument and outputs an absolute path. +get_script_path() ( + script_path="$1" + while [ -L "$script_path" ] ; do + script_dir=$(command dirname -- "$script_path") + script_dir=$(cd "$script_dir" && command pwd -P) + script_path="$(readlink "$script_path")" + case $script_path in + (/*) ;; + (*) script_path="$script_dir/$script_path" ;; + esac + done + script_dir=$(command dirname -- "$script_path") + script_dir=$(cd "$script_dir" && command pwd -P) + printf "%s" "$script_dir" +) + +_vars_get_proc_name() { + if [ -n "${ZSH_VERSION:-}" ] ; then + script="$(ps -p "$$" -o comm=)" + else + script="$1" + while [ -L "$script" ] ; do + script="$(readlink "$script")" + done + fi + basename -- "$script" +} + +_vars_this_script_name="vars.sh" +if [ "$_vars_this_script_name" = "$(_vars_get_proc_name "$0")" ] ; then + echo ":: ERROR: Incorrect usage: this script must be sourced." + echo " Usage: . path/to/${_vars_this_script_name}" + return 255 2>/dev/null || exit 255 +fi + +# Prepend path segment(s) to path-like env vars (PATH, CPATH, etc.). + +# prepend_path() avoids dangling ":" that affects some env vars (PATH and CPATH) +# PATH > https://www.gnu.org/software/libc/manual/html_node/Standard-Environment.html + +# Usage: +# env_var=$(prepend_path "$prepend_to_var" "$existing_env_var") +# export env_var +# +# Inputs: +# $1 == path segment to be prepended to $2 +# $2 == value of existing path-like environment variable + +prepend_path() ( + path_to_add="$1" + path_is_now="$2" + + if [ "" = "${path_is_now}" ] ; then # avoid dangling ":" + printf "%s" "${path_to_add}" + else + printf "%s" "${path_to_add}:${path_is_now}" + fi +) + +# Extract the name and location of this sourced script. + +# Generally, "ps -o comm=" is limited to a 15 character result, but it works +# fine for this usage, because we are primarily interested in finding the name +# of the execution shell, not the name of any calling script. + +vars_script_name="" +vars_script_shell="$(ps -p "$$" -o comm=)" +# ${var:-} needed to pass "set -eu" checks +if [ -n "${ZSH_VERSION:-}" ] && [ -n "${ZSH_EVAL_CONTEXT:-}" ] ; then # zsh 5.x and later + # shellcheck disable=2249 + case $ZSH_EVAL_CONTEXT in (*:file*) vars_script_name="${(%):-%x}" ;; esac ; +elif [ -n "${KSH_VERSION:-}" ] ; then # ksh, mksh or lksh + if [ "$(set | grep -Fq "KSH_VERSION=.sh.version" ; echo $?)" -eq 0 ] ; then # ksh + vars_script_name="${.sh.file}" ; + else # mksh or lksh or [lm]ksh masquerading as ksh or sh + # force [lm]ksh to issue error msg; which contains this script's path/filename, e.g.: + # mksh: /home/ubuntu/intel/oneapi/vars.sh[137]: ${.sh.file}: bad substitution + vars_script_name="$( (echo "${.sh.file}") 2>&1 )" || : ; + vars_script_name="$(expr "${vars_script_name:-}" : '^.*sh: \(.*\)\[[0-9]*\]:')" ; + fi +elif [ -n "${BASH_VERSION:-}" ] ; then # bash + # shellcheck disable=2128 + (return 0 2>/dev/null) && vars_script_name="${BASH_SOURCE}" ; +elif [ "dash" = "$vars_script_shell" ] ; then # dash + # force dash to issue error msg; which contains this script's rel/path/filename, e.g.: + # dash: 146: /home/ubuntu/intel/oneapi/vars.sh: Bad substitution + vars_script_name="$( (echo "${.sh.file}") 2>&1 )" || : ; + vars_script_name="$(expr "${vars_script_name:-}" : '^.*dash: [0-9]*: \(.*\):')" ; +elif [ "sh" = "$vars_script_shell" ] ; then # could be dash masquerading as /bin/sh + # force a shell error msg; which should contain this script's path/filename + # sample error msg shown; assume this file is named "vars.sh"; as required by setvars.sh + vars_script_name="$( (echo "${.sh.file}") 2>&1 )" || : ; + if [ "$(printf "%s" "$vars_script_name" | grep -Eq "sh: [0-9]+: .*vars\.sh: " ; echo $?)" -eq 0 ] ; then # dash as sh + # sh: 155: /home/ubuntu/intel/oneapi/vars.sh: Bad substitution + vars_script_name="$(expr "${vars_script_name:-}" : '^.*sh: [0-9]*: \(.*\):')" ; + fi +else # unrecognized shell or dash being sourced from within a user's script + # force a shell error msg; which should contain this script's path/filename + # sample error msg shown; assume this file is named "vars.sh"; as required by setvars.sh + vars_script_name="$( (echo "${.sh.file}") 2>&1 )" || : ; + if [ "$(printf "%s" "$vars_script_name" | grep -Eq "^.+: [0-9]+: .*vars\.sh: " ; echo $?)" -eq 0 ] ; then # dash + # .*: 164: intel/oneapi/vars.sh: Bad substitution + vars_script_name="$(expr "${vars_script_name:-}" : '^.*: [0-9]*: \(.*\):')" ; + else + vars_script_name="" ; + fi +fi + +if [ "" = "$vars_script_name" ] ; then + >&2 echo ":: ERROR: Unable to proceed: possible causes listed below." + >&2 echo " This script must be sourced. Did you execute or source this script?" ; + >&2 echo " Unrecognized/unsupported shell (supported: bash, zsh, ksh, m/lksh, dash)." ; + >&2 echo " Can be caused by sourcing from ZSH version 4.x or older." ; + return 255 2>/dev/null || exit 255 +fi + +TBBROOT=$(get_script_path "${vars_script_name:-}")/.. + +TBB_TARGET_ARCH="intel64" +TBB_ARCH_SUFFIX="" + +if [ -n "${SETVARS_ARGS:-}" ]; then + tbb_arg_ia32="$(expr "${SETVARS_ARGS:-}" : '^.*\(ia32\)')" || true + if [ -n "${tbb_arg_ia32:-}" ]; then + TBB_TARGET_ARCH="ia32" + fi +else + for arg do + case "$arg" in + (intel64|ia32) + TBB_TARGET_ARCH="${arg}" + ;; + (*) ;; + esac + done +fi + +TBB_LIB_NAME="libtbb.so.12" + +# Parse layout +if [ -e "$TBBROOT/lib/$TBB_TARGET_ARCH" ]; then + TBB_LIB_DIR="$TBB_TARGET_ARCH/gcc4.8" +else + if [ "$TBB_TARGET_ARCH" = "ia32" ] ; then + TBB_ARCH_SUFFIX="32" + fi + TBB_LIB_DIR="" +fi + +if [ -e "$TBBROOT/lib$TBB_ARCH_SUFFIX/$TBB_LIB_DIR/$TBB_LIB_NAME" ]; then + export TBBROOT + + LIBRARY_PATH=$(prepend_path "${TBBROOT}/lib$TBB_ARCH_SUFFIX/$TBB_LIB_DIR" "${LIBRARY_PATH:-}") ; export LIBRARY_PATH + LD_LIBRARY_PATH=$(prepend_path "${TBBROOT}/lib$TBB_ARCH_SUFFIX/$TBB_LIB_DIR" "${LD_LIBRARY_PATH:-}") ; export LD_LIBRARY_PATH + CPATH=$(prepend_path "${TBBROOT}/include" "${CPATH:-}") ; export CPATH + CMAKE_PREFIX_PATH=$(prepend_path "${TBBROOT}" "${CMAKE_PREFIX_PATH:-}") ; export CMAKE_PREFIX_PATH + PKG_CONFIG_PATH=$(prepend_path "${TBBROOT}/lib$TBB_ARCH_SUFFIX/pkgconfig" "${PKG_CONFIG_PATH:-}") ; export PKG_CONFIG_PATH +else + >&2 echo "ERROR: $TBB_LIB_NAME library does not exist in $TBBROOT/lib$TBB_ARCH_SUFFIX/$TBB_LIB_DIR." + return 255 2>/dev/null || exit 255 +fi diff --git a/src/tbb/integration/linux/env/vars.sh.in b/src/tbb/integration/linux/env/vars.sh.in new file mode 100644 index 000000000..18774e0d2 --- /dev/null +++ b/src/tbb/integration/linux/env/vars.sh.in @@ -0,0 +1,24 @@ +#!/bin/sh +# +# Copyright (c) 2005-2021 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +export TBBROOT=@TBBROOT_REPLACEMENT@ + +LD_LIBRARY_PATH="@LIBRARY_PATH_REPLACEMENT@:${LD_LIBRARY_PATH}"; export LD_LIBRARY_PATH +LIBRARY_PATH="@LIBRARY_PATH_REPLACEMENT@:${LIBRARY_PATH}"; export LIBRARY_PATH +CPATH="${TBBROOT}/include:${CPATH}"; export CPATH +PKG_CONFIG_PATH="@LIBRARY_PATH_REPLACEMENT@/pkgconfig:${PKG_CONFIG_PATH}"; export PKG_CONFIG_PATH + +@CMAKE_ENVIRONMENT_SOURCING_STRING@ diff --git a/src/tbb/integration/linux/modulefiles/tbb b/src/tbb/integration/linux/modulefiles/tbb new file mode 100644 index 000000000..b8c695ed2 --- /dev/null +++ b/src/tbb/integration/linux/modulefiles/tbb @@ -0,0 +1,69 @@ +#%Module1.0################################################################### +# +# Copyright (c) 2020-2023 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This modulefile requires Environment Modules 4.1 or later. +# Type `module --version` to determine the current installed version. + +############################################################################## + +set min_tcl_ver 8.4 +if { $tcl_version < $min_tcl_ver } { + puts stderr " " + puts stderr "ERROR: This modulefile requires tcl $min_tcl_ver or greater." + puts stderr "Your system reports that tclsh version $tcl_version is installed." + exit 1 +} + +# if modulefile script name is a symlink, resolve it to get the fully +# qualified pathname that points to the actual modulefile script +# see: https://wiki.tcl-lang.org/page/file+normalize +set scriptpath "${ModulesCurrentModulefile}" +set scriptpath "[file dirname [file normalize "$scriptpath/___"]]" + +# define componentroot, modulefilepath, modulefilename and modulefilever +set modulefilename "[file tail [file dirname "${scriptpath}"]]" +set modulefilever "[file tail "${scriptpath}"]" +set modulefilepath "${scriptpath}" +set componentroot "[file dirname [file dirname [file dirname [file dirname "${scriptpath}"]]]]" + +############################################################################## + +module-whatis "Name: Intel(R) oneAPI Threading Building Blocks" +module-whatis "Version: $modulefilename/$modulefilever" +module-whatis "Description: Flexible threading library for adding parallelism to complex applications across accelerated architectures." +module-whatis "URL: https://www.intel.com/content/www/us/en/developer/tools/oneapi/onetbb.html" +module-whatis "Dependencies: none" + +proc ModulesHelp { } { + global modulefilename + global modulefilever + module whatis "${modulefilename}/${modulefilever}" +} + +############################################################################## + +# Define environment variables needed for an isolated component install. + +set tbbroot "$componentroot" +set tbb_target_arch "intel64" + +setenv TBBROOT "$tbbroot" + +prepend-path CPATH "$tbbroot/include" +prepend-path LIBRARY_PATH "$tbbroot/lib" +prepend-path LD_LIBRARY_PATH "$tbbroot/lib" +prepend-path CMAKE_PREFIX_PATH "$tbbroot" +prepend-path PKG_CONFIG_PATH "$tbbroot/lib/pkgconfig" diff --git a/src/tbb/integration/linux/modulefiles/tbb32 b/src/tbb/integration/linux/modulefiles/tbb32 new file mode 100644 index 000000000..db3413517 --- /dev/null +++ b/src/tbb/integration/linux/modulefiles/tbb32 @@ -0,0 +1,69 @@ +#%Module1.0################################################################### +# +# Copyright (c) 2020-2023 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This modulefile requires Environment Modules 4.1 or later. +# Type `module --version` to determine the current installed version. + +############################################################################## + +set min_tcl_ver 8.4 +if { $tcl_version < $min_tcl_ver } { + puts stderr " " + puts stderr "ERROR: This modulefile requires tcl $min_tcl_ver or greater." + puts stderr "Your system reports that tclsh version $tcl_version is installed." + exit 1 +} + +# if modulefile script name is a symlink, resolve it to get the fully +# qualified pathname that points to the actual modulefile script +# see: https://wiki.tcl-lang.org/page/file+normalize +set scriptpath "${ModulesCurrentModulefile}" +set scriptpath "[file dirname [file normalize "$scriptpath/___"]]" + +# define componentroot, modulefilepath, modulefilename and modulefilever +set modulefilename "[file tail [file dirname "${scriptpath}"]]" +set modulefilever "[file tail "${scriptpath}"]" +set modulefilepath "${scriptpath}" +set componentroot "[file dirname [file dirname [file dirname [file dirname "${scriptpath}"]]]]" + +############################################################################## + +module-whatis "Name: Intel(R) oneAPI Threading Building Blocks" +module-whatis "Version: $modulefilename/$modulefilever" +module-whatis "Description: Flexible threading library for adding parallelism to complex applications across accelerated architectures." +module-whatis "URL: https://www.intel.com/content/www/us/en/developer/tools/oneapi/onetbb.html" +module-whatis "Dependencies: none" + +proc ModulesHelp { } { + global modulefilename + global modulefilever + module whatis "${modulefilename}/${modulefilever}" +} + +############################################################################## + +# Define environment variables needed for an isolated component install. + +set tbbroot "$componentroot" +set tbb_target_arch "ia32" + +setenv TBBROOT "$tbbroot" + +prepend-path CPATH "$tbbroot/include32:$tbbroot/include" +prepend-path LIBRARY_PATH "$tbbroot/lib32" +prepend-path LD_LIBRARY_PATH "$tbbroot/lib32" +prepend-path CMAKE_PREFIX_PATH "$tbbroot" +prepend-path PKG_CONFIG_PATH "$tbbroot/lib32/pkgconfig" diff --git a/src/tbb/integration/linux/oneapi/vars.sh b/src/tbb/integration/linux/oneapi/vars.sh new file mode 100644 index 000000000..ffcf56a55 --- /dev/null +++ b/src/tbb/integration/linux/oneapi/vars.sh @@ -0,0 +1,34 @@ +#!/bin/sh +# shellcheck shell=sh +# +# Copyright (c) 2023 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ -z "${SETVARS_CALL:-}" ] ; then + >&2 echo " " + >&2 echo ":: ERROR: This script must be sourced by setvars.sh." + >&2 echo " Try 'source /setvars.sh --help' for help." + >&2 echo " " + return 255 +fi + +if [ -z "${ONEAPI_ROOT:-}" ] ; then + >&2 echo " " + >&2 echo ":: ERROR: This script requires that the ONEAPI_ROOT env variable is set." + >&2 echo " Try 'source \setvars.sh --help' for help." + >&2 echo " " + return 254 +fi + +TBBROOT="${ONEAPI_ROOT}"; export TBBROOT diff --git a/src/tbb/build/ios.clang.inc b/src/tbb/integration/linux/sys_check/sys_check.sh similarity index 71% rename from src/tbb/build/ios.clang.inc rename to src/tbb/integration/linux/sys_check/sys_check.sh index 265e9fa16..a279c58ee 100644 --- a/src/tbb/build/ios.clang.inc +++ b/src/tbb/integration/linux/sys_check/sys_check.sh @@ -1,4 +1,6 @@ -# Copyright (c) 2005-2017 Intel Corporation +#!/bin/sh +# +# Copyright (c) 2019-2021 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,13 +13,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# -# -# -include $(tbb_root)/build/macos.clang.inc +LOC=$(realpath $(dirname "${BASH_SOURCE[0]}")) +source $LOC/../../../common.sh $@ -CPLUS_FLAGS += -miphoneos-version-min=8.0 -LINK_FLAGS += -miphoneos-version-min=8.0 -LIB_LINK_FLAGS += -miphoneos-version-min=8.0 +ERRORSTATE=0 +return $ERRORSTATE diff --git a/src/tbb/integration/mac/env/vars.sh b/src/tbb/integration/mac/env/vars.sh new file mode 100644 index 000000000..12be0e52a --- /dev/null +++ b/src/tbb/integration/mac/env/vars.sh @@ -0,0 +1,149 @@ +#!/bin/sh +# shellcheck shell=sh +# +# Copyright (c) 2005-2021 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Get absolute path to script. Gets a relative path as argument and outputs an absolute path. +get_script_path() ( + script_path="$1" + while [ -L "$script_path" ] ; do + script_dir=$(command dirname -- "$script_path") + script_dir=$(cd "$script_dir" && command pwd -P) + script_path="$(readlink "$script_path")" + case $script_path in + (/*) ;; + (*) script_path="$script_dir/$script_path" ;; + esac + done + script_dir=$(command dirname -- "$script_path") + script_dir=$(cd "$script_dir" && command pwd -P) + printf "%s" "$script_dir" +) + +_vars_get_proc_name() { + if [ -n "${ZSH_VERSION:-}" ] ; then + script="$(ps -p "$$" -o comm=)" + else + script="$1" + while [ -L "$script" ] ; do + script="$(readlink "$script")" + done + fi + basename -- "$script" +} + +_vars_this_script_name="vars.sh" +if [ "$_vars_this_script_name" = "$(_vars_get_proc_name "$0")" ] ; then + echo ":: ERROR: Incorrect usage: this script must be sourced." + echo " Usage: . path/to/${_vars_this_script_name}" + return 255 2>/dev/null || exit 255 +fi + +# Prepend path segment(s) to path-like env vars (PATH, CPATH, etc.). + +# prepend_path() avoids dangling ":" that affects some env vars (PATH and CPATH) +# PATH > https://www.gnu.org/software/libc/manual/html_node/Standard-Environment.html + +# Usage: +# env_var=$(prepend_path "$prepend_to_var" "$existing_env_var") +# export env_var +# +# Inputs: +# $1 == path segment to be prepended to $2 +# $2 == value of existing path-like environment variable + +prepend_path() ( + path_to_add="$1" + path_is_now="$2" + + if [ "" = "${path_is_now}" ] ; then # avoid dangling ":" + printf "%s" "${path_to_add}" + else + printf "%s" "${path_to_add}:${path_is_now}" + fi +) + +# Extract the name and location of this sourced script. + +# Generally, "ps -o comm=" is limited to a 15 character result, but it works +# fine for this usage, because we are primarily interested in finding the name +# of the execution shell, not the name of any calling script. + +vars_script_name="" +vars_script_shell="$(ps -p "$$" -o comm=)" +# ${var:-} needed to pass "set -eu" checks +if [ -n "${ZSH_VERSION:-}" ] && [ -n "${ZSH_EVAL_CONTEXT:-}" ] ; then # zsh 5.x and later + # shellcheck disable=2249 + case $ZSH_EVAL_CONTEXT in (*:file*) vars_script_name="${(%):-%x}" ;; esac ; +elif [ -n "${KSH_VERSION:-}" ] ; then # ksh, mksh or lksh + if [ "$(set | grep -Fq "KSH_VERSION=.sh.version" ; echo $?)" -eq 0 ] ; then # ksh + vars_script_name="${.sh.file}" ; + else # mksh or lksh or [lm]ksh masquerading as ksh or sh + # force [lm]ksh to issue error msg; which contains this script's path/filename, e.g.: + # mksh: /home/ubuntu/intel/oneapi/vars.sh[137]: ${.sh.file}: bad substitution + vars_script_name="$( (echo "${.sh.file}") 2>&1 )" || : ; + vars_script_name="$(expr "${vars_script_name:-}" : '^.*sh: \(.*\)\[[0-9]*\]:')" ; + fi +elif [ -n "${BASH_VERSION:-}" ] ; then # bash + # shellcheck disable=2128 + (return 0 2>/dev/null) && vars_script_name="${BASH_SOURCE}" ; +elif [ "dash" = "$vars_script_shell" ] ; then # dash + # force dash to issue error msg; which contains this script's rel/path/filename, e.g.: + # dash: 146: /home/ubuntu/intel/oneapi/vars.sh: Bad substitution + vars_script_name="$( (echo "${.sh.file}") 2>&1 )" || : ; + vars_script_name="$(expr "${vars_script_name:-}" : '^.*dash: [0-9]*: \(.*\):')" ; +elif [ "sh" = "$vars_script_shell" ] ; then # could be dash masquerading as /bin/sh + # force a shell error msg; which should contain this script's path/filename + # sample error msg shown; assume this file is named "vars.sh"; as required by setvars.sh + vars_script_name="$( (echo "${.sh.file}") 2>&1 )" || : ; + if [ "$(printf "%s" "$vars_script_name" | grep -Eq "sh: [0-9]+: .*vars\.sh: " ; echo $?)" -eq 0 ] ; then # dash as sh + # sh: 155: /home/ubuntu/intel/oneapi/vars.sh: Bad substitution + vars_script_name="$(expr "${vars_script_name:-}" : '^.*sh: [0-9]*: \(.*\):')" ; + fi +else # unrecognized shell or dash being sourced from within a user's script + # force a shell error msg; which should contain this script's path/filename + # sample error msg shown; assume this file is named "vars.sh"; as required by setvars.sh + vars_script_name="$( (echo "${.sh.file}") 2>&1 )" || : ; + if [ "$(printf "%s" "$vars_script_name" | grep -Eq "^.+: [0-9]+: .*vars\.sh: " ; echo $?)" -eq 0 ] ; then # dash + # .*: 164: intel/oneapi/vars.sh: Bad substitution + vars_script_name="$(expr "${vars_script_name:-}" : '^.*: [0-9]*: \(.*\):')" ; + else + vars_script_name="" ; + fi +fi + +if [ "" = "$vars_script_name" ] ; then + >&2 echo ":: ERROR: Unable to proceed: possible causes listed below." + >&2 echo " This script must be sourced. Did you execute or source this script?" ; + >&2 echo " Unrecognized/unsupported shell (supported: bash, zsh, ksh, m/lksh, dash)." ; + >&2 echo " Can be caused by sourcing from ZSH version 4.x or older." ; + return 255 2>/dev/null || exit 255 +fi + +TBBROOT=$(get_script_path "${vars_script_name:-}")/.. +LIBTBB_NAME="libtbb.dylib" + +if [ -e "$TBBROOT/lib/$LIBTBB_NAME" ]; then + export TBBROOT + + LIBRARY_PATH=$(prepend_path "${TBBROOT}/lib" "${LIBRARY_PATH:-}") ; export LIBRARY_PATH + DYLD_LIBRARY_PATH=$(prepend_path "${TBBROOT}/lib" "${DYLD_LIBRARY_PATH:-}") ; export DYLD_LIBRARY_PATH + CPATH=$(prepend_path "${TBBROOT}/include" "${CPATH:-}") ; export CPATH + CMAKE_PREFIX_PATH=$(prepend_path "${TBBROOT}" "${CMAKE_PREFIX_PATH:-}") ; export CMAKE_PREFIX_PATH + PKG_CONFIG_PATH=$(prepend_path "${TBBROOT}/lib/pkgconfig" "${PKG_CONFIG_PATH:-}") ; export PKG_CONFIG_PATH +else + >&2 echo "ERROR: $LIBTBB_NAME library does not exist in $TBBROOT/lib." + return 255 2>/dev/null || exit 255 +fi diff --git a/src/tbb/integration/mac/env/vars.sh.in b/src/tbb/integration/mac/env/vars.sh.in new file mode 100644 index 000000000..2a4ca33cb --- /dev/null +++ b/src/tbb/integration/mac/env/vars.sh.in @@ -0,0 +1,24 @@ +#!/bin/sh +# +# Copyright (c) 2005-2021 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +export TBBROOT=@TBBROOT_REPLACEMENT@ + +DYLD_LIBRARY_PATH="@LIBRARY_PATH_REPLACEMENT@:${DYLD_LIBRARY_PATH}"; export DYLD_LIBRARY_PATH +LIBRARY_PATH="@LIBRARY_PATH_REPLACEMENT@:${LIBRARY_PATH}"; export LIBRARY_PATH +CPATH="${TBBROOT}/include:${CPATH}"; export CPATH +PKG_CONFIG_PATH="@LIBRARY_PATH_REPLACEMENT@/pkgconfig:${PKG_CONFIG_PATH}"; export PKG_CONFIG_PATH + +@CMAKE_ENVIRONMENT_SOURCING_STRING@ diff --git a/src/tbb/integration/pkg-config/tbb.pc.in b/src/tbb/integration/pkg-config/tbb.pc.in new file mode 100644 index 000000000..34ea3bea1 --- /dev/null +++ b/src/tbb/integration/pkg-config/tbb.pc.in @@ -0,0 +1,24 @@ +# Copyright (c) 2021-2023 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +prefix=@_prefix_for_pc_file@ +libdir=@_libdir_for_pc_file@ +includedir=@_includedir_for_pc_file@ + +Name: oneAPI Threading Building Blocks (oneTBB) +Description: C++ library for parallel programming on multi-core processors. +URL: https://github.com/oneapi-src/oneTBB +Version: @TBB_VERSION@ +Libs: -L${libdir} @_tbb_pc_extra_libdir@ -l@_tbb_pc_lib_name@ +Cflags: -I${includedir} diff --git a/src/tbb/integration/windows/env/vars.bat b/src/tbb/integration/windows/env/vars.bat new file mode 100644 index 000000000..c5ec0ddc4 --- /dev/null +++ b/src/tbb/integration/windows/env/vars.bat @@ -0,0 +1,93 @@ +@echo off +REM +REM Copyright (c) 2005-2023 Intel Corporation +REM +REM Licensed under the Apache License, Version 2.0 (the "License"); +REM you may not use this file except in compliance with the License. +REM You may obtain a copy of the License at +REM +REM http://www.apache.org/licenses/LICENSE-2.0 +REM +REM Unless required by applicable law or agreed to in writing, software +REM distributed under the License is distributed on an "AS IS" BASIS, +REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +REM See the License for the specific language governing permissions and +REM limitations under the License. +REM + +REM Syntax: +REM %SCRIPT_NAME% [^] [^] +REM ^ should be one of the following +REM ia32 : Set up for IA-32 architecture +REM intel64 : Set up for Intel(R) 64 architecture +REM if ^ is not set Intel(R) 64 architecture will be used +REM ^ should be one of the following +REM vs2019 : Set to use with Microsoft Visual Studio 2019 runtime DLLs +REM vs2022 : Set to use with Microsoft Visual Studio 2022 runtime DLLs +REM all : Set to use oneTBB statically linked with Microsoft Visual C++ runtime +REM if ^ is not set oneTBB dynamically linked with Microsoft Visual C++ runtime will be used. + +set "SCRIPT_NAME=%~nx0" +set "TBB_SCRIPT_DIR=%~d0%~p0" +set "TBBROOT=%TBB_SCRIPT_DIR%.." + +:: Set the default arguments +set TBB_TARGET_ARCH=intel64 +set TBB_ARCH_SUFFIX= +set TBB_TARGET_VS=vc14 + +:ParseArgs +:: Parse the incoming arguments +if /i "%1"=="" goto ParseLayout +if /i "%1"=="ia32" (set TBB_TARGET_ARCH=ia32) & shift & goto ParseArgs +if /i "%1"=="intel64" (set TBB_TARGET_ARCH=intel64) & shift & goto ParseArgs +if /i "%1"=="vs2019" (set TBB_TARGET_VS=vc14) & shift & goto ParseArgs +if /i "%1"=="vs2022" (set TBB_TARGET_VS=vc14) & shift & goto ParseArgs +if /i "%1"=="all" (set TBB_TARGET_VS=vc_mt) & shift & goto ParseArgs + +:ParseLayout +if exist "%TBBROOT%\redist\" ( + set "TBB_BIN_DIR=%TBBROOT%\redist" + set "TBB_SUBDIR=%TBB_TARGET_ARCH%" + goto SetEnv +) + +if "%TBB_TARGET_ARCH%" == "ia32" ( + set TBB_ARCH_SUFFIX=32 +) +if exist "%TBBROOT%\bin%TBB_ARCH_SUFFIX%" ( + set "TBB_BIN_DIR=%TBBROOT%\bin%TBB_ARCH_SUFFIX%" + if "%TBB_TARGET_VS%" == "vc14" ( + set TBB_TARGET_VS= + ) + goto SetEnv +) +:: Couldn't parse TBBROOT/bin, unset variable +set TBB_ARCH_SUFFIX= + +if exist "%TBBROOT%\..\redist\" ( + set "TBB_BIN_DIR=%TBBROOT%\..\redist" + set "TBB_SUBDIR=%TBB_TARGET_ARCH%\tbb" + goto SetEnv +) + +:SetEnv +if exist "%TBB_BIN_DIR%\%TBB_SUBDIR%\%TBB_TARGET_VS%\tbb12.dll" ( + set "TBB_DLL_PATH=%TBB_BIN_DIR%\%TBB_SUBDIR%\%TBB_TARGET_VS%" +) else ( + echo: + echo :: ERROR: tbb12.dll library does not exist in "%TBB_BIN_DIR%\%TBB_SUBDIR%\%TBB_TARGET_VS%\" + echo: + exit /b 255 +) + +set "PATH=%TBB_DLL_PATH%;%PATH%" + +set "LIB=%TBBROOT%\lib%TBB_ARCH_SUFFIX%\%TBB_SUBDIR%\%TBB_TARGET_VS%;%LIB%" +set "INCLUDE=%TBBROOT%\include;%INCLUDE%" +set "CPATH=%TBBROOT%\include;%CPATH%" +set "CMAKE_PREFIX_PATH=%TBBROOT%;%CMAKE_PREFIX_PATH%" +set "PKG_CONFIG_PATH=%TBBROOT%\lib%TBB_ARCH_SUFFIX%\pkgconfig;%PKG_CONFIG_PATH%" + +:End +exit /B 0 diff --git a/src/tbb/integration/windows/env/vars.bat.in b/src/tbb/integration/windows/env/vars.bat.in new file mode 100644 index 000000000..5d94588fa --- /dev/null +++ b/src/tbb/integration/windows/env/vars.bat.in @@ -0,0 +1,29 @@ +@echo off +REM +REM Copyright (c) 2005-2021 Intel Corporation +REM +REM Licensed under the Apache License, Version 2.0 (the "License"); +REM you may not use this file except in compliance with the License. +REM You may obtain a copy of the License at +REM +REM http://www.apache.org/licenses/LICENSE-2.0 +REM +REM Unless required by applicable law or agreed to in writing, software +REM distributed under the License is distributed on an "AS IS" BASIS, +REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +REM See the License for the specific language governing permissions and +REM limitations under the License. +REM + +@echo off + +set "TBBROOT=@TBBROOT_REPLACEMENT@" +set "TBB_DLL_PATH=@BINARY_PATH_REPLACEMENT@" + +set "INCLUDE=%TBBROOT%\include;%INCLUDE%" +set "CPATH=%TBBROOT%\include;%CPATH%" +set "LIB=@LIBRARY_PATH_REPLACEMENT@;%LIB%" +set "PATH=@BINARY_PATH_REPLACEMENT@;%PATH%" +set "PKG_CONFIG_PATH=@LIBRARY_PATH_REPLACEMENT@\pkgconfig;%PKG_CONFIG_PATH%" + +@CMAKE_ENVIRONMENT_SOURCING_STRING@ diff --git a/src/tbb/integration/windows/nuget/inteltbb.devel.win.targets b/src/tbb/integration/windows/nuget/inteltbb.devel.win.targets new file mode 100644 index 000000000..1c94a12c1 --- /dev/null +++ b/src/tbb/integration/windows/nuget/inteltbb.devel.win.targets @@ -0,0 +1,64 @@ + + + + + + + + $(MSBuildThisFileDirectory)..\..\build\native\include;%(AdditionalIncludeDirectories) + TBB_USE_DEBUG;%(PreprocessorDefinitions) + + + + + + + $(MSBuildThisFileDirectory)..\..\build\native\win-x86;%(AdditionalLibraryDirectories) + tbb12.lib;tbbmalloc.lib;tbbmalloc_proxy.lib;%(AdditionalDependencies) + + + + + $(MSBuildThisFileDirectory)..\..\build\native\win-x64;%(AdditionalLibraryDirectories) + tbb12.lib;tbbmalloc.lib;tbbmalloc_proxy.lib;%(AdditionalDependencies) + + + + + $(MSBuildThisFileDirectory)..\..\build\native\win-x86;%(AdditionalLibraryDirectories) + tbb12_debug.lib;tbbmalloc_debug.lib;tbbmalloc_proxy_debug.lib;%(AdditionalDependencies) + + + + + $(MSBuildThisFileDirectory)..\..\build\native\win-x64;%(AdditionalLibraryDirectories) + tbb12_debug.lib;tbbmalloc_debug.lib;tbbmalloc_proxy_debug.lib;%(AdditionalDependencies) + + + + + + + + + + + + + diff --git a/src/tbb/integration/windows/nuget/inteltbb.redist.win.targets b/src/tbb/integration/windows/nuget/inteltbb.redist.win.targets new file mode 100644 index 000000000..29b61acfa --- /dev/null +++ b/src/tbb/integration/windows/nuget/inteltbb.redist.win.targets @@ -0,0 +1,30 @@ + + + + + + + + + + + + + + diff --git a/src/tbb/integration/windows/oneapi/vars.bat b/src/tbb/integration/windows/oneapi/vars.bat new file mode 100644 index 000000000..9c53c7105 --- /dev/null +++ b/src/tbb/integration/windows/oneapi/vars.bat @@ -0,0 +1,56 @@ +@echo off +REM +REM Copyright (c) 2023 Intel Corporation +REM +REM Licensed under the Apache License, Version 2.0 (the "License"); +REM you may not use this file except in compliance with the License. +REM You may obtain a copy of the License at +REM +REM http://www.apache.org/licenses/LICENSE-2.0 +REM +REM Unless required by applicable law or agreed to in writing, software +REM distributed under the License is distributed on an "AS IS" BASIS, +REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +REM See the License for the specific language governing permissions and +REM limitations under the License. +REM + +if not defined SETVARS_CALL ( + echo: + echo :: ERROR: This script must be executed by setvars.bat. + echo: Try '[install-dir]\setvars.bat --help' for help. + echo: + exit /b 255 +) + +if not defined ONEAPI_ROOT ( + echo: + echo :: ERROR: This script requires that the ONEAPI_ROOT env variable is set." + echo: Try '[install-dir]\setvars.bat --help' for help. + echo: + exit /b 254 +) + +set "TBBROOT=%ONEAPI_ROOT%" + +:: Set the default arguments +set "TBB_TARGET_ARCH=%INTEL_TARGET_ARCH%" +set TBB_TARGET_VS= +set ARCH_SUFFIX= + +:ParseArgs +:: Parse the incoming arguments +if /i "%1"=="" goto SetEnv +if /i "%1"=="vs2019" (set TBB_TARGET_VS= ) & shift & goto ParseArgs +if /i "%1"=="vs2022" (set TBB_TARGET_VS= ) & shift & goto ParseArgs +if /i "%1"=="all" (set TBB_TARGET_VS=vc_mt) & shift & goto ParseArgs + +if "%TBB_TARGET_ARCH%"=="ia32" set ARCH_SUFFIX=32 + +:SetEnv +if exist "%TBBROOT%\bin%ARCH_SUFFIX%\%TBB_TARGET_VS%\tbb12.dll" ( + set "TBB_DLL_PATH=%TBBROOT%\bin%ARCH_SUFFIX%\%TBB_TARGET_VS%" +) + +:End +exit /B 0 diff --git a/src/tbb/integration/windows/sys_check/sys_check.bat b/src/tbb/integration/windows/sys_check/sys_check.bat new file mode 100644 index 000000000..202bf72c1 --- /dev/null +++ b/src/tbb/integration/windows/sys_check/sys_check.bat @@ -0,0 +1,18 @@ +@echo off +REM +REM Copyright (c) 2019-2021 Intel Corporation +REM +REM Licensed under the Apache License, Version 2.0 (the "License"); +REM you may not use this file except in compliance with the License. +REM You may obtain a copy of the License at +REM +REM http://www.apache.org/licenses/LICENSE-2.0 +REM +REM Unless required by applicable law or agreed to in writing, software +REM distributed under the License is distributed on an "AS IS" BASIS, +REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +REM See the License for the specific language governing permissions and +REM limitations under the License. +REM + +exit /B 0 diff --git a/src/tbb/src/Makefile b/src/tbb/src/Makefile deleted file mode 100644 index 195304d5f..000000000 --- a/src/tbb/src/Makefile +++ /dev/null @@ -1,269 +0,0 @@ -# Copyright (c) 2005-2017 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -# -# - -tbb_root?=.. -examples_root:=$(tbb_root)/examples -include $(tbb_root)/build/common.inc - -#workaround for non-depend targets tbb and tbbmalloc which both depend on version_string.ver -#According to documentation, recursively invoked make commands can process their targets in parallel -.NOTPARALLEL: - -.PHONY: all tbb tbbmalloc tbbproxy test test_no_depends release debug examples clean - -all: release debug examples - -tbb: tbb_release tbb_debug - -tbbmalloc: tbbmalloc_release tbbmalloc_debug - -tbbproxy: tbbproxy_release tbbproxy_debug - -rml: rml_release rml_debug - -test: tbbmalloc_test_release $(if $(use_proxy),tbbproxy_test_release) tbb_test_release tbbmalloc_test_debug $(if $(use_proxy),tbbproxy_test_debug) tbb_test_debug -ifeq (,$(findstring skip,$(target:android=skip) $(offload:mic=skip))) -test: rml_test_debug rml_test_release -endif - -test_no_depends: tbbmalloc_test_release_no_depends $(if $(use_proxy),tbbproxy_test_release_no_depends) tbb_test_release_no_depends tbbmalloc_test_debug_no_depends $(if $(use_proxy),tbbproxy_test_debug_no_depends) tbb_test_debug_no_depends - @echo done - -release: tbb_release tbbmalloc_release $(if $(use_proxy),tbbproxy_release) -release: $(call cross_cfg,tbbmalloc_test_release) $(call cross_cfg,test_release) - -debug: tbb_debug tbbmalloc_debug $(if $(use_proxy),tbbproxy_debug) -debug: $(call cross_cfg,tbbmalloc_test_debug) $(call cross_cfg, test_debug) - -examples: tbb tbbmalloc examples_debug clean_examples examples_release - -examples_no_depends: examples_release_no_depends examples_debug_no_depends - -clean: clean_release clean_debug clean_examples - @echo clean done - -.PHONY: full -full: - $(MAKE) -sir --no-print-directory -f Makefile tbb_root=.. clean all -ifeq ($(tbb_os),windows) - $(MAKE) -sir --no-print-directory -f Makefile tbb_root=.. compiler=icl clean all native_examples -else - $(MAKE) -sir --no-print-directory -f Makefile tbb_root=.. compiler=icc clean all native_examples -endif -ifeq ($(arch),intel64) - $(MAKE) -sir --no-print-directory -f Makefile tbb_root=.. arch=ia32 clean all -endif -# it doesn't test compiler=icc arch=ia32 on intel64 systems due to environment settings of icc - -native_examples: tbb tbbmalloc - $(MAKE) -C $(examples_root) -r -f Makefile tbb_root=.. compiler=$(native_compiler) debug test - $(MAKE) -C $(examples_root) -r -f Makefile tbb_root=.. compiler=$(native_compiler) clean release test - -../examples/% examples/%:: - $(MAKE) -C $(examples_root) -r -f Makefile tbb_root=.. $(subst examples/,,$(subst ../,,$@)) - -debug_%:: cfg:=$(if $(findstring file,$(origin cfg)),debug,$(cfg)) -debug_%:: export run_cmd=$(debugger) -debug_malloc_% test_malloc_% debug_ScalableAlloc% test_ScalableAlloc%:: TESTFILE=tbbmalloc -debug_rml_% test_rml_%:: TESTFILE=rml -debug_runtime_load% test_runtime_load%:: TESTFILE=tbbproxy -debug_% test_% stress_% time_% perf_%:: TESTFILE?=test -debug_% test_% stress_% time_% perf_%:: - $(MAKE) -C "$(work_dir)_$(cfg)" -r -f $(tbb_root)/build/Makefile.$(TESTFILE) cfg=$(cfg) $(subst .cpp,,$@) - -clean_%:: -ifeq ($(origin cfg),file) - @$(MAKE) -C "$(work_dir)_release" -r -f $(tbb_root)/build/Makefile.test cfg=release $@ - @$(MAKE) -C "$(work_dir)_debug" -r -f $(tbb_root)/build/Makefile.test cfg=debug $@ -else - @$(MAKE) -C "$(work_dir)_$(cfg)" -r -f $(tbb_root)/build/Makefile.test $@ -endif - -python_%: tbb_release - bash -c ". $(work_dir)_release$(SLASH)tbbvars.sh && $(MAKE) -rC '$(full_tbb_root)/python' CXX=$(compiler) $(subst python_,,$@)" - -.PHONY: test_release test_debug test_release_no_depends test_debug_no_depends -.PHONY: tbb_release tbb_debug tbb_test_release tbb_test_debug tbb_test_release_no_depends tbb_test_debug_no_depends -# do not delete double-space after -C option -tbb_release: mkdir_release - $(MAKE) -C "$(work_dir)_release" -r -f $(tbb_root)/build/Makefile.tbb cfg=release - -tbb_debug: mkdir_debug - $(MAKE) -C "$(work_dir)_debug" -r -f $(tbb_root)/build/Makefile.tbb cfg=debug - -tbb_test: tbb_test_release tbb_test_debug - -tbb_test_release: $(call cross_cfg,tbb_release) $(if $(use_proxy),$(call cross_cfg,tbbproxy_release)) tbb_test_release_no_depends -tbb_test_release_no_depends:$(call cross_cfg,mkdir_release) - $(MAKE) -C "$(call cross_cfg,$(work_dir)_release)" -r -f $(tbb_root)/build/Makefile.test cfg=release - -tbb_test_debug: $(call cross_cfg,tbb_debug) $(if $(use_proxy),$(call cross_cfg,tbbproxy_debug)) tbb_test_debug_no_depends -tbb_test_debug_no_depends:$(call cross_cfg,mkdir_debug) - $(MAKE) -C "$(call cross_cfg,$(work_dir)_debug)" -r -f $(tbb_root)/build/Makefile.test cfg=debug -# backward compatibility -test_release: tbb_test_release -test_debug: tbb_test_debug -test_release_no_depends: tbb_test_release_no_depends -test_debug_no_depends: tbb_test_debug_no_depends - -.PHONY: tbbmalloc_release tbbmalloc_debug -.PHONY: tbbmalloc_dll_release tbbmalloc_dll_debug tbbmalloc_proxy_dll_release tbbmalloc_proxy_dll_debug -.PHONY: tbbmalloc_test tbbmalloc_test_release tbbmalloc_test_debug tbbmalloc_test_release_no_depends tbbmalloc_test_debug_no_depends - -tbbmalloc_release: mkdir_release - $(MAKE) -C "$(work_dir)_release" -r -f $(tbb_root)/build/Makefile.tbbmalloc cfg=release malloc - -tbbmalloc_debug: mkdir_debug - $(MAKE) -C "$(work_dir)_debug" -r -f $(tbb_root)/build/Makefile.tbbmalloc cfg=debug malloc - -tbbmalloc_dll_release: mkdir_release - $(MAKE) -C "$(work_dir)_release" -r -f $(tbb_root)/build/Makefile.tbbmalloc cfg=release malloc_dll - -tbbmalloc_proxy_dll_release: mkdir_release - $(MAKE) -C "$(work_dir)_release" -r -f $(tbb_root)/build/Makefile.tbbmalloc cfg=release malloc_proxy_dll - -tbbmalloc_dll_debug: mkdir_debug - $(MAKE) -C "$(work_dir)_debug" -r -f $(tbb_root)/build/Makefile.tbbmalloc cfg=debug malloc_dll - -tbbmalloc_proxy_dll_debug: mkdir_debug - $(MAKE) -C "$(work_dir)_debug" -r -f $(tbb_root)/build/Makefile.tbbmalloc cfg=debug malloc_proxy_dll - -tbbmalloc_test: tbbmalloc_test_release tbbmalloc_test_debug - -tbbmalloc_test_release: $(call cross_cfg,tbbmalloc_release) tbbmalloc_test_release_no_depends -tbbmalloc_test_release_no_depends: $(call cross_cfg,mkdir_release) - $(MAKE) -C "$(call cross_cfg,$(work_dir)_release)" -r -f $(tbb_root)/build/Makefile.tbbmalloc cfg=release malloc_test_no_depends - -tbbmalloc_test_debug: $(call cross_cfg,tbbmalloc_debug) tbbmalloc_test_debug_no_depends -tbbmalloc_test_debug_no_depends: $(call cross_cfg,mkdir_debug) - $(MAKE) -C "$(call cross_cfg,$(work_dir)_debug)" -r -f $(tbb_root)/build/Makefile.tbbmalloc cfg=debug malloc_test_no_depends - -.PHONY: tbbproxy_release tbbproxy_debug -.PHONY: tbbproxy_test tbbproxy_test_release tbbproxy_test_debug tbbproxy_test_release_no_depends tbbproxy_test_debug_no_depends - -tbbproxy_release: mkdir_release tbb_release - $(MAKE) -C "$(work_dir)_release" -r -f $(tbb_root)/build/Makefile.tbbproxy cfg=release tbbproxy - -tbbproxy_debug: mkdir_debug tbb_debug - $(MAKE) -C "$(work_dir)_debug" -r -f $(tbb_root)/build/Makefile.tbbproxy cfg=debug tbbproxy - -tbbproxy_test: tbbproxy_test_release tbbproxy_test_debug - -tbbproxy_test_release: $(call cross_cfg,tbb_release) $(call cross_cfg,tbbproxy_release) tbbproxy_test_release_no_depends -tbbproxy_test_release_no_depends:$(call cross_cfg,mkdir_release) - $(MAKE) -C "$(call cross_cfg,$(work_dir)_release)" -r -f $(tbb_root)/build/Makefile.tbbproxy cfg=release tbbproxy_test - -tbbproxy_test_debug: $(call cross_cfg,tbb_debug) $(call cross_cfg,tbbproxy_debug) tbbproxy_test_debug_no_depends -tbbproxy_test_debug_no_depends: $(call cross_cfg,mkdir_debug) - $(MAKE) -C "$(call cross_cfg,$(work_dir)_debug)" -r -f $(tbb_root)/build/Makefile.tbbproxy cfg=debug tbbproxy_test - -.PHONY: rml_release rml_debug rml_test_release rml_test_debug -.PHONY: rml_test_release_no_depends rml_test_debug_no_depends - -rml_release: mkdir_release - $(MAKE) -C "$(work_dir)_release" -r -f $(tbb_root)/build/Makefile.rml cfg=release rml - -rml_debug: mkdir_debug - $(MAKE) -C "$(work_dir)_debug" -r -f $(tbb_root)/build/Makefile.rml cfg=debug rml - -rml_test: rml_test_release rml_test_debug - -rml_test_release: $(call cross_cfg,rml_release) rml_test_release_no_depends -rml_test_release_no_depends: $(call cross_cfg,mkdir_release) - $(MAKE) -C "$(call cross_cfg,$(work_dir)_release)" -r -f $(tbb_root)/build/Makefile.rml cfg=release rml_test - -rml_test_debug: $(call cross_cfg,rml_debug) rml_test_debug_no_depends -rml_test_debug_no_depends: $(call cross_cfg,mkdir_debug) - $(MAKE) -C "$(call cross_cfg,$(work_dir)_debug)" -r -f $(tbb_root)/build/Makefile.rml cfg=debug rml_test - -.PHONY: examples_release examples_debug examples_release_no_depends examples_debug_no_depends - -examples_release: tbb_release tbbmalloc_release examples_release_no_depends -examples_release_no_depends: - $(MAKE) -C $(examples_root) -r -f Makefile tbb_root=.. release test UI=con - -examples_debug: tbb_debug tbbmalloc_debug examples_debug_no_depends -examples_debug_no_depends: - $(MAKE) -C $(examples_root) -r -f Makefile tbb_root=.. debug test UI=con - -.PHONY: clean_release clean_debug clean_examples - -clean_release: - $(shell $(RM) $(work_dir)_release$(SLASH)*.* >$(NUL) 2>$(NUL)) - $(shell $(RD) $(work_dir)_release >$(NUL) 2>$(NUL)) - -clean_debug: - $(shell $(RM) $(work_dir)_debug$(SLASH)*.* >$(NUL) 2>$(NUL)) - $(shell $(RD) $(work_dir)_debug >$(NUL) 2>$(NUL)) - -clean_examples: - $(shell $(MAKE) -s -i -r -C $(examples_root) -f Makefile tbb_root=.. clean >$(NUL) 2>$(NUL)) - -.PHONY: mkdir_release mkdir_debug codecov do_codecov info tbbvars shell - -mkdir_release: - $(shell $(MD) "$(work_dir)_release" >$(NUL) 2>$(NUL)) - @echo Created $(work_dir)_release directory - -mkdir_debug: - $(shell $(MD) "$(work_dir)_debug" >$(NUL) 2>$(NUL)) - @echo Created $(work_dir)_debug directory - -ifeq ($(compiler),$(if $(findstring windows,$(tbb_os)),icl,icc)) -codecov: codecov=yes -codecov: do_codecov - $(MAKE) -C "$(work_dir)_release" -r -f $(tbb_root)/build/Makefile.test cfg=release codecov_gen -else -codecov: - $(error Only Intel(R) C++ Compiler is supported for code coverage) -endif - -export codecov - -do_codecov: tbb_root=.. -do_codecov: - $(MAKE) RML=yes tbbmalloc_test_release test_release - $(MAKE) clean_test_* cfg=release - $(MAKE) RML=yes crosstest=yes tbbmalloc_test_debug test_debug - $(MAKE) clean_test_* cfg=release - $(MAKE) rml_test_release - $(MAKE) clean_test_* cfg=release - $(MAKE) crosstest=yes rml_test_debug - -info: - @echo OS: $(tbb_os) - @echo arch=$(arch) - @echo compiler=$(compiler) - @echo runtime=$(runtime) - @echo tbb_build_prefix=$(tbb_build_prefix) - @echo work_dir=$(abspath $(tbb_build_dir)$(SLASH)$(tbb_build_prefix)_$(cfg)) - -# [usage]$ source `make tbbvars`.sh -tbbvars: - @echo $(tbb_build_dir)$(SLASH)$(tbb_build_prefix)_$(cfg)$(SLASH)tbbvars - -symbols: args=$(if $(findstring cl,$(compiler)), dumpbin /section:.text *.obj|findstr COMDAT , nm -Pg *.o|grep ' T '|cut -f1 -d' ') -symbols: shell - -shell: -ifdef BUILDING_PHASE - -$(run_cmd) $(shell_cmd) -else - @$(MAKE) -C "$(work_dir)_$(cfg)" -rf $(tbb_root)/src/Makefile BUILDING_PHASE=1 shell shell_cmd="$(if $(args),$(args),$(SHELL))" -endif - diff --git a/src/tbb/src/index.html b/src/tbb/src/index.html deleted file mode 100644 index e828987c9..000000000 --- a/src/tbb/src/index.html +++ /dev/null @@ -1,76 +0,0 @@ - - - -

Overview

-This directory contains the source code and unit tests for Intel® Threading Building Blocks. - -

Directories

-
-
tbb -
Source code of the TBB library core. -
tbbmalloc -
Source code of the TBB scalable memory allocator. -
test -
Source code of the TBB unit tests. -
rml -
Source code of the Resource Management Layer (RML). -
perf -
Source code of microbenchmarks. -
old -
Source code of deprecated TBB entities that are still shipped as part of the TBB library for the sake of backward compatibility. -
- -

Files

-
-
Makefile -
Advanced Makefile for developing and debugging of TBB. See the basic build directions. Additional targets and options: -
make test_{name} time_{name} -
Make and run individual test or benchmark. -
make stress_{name} -
Equivalent to 'make test_{name}' but runs until a failure detected or terminated by user. -
make run_cmd="{command}" [(above options or targets)] -
Command prefix for tests execution. Also, "run_cmd=-" will ignore test execution failures. See also -k and -i options of the GNU make for more options to keep building and testing despite of failures. -
make debug_{name} -
Equivalent to 'make test_{name}' but compiles in debug mode and runs under debugger ("run_cmd=$(debugger)"). -
make args="{command-line arguments}" [(above options or targets)] -
Additional arguments for the run. -
make repeat="{N}" [(above options or targets)] -
Repeats execution N times. -
make clean_{filename} -
Removes executable, object, and other intermediate files with specified filename ('*' also works). -
make cfg={debug|release} [(above options or targets)] -
Specifies a build mode or corresponding directory to work in. -
make tbb_strict=1 [(above options or targets)] -
Enables warnings as errors. -
make examples/{target} -
Invokes examples/Makefile with specified target. Available in the open-source version only. - For the commercial version, you can download Intel TBB Samples at the Intel® Software Product Samples and Tutorials website. -
make python_{target} [compiler={icl, icc}] -
Invokes Makefile with the specified target in python directory. E.g. 'python_install' target builds and installs the module into Python. -
make clean_release clean_debug clean_examples -
Removes release or debug build directories, or cleans all examples. The target clean_examples is available in the open-source version only. -
make test_no_depends -
Equivalent to 'make test' but does not check for libraries updates. -
make info -
Output information about build configuration and directories. -
make cpp0x=1 [(above options or targets)] -
Enables C++0x extensions like lambdas for compilers that implement them as experimental features. -
make CXXFLAGS={Flags} [(above options or targets)] -
Specifies additional options for compiler. -
make target={name} [(above options or targets)] -
Includes additional build/{name}.inc file after OS-specific one. -
make extra_inc={filename} [(above options or targets)] -
Includes additional makefile. -
- -
-Up to parent directory -

-Copyright © 2005-2017 Intel Corporation. All Rights Reserved. -

-Intel is a registered trademark or trademark of Intel Corporation -or its subsidiaries in the United States and other countries. -

-* Other names and brands may be claimed as the property of others. - - diff --git a/src/tbb/src/old/concurrent_queue_v2.cpp b/src/tbb/src/old/concurrent_queue_v2.cpp deleted file mode 100644 index bf7e6e633..000000000 --- a/src/tbb/src/old/concurrent_queue_v2.cpp +++ /dev/null @@ -1,367 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#include "concurrent_queue_v2.h" -#include "tbb/cache_aligned_allocator.h" -#include "tbb/spin_mutex.h" -#include "tbb/atomic.h" -#include -#include - -#if defined(_MSC_VER) && defined(_Wp64) - // Workaround for overzealous compiler warnings in /Wp64 mode - #pragma warning (disable: 4267) -#endif - -#define RECORD_EVENTS 0 - -using namespace std; - -namespace tbb { - -namespace internal { - -class concurrent_queue_rep; - -//! A queue using simple locking. -/** For efficiency, this class has no constructor. - The caller is expected to zero-initialize it. */ -struct micro_queue { - typedef concurrent_queue_base::page page; - typedef size_t ticket; - - atomic head_page; - atomic head_counter; - - atomic tail_page; - atomic tail_counter; - - spin_mutex page_mutex; - - class push_finalizer: no_copy { - ticket my_ticket; - micro_queue& my_queue; - public: - push_finalizer( micro_queue& queue, ticket k ) : - my_ticket(k), my_queue(queue) - {} - ~push_finalizer() { - my_queue.tail_counter = my_ticket; - } - }; - - void push( const void* item, ticket k, concurrent_queue_base& base ); - - class pop_finalizer: no_copy { - ticket my_ticket; - micro_queue& my_queue; - page* my_page; - public: - pop_finalizer( micro_queue& queue, ticket k, page* p ) : - my_ticket(k), my_queue(queue), my_page(p) - {} - ~pop_finalizer() { - page* p = my_page; - if( p ) { - spin_mutex::scoped_lock lock( my_queue.page_mutex ); - page* q = p->next; - my_queue.head_page = q; - if( !q ) { - my_queue.tail_page = NULL; - } - } - my_queue.head_counter = my_ticket; - if( p ) - operator delete(p); - } - }; - - bool pop( void* dst, ticket k, concurrent_queue_base& base ); -}; - -//! Internal representation of a ConcurrentQueue. -/** For efficiency, this class has no constructor. - The caller is expected to zero-initialize it. */ -class concurrent_queue_rep { -public: - typedef size_t ticket; - -private: - friend struct micro_queue; - - //! Approximately n_queue/golden ratio - static const size_t phi = 3; - -public: - //! Must be power of 2 - static const size_t n_queue = 8; - - //! Map ticket to an array index - static size_t index( ticket k ) { - return k*phi%n_queue; - } - - atomic head_counter; - char pad1[NFS_MaxLineSize-sizeof(atomic)]; - - atomic tail_counter; - char pad2[NFS_MaxLineSize-sizeof(atomic)]; - micro_queue array[n_queue]; - - micro_queue& choose( ticket k ) { - // The formula here approximates LRU in a cache-oblivious way. - return array[index(k)]; - } - - //! Value for effective_capacity that denotes unbounded queue. - static const ptrdiff_t infinite_capacity = ptrdiff_t(~size_t(0)/2); -}; - -#if _MSC_VER && !defined(__INTEL_COMPILER) - // unary minus operator applied to unsigned type, result still unsigned - #pragma warning( push ) - #pragma warning( disable: 4146 ) -#endif - -//------------------------------------------------------------------------ -// micro_queue -//------------------------------------------------------------------------ -void micro_queue::push( const void* item, ticket k, concurrent_queue_base& base ) { - k &= -concurrent_queue_rep::n_queue; - page* p = NULL; - size_t index = modulo_power_of_two( k/concurrent_queue_rep::n_queue, base.items_per_page ); - if( !index ) { - size_t n = sizeof(page) + base.items_per_page*base.item_size; - p = static_cast(operator new( n )); - p->mask = 0; - p->next = NULL; - } - { - push_finalizer finalizer( *this, k+concurrent_queue_rep::n_queue ); - spin_wait_until_eq( tail_counter, k ); - if( p ) { - spin_mutex::scoped_lock lock( page_mutex ); - if( page* q = tail_page ) - q->next = p; - else - head_page = p; - tail_page = p; - } else { - p = tail_page; - } - base.copy_item( *p, index, item ); - // If no exception was thrown, mark item as present. - p->mask |= uintptr_t(1)<mask & uintptr_t(1)<1 ? item_sz : 2); - my_rep = cache_aligned_allocator().allocate(1); - __TBB_ASSERT( (size_t)my_rep % NFS_GetLineSize()==0, "alignment error" ); - __TBB_ASSERT( (size_t)&my_rep->head_counter % NFS_GetLineSize()==0, "alignment error" ); - __TBB_ASSERT( (size_t)&my_rep->tail_counter % NFS_GetLineSize()==0, "alignment error" ); - __TBB_ASSERT( (size_t)&my_rep->array % NFS_GetLineSize()==0, "alignment error" ); - memset(my_rep,0,sizeof(concurrent_queue_rep)); - this->item_size = item_sz; -} - -concurrent_queue_base::~concurrent_queue_base() { - size_t nq = my_rep->n_queue; - for( size_t i=0; iarray[i].tail_page; - __TBB_ASSERT( my_rep->array[i].head_page==tp, "at most one page should remain" ); - if( tp!=NULL ) - delete tp; - } - cache_aligned_allocator().deallocate(my_rep,1); -} - -void concurrent_queue_base::internal_push( const void* src ) { - concurrent_queue_rep& r = *my_rep; - concurrent_queue_rep::ticket k = r.tail_counter++; - if( my_capacity=const_cast(my_capacity) ) - backoff.pause(); - } - r.choose(k).push(src,k,*this); -} - -void concurrent_queue_base::internal_pop( void* dst ) { - concurrent_queue_rep& r = *my_rep; - concurrent_queue_rep::ticket k; - do { - k = r.head_counter++; - } while( !r.choose(k).pop(dst,k,*this) ); -} - -bool concurrent_queue_base::internal_pop_if_present( void* dst ) { - concurrent_queue_rep& r = *my_rep; - concurrent_queue_rep::ticket k; - do { - for( atomic_backoff b;;b.pause() ) { - k = r.head_counter; - if( r.tail_counter<=k ) { - // Queue is empty - return false; - } - // Queue had item with ticket k when we looked. Attempt to get that item. - if( r.head_counter.compare_and_swap(k+1,k)==k ) { - break; - } - // Another thread snatched the item, so pause and retry. - } - } while( !r.choose(k).pop(dst,k,*this) ); - return true; -} - -bool concurrent_queue_base::internal_push_if_not_full( const void* src ) { - concurrent_queue_rep& r = *my_rep; - concurrent_queue_rep::ticket k; - for( atomic_backoff b;;b.pause() ) { - k = r.tail_counter; - if( (ptrdiff_t)(k-r.head_counter)>=my_capacity ) { - // Queue is full - return false; - } - // Queue had empty slot with ticket k when we looked. Attempt to claim that slot. - if( r.tail_counter.compare_and_swap(k+1,k)==k ) - break; - // Another thread claimed the slot, so pause and retry. - } - r.choose(k).push(src,k,*this); - return true; -} - -ptrdiff_t concurrent_queue_base::internal_size() const { - __TBB_ASSERT( sizeof(ptrdiff_t)<=sizeof(size_t), NULL ); - return ptrdiff_t(my_rep->tail_counter-my_rep->head_counter); -} - -void concurrent_queue_base::internal_set_capacity( ptrdiff_t capacity, size_t /*item_sz*/ ) { - my_capacity = capacity<0 ? concurrent_queue_rep::infinite_capacity : capacity; -} - -//------------------------------------------------------------------------ -// concurrent_queue_iterator_rep -//------------------------------------------------------------------------ -class concurrent_queue_iterator_rep: no_assign { -public: - typedef concurrent_queue_rep::ticket ticket; - ticket head_counter; - const concurrent_queue_base& my_queue; - concurrent_queue_base::page* array[concurrent_queue_rep::n_queue]; - concurrent_queue_iterator_rep( const concurrent_queue_base& queue ) : - head_counter(queue.my_rep->head_counter), - my_queue(queue) - { - const concurrent_queue_rep& rep = *queue.my_rep; - for( size_t k=0; ktail_counter ) - return NULL; - else { - concurrent_queue_base::page* p = array[concurrent_queue_rep::index(k)]; - __TBB_ASSERT(p,NULL); - size_t i = modulo_power_of_two( k/concurrent_queue_rep::n_queue, my_queue.items_per_page ); - return static_cast(static_cast(p+1)) + my_queue.item_size*i; - } - } -}; - -//------------------------------------------------------------------------ -// concurrent_queue_iterator_base -//------------------------------------------------------------------------ -concurrent_queue_iterator_base::concurrent_queue_iterator_base( const concurrent_queue_base& queue ) { - my_rep = new concurrent_queue_iterator_rep(queue); - my_item = my_rep->choose(my_rep->head_counter); -} - -void concurrent_queue_iterator_base::assign( const concurrent_queue_iterator_base& other ) { - if( my_rep!=other.my_rep ) { - if( my_rep ) { - delete my_rep; - my_rep = NULL; - } - if( other.my_rep ) { - my_rep = new concurrent_queue_iterator_rep( *other.my_rep ); - } - } - my_item = other.my_item; -} - -void concurrent_queue_iterator_base::advance() { - __TBB_ASSERT( my_item, "attempt to increment iterator past end of queue" ); - size_t k = my_rep->head_counter; - const concurrent_queue_base& queue = my_rep->my_queue; - __TBB_ASSERT( my_item==my_rep->choose(k), NULL ); - size_t i = modulo_power_of_two( k/concurrent_queue_rep::n_queue, queue.items_per_page ); - if( i==queue.items_per_page-1 ) { - concurrent_queue_base::page*& root = my_rep->array[concurrent_queue_rep::index(k)]; - root = root->next; - } - my_rep->head_counter = k+1; - my_item = my_rep->choose(k+1); -} - -concurrent_queue_iterator_base::~concurrent_queue_iterator_base() { - delete my_rep; - my_rep = NULL; -} - -} // namespace internal - -} // namespace tbb diff --git a/src/tbb/src/old/concurrent_queue_v2.h b/src/tbb/src/old/concurrent_queue_v2.h deleted file mode 100644 index 54dd8223b..000000000 --- a/src/tbb/src/old/concurrent_queue_v2.h +++ /dev/null @@ -1,324 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_concurrent_queue_H -#define __TBB_concurrent_queue_H - -#include "tbb/tbb_stddef.h" -#include - -namespace tbb { - -template class concurrent_queue; - -//! @cond INTERNAL -namespace internal { - -class concurrent_queue_rep; -class concurrent_queue_iterator_rep; -template class concurrent_queue_iterator; - -//! For internal use only. -/** Type-independent portion of concurrent_queue. - @ingroup containers */ -class concurrent_queue_base: no_copy { - //! Internal representation - concurrent_queue_rep* my_rep; - - friend class concurrent_queue_rep; - friend struct micro_queue; - friend class concurrent_queue_iterator_rep; - friend class concurrent_queue_iterator_base; - - // In C++ 1998/2003 (but quite likely not beyond), friend micro_queue's rights - // do not apply to the declaration of micro_queue::pop_finalizer::my_page, - // as a member of a class nested within that friend class, so... -public: - //! Prefix on a page - struct page { - page* next; - uintptr_t mask; - }; - -protected: - //! Capacity of the queue - ptrdiff_t my_capacity; - - //! Always a power of 2 - size_t items_per_page; - - //! Size of an item - size_t item_size; -private: - virtual void copy_item( page& dst, size_t index, const void* src ) = 0; - virtual void assign_and_destroy_item( void* dst, page& src, size_t index ) = 0; -protected: - __TBB_EXPORTED_METHOD concurrent_queue_base( size_t item_size ); - virtual __TBB_EXPORTED_METHOD ~concurrent_queue_base(); - - //! Enqueue item at tail of queue - void __TBB_EXPORTED_METHOD internal_push( const void* src ); - - //! Dequeue item from head of queue - void __TBB_EXPORTED_METHOD internal_pop( void* dst ); - - //! Attempt to enqueue item onto queue. - bool __TBB_EXPORTED_METHOD internal_push_if_not_full( const void* src ); - - //! Attempt to dequeue item from queue. - /** NULL if there was no item to dequeue. */ - bool __TBB_EXPORTED_METHOD internal_pop_if_present( void* dst ); - - //! Get size of queue - ptrdiff_t __TBB_EXPORTED_METHOD internal_size() const; - - void __TBB_EXPORTED_METHOD internal_set_capacity( ptrdiff_t capacity, size_t element_size ); -}; - -//! Type-independent portion of concurrent_queue_iterator. -/** @ingroup containers */ -class concurrent_queue_iterator_base : no_assign{ - //! concurrent_queue over which we are iterating. - /** NULL if one past last element in queue. */ - concurrent_queue_iterator_rep* my_rep; - - template - friend bool operator==( const concurrent_queue_iterator& i, const concurrent_queue_iterator& j ); - - template - friend bool operator!=( const concurrent_queue_iterator& i, const concurrent_queue_iterator& j ); -protected: - //! Pointer to current item - mutable void* my_item; - - //! Default constructor - __TBB_EXPORTED_METHOD concurrent_queue_iterator_base() : my_rep(NULL), my_item(NULL) {} - - //! Copy constructor - concurrent_queue_iterator_base( const concurrent_queue_iterator_base& i ) : my_rep(NULL), my_item(NULL) { - assign(i); - } - - //! Construct iterator pointing to head of queue. - concurrent_queue_iterator_base( const concurrent_queue_base& queue ); - - //! Assignment - void __TBB_EXPORTED_METHOD assign( const concurrent_queue_iterator_base& i ); - - //! Advance iterator one step towards tail of queue. - void __TBB_EXPORTED_METHOD advance(); - - //! Destructor - __TBB_EXPORTED_METHOD ~concurrent_queue_iterator_base(); -}; - -//! Meets requirements of a forward iterator for STL. -/** Value is either the T or const T type of the container. - @ingroup containers */ -template -class concurrent_queue_iterator: public concurrent_queue_iterator_base { -#if !defined(_MSC_VER) || defined(__INTEL_COMPILER) - template - friend class ::tbb::concurrent_queue; -#else -public: // workaround for MSVC -#endif - //! Construct iterator pointing to head of queue. - concurrent_queue_iterator( const concurrent_queue_base& queue ) : - concurrent_queue_iterator_base(queue) - { - } -public: - concurrent_queue_iterator() {} - - /** If Value==Container::value_type, then this routine is the copy constructor. - If Value==const Container::value_type, then this routine is a conversion constructor. */ - concurrent_queue_iterator( const concurrent_queue_iterator& other ) : - concurrent_queue_iterator_base(other) - {} - - //! Iterator assignment - concurrent_queue_iterator& operator=( const concurrent_queue_iterator& other ) { - assign(other); - return *this; - } - - //! Reference to current item - Value& operator*() const { - return *static_cast(my_item); - } - - Value* operator->() const {return &operator*();} - - //! Advance to next item in queue - concurrent_queue_iterator& operator++() { - advance(); - return *this; - } - - //! Post increment - Value* operator++(int) { - Value* result = &operator*(); - operator++(); - return result; - } -}; // concurrent_queue_iterator - -template -bool operator==( const concurrent_queue_iterator& i, const concurrent_queue_iterator& j ) { - return i.my_item==j.my_item; -} - -template -bool operator!=( const concurrent_queue_iterator& i, const concurrent_queue_iterator& j ) { - return i.my_item!=j.my_item; -} - -} // namespace internal; -//! @endcond - -//! A high-performance thread-safe queue. -/** Multiple threads may each push and pop concurrently. - Assignment and copy construction are not allowed. - @ingroup containers */ -template -class concurrent_queue: public internal::concurrent_queue_base { - template friend class internal::concurrent_queue_iterator; - - //! Class used to ensure exception-safety of method "pop" - class destroyer { - T& my_value; - public: - destroyer( T& value ) : my_value(value) {} - ~destroyer() {my_value.~T();} - }; - - T& get_ref( page& pg, size_t index ) { - __TBB_ASSERT( index(static_cast(&pg+1))[index]; - } - - virtual void copy_item( page& dst, size_t index, const void* src ) __TBB_override { - new( &get_ref(dst,index) ) T(*static_cast(src)); - } - - virtual void assign_and_destroy_item( void* dst, page& src, size_t index ) __TBB_override { - T& from = get_ref(src,index); - destroyer d(from); - *static_cast(dst) = from; - } - -public: - //! Element type in the queue. - typedef T value_type; - - //! Reference type - typedef T& reference; - - //! Const reference type - typedef const T& const_reference; - - //! Integral type for representing size of the queue. - /** Note that the size_type is a signed integral type. - This is because the size can be negative if there are pending pops without corresponding pushes. */ - typedef std::ptrdiff_t size_type; - - //! Difference type for iterator - typedef std::ptrdiff_t difference_type; - - //! Construct empty queue - concurrent_queue() : - concurrent_queue_base( sizeof(T) ) - { - } - - //! Destroy queue - ~concurrent_queue(); - - //! Enqueue an item at tail of queue. - void push( const T& source ) { - internal_push( &source ); - } - - //! Dequeue item from head of queue. - /** Block until an item becomes available, and then dequeue it. */ - void pop( T& destination ) { - internal_pop( &destination ); - } - - //! Enqueue an item at tail of queue if queue is not already full. - /** Does not wait for queue to become not full. - Returns true if item is pushed; false if queue was already full. */ - bool push_if_not_full( const T& source ) { - return internal_push_if_not_full( &source ); - } - - //! Attempt to dequeue an item from head of queue. - /** Does not wait for item to become available. - Returns true if successful; false otherwise. */ - bool pop_if_present( T& destination ) { - return internal_pop_if_present( &destination ); - } - - //! Return number of pushes minus number of pops. - /** Note that the result can be negative if there are pops waiting for the - corresponding pushes. The result can also exceed capacity() if there - are push operations in flight. */ - size_type size() const {return internal_size();} - - //! Equivalent to size()<=0. - bool empty() const {return size()<=0;} - - //! Maximum number of allowed elements - size_type capacity() const { - return my_capacity; - } - - //! Set the capacity - /** Setting the capacity to 0 causes subsequent push_if_not_full operations to always fail, - and subsequent push operations to block forever. */ - void set_capacity( size_type new_capacity ) { - internal_set_capacity( new_capacity, sizeof(T) ); - } - - typedef internal::concurrent_queue_iterator iterator; - typedef internal::concurrent_queue_iterator const_iterator; - - //------------------------------------------------------------------------ - // The iterators are intended only for debugging. They are slow and not thread safe. - //------------------------------------------------------------------------ - iterator begin() {return iterator(*this);} - iterator end() {return iterator();} - const_iterator begin() const {return const_iterator(*this);} - const_iterator end() const {return const_iterator();} - -}; - -template -concurrent_queue::~concurrent_queue() { - while( !empty() ) { - T value; - internal_pop(&value); - } -} - -} // namespace tbb - -#endif /* __TBB_concurrent_queue_H */ diff --git a/src/tbb/src/old/concurrent_vector_v2.cpp b/src/tbb/src/old/concurrent_vector_v2.cpp deleted file mode 100644 index d31b5b3a6..000000000 --- a/src/tbb/src/old/concurrent_vector_v2.cpp +++ /dev/null @@ -1,256 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#include "concurrent_vector_v2.h" -#include "tbb/tbb_machine.h" -#include "../tbb/itt_notify.h" -#include "tbb/task.h" - -#include // std::length_error -#include - -#if defined(_MSC_VER) && defined(_Wp64) - // Workaround for overzealous compiler warnings in /Wp64 mode - #pragma warning (disable: 4267) -#endif - -namespace tbb { - -namespace internal { - -void concurrent_vector_base::internal_grow_to_at_least( size_type new_size, size_type element_size, internal_array_op1 init ) { - size_type e = my_early_size; - while( e=pointers_per_short_segment && v.my_segment==v.my_storage ) { - extend_segment(v); - } - } -}; - -void concurrent_vector_base::helper::extend_segment( concurrent_vector_base& v ) { - const size_t pointers_per_long_segment = sizeof(void*)==4 ? 32 : 64; - segment_t* s = (segment_t*)NFS_Allocate( pointers_per_long_segment, sizeof(segment_t), NULL ); - std::memset( (void*) s, 0, pointers_per_long_segment*sizeof(segment_t) ); - // If other threads are trying to set pointers in the short segment, wait for them to finish their - // assignments before we copy the short segment to the long segment. - atomic_backoff backoff; - while( !v.my_storage[0].array || !v.my_storage[1].array ) backoff.pause(); - s[0] = v.my_storage[0]; - s[1] = v.my_storage[1]; - if( v.my_segment.compare_and_swap( s, v.my_storage )!=v.my_storage ) - NFS_Free(s); -} - -concurrent_vector_base::size_type concurrent_vector_base::internal_capacity() const { - return segment_base( helper::find_segment_end(*this) ); -} - -void concurrent_vector_base::internal_reserve( size_type n, size_type element_size, size_type max_size ) { - if( n>max_size ) { - __TBB_THROW( std::length_error("argument to concurrent_vector::reserve exceeds concurrent_vector::max_size()") ); - } - for( segment_index_t k = helper::find_segment_end(*this); segment_base(k)n-b ) m = n-b; - copy( my_segment[k].array, src.my_segment[k].array, m ); - } - } -} - -void concurrent_vector_base::internal_assign( const concurrent_vector_base& src, size_type element_size, internal_array_op1 destroy, internal_array_op2 assign, internal_array_op2 copy ) { - size_type n = src.my_early_size; - while( my_early_size>n ) { - segment_index_t k = segment_index_of( my_early_size-1 ); - size_type b=segment_base(k); - size_type new_end = b>=n ? b : n; - __TBB_ASSERT( my_early_size>new_end, NULL ); - destroy( (char*)my_segment[k].array+element_size*(new_end-b), my_early_size-new_end ); - my_early_size = new_end; - } - size_type dst_initialized_size = my_early_size; - my_early_size = n; - size_type b; - for( segment_index_t k=0; (b=segment_base(k))n-b ) m = n-b; - size_type a = 0; - if( dst_initialized_size>b ) { - a = dst_initialized_size-b; - if( a>m ) a = m; - assign( my_segment[k].array, src.my_segment[k].array, a ); - m -= a; - a *= element_size; - } - if( m>0 ) - copy( (char*)my_segment[k].array+a, (char*)src.my_segment[k].array+a, m ); - } - __TBB_ASSERT( src.my_early_size==n, "detected use of concurrent_vector::operator= with right side that was concurrently modified" ); -} - -void* concurrent_vector_base::internal_push_back( size_type element_size, size_type& index ) { - __TBB_ASSERT( sizeof(my_early_size)==sizeof(reference_count), NULL ); - //size_t tmp = __TBB_FetchAndIncrementWacquire(*(tbb::internal::reference_count*)&my_early_size); - size_t tmp = __TBB_FetchAndIncrementWacquire((tbb::internal::reference_count*)&my_early_size); - index = tmp; - segment_index_t k_old = segment_index_of( tmp ); - size_type base = segment_base(k_old); - helper::extend_segment_if_necessary(*this,k_old); - segment_t& s = my_segment[k_old]; - void* array = s.array; - if( !array ) { - // FIXME - consider factoring this out and share with internal_grow_by - if( base==tmp ) { - __TBB_ASSERT( !s.array, NULL ); - size_t n = segment_size(k_old); - array = NFS_Allocate( n, element_size, NULL ); - ITT_NOTIFY( sync_releasing, &s.array ); - s.array = array; - } else { - ITT_NOTIFY(sync_prepare, &s.array); - spin_wait_while_eq( s.array, (void*)0 ); - ITT_NOTIFY(sync_acquired, &s.array); - array = s.array; - } - } - size_type j_begin = tmp-base; - return (void*)((char*)array+element_size*j_begin); -} - -concurrent_vector_base::size_type concurrent_vector_base::internal_grow_by( size_type delta, size_type element_size, internal_array_op1 init ) { - size_type result = my_early_size.fetch_and_add(delta); - internal_grow( result, result+delta, element_size, init ); - return result; -} - -void concurrent_vector_base::internal_grow( const size_type start, size_type finish, size_type element_size, internal_array_op1 init ) { - __TBB_ASSERT( start finish-base ? finish-base : n; - (*init)( (void*)((char*)array+element_size*j_begin), j_end-j_begin ); - tmp = base+j_end; - } while( tmp0 ) { - segment_index_t k_old = segment_index_of(finish-1); - segment_t& s = my_segment[k_old]; - __TBB_ASSERT( s.array, NULL ); - size_type base = segment_base(k_old); - size_type j_end = finish-base; - __TBB_ASSERT( j_end, NULL ); - (*destroy)( s.array, j_end ); - finish = base; - } - - // Free the arrays - if( reclaim_storage ) { - size_t k = helper::find_segment_end(*this); - while( k>0 ) { - --k; - segment_t& s = my_segment[k]; - void* array = s.array; - s.array = NULL; - NFS_Free( array ); - } - // Clear short segment. - my_storage[0].array = NULL; - my_storage[1].array = NULL; - segment_t* s = my_segment; - if( s!=my_storage ) { - my_segment = my_storage; - NFS_Free( s ); - } - } -} - -} // namespace internal - -} // tbb diff --git a/src/tbb/src/old/concurrent_vector_v2.h b/src/tbb/src/old/concurrent_vector_v2.h deleted file mode 100644 index 6dcae5953..000000000 --- a/src/tbb/src/old/concurrent_vector_v2.h +++ /dev/null @@ -1,512 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_concurrent_vector_H -#define __TBB_concurrent_vector_H - -#include "tbb/tbb_stddef.h" -#include "tbb/atomic.h" -#include "tbb/cache_aligned_allocator.h" -#include "tbb/blocked_range.h" -#include "tbb/tbb_machine.h" -#include -#include - -namespace tbb { - -template -class concurrent_vector; - -//! @cond INTERNAL -namespace internal { - - //! Base class of concurrent vector implementation. - /** @ingroup containers */ - class concurrent_vector_base { - protected: - - // Basic types declarations - typedef unsigned long segment_index_t; - typedef size_t size_type; - - //! Log2 of "min_segment_size". - static const int lg_min_segment_size = 4; - - //! Minimum size (in physical items) of a segment. - static const int min_segment_size = segment_index_t(1)<>1< my_early_size; - - /** Can be zero-initialized. */ - struct segment_t { - /** Declared volatile because in weak memory model, must have ld.acq/st.rel */ - void* volatile array; -#if TBB_USE_ASSERT - ~segment_t() { - __TBB_ASSERT( !array, "should have been set to NULL by clear" ); - } -#endif /* TBB_USE_ASSERT */ - }; - - // Data fields - - //! Pointer to the segments table - atomic my_segment; - - //! embedded storage of segment pointers - segment_t my_storage[2]; - - // Methods - - concurrent_vector_base() { - my_early_size = 0; - my_storage[0].array = NULL; - my_storage[1].array = NULL; - my_segment = my_storage; - } - - //! An operation on an n-element array starting at begin. - typedef void(__TBB_EXPORTED_FUNC *internal_array_op1)(void* begin, size_type n ); - - //! An operation on n-element destination array and n-element source array. - typedef void(__TBB_EXPORTED_FUNC *internal_array_op2)(void* dst, const void* src, size_type n ); - - void __TBB_EXPORTED_METHOD internal_grow_to_at_least( size_type new_size, size_type element_size, internal_array_op1 init ); - void internal_grow( size_type start, size_type finish, size_type element_size, internal_array_op1 init ); - size_type __TBB_EXPORTED_METHOD internal_grow_by( size_type delta, size_type element_size, internal_array_op1 init ); - void* __TBB_EXPORTED_METHOD internal_push_back( size_type element_size, size_type& index ); - void __TBB_EXPORTED_METHOD internal_clear( internal_array_op1 destroy, bool reclaim_storage ); - void __TBB_EXPORTED_METHOD internal_copy( const concurrent_vector_base& src, size_type element_size, internal_array_op2 copy ); - void __TBB_EXPORTED_METHOD internal_assign( const concurrent_vector_base& src, size_type element_size, - internal_array_op1 destroy, internal_array_op2 assign, internal_array_op2 copy ); -private: - //! Private functionality that does not cross DLL boundary. - class helper; - friend class helper; - }; - - //! Meets requirements of a forward iterator for STL and a Value for a blocked_range.*/ - /** Value is either the T or const T type of the container. - @ingroup containers */ - template - class vector_iterator -#if defined(_WIN64) && defined(_MSC_VER) - // Ensure that Microsoft's internal template function _Val_type works correctly. - : public std::iterator -#endif /* defined(_WIN64) && defined(_MSC_VER) */ - { - //! concurrent_vector over which we are iterating. - Container* my_vector; - - //! Index into the vector - size_t my_index; - - //! Caches my_vector->internal_subscript(my_index) - /** NULL if cached value is not available */ - mutable Value* my_item; - - template - friend bool operator==( const vector_iterator& i, const vector_iterator& j ); - - template - friend bool operator<( const vector_iterator& i, const vector_iterator& j ); - - template - friend ptrdiff_t operator-( const vector_iterator& i, const vector_iterator& j ); - - template - friend class internal::vector_iterator; - -#if !defined(_MSC_VER) || defined(__INTEL_COMPILER) - template - friend class tbb::concurrent_vector; -#else -public: // workaround for MSVC -#endif - - vector_iterator( const Container& vector, size_t index ) : - my_vector(const_cast(&vector)), - my_index(index), - my_item(NULL) - {} - - public: - //! Default constructor - vector_iterator() : my_vector(NULL), my_index(~size_t(0)), my_item(NULL) {} - - vector_iterator( const vector_iterator& other ) : - my_vector(other.my_vector), - my_index(other.my_index), - my_item(other.my_item) - {} - - vector_iterator operator+( ptrdiff_t offset ) const { - return vector_iterator( *my_vector, my_index+offset ); - } - friend vector_iterator operator+( ptrdiff_t offset, const vector_iterator& v ) { - return vector_iterator( *v.my_vector, v.my_index+offset ); - } - vector_iterator operator+=( ptrdiff_t offset ) { - my_index+=offset; - my_item = NULL; - return *this; - } - vector_iterator operator-( ptrdiff_t offset ) const { - return vector_iterator( *my_vector, my_index-offset ); - } - vector_iterator operator-=( ptrdiff_t offset ) { - my_index-=offset; - my_item = NULL; - return *this; - } - Value& operator*() const { - Value* item = my_item; - if( !item ) { - item = my_item = &my_vector->internal_subscript(my_index); - } - __TBB_ASSERT( item==&my_vector->internal_subscript(my_index), "corrupt cache" ); - return *item; - } - Value& operator[]( ptrdiff_t k ) const { - return my_vector->internal_subscript(my_index+k); - } - Value* operator->() const {return &operator*();} - - //! Pre increment - vector_iterator& operator++() { - size_t k = ++my_index; - if( my_item ) { - // Following test uses 2's-complement wizardry and fact that - // min_segment_size is a power of 2. - if( (k& k-concurrent_vector::min_segment_size)==0 ) { - // k is a power of two that is at least k-min_segment_size - my_item= NULL; - } else { - ++my_item; - } - } - return *this; - } - - //! Pre decrement - vector_iterator& operator--() { - __TBB_ASSERT( my_index>0, "operator--() applied to iterator already at beginning of concurrent_vector" ); - size_t k = my_index--; - if( my_item ) { - // Following test uses 2's-complement wizardry and fact that - // min_segment_size is a power of 2. - if( (k& k-concurrent_vector::min_segment_size)==0 ) { - // k is a power of two that is at least k-min_segment_size - my_item= NULL; - } else { - --my_item; - } - } - return *this; - } - - //! Post increment - vector_iterator operator++(int) { - vector_iterator result = *this; - operator++(); - return result; - } - - //! Post decrement - vector_iterator operator--(int) { - vector_iterator result = *this; - operator--(); - return result; - } - - // STL support - - typedef ptrdiff_t difference_type; - typedef Value value_type; - typedef Value* pointer; - typedef Value& reference; - typedef std::random_access_iterator_tag iterator_category; - }; - - template - bool operator==( const vector_iterator& i, const vector_iterator& j ) { - return i.my_index==j.my_index; - } - - template - bool operator!=( const vector_iterator& i, const vector_iterator& j ) { - return !(i==j); - } - - template - bool operator<( const vector_iterator& i, const vector_iterator& j ) { - return i.my_index - bool operator>( const vector_iterator& i, const vector_iterator& j ) { - return j - bool operator>=( const vector_iterator& i, const vector_iterator& j ) { - return !(i - bool operator<=( const vector_iterator& i, const vector_iterator& j ) { - return !(j - ptrdiff_t operator-( const vector_iterator& i, const vector_iterator& j ) { - return ptrdiff_t(i.my_index)-ptrdiff_t(j.my_index); - } - -} // namespace internal -//! @endcond - -//! Concurrent vector -/** @ingroup containers */ -template -class concurrent_vector: private internal::concurrent_vector_base { -public: - using internal::concurrent_vector_base::size_type; -private: - template - class generic_range_type: public blocked_range { - public: - typedef T value_type; - typedef T& reference; - typedef const T& const_reference; - typedef I iterator; - typedef ptrdiff_t difference_type; - generic_range_type( I begin_, I end_, size_t grainsize_ ) : blocked_range(begin_,end_,grainsize_) {} - generic_range_type( generic_range_type& r, split ) : blocked_range(r,split()) {} - }; - - template - friend class internal::vector_iterator; -public: - typedef T& reference; - typedef const T& const_reference; - typedef T value_type; - typedef ptrdiff_t difference_type; - - //! Construct empty vector. - concurrent_vector() {} - - //! Copy a vector. - concurrent_vector( const concurrent_vector& vector ) : internal::concurrent_vector_base() - { internal_copy(vector,sizeof(T),©_array); } - - //! Assignment - concurrent_vector& operator=( const concurrent_vector& vector ) { - if( this!=&vector ) - internal_assign(vector,sizeof(T),&destroy_array,&assign_array,©_array); - return *this; - } - - //! Clear and destroy vector. - ~concurrent_vector() {internal_clear(&destroy_array,/*reclaim_storage=*/true);} - - //------------------------------------------------------------------------ - // Concurrent operations - //------------------------------------------------------------------------ - //! Grow by "delta" elements. - /** Returns old size. */ - size_type grow_by( size_type delta ) { - return delta ? internal_grow_by( delta, sizeof(T), &initialize_array ) : my_early_size.load(); - } - - //! Grow array until it has at least n elements. - void grow_to_at_least( size_type n ) { - if( my_early_size iterator; - typedef internal::vector_iterator const_iterator; - -#if !defined(_MSC_VER) || _CPPLIB_VER>=300 - // Assume ISO standard definition of std::reverse_iterator - typedef std::reverse_iterator reverse_iterator; - typedef std::reverse_iterator const_reverse_iterator; -#else - // Use non-standard std::reverse_iterator - typedef std::reverse_iterator reverse_iterator; - typedef std::reverse_iterator const_reverse_iterator; -#endif /* defined(_MSC_VER) && (_MSC_VER<1300) */ - - // Forward sequence - iterator begin() {return iterator(*this,0);} - iterator end() {return iterator(*this,size());} - const_iterator begin() const {return const_iterator(*this,0);} - const_iterator end() const {return const_iterator(*this,size());} - - // Reverse sequence - reverse_iterator rbegin() {return reverse_iterator(end());} - reverse_iterator rend() {return reverse_iterator(begin());} - const_reverse_iterator rbegin() const {return const_reverse_iterator(end());} - const_reverse_iterator rend() const {return const_reverse_iterator(begin());} - - //------------------------------------------------------------------------ - // Support for TBB algorithms (ranges) - //------------------------------------------------------------------------ - typedef generic_range_type range_type; - typedef generic_range_type const_range_type; - - //! Get range to use with parallel algorithms - range_type range( size_t grainsize = 1 ) { - return range_type( begin(), end(), grainsize ); - } - - //! Get const range for iterating with parallel algorithms - const_range_type range( size_t grainsize = 1 ) const { - return const_range_type( begin(), end(), grainsize ); - } - - //------------------------------------------------------------------------ - // Size and capacity - //------------------------------------------------------------------------ - //! Return size of vector. - size_type size() const {return my_early_size;} - - //! Return false if vector is not empty. - bool empty() const {return !my_early_size;} - - //! Maximum size to which array can grow without allocating more memory. - size_type capacity() const {return internal_capacity();} - - //! Allocate enough space to grow to size n without having to allocate more memory later. - /** Like most of the methods provided for STL compatibility, this method is *not* thread safe. - The capacity afterwards may be bigger than the requested reservation. */ - void reserve( size_type n ) { - if( n ) - internal_reserve(n, sizeof(T), max_size()); - } - - //! Upper bound on argument to reserve. - size_type max_size() const {return (~size_t(0))/sizeof(T);} - - //! Not thread safe - /** Does not change capacity. */ - void clear() {internal_clear(&destroy_array,/*reclaim_storage=*/false);} -private: - //! Get reference to element at given index. - T& internal_subscript( size_type index ) const; - - //! Construct n instances of T, starting at "begin". - static void __TBB_EXPORTED_FUNC initialize_array( void* begin, size_type n ); - - //! Construct n instances of T, starting at "begin". - static void __TBB_EXPORTED_FUNC copy_array( void* dst, const void* src, size_type n ); - - //! Assign n instances of T, starting at "begin". - static void __TBB_EXPORTED_FUNC assign_array( void* dst, const void* src, size_type n ); - - //! Destroy n instances of T, starting at "begin". - static void __TBB_EXPORTED_FUNC destroy_array( void* begin, size_type n ); -}; - -template -T& concurrent_vector::internal_subscript( size_type index ) const { - __TBB_ASSERT( index(my_segment[k].array)[j]; -} - -template -void concurrent_vector::initialize_array( void* begin, size_type n ) { - T* array = static_cast(begin); - for( size_type j=0; j -void concurrent_vector::copy_array( void* dst, const void* src, size_type n ) { - T* d = static_cast(dst); - const T* s = static_cast(src); - for( size_type j=0; j -void concurrent_vector::assign_array( void* dst, const void* src, size_type n ) { - T* d = static_cast(dst); - const T* s = static_cast(src); - for( size_type j=0; j -void concurrent_vector::destroy_array( void* begin, size_type n ) { - T* array = static_cast(begin); - for( size_type j=n; j>0; --j ) - array[j-1].~T(); -} - -} // namespace tbb - -#endif /* __TBB_concurrent_vector_H */ diff --git a/src/tbb/src/old/spin_rw_mutex_v2.cpp b/src/tbb/src/old/spin_rw_mutex_v2.cpp deleted file mode 100644 index 02cf633a0..000000000 --- a/src/tbb/src/old/spin_rw_mutex_v2.cpp +++ /dev/null @@ -1,156 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#include "spin_rw_mutex_v2.h" -#include "tbb/tbb_machine.h" -#include "../tbb/itt_notify.h" -#include "tbb/atomic.h" - -namespace tbb { - -using namespace internal; - -static inline bool CAS(volatile uintptr_t &addr, uintptr_t newv, uintptr_t oldv) { - return as_atomic(addr).compare_and_swap(newv, oldv) == oldv; -} - -//! Signal that write lock is released -void spin_rw_mutex::internal_itt_releasing(spin_rw_mutex *mutex) { - __TBB_ASSERT_EX(mutex, NULL); // To prevent compiler warnings - ITT_NOTIFY(sync_releasing, mutex); -} - -//! Acquire write (exclusive) lock on the given mutex. -bool spin_rw_mutex::internal_acquire_writer(spin_rw_mutex *mutex) -{ - ITT_NOTIFY(sync_prepare, mutex); - for( atomic_backoff backoff;;backoff.pause() ) { - state_t s = mutex->state; - if( !(s & BUSY) ) { // no readers, no writers - if( CAS(mutex->state, WRITER, s) ) - break; // successfully stored writer flag - backoff.reset(); // we could be very close to complete op. - } else if( !(s & WRITER_PENDING) ) { // no pending writers - __TBB_AtomicOR(&mutex->state, WRITER_PENDING); - } - } - ITT_NOTIFY(sync_acquired, mutex); - __TBB_ASSERT( (mutex->state & BUSY)==WRITER, "invalid state of a write lock" ); - return false; -} - -//! Release write lock on the given mutex -void spin_rw_mutex::internal_release_writer(spin_rw_mutex *mutex) { - __TBB_ASSERT( (mutex->state & BUSY)==WRITER, "invalid state of a write lock" ); - ITT_NOTIFY(sync_releasing, mutex); - mutex->state = 0; -} - -//! Acquire read (shared) lock on the given mutex. -void spin_rw_mutex::internal_acquire_reader(spin_rw_mutex *mutex) { - ITT_NOTIFY(sync_prepare, mutex); - for( atomic_backoff backoff;;backoff.pause() ) { - state_t s = mutex->state; - if( !(s & (WRITER|WRITER_PENDING)) ) { // no writer or write requests - if( CAS(mutex->state, s+ONE_READER, s) ) - break; // successfully stored increased number of readers - backoff.reset(); // we could be very close to complete op. - } - } - ITT_NOTIFY(sync_acquired, mutex); - __TBB_ASSERT( mutex->state & READERS, "invalid state of a read lock: no readers" ); - __TBB_ASSERT( !(mutex->state & WRITER), "invalid state of a read lock: active writer" ); -} - -//! Upgrade reader to become a writer. -/** Returns whether the upgrade happened without releasing and re-acquiring the lock */ -bool spin_rw_mutex::internal_upgrade(spin_rw_mutex *mutex) { - state_t s = mutex->state; - __TBB_ASSERT( s & READERS, "invalid state before upgrade: no readers " ); - __TBB_ASSERT( !(s & WRITER), "invalid state before upgrade: active writer " ); - // check and set writer-pending flag - // required conditions: either no pending writers, or we are the only reader - // (with multiple readers and pending writer, another upgrade could have been requested) - while( (s & READERS)==ONE_READER || !(s & WRITER_PENDING) ) { - if( CAS(mutex->state, s | WRITER_PENDING, s) ) - { - ITT_NOTIFY(sync_prepare, mutex); - atomic_backoff backoff; - while( (mutex->state & READERS) != ONE_READER ) backoff.pause(); - __TBB_ASSERT(mutex->state == (ONE_READER | WRITER_PENDING),"invalid state when upgrading to writer"); - // both new readers and writers are blocked at this time - mutex->state = WRITER; - ITT_NOTIFY(sync_acquired, mutex); - __TBB_ASSERT( (mutex->state & BUSY) == WRITER, "invalid state after upgrade" ); - return true; // successfully upgraded - } else { - s = mutex->state; // re-read - } - } - // slow reacquire - internal_release_reader(mutex); - return internal_acquire_writer(mutex); // always returns false -} - -//! Downgrade writer to a reader -void spin_rw_mutex::internal_downgrade(spin_rw_mutex *mutex) { - __TBB_ASSERT( (mutex->state & BUSY) == WRITER, "invalid state before downgrade" ); - ITT_NOTIFY(sync_releasing, mutex); - mutex->state = ONE_READER; - __TBB_ASSERT( mutex->state & READERS, "invalid state after downgrade: no readers" ); - __TBB_ASSERT( !(mutex->state & WRITER), "invalid state after downgrade: active writer" ); -} - -//! Release read lock on the given mutex -void spin_rw_mutex::internal_release_reader(spin_rw_mutex *mutex) -{ - __TBB_ASSERT( mutex->state & READERS, "invalid state of a read lock: no readers" ); - __TBB_ASSERT( !(mutex->state & WRITER), "invalid state of a read lock: active writer" ); - ITT_NOTIFY(sync_releasing, mutex); // release reader - __TBB_FetchAndAddWrelease((volatile void *)&(mutex->state),-(intptr_t)ONE_READER); -} - -//! Try to acquire write lock on the given mutex -bool spin_rw_mutex::internal_try_acquire_writer( spin_rw_mutex * mutex ) -{ - // for a writer: only possible to acquire if no active readers or writers - state_t s = mutex->state; // on IA-64 architecture, this volatile load has acquire semantic - if( !(s & BUSY) ) // no readers, no writers; mask is 1..1101 - if( CAS(mutex->state, WRITER, s) ) { - ITT_NOTIFY(sync_acquired, mutex); - return true; // successfully stored writer flag - } - return false; -} - -//! Try to acquire read lock on the given mutex -bool spin_rw_mutex::internal_try_acquire_reader( spin_rw_mutex * mutex ) -{ - // for a reader: acquire if no active or waiting writers - state_t s = mutex->state; // on IA-64 architecture, a load of volatile variable has acquire semantic - while( !(s & (WRITER|WRITER_PENDING)) ) // no writers - if( CAS(mutex->state, s+ONE_READER, s) ) { - ITT_NOTIFY(sync_acquired, mutex); - return true; // successfully stored increased number of readers - } - return false; -} - -} // namespace tbb diff --git a/src/tbb/src/old/spin_rw_mutex_v2.h b/src/tbb/src/old/spin_rw_mutex_v2.h deleted file mode 100644 index 12e097d65..000000000 --- a/src/tbb/src/old/spin_rw_mutex_v2.h +++ /dev/null @@ -1,175 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TBB_spin_rw_mutex_H -#define __TBB_spin_rw_mutex_H - -#include "tbb/tbb_stddef.h" - -namespace tbb { - -//! Fast, unfair, spinning reader-writer lock with backoff and writer-preference -/** @ingroup synchronization */ -class spin_rw_mutex { - //! @cond INTERNAL - - //! Present so that 1.0 headers work with 1.1 dynamic library. - static void __TBB_EXPORTED_FUNC internal_itt_releasing(spin_rw_mutex *); - - //! Internal acquire write lock. - static bool __TBB_EXPORTED_FUNC internal_acquire_writer(spin_rw_mutex *); - - //! Out of line code for releasing a write lock. - /** This code has debug checking and instrumentation for Intel(R) Thread Checker and Intel(R) Thread Profiler. */ - static void __TBB_EXPORTED_FUNC internal_release_writer(spin_rw_mutex *); - - //! Internal acquire read lock. - static void __TBB_EXPORTED_FUNC internal_acquire_reader(spin_rw_mutex *); - - //! Internal upgrade reader to become a writer. - static bool __TBB_EXPORTED_FUNC internal_upgrade(spin_rw_mutex *); - - //! Out of line code for downgrading a writer to a reader. - /** This code has debug checking and instrumentation for Intel(R) Thread Checker and Intel(R) Thread Profiler. */ - static void __TBB_EXPORTED_FUNC internal_downgrade(spin_rw_mutex *); - - //! Internal release read lock. - static void __TBB_EXPORTED_FUNC internal_release_reader(spin_rw_mutex *); - - //! Internal try_acquire write lock. - static bool __TBB_EXPORTED_FUNC internal_try_acquire_writer(spin_rw_mutex *); - - //! Internal try_acquire read lock. - static bool __TBB_EXPORTED_FUNC internal_try_acquire_reader(spin_rw_mutex *); - - //! @endcond -public: - //! Construct unacquired mutex. - spin_rw_mutex() : state(0) {} - -#if TBB_USE_ASSERT - //! Destructor asserts if the mutex is acquired, i.e. state is zero. - ~spin_rw_mutex() { - __TBB_ASSERT( !state, "destruction of an acquired mutex"); - }; -#endif /* TBB_USE_ASSERT */ - - //! The scoped locking pattern - /** It helps to avoid the common problem of forgetting to release lock. - It also nicely provides the "node" for queuing locks. */ - class scoped_lock : internal::no_copy { - public: - //! Construct lock that has not acquired a mutex. - /** Equivalent to zero-initialization of *this. */ - scoped_lock() : mutex(NULL) {} - - //! Construct and acquire lock on given mutex. - scoped_lock( spin_rw_mutex& m, bool write = true ) : mutex(NULL) { - acquire(m, write); - } - - //! Release lock (if lock is held). - ~scoped_lock() { - if( mutex ) release(); - } - - //! Acquire lock on given mutex. - void acquire( spin_rw_mutex& m, bool write = true ) { - __TBB_ASSERT( !mutex, "holding mutex already" ); - mutex = &m; - is_writer = write; - if( write ) internal_acquire_writer(mutex); - else internal_acquire_reader(mutex); - } - - //! Upgrade reader to become a writer. - /** Returns whether the upgrade happened without releasing and re-acquiring the lock */ - bool upgrade_to_writer() { - __TBB_ASSERT( mutex, "lock is not acquired" ); - __TBB_ASSERT( !is_writer, "not a reader" ); - is_writer = true; - return internal_upgrade(mutex); - } - - //! Release lock. - void release() { - __TBB_ASSERT( mutex, "lock is not acquired" ); - spin_rw_mutex *m = mutex; - mutex = NULL; - if( is_writer ) { -#if TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT - internal_release_writer(m); -#else - m->state = 0; -#endif /* TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT */ - } else { - internal_release_reader(m); - } - }; - - //! Downgrade writer to become a reader. - bool downgrade_to_reader() { - __TBB_ASSERT( mutex, "lock is not acquired" ); - __TBB_ASSERT( is_writer, "not a writer" ); -#if TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT - internal_downgrade(mutex); -#else - mutex->state = 4; // Bit 2 - reader, 00..00100 -#endif - is_writer = false; - return true; - } - - //! Try acquire lock on given mutex. - bool try_acquire( spin_rw_mutex& m, bool write = true ) { - __TBB_ASSERT( !mutex, "holding mutex already" ); - bool result; - is_writer = write; - result = write? internal_try_acquire_writer(&m) - : internal_try_acquire_reader(&m); - if( result ) mutex = &m; - return result; - } - - private: - //! The pointer to the current mutex that is held, or NULL if no mutex is held. - spin_rw_mutex* mutex; - - //! If mutex!=NULL, then is_writer is true if holding a writer lock, false if holding a reader lock. - /** Not defined if not holding a lock. */ - bool is_writer; - }; - -private: - typedef uintptr_t state_t; - static const state_t WRITER = 1; - static const state_t WRITER_PENDING = 2; - static const state_t READERS = ~(WRITER | WRITER_PENDING); - static const state_t ONE_READER = 4; - static const state_t BUSY = WRITER | READERS; - /** Bit 0 = writer is holding lock - Bit 1 = request by a writer to acquire lock (hint to readers to wait) - Bit 2..N = number of readers holding lock */ - volatile state_t state; -}; - -} // namespace tbb - -#endif /* __TBB_spin_rw_mutex_H */ diff --git a/src/tbb/src/old/task_v2.cpp b/src/tbb/src/old/task_v2.cpp deleted file mode 100644 index 9b569c3bb..000000000 --- a/src/tbb/src/old/task_v2.cpp +++ /dev/null @@ -1,38 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -/* This compilation unit provides definition of task::destroy( task& ) - that is binary compatible with TBB 2.x. In TBB 3.0, the method became - static, and its name decoration changed, though the definition remained. - - The macro switch should be set prior to including task.h - or any TBB file that might bring task.h up. -*/ -#define __TBB_DEPRECATED_TASK_INTERFACE 1 -#include "tbb/task.h" - -namespace tbb { - -void task::destroy( task& victim ) { - // Forward to static version - task_base::destroy( victim ); -} - -} // namespace tbb diff --git a/src/tbb/src/old/test_concurrent_queue_v2.cpp b/src/tbb/src/old/test_concurrent_queue_v2.cpp deleted file mode 100644 index b0bb56ba5..000000000 --- a/src/tbb/src/old/test_concurrent_queue_v2.cpp +++ /dev/null @@ -1,348 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#include "old/concurrent_queue_v2.h" -#include "tbb/atomic.h" -#include "tbb/tick_count.h" - -#include "../test/harness_assert.h" -#include "../test/harness.h" - -static tbb::atomic FooConstructed; -static tbb::atomic FooDestroyed; - -enum state_t{ - LIVE=0x1234, - DEAD=0xDEAD -}; - -class Foo { - state_t state; -public: - int thread_id; - int serial; - Foo() : state(LIVE) { - ++FooConstructed; - } - Foo( const Foo& item ) : state(LIVE) { - ASSERT( item.state==LIVE, NULL ); - ++FooConstructed; - thread_id = item.thread_id; - serial = item.serial; - } - ~Foo() { - ASSERT( state==LIVE, NULL ); - ++FooDestroyed; - state=DEAD; - thread_id=DEAD; - serial=DEAD; - } - void operator=( Foo& item ) { - ASSERT( item.state==LIVE, NULL ); - ASSERT( state==LIVE, NULL ); - thread_id = item.thread_id; - serial = item.serial; - } - bool is_const() {return false;} - bool is_const() const {return true;} -}; - -const size_t MAXTHREAD = 256; - -static int Sum[MAXTHREAD]; - -//! Count of various pop operations -/** [0] = pop_if_present that failed - [1] = pop_if_present that succeeded - [2] = pop */ -static tbb::atomic PopKind[3]; - -const int M = 10000; - -struct Body: NoAssign { - tbb::concurrent_queue* queue; - const int nthread; - Body( int nthread_ ) : nthread(nthread_) {} - void operator()( long thread_id ) const { - long pop_kind[3] = {0,0,0}; - int serial[MAXTHREAD+1]; - memset( serial, 0, nthread*sizeof(unsigned) ); - ASSERT( thread_idpop_if_present(f); - ++pop_kind[prepopped]; - } - Foo g; - g.thread_id = thread_id; - g.serial = j+1; - queue->push( g ); - if( !prepopped ) { - queue->pop(f); - ++pop_kind[2]; - } - ASSERT( f.thread_id<=nthread, NULL ); - ASSERT( f.thread_id==nthread || serial[f.thread_id]0, "nthread must be positive" ); - if( prefill+1>=capacity ) - return; - bool success = false; - for( int k=0; k<3; ++k ) - PopKind[k] = 0; - for( int trial=0; !success; ++trial ) { - FooConstructed = 0; - FooDestroyed = 0; - Body body(nthread); - tbb::concurrent_queue queue; - queue.set_capacity( capacity ); - body.queue = &queue; - for( int i=0; i=0; ) { - ASSERT( !queue.empty(), NULL ); - Foo f; - queue.pop(f); - ASSERT( queue.size()==i, NULL ); - sum += f.serial-1; - } - ASSERT( queue.empty(), NULL ); - ASSERT( queue.size()==0, NULL ); - if( sum!=expected ) - printf("sum=%d expected=%d\n",sum,expected); - ASSERT( FooConstructed==FooDestroyed, NULL ); - - success = true; - if( nthread>1 && prefill==0 ) { - // Check that pop_if_present got sufficient exercise - for( int k=0; k<2; ++k ) { -#if (_WIN32||_WIN64) - // The TBB library on Windows seems to have a tough time generating - // the desired interleavings for pop_if_present, so the code tries longer, and settles - // for fewer desired interleavings. - const int max_trial = 100; - const int min_requirement = 20; -#else - const int min_requirement = 100; - const int max_trial = 20; -#endif /* _WIN32||_WIN64 */ - if( PopKind[k]=max_trial ) { - if( Verbose ) - printf("Warning: %d threads had only %ld pop_if_present operations %s after %d trials (expected at least %d). " - "This problem may merely be unlucky scheduling. " - "Investigate only if it happens repeatedly.\n", - nthread, long(PopKind[k]), k==0?"failed":"succeeded", max_trial, min_requirement); - else - printf("Warning: the number of %s pop_if_present operations is less than expected for %d threads. Investigate if it happens repeatedly.\n", - k==0?"failed":"succeeded", nthread ); - } else { - success = false; - } - } - } - } - } -} - -template -void TestIteratorAux( Iterator1 i, Iterator2 j, int size ) { - Iterator1 old_i; // assigned at first iteration below - for( int k=0; k" - ASSERT( k+1==i->serial, NULL ); - if( k&1 ) { - // Test post-increment - Foo f = *old_i++; - ASSERT( k+1==f.serial, NULL ); - // Test assignment - i = old_i; - } else { - // Test pre-increment - if( k -void TestIteratorAssignment( Iterator2 j ) { - Iterator1 i(j); - ASSERT( i==j, NULL ); - ASSERT( !(i!=j), NULL ); - Iterator1 k; - k = j; - ASSERT( k==j, NULL ); - ASSERT( !(k!=j), NULL ); -} - -//! Test the iterators for concurrent_queue -void TestIterator() { - tbb::concurrent_queue queue; - tbb::concurrent_queue& const_queue = queue; - for( int j=0; j<500; ++j ) { - TestIteratorAux( queue.begin(), queue.end(), j ); - TestIteratorAux( const_queue.begin(), const_queue.end(), j ); - TestIteratorAux( const_queue.begin(), queue.end(), j ); - TestIteratorAux( queue.begin(), const_queue.end(), j ); - Foo f; - f.serial = j+1; - queue.push(f); - } - TestIteratorAssignment::const_iterator>( const_queue.begin() ); - TestIteratorAssignment::const_iterator>( queue.begin() ); - TestIteratorAssignment:: iterator>( queue.begin() ); -} - -void TestConcurrentQueueType() { - AssertSameType( tbb::concurrent_queue::value_type(), Foo() ); - Foo f; - const Foo g; - tbb::concurrent_queue::reference r = f; - ASSERT( &r==&f, NULL ); - ASSERT( !r.is_const(), NULL ); - tbb::concurrent_queue::const_reference cr = g; - ASSERT( &cr==&g, NULL ); - ASSERT( cr.is_const(), NULL ); -} - -template -void TestEmptyQueue() { - const tbb::concurrent_queue queue; - ASSERT( queue.size()==0, NULL ); - ASSERT( queue.capacity()>0, NULL ); - ASSERT( size_t(queue.capacity())>=size_t(-1)/(sizeof(void*)+sizeof(T)), NULL ); -} - -void TestFullQueue() { - for( int n=0; n<10; ++n ) { - FooConstructed = 0; - FooDestroyed = 0; - tbb::concurrent_queue queue; - queue.set_capacity(n); - for( int i=0; i<=n; ++i ) { - Foo f; - f.serial = i; - bool result = queue.push_if_not_full( f ); - ASSERT( result==(i -struct TestNegativeQueueBody: NoAssign { - tbb::concurrent_queue& queue; - const int nthread; - TestNegativeQueueBody( tbb::concurrent_queue& q, int n ) : queue(q), nthread(n) {} - void operator()( int k ) const { - if( k==0 ) { - int number_of_pops = nthread-1; - // Wait for all pops to pend. - while( queue.size()>-number_of_pops ) { - __TBB_Yield(); - } - for( int i=0; ; ++i ) { - ASSERT( queue.size()==i-number_of_pops, NULL ); - ASSERT( queue.empty()==(queue.size()<=0), NULL ); - if( i==number_of_pops ) break; - // Satisfy another pop - queue.push( T() ); - } - } else { - // Pop item from queue - T item; - queue.pop(item); - } - } -}; - -//! Test a queue with a negative size. -template -void TestNegativeQueue( int nthread ) { - tbb::concurrent_queue queue; - NativeParallelFor( nthread, TestNegativeQueueBody(queue,nthread) ); -} - -int TestMain () { - TestEmptyQueue(); - TestEmptyQueue(); - TestFullQueue(); - TestConcurrentQueueType(); - TestIterator(); - - // Test concurrent operations - for( int nthread=MinThread; nthread<=MaxThread; ++nthread ) { - TestNegativeQueue(nthread); - for( int prefill=0; prefill<64; prefill+=(1+prefill/3) ) { - TestPushPop(prefill,ptrdiff_t(-1),nthread); - TestPushPop(prefill,ptrdiff_t(1),nthread); - TestPushPop(prefill,ptrdiff_t(2),nthread); - TestPushPop(prefill,ptrdiff_t(10),nthread); - TestPushPop(prefill,ptrdiff_t(100),nthread); - } - } - return Harness::Done; -} diff --git a/src/tbb/src/old/test_concurrent_vector_v2.cpp b/src/tbb/src/old/test_concurrent_vector_v2.cpp deleted file mode 100644 index 1d0fd2a79..000000000 --- a/src/tbb/src/old/test_concurrent_vector_v2.cpp +++ /dev/null @@ -1,558 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#include "concurrent_vector_v2.h" -#include -#include -#include "../test/harness_assert.h" - -tbb::atomic FooCount; - -//! Problem size -const size_t N = 500000; - -struct Foo { - int my_bar; -public: - enum State { - DefaultInitialized=0x1234, - CopyInitialized=0x89ab, - Destroyed=0x5678 - } state; - int& bar() { - ASSERT( state==DefaultInitialized||state==CopyInitialized, NULL ); - return my_bar; - } - int bar() const { - ASSERT( state==DefaultInitialized||state==CopyInitialized, NULL ); - return my_bar; - } - static const int initial_value_of_bar = 42; - Foo() { - state = DefaultInitialized; - ++FooCount; - my_bar = initial_value_of_bar; - } - Foo( const Foo& foo ) { - state = CopyInitialized; - ++FooCount; - my_bar = foo.my_bar; - } - ~Foo() { - ASSERT( state==DefaultInitialized||state==CopyInitialized, NULL ); - state = Destroyed; - my_bar = ~initial_value_of_bar; - --FooCount; - } - bool is_const() const {return true;} - bool is_const() {return false;} -}; - -class FooWithAssign: public Foo { -public: - void operator=( const FooWithAssign& x ) { - ASSERT( x.state==DefaultInitialized||x.state==CopyInitialized, NULL ); - ASSERT( state==DefaultInitialized||state==CopyInitialized, NULL ); - my_bar = x.my_bar; - } -}; - -inline void NextSize( int& s ) { - if( s<=32 ) ++s; - else s += s/10; -} - -static void CheckVector( const tbb::concurrent_vector& cv, size_t expected_size, size_t old_size ) { - ASSERT( cv.size()==expected_size, NULL ); - ASSERT( cv.empty()==(expected_size==0), NULL ); - for( int j=0; j vector_t; - for( int old_size=0; old_size<=128; NextSize( old_size ) ) { - for( int new_size=old_size; new_size<=128; NextSize( new_size ) ) { - long count = FooCount; - vector_t v; - ASSERT( count==FooCount, NULL ); - v.grow_by(old_size); - ASSERT( count+old_size==FooCount, NULL ); - for( int j=0; j vector_t; - vector_t v; - v.reserve( old_size ); - ASSERT( v.capacity()>=old_size, NULL ); - v.reserve( new_size ); - ASSERT( v.capacity()>=old_size, NULL ); - ASSERT( v.capacity()>=new_size, NULL ); - for( size_t i=0; i<2*new_size; ++i ) { - ASSERT( size_t(FooCount)==count+i, NULL ); - size_t j = v.grow_by(1); - ASSERT( j==i, NULL ); - } - } - ASSERT( FooCount==count, NULL ); - } - } -} - -struct AssignElement { - typedef tbb::concurrent_vector::range_type::iterator iterator; - iterator base; - void operator()( const tbb::concurrent_vector::range_type& range ) const { - for( iterator i=range.begin(); i!=range.end(); ++i ) { - if( *i!=0 ) - std::printf("ERROR for v[%ld]\n", long(i-base)); - *i = int(i-base); - } - } - AssignElement( iterator base_ ) : base(base_) {} -}; - -struct CheckElement { - typedef tbb::concurrent_vector::const_range_type::iterator iterator; - iterator base; - void operator()( const tbb::concurrent_vector::const_range_type& range ) const { - for( iterator i=range.begin(); i!=range.end(); ++i ) - if( *i != int(i-base) ) - std::printf("ERROR for v[%ld]\n", long(i-base)); - } - CheckElement( iterator base_ ) : base(base_) {} -}; - -#include "tbb/tick_count.h" -#include "tbb/parallel_for.h" -#include "../test/harness.h" - -//! Test parallel access by iterators -void TestParallelFor( int nthread ) { - typedef tbb::concurrent_vector vector_t; - vector_t v; - v.grow_to_at_least(N); - tbb::tick_count t0 = tbb::tick_count::now(); - if( Verbose ) - std::printf("Calling parallel_for.h with %ld threads\n",long(nthread)); - tbb::parallel_for( v.range(10000), AssignElement(v.begin()) ); - tbb::tick_count t1 = tbb::tick_count::now(); - const vector_t& u = v; - tbb::parallel_for( u.range(10000), CheckElement(u.begin()) ); - tbb::tick_count t2 = tbb::tick_count::now(); - if( Verbose ) - std::printf("Time for parallel_for.h: assign time = %8.5f, check time = %8.5f\n", - (t1-t0).seconds(),(t2-t1).seconds()); - for( long i=0; size_t(i) -void TestIteratorAssignment( Iterator2 j ) { - Iterator1 i(j); - ASSERT( i==j, NULL ); - ASSERT( !(i!=j), NULL ); - Iterator1 k; - k = j; - ASSERT( k==j, NULL ); - ASSERT( !(k!=j), NULL ); -} - -template -void TestIteratorTraits() { - AssertSameType( static_cast(0), static_cast(0) ); - AssertSameType( static_cast(0), static_cast(0) ); - AssertSameType( static_cast(0), static_cast(0) ); - AssertSameType( static_cast(0), static_cast(0) ); - T x; - typename Iterator::reference xr = x; - typename Iterator::pointer xp = &x; - ASSERT( &xr==xp, NULL ); -} - -template -void CheckConstIterator( const Vector& u, int i, const Iterator& cp ) { - typename Vector::const_reference pref = *cp; - if( pref.bar()!=i ) - std::printf("ERROR for u[%ld] using const_iterator\n", long(i)); - typename Vector::difference_type delta = cp-u.begin(); - ASSERT( delta==i, NULL ); - if( u[i].bar()!=i ) - std::printf("ERROR for u[%ld] using subscripting\n", long(i)); - ASSERT( u.begin()[i].bar()==i, NULL ); -} - -template -void CheckIteratorComparison( V& u ) { - Iterator1 i = u.begin(); - for( int i_count=0; i_count<100; ++i_count ) { - Iterator2 j = u.begin(); - for( int j_count=0; j_count<100; ++j_count ) { - ASSERT( (i==j)==(i_count==j_count), NULL ); - ASSERT( (i!=j)==(i_count!=j_count), NULL ); - ASSERT( (i-j)==(i_count-j_count), NULL ); - ASSERT( (ij)==(i_count>j_count), NULL ); - ASSERT( (i<=j)==(i_count<=j_count), NULL ); - ASSERT( (i>=j)==(i_count>=j_count), NULL ); - ++j; - } - ++i; - } -} - -//! Test sequential iterators for vector type V. -/** Also does timing. */ -template -void TestSequentialFor() { - V v; - v.grow_by(N); - - // Check iterator - tbb::tick_count t0 = tbb::tick_count::now(); - typename V::iterator p = v.begin(); - ASSERT( !(*p).is_const(), NULL ); - ASSERT( !p->is_const(), NULL ); - for( int i=0; size_t(i)is_const(), NULL ); - for( int i=0; size_t(i)0; ) { - --i; - --cp; - if( i>0 ) { - typename V::const_iterator cp_old = cp--; - int here = (*cp_old).bar(); - ASSERT( here==u[i].bar(), NULL ); - typename V::const_iterator cp_new = cp++; - int prev = (*cp_new).bar(); - ASSERT( prev==u[i-1].bar(), NULL ); - } - CheckConstIterator(u,i,cp); - } - - // Now go forwards and backwards - cp = u.begin(); - ptrdiff_t k = 0; - for( size_t i=0; i(v); - CheckIteratorComparison(v); - CheckIteratorComparison(v); - CheckIteratorComparison(v); - - TestIteratorAssignment( u.begin() ); - TestIteratorAssignment( v.begin() ); - TestIteratorAssignment( v.begin() ); - - // Check reverse_iterator - typename V::reverse_iterator rp = v.rbegin(); - for( size_t i=v.size(); i>0; --i, ++rp ) { - typename V::reference pref = *rp; - ASSERT( size_t(pref.bar())==i-1, NULL ); - ASSERT( rp!=v.rend(), NULL ); - } - ASSERT( rp==v.rend(), NULL ); - - // Check const_reverse_iterator - typename V::const_reverse_iterator crp = u.rbegin(); - for( size_t i=v.size(); i>0; --i, ++crp ) { - typename V::const_reference cpref = *crp; - ASSERT( size_t(cpref.bar())==i-1, NULL ); - ASSERT( crp!=u.rend(), NULL ); - } - ASSERT( crp==u.rend(), NULL ); - - TestIteratorAssignment( u.rbegin() ); - TestIteratorAssignment( v.rbegin() ); -} - -static const size_t Modulus = 7; - -typedef tbb::concurrent_vector MyVector; - -class GrowToAtLeast { - MyVector& my_vector; -public: - void operator()( const tbb::blocked_range& range ) const { - for( size_t i=range.begin(); i!=range.end(); ++i ) { - size_t n = my_vector.size(); - size_t k = n==0 ? 0 : i % (2*n+1); - my_vector.grow_to_at_least(k+1); - ASSERT( my_vector.size()>=k+1, NULL ); - } - } - GrowToAtLeast( MyVector& vector ) : my_vector(vector) {} -}; - -void TestConcurrentGrowToAtLeast() { - MyVector v; - for( size_t s=1; s<1000; s*=10 ) { - tbb::parallel_for( tbb::blocked_range(0,1000000,100), GrowToAtLeast(v) ); - } -} - -//! Test concurrent invocations of method concurrent_vector::grow_by -class GrowBy { - MyVector& my_vector; -public: - void operator()( const tbb::blocked_range& range ) const { - for( int i=range.begin(); i!=range.end(); ++i ) { - if( i%3 ) { - Foo& element = my_vector[my_vector.grow_by(1)]; - element.bar() = i; - } else { - Foo f; - f.bar() = i; - size_t k = my_vector.push_back( f ); - ASSERT( my_vector[k].bar()==i, NULL ); - } - } - } - GrowBy( MyVector& vector ) : my_vector(vector) {} -}; - -//! Test concurrent invocations of method concurrent_vector::grow_by -void TestConcurrentGrowBy( int nthread ) { - int m = 100000; - MyVector v; - tbb::parallel_for( tbb::blocked_range(0,m,1000), GrowBy(v) ); - ASSERT( v.size()==size_t(m), NULL ); - - // Verify that v is a permutation of 0..m - int inversions = 0; - bool* found = new bool[m]; - memset( found, 0, m ); - for( int i=0; i0 ) - inversions += v[i].bar()1 || v[i].bar()==i, "sequential execution is wrong" ); - } - delete[] found; - if( nthread>1 && inversions vector_t; - for( int dst_size=1; dst_size<=128; NextSize( dst_size ) ) { - for( int src_size=2; src_size<=128; NextSize( src_size ) ) { - vector_t u; - u.grow_to_at_least(src_size); - for( int i=0; i - -typedef unsigned long Number; - -static tbb::concurrent_vector Primes; - -class FindPrimes { - bool is_prime( Number val ) const { - int limit, factor = 3; - if( val<5u ) - return val==2; - else { - limit = long(sqrtf(float(val))+0.5f); - while( factor<=limit && val % factor ) - ++factor; - return factor>limit; - } - } -public: - void operator()( const tbb::blocked_range& r ) const { - for( Number i=r.begin(); i!=r.end(); ++i ) { - if( i%2 && is_prime(i) ) { - Primes[Primes.grow_by(1)] = i; - } - } - } -}; - -static double TimeFindPrimes( int nthread ) { - Primes.clear(); - tbb::task_scheduler_init init(nthread); - tbb::tick_count t0 = tbb::tick_count::now(); - tbb::parallel_for( tbb::blocked_range(0,1000000,500), FindPrimes() ); - tbb::tick_count t1 = tbb::tick_count::now(); - return (t1-t0).seconds(); -} - -static void TestFindPrimes() { - // Time fully subscribed run. - double t2 = TimeFindPrimes( tbb::task_scheduler_init::automatic ); - - // Time parallel run that is very likely oversubscribed. - double t128 = TimeFindPrimes(128); - - if( Verbose ) - std::printf("TestFindPrimes: t2==%g t128=%g\n", t2, t128 ); - - // We allow the 128-thread run a little extra time to allow for thread overhead. - // Theoretically, following test will fail on machine with >128 processors. - // But that situation is not going to come up in the near future, - // and the generalization to fix the issue is not worth the trouble. - if( t128>1.10*t2 ) { - std::printf("Warning: grow_by is pathetically slow: t2==%g t128=%g\n", t2, t128); - } -} - -//------------------------------------------------------------------------ -// Test compatibility with STL sort. -//------------------------------------------------------------------------ - -#include - -void TestSort() { - for( int n=1; n<100; n*=3 ) { - tbb::concurrent_vector array; - array.grow_by( n ); - for( int i=0; i::iterator,Foo>(); - TestIteratorTraits::const_iterator,const Foo>(); - TestSequentialFor > (); - TestResizeAndCopy(); - TestAssign(); - TestCapacity(); - for( int nthread=MinThread; nthread<=MaxThread; ++nthread ) { - tbb::task_scheduler_init init( nthread ); - TestParallelFor( nthread ); - TestConcurrentGrowToAtLeast(); - TestConcurrentGrowBy( nthread ); - } - TestFindPrimes(); - TestSort(); - return Harness::Done; -} diff --git a/src/tbb/src/old/test_mutex_v2.cpp b/src/tbb/src/old/test_mutex_v2.cpp deleted file mode 100644 index e6d9c3d5c..000000000 --- a/src/tbb/src/old/test_mutex_v2.cpp +++ /dev/null @@ -1,242 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#define HARNESS_DEFAULT_MIN_THREADS 1 -#define HARNESS_DEFAULT_MAX_THREADS 3 - -//------------------------------------------------------------------------ -// Test TBB mutexes when used with parallel_for.h -// -// Usage: test_Mutex.exe [-v] nthread -// -// The -v option causes timing information to be printed. -// -// Compile with _OPENMP and -openmp -//------------------------------------------------------------------------ -#include "../test/harness_defs.h" -#include "tbb/atomic.h" -#include "tbb/blocked_range.h" -#include "tbb/parallel_for.h" -#include "tbb/tick_count.h" -#include "../test/harness.h" -#include "spin_rw_mutex_v2.h" -#include -#include - -// This test deliberately avoids a "using tbb" statement, -// so that the error of putting types in the wrong namespace will be caught. - -template -struct Counter { - typedef M mutex_type; - M mutex; - volatile long value; -}; - -//! Function object for use with parallel_for.h. -template -struct AddOne: NoAssign { - C& counter; - /** Increments counter once for each iteration in the iteration space. */ - void operator()( tbb::blocked_range& range ) const { - for( size_t i=range.begin(); i!=range.end(); ++i ) { - if( i&1 ) { - // Try implicit acquire and explicit release - typename C::mutex_type::scoped_lock lock(counter.mutex); - counter.value = counter.value+1; - lock.release(); - } else { - // Try explicit acquire and implicit release - typename C::mutex_type::scoped_lock lock; - lock.acquire(counter.mutex); - counter.value = counter.value+1; - } - } - } - AddOne( C& counter_ ) : counter(counter_) {} -}; - -//! Generic test of a TBB mutex type M. -/** Does not test features specific to reader-writer locks. */ -template -void Test( const char * name ) { - if( Verbose ) { - printf("%s time = ",name); - fflush(stdout); - } - Counter counter; - counter.value = 0; - const int n = 100000; - tbb::tick_count t0 = tbb::tick_count::now(); - tbb::parallel_for(tbb::blocked_range(0,n,n/10),AddOne >(counter)); - tbb::tick_count t1 = tbb::tick_count::now(); - if( Verbose ) - printf("%g usec\n",(t1-t0).seconds()); - if( counter.value!=n ) - printf("ERROR for %s: counter.value=%ld\n",name,counter.value); -} - -template -struct Invariant { - typedef M mutex_type; - M mutex; - const char* mutex_name; - volatile long value[N]; - Invariant( const char* mutex_name_ ) : - mutex_name(mutex_name_) - { - for( size_t k=0; k -struct TwiddleInvariant: NoAssign { - I& invariant; - TwiddleInvariant( I& invariant_ ) : invariant(invariant_) {} - - /** Increments counter once for each iteration in the iteration space. */ - void operator()( tbb::blocked_range& range ) const { - for( size_t i=range.begin(); i!=range.end(); ++i ) { - //! Every 8th access is a write access - const bool write = (i%8)==7; - bool okay = true; - bool lock_kept = true; - if( (i/8)&1 ) { - // Try implicit acquire and explicit release - typename I::mutex_type::scoped_lock lock(invariant.mutex,write); - execute_aux(lock, i, write, okay, lock_kept); - lock.release(); - } else { - // Try explicit acquire and implicit release - typename I::mutex_type::scoped_lock lock; - lock.acquire(invariant.mutex,write); - execute_aux(lock, i, write, okay, lock_kept); - } - if( !okay ) { - printf( "ERROR for %s at %ld: %s %s %s %s\n",invariant.mutex_name, long(i), - write ? "write," : "read,", - write ? (i%16==7?"downgrade,":"") : (i%8==3?"upgrade,":""), - lock_kept ? "lock kept," : "lock not kept,", // TODO: only if downgrade/upgrade - (i/8)&1 ? "impl/expl" : "expl/impl" ); - } - } - } -private: - void execute_aux(typename I::mutex_type::scoped_lock & lock, const size_t i, const bool write, bool & okay, bool & lock_kept) const { - if( write ) { - long my_value = invariant.value[0]; - invariant.update(); - if( i%16==7 ) { - lock_kept = lock.downgrade_to_reader(); - if( !lock_kept ) - my_value = invariant.value[0] - 1; - okay = invariant.value_is(my_value+1); - } - } else { - okay = invariant.is_okay(); - if( i%8==3 ) { - long my_value = invariant.value[0]; - lock_kept = lock.upgrade_to_writer(); - if( !lock_kept ) - my_value = invariant.value[0]; - invariant.update(); - okay = invariant.value_is(my_value+1); - } - } - } -}; - -/** This test is generic so that we can test any other kinds of ReaderWriter locks we write later. */ -template -void TestReaderWriterLock( const char * mutex_name ) { - if( Verbose ) { - printf("%s readers & writers time = ",mutex_name); - fflush(stdout); - } - Invariant invariant(mutex_name); - const size_t n = 500000; - tbb::tick_count t0 = tbb::tick_count::now(); - tbb::parallel_for(tbb::blocked_range(0,n,n/100),TwiddleInvariant >(invariant)); - tbb::tick_count t1 = tbb::tick_count::now(); - // There is either a writer or a reader upgraded to a writer for each 4th iteration - long expected_value = n/4; - if( !invariant.value_is(expected_value) ) - printf("ERROR for %s: final invariant value is wrong\n",mutex_name); - if( Verbose ) - printf("%g usec\n", (t1-t0).seconds()); -} - -/** Test try_acquire functionality of a non-reenterable mutex */ -template -void TestTryAcquire_OneThread( const char * mutex_name ) { - M tested_mutex; - typename M::scoped_lock lock1; - if( lock1.try_acquire(tested_mutex) ) - lock1.release(); - else - printf("ERROR for %s: try_acquire failed though it should not\n", mutex_name); - { - typename M::scoped_lock lock2(tested_mutex); - if( lock1.try_acquire(tested_mutex) ) - printf("ERROR for %s: try_acquire succeeded though it should not\n", mutex_name); - } - if( lock1.try_acquire(tested_mutex) ) - lock1.release(); - else - printf("ERROR for %s: try_acquire failed though it should not\n", mutex_name); -} - -#include "tbb/task_scheduler_init.h" - -int TestMain () { - for( int p=MinThread; p<=MaxThread; ++p ) { - tbb::task_scheduler_init init( p ); - if( Verbose ) - printf( "testing with %d workers\n", static_cast(p) ); - const int n = 3; - // Run each test several times. - for( int i=0; i( "Spin RW Mutex" ); - TestTryAcquire_OneThread("Spin RW Mutex"); // only tests try_acquire for writers - TestReaderWriterLock( "Spin RW Mutex" ); - } - if( Verbose ) - printf( "calling destructor for task_scheduler_init\n" ); - } - return Harness::Done; -} diff --git a/src/tbb/src/old/test_task_scheduler_observer_v3.cpp b/src/tbb/src/old/test_task_scheduler_observer_v3.cpp deleted file mode 100644 index cfd772be7..000000000 --- a/src/tbb/src/old/test_task_scheduler_observer_v3.cpp +++ /dev/null @@ -1,121 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -//TODO: when removing TBB_PREVIEW_LOCAL_OBSERVER, change the header or defines here -#include "tbb/task_scheduler_observer.h" - -typedef uintptr_t FlagType; -const int MaxFlagIndex = sizeof(FlagType)*8-1; - -class MyObserver: public tbb::task_scheduler_observer { - FlagType flags; - void on_scheduler_entry( bool is_worker ) __TBB_override; - void on_scheduler_exit( bool is_worker ) __TBB_override; -public: - MyObserver( FlagType flags_ ) : flags(flags_) { - observe(true); - } -}; - -#include "harness_assert.h" -#include "tbb/atomic.h" - -tbb::atomic EntryCount; -tbb::atomic ExitCount; - -struct State { - FlagType MyFlags; - bool IsMaster; - State() : MyFlags(), IsMaster() {} -}; - -#include "../tbb/tls.h" -tbb::internal::tls LocalState; - -void MyObserver::on_scheduler_entry( bool is_worker ) { - State& state = *LocalState; - ASSERT( is_worker==!state.IsMaster, NULL ); - ++EntryCount; - state.MyFlags |= flags; -} - -void MyObserver::on_scheduler_exit( bool is_worker ) { - State& state = *LocalState; - ASSERT( is_worker==!state.IsMaster, NULL ); - ++ExitCount; - state.MyFlags &= ~flags; -} - -#include "tbb/task.h" - -class FibTask: public tbb::task { - const int n; - FlagType flags; -public: - FibTask( int n_, FlagType flags_ ) : n(n_), flags(flags_) {} - tbb::task* execute() __TBB_override { - ASSERT( !(~LocalState->MyFlags & flags), NULL ); - if( n>=2 ) { - set_ref_count(3); - spawn(*new( allocate_child() ) FibTask(n-1,flags)); - spawn_and_wait_for_all(*new( allocate_child() ) FibTask(n-2,flags)); - } - return NULL; - } -}; - -void DoFib( FlagType flags ) { - tbb::task* t = new( tbb::task::allocate_root() ) FibTask(10,flags); - tbb::task::spawn_root_and_wait(*t); -} - -#include "tbb/task_scheduler_init.h" -#include "harness.h" - -class DoTest { - int nthread; -public: - DoTest( int n ) : nthread(n) {} - void operator()( int i ) const { - LocalState->IsMaster = true; - if( i==0 ) { - tbb::task_scheduler_init init(nthread); - DoFib(0); - } else { - FlagType f = i<=MaxFlagIndex? 1<0, "on_scheduler_entry not exercised" ); - ASSERT( ExitCount>0, "on_scheduler_exit not exercised" ); - return Harness::Done; -} diff --git a/src/tbb/src/perf/coarse_grained_raii_lru_cache.h b/src/tbb/src/perf/coarse_grained_raii_lru_cache.h deleted file mode 100644 index 9b11400e2..000000000 --- a/src/tbb/src/perf/coarse_grained_raii_lru_cache.h +++ /dev/null @@ -1,147 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef coarse_grained_raii_lru_cache_H -#define coarse_grained_raii_lru_cache_H - -#include -#include -#include -#include - -#include "tbb/spin_mutex.h" -#include "tbb/tbb_stddef.h" -template -class coarse_grained_raii_lru_cache : tbb::internal::no_assign{ - typedef value_functor_type value_function_type; - - typedef std::size_t ref_counter_type; - struct map_value_type; - typedef std::map map_storage_type; - typedef std::list lru_list_type; - struct map_value_type { - value_type my_value; - ref_counter_type my_ref_counter; - typename lru_list_type::iterator my_lru_list_iterator; - bool my_is_ready; - - map_value_type (value_type const& a_value, ref_counter_type a_ref_counter, typename lru_list_type::iterator a_lru_list_iterator, bool a_is_ready) - : my_value(a_value), my_ref_counter(a_ref_counter), my_lru_list_iterator (a_lru_list_iterator) - ,my_is_ready(a_is_ready) - {} - }; - - class handle_object; -public: - typedef handle_object handle; - - coarse_grained_raii_lru_cache(value_function_type f, std::size_t number_of_lru_history_items): my_value_function(f),my_number_of_lru_history_items(number_of_lru_history_items){} - handle_object operator[](key_type k){ - tbb::spin_mutex::scoped_lock lock(my_mutex); - bool is_new_value_needed = false; - typename map_storage_type::iterator it = my_map_storage.find(k); - if (it == my_map_storage.end()){ - it = my_map_storage.insert(it,std::make_pair(k,map_value_type(value_type(),0,my_lru_list.end(),false))); - is_new_value_needed = true; - }else { - typename lru_list_type::iterator list_it = it->second.my_lru_list_iterator; - if (list_it!=my_lru_list.end()) { - my_lru_list.erase(list_it); - it->second.my_lru_list_iterator= my_lru_list.end(); - } - } - typename map_storage_type::reference value_ref = *it; - //increase ref count - ++(value_ref.second.my_ref_counter); - if (is_new_value_needed){ - lock.release(); - value_ref.second.my_value = my_value_function(k); - __TBB_store_with_release(value_ref.second.my_is_ready, true); - - }else{ - if (!value_ref.second.my_is_ready){ - lock.release(); - tbb::internal::spin_wait_while_eq(value_ref.second.my_is_ready,false); - } - } - return handle_object(*this,(value_ref)); - } -private: - void signal_end_of_usage(typename map_storage_type::reference value_ref){ - tbb::spin_mutex::scoped_lock lock(my_mutex); - typename map_storage_type::iterator it = my_map_storage.find(value_ref.first); - __TBB_ASSERT(it!=my_map_storage.end(),"cache should not return past-end iterators to outer world"); - __TBB_ASSERT(&(*it) == &value_ref,"dangling reference has been returned to outside world? data race ?"); - __TBB_ASSERT( my_lru_list.end()== std::find(my_lru_list.begin(),my_lru_list.end(),it), - "object in use should not be in list of unused objects "); - if (! --(it->second.my_ref_counter)){ //decrease ref count, and check if it was the last reference - if (my_lru_list.size()>=my_number_of_lru_history_items){ - size_t number_of_elements_to_evict = 1 + my_lru_list.size() - my_number_of_lru_history_items; - for (size_t i=0; isecond.my_lru_list_iterator = my_lru_list.begin(); - } - } -private: - value_function_type my_value_function; - std::size_t const my_number_of_lru_history_items; - map_storage_type my_map_storage; - lru_list_type my_lru_list; - tbb::spin_mutex my_mutex; -private: - struct handle_move_t:tbb::internal::no_assign{ - coarse_grained_raii_lru_cache & my_cache_ref; - typename map_storage_type::reference my_value_ref; - handle_move_t(coarse_grained_raii_lru_cache & cache_ref, typename map_storage_type::reference value_ref):my_cache_ref(cache_ref),my_value_ref(value_ref) {}; - }; - class handle_object { - coarse_grained_raii_lru_cache * my_cache_pointer; - typename map_storage_type::reference my_value_ref; - public: - handle_object(coarse_grained_raii_lru_cache & cache_ref, typename map_storage_type::reference value_ref):my_cache_pointer(&cache_ref), my_value_ref(value_ref) {} - handle_object(handle_move_t m):my_cache_pointer(&m.my_cache_ref), my_value_ref(m.my_value_ref){} - operator handle_move_t(){ return move(*this);} - value_type& value(){return my_value_ref.second.my_value;} - ~handle_object(){ - if (my_cache_pointer){ - my_cache_pointer->signal_end_of_usage(my_value_ref); - } - } - private: - friend handle_move_t move(handle_object& h){ - return handle_object::move(h); - } - static handle_move_t move(handle_object& h){ - __TBB_ASSERT(h.my_cache_pointer,"move from the same object twice ?"); - coarse_grained_raii_lru_cache * cache_pointer = NULL; - std::swap(cache_pointer,h.my_cache_pointer); - return handle_move_t(*cache_pointer,h.my_value_ref); - } - private: - void operator=(handle_object&); - handle_object(handle_object &); - }; -}; -#endif //coarse_grained_raii_lru_cache_H diff --git a/src/tbb/src/perf/cpq_pdes.cpp b/src/tbb/src/perf/cpq_pdes.cpp deleted file mode 100644 index c54eb8fce..000000000 --- a/src/tbb/src/perf/cpq_pdes.cpp +++ /dev/null @@ -1,230 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#include -#include -#include -#include "tbb/tbb_stddef.h" -#include "tbb/spin_mutex.h" -#include "tbb/task_scheduler_init.h" -#include "tbb/parallel_for.h" -#include "tbb/tick_count.h" -#include "tbb/blocked_range.h" -#include "../test/harness.h" -#include "tbb/concurrent_priority_queue.h" - -#pragma warning(disable: 4996) - -#define IMPL_STL 0 -#define IMPL_CPQ 1 - -using namespace tbb; - -//const int contention = 75; // degree contention. 100 = 0 us busy_wait, 50 = 50*contention_unit us -const double contention_unit = 0.025; // in microseconds (us) -const double throughput_window = 30; // in seconds -const int num_initial_events = 10000; // number of initial events in the queue -const int min_elapse = 20; // min contention_units to elapse between event spawns -const int max_elapse = 40; // max contention_units to elapse between event spawns -const int min_spawn = 0; // min number of events to spawn -const int max_spawn = 2; // max number of events to spawn - -tbb::atomic operation_count; -tbb::tick_count start; -bool done; - -class event { -public: - int timestamp; - int elapse; - int spawn; -}; - -class timestamp_compare { -public: - bool operator()(event e1, event e2) { - return e2.timestamp, timestamp_compare > *stl_cpq; -concurrent_priority_queue *lfc_pq; - -unsigned int one_us_iters = 429; // default value - -// if user wants to calibrate to microseconds on particular machine, call this at beginning of program -// sets one_us_iters to number of iters to busy_wait for approx. 1 us -void calibrate_busy_wait() { - const unsigned niter = 1000000; - tbb::tick_count t0 = tbb::tick_count::now(); - for (volatile unsigned int i=0; ipush(elem); - } - else { - tbb::spin_mutex::scoped_lock myLock(*my_mutex); - stl_cpq->push(elem); - } - } - else { - lfc_pq->push(elem); - } -} - -bool do_pop(event& elem, int nThr, int impl) { - if (impl == IMPL_STL) { - if (nThr == 1) { - if (!stl_cpq->empty()) { - elem = stl_cpq->top(); - stl_cpq->pop(); - return true; - } - } - else { - tbb::spin_mutex::scoped_lock myLock(*my_mutex); - if (!stl_cpq->empty()) { - elem = stl_cpq->top(); - stl_cpq->pop(); - return true; - } - } - } - else { - if (lfc_pq->try_pop(elem)) { - return true; - } - } - return false; -} - -struct TestPDESloadBody : NoAssign { - int nThread; - int implementation; - - TestPDESloadBody(int nThread_, int implementation_) : - nThread(nThread_), implementation(implementation_) {} - - void operator()(const int threadID) const { - if (threadID == nThread) { - sleep(throughput_window); - done = true; - } - else { - event e, tmp; - unsigned int num_operations = 0; - for (;;) { - // pop an event - if (do_pop(e, nThread, implementation)) { - num_operations++; - // do the event - busy_wait(e.elapse*contention_unit); - while (e.spawn > 0) { - tmp.spawn = ((e.spawn+1-min_spawn) % ((max_spawn-min_spawn)+1))+min_spawn; - tmp.timestamp = e.timestamp + e.elapse; - e.timestamp = tmp.timestamp; - e.elapse = ((e.elapse+1-min_elapse) % ((max_elapse-min_elapse)+1))+min_elapse; - tmp.elapse = e.elapse; - do_push(tmp, nThread, implementation); - num_operations++; - e.spawn--; - busy_wait(e.elapse*contention_unit); - if (done) break; - } - } - if (done) break; - } - operation_count += num_operations; - } - } -}; - -void preload_queue(int nThr, int impl) { - event an_event; - for (int i=0; i, timestamp_compare >; - preload_queue(nThreads, IMPL_STL); - TestPDESloadBody my_stl_test(nThreads, IMPL_STL); - start = tbb::tick_count::now(); - NativeParallelFor(nThreads+1, my_stl_test); - delete stl_cpq; - - REPORT(" %10d", operation_count/throughput_window); - - operation_count = 0; - done = false; - lfc_pq = new concurrent_priority_queue; - preload_queue(nThreads, IMPL_CPQ); - TestPDESloadBody my_cpq_test(nThreads, IMPL_CPQ); - start = tbb::tick_count::now(); - NativeParallelFor(nThreads+1, my_cpq_test); - delete lfc_pq; - - REPORT(" %10d\n", operation_count/throughput_window); -} - -int TestMain() { - srand(42); - if (MinThread < 1) - MinThread = 1; - //calibrate_busy_wait(); - cache_aligned_allocator my_mutex_allocator; - my_mutex = (spin_mutex *)my_mutex_allocator.allocate(1); - - REPORT("#Thr "); - REPORT("STL "); -#ifdef LINEARIZABLE - REPORT("CPQ_L\n"); -#else - REPORT("CPQ_N\n"); -#endif - for (int p = MinThread; p <= MaxThread; ++p) { - TestPDESload(p); - } - - return Harness::Done; -} diff --git a/src/tbb/src/perf/fibonacci_impl_tbb.cpp b/src/tbb/src/perf/fibonacci_impl_tbb.cpp deleted file mode 100644 index 158ddb49b..000000000 --- a/src/tbb/src/perf/fibonacci_impl_tbb.cpp +++ /dev/null @@ -1,78 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#include -#include - -#include "tbb/task_scheduler_init.h" -#include "tbb/task.h" -#include "tbb/tick_count.h" - -extern long CutOff; - -long SerialFib( const long n ) { - if( n<2 ) - return n; - else - return SerialFib(n-1)+SerialFib(n-2); -} - -struct FibContinuation: public tbb::task { - long* const sum; - long x, y; - FibContinuation( long* sum_ ) : sum(sum_) {} - tbb::task* execute() { - *sum = x+y; - return NULL; - } -}; - -struct FibTask: public tbb::task { - long n; - long * sum; - FibTask( const long n_, long * const sum_ ) : - n(n_), sum(sum_) - {} - tbb::task* execute() { - if( n -#include -#include -#include -#include - -#include "tbb/tick_count.h" - -#define HARNESS_CUSTOM_MAIN 1 -#include "../src/test/harness.h" -#include "../src/test/harness_barrier.h" - -#include "tbb/task_scheduler_init.h" -#include "tbb/task.h" -#include "tbb/atomic.h" - -#if __linux__ || __APPLE__ || __FreeBSD__ || __NetBSD__ - #include -#endif - -__TBB_PERF_API int NumCpus = tbb::task_scheduler_init::default_num_threads(), - NumThreads, - MaxConcurrency; - -namespace Perf { - -SessionSettings theSettings; - -namespace internal { - - typedef std::vector durations_t; - - static uintptr_t NumRuns = 7; - static duration_t RunDuration = 0.01; - - static const int RateFieldLen = 10; - static const int OvhdFieldLen = 12; - - const char* TestNameColumnTitle = "Test name"; - const char* WorkloadNameColumnTitle = "Workload"; - - size_t TitleFieldLen = 0; - size_t WorkloadFieldLen = 0; - - int TotalConfigs = 0; - int MaxTbbMasters = 1; - - //! Defines the mapping between threads and cores in the undersubscription mode - /** When adding new enumerator, insert it before amLast, and do not specify - its value explicitly. **/ - enum AffinitizationMode { - amFirst = 0, - amDense = amFirst, - amSparse, - //! Used to track the number of supported affinitization modes - amLast - }; - - static const int NumAffinitizationModes = amLast - amFirst; - - const char* AffinitizationModeNames[] = { "dense", "sparse" }; - - int NumActiveAffModes = 1; - - //! Settings of a test run configuration - struct RunConfig { - int my_maxConcurrency; - int my_numThreads; // For task scheduler tests this is number of workers + 1 - int my_numMasters; // Used for task scheduler tests only - int my_affinityMode; // Used for task scheduler tests only - int my_workloadID; - - int NumMasters () const { - return theSettings.my_opts & UseTaskScheduler ? my_numMasters : my_numThreads; - } - }; - - double StandardDeviation ( double avg, const durations_t& d ) { - double std_dev = 0; - for ( uintptr_t i = 0; i < d.size(); ++i ) { - double dev = fabs(d[i] - avg); - std_dev += dev * dev; - } - std_dev = sqrt(std_dev / d.size()); - return std_dev / avg * 100; - } - - void Statistics ( const durations_t& d, - duration_t& avgTime, double& stdDev, - duration_t& minTime, duration_t& maxTime ) - { - minTime = maxTime = avgTime = d[0]; - for ( size_t i = 1; i < d.size(); ++i ) { - avgTime += d[i]; - if ( minTime > d[i] ) - minTime = d[i]; - else if ( maxTime < d[i] ) - maxTime = d[i]; - } - avgTime = avgTime / d.size(); - stdDev = StandardDeviation( avgTime, d ); - } - - //! Timing data for the series of repeated runs and results of their statistical processing - struct TimingSeries { - //! Statistical timing series - durations_t my_durations; - - //! Average time obtained from my_durations data - duration_t my_avgTime; - - //! Minimal time obtained from my_durations data - duration_t my_minTime; - - //! Minimal time obtained from my_durations data - duration_t my_maxTime; - - //! Standard deviation of my_avgTime value (per cent) - double my_stdDev; - - TimingSeries ( uintptr_t nruns = NumRuns ) - : my_durations(nruns), my_avgTime(0), my_minTime(0), my_maxTime(0) - {} - - void CalculateStatistics () { - Statistics( my_durations, my_avgTime, my_stdDev, my_minTime, my_maxTime ); - } - }; // struct TimingSeries - - //! Settings and timing results for a test run configuration - struct RunResults { - //! Run configuration settings - RunConfig my_config; - - //! Timing results for this run configuration - TimingSeries my_timing; - }; - - typedef std::vector names_t; - typedef std::vector timings_t; - typedef std::vector test_results_t; - - enum TestMethods { - idRunSerial = 0x01, - idOnStart = 0x02, - idOnFinish = 0x04, - idPrePostProcess = idOnStart | idOnFinish - }; - - //! Set of flags identifying methods not overridden by the currently active test - /** Used as a scratch var. **/ - uintptr_t g_absentMethods; - - //! Test object and timing results for all of its configurations - struct TestResults { - //! Pointer to the test object interface - Test* my_test; - - //! Set of flags identifying optional methods overridden by my_test - /** A set of ORed TestMethods flags **/ - uintptr_t my_availableMethods; - - //! Vector of serial times for each workload supported by this test - /** Element index in the vector serves as a zero based workload ID. **/ - timings_t my_serialBaselines; - - //! Common baselines for both parallel and serial variants - /** Element index in the vector serves as a zero based workload ID. **/ - timings_t my_baselines; - - //! Strings identifying workloads to be used in output - names_t my_workloadNames; - - //! Vector of timings for all run configurations of my_test - test_results_t my_results; - - const char* my_testName; - - mutable bool my_hasOwnership; - - TestResults ( Test* t, const char* className, bool takeOwnership ) - : my_test(t), my_availableMethods(0), my_testName(className), my_hasOwnership(takeOwnership) - {} - - TestResults ( const TestResults& tr ) - : my_test(tr.my_test) - , my_availableMethods(0) - , my_testName(tr.my_testName) - , my_hasOwnership(tr.my_hasOwnership) - { - tr.my_hasOwnership = false; - } - - ~TestResults () { - for ( size_t i = 0; i < my_workloadNames.size(); ++i ) - delete my_workloadNames[i]; - if ( my_hasOwnership ) - delete my_test; - } - }; // struct TestResults - - typedef std::vector session_t; - - session_t theSession; - - TimingSeries CalibrationTiming; - - const uintptr_t CacheSize = 8*1024*1024; - volatile intptr_t W[CacheSize]; - - struct WiperBody { - void operator()( int ) const { - volatile intptr_t sink = 0; - for ( uintptr_t i = 0; i < CacheSize; ++i ) - sink += W[i]; - } - }; - - void TraceHistogram ( const durations_t& t, const char* histogramFileName ) { - FILE* f = histogramFileName ? fopen(histogramFileName, "wt") : stdout; - uintptr_t n = t.size(); - const uintptr_t num_buckets = 100; - double min_val = *std::min_element(t.begin(), t.end()), - max_val = *std::max_element(t.begin(), t.end()), - bucket_size = (max_val - min_val) / num_buckets; - std::vector hist(num_buckets + 1, 0); - for ( uintptr_t i = 0; i < n; ++i ) - ++hist[uintptr_t((t[i]-min_val)/bucket_size)]; - ASSERT (hist[num_buckets] == 1, ""); - ++hist[num_buckets - 1]; - hist.resize(num_buckets); - fprintf (f, "Histogram: nvals = %u, min = %g, max = %g, nbuckets = %u\n", (unsigned)n, min_val, max_val, (unsigned)num_buckets); - double bucket = min_val; - for ( uintptr_t i = 0; i < num_buckets; ++i, bucket+=bucket_size ) - fprintf (f, "%12g\t%u\n", bucket, (unsigned)hist[i]); - fclose(f); - } - -#if _MSC_VER - typedef DWORD_PTR cpu_set_t; - - class AffinityHelper { - static const unsigned MaxAffinitySetSize = sizeof(cpu_set_t) * 8; - static unsigned AffinitySetSize; - - //! Mapping from a CPU index to a valid affinity cpu_mask - /** The first element is not used. **/ - static cpu_set_t m_affinities[MaxAffinitySetSize + 1]; - - static cpu_set_t m_processMask; - - class Initializer { - public: - Initializer () { - SYSTEM_INFO si; - GetNativeSystemInfo(&si); - ASSERT( si.dwNumberOfProcessors <= MaxAffinitySetSize, "Too many CPUs" ); - AffinitySetSize = min (si.dwNumberOfProcessors, MaxAffinitySetSize); - cpu_set_t systemMask = 0; - GetProcessAffinityMask( GetCurrentProcess(), &m_processMask, &systemMask ); - cpu_set_t cpu_mask = 1; - for ( DWORD i = 0; i < AffinitySetSize; ++i ) { - while ( !(cpu_mask & m_processMask) && cpu_mask ) - cpu_mask <<= 1; - ASSERT( cpu_mask != 0, "Process affinity set is culled?" ); - m_affinities[i] = cpu_mask; - cpu_mask <<= 1; - } - } - }; // class AffinityHelper::Initializer - - static Initializer m_initializer; - - public: - static cpu_set_t CpuAffinity ( int cpuIndex ) { - return m_affinities[cpuIndex % AffinitySetSize]; - } - - static const cpu_set_t& ProcessMask () { return m_processMask; } - }; // class AffinityHelper - - unsigned AffinityHelper::AffinitySetSize = 0; - cpu_set_t AffinityHelper::m_affinities[AffinityHelper::MaxAffinitySetSize + 1] = {0}; - cpu_set_t AffinityHelper::m_processMask = 0; - AffinityHelper::Initializer AffinityHelper::m_initializer; - - #define CPU_ZERO(cpu_mask) (*cpu_mask = 0) - #define CPU_SET(cpu_idx, cpu_mask) (*cpu_mask |= AffinityHelper::CpuAffinity(cpu_idx)) - #define CPU_CLR(cpu_idx, cpu_mask) (*cpu_mask &= ~AffinityHelper::CpuAffinity(cpu_idx)) - #define CPU_ISSET(cpu_idx, cpu_mask) ((*cpu_mask & AffinityHelper::CpuAffinity(cpu_idx)) != 0) - -#elif __linux__ /* end of _MSC_VER */ - - #include - #include - #include - - pid_t gettid() { return (pid_t)syscall(__NR_gettid); } - - #define GET_MASK(cpu_set) (*(unsigned*)(void*)&cpu_set) - #define RES_STAT(res) (res != 0 ? "failed" : "ok") - - class AffinityHelper { - static cpu_set_t m_processMask; - - class Initializer { - public: - Initializer () { - CPU_ZERO (&m_processMask); - int res = sched_getaffinity( getpid(), sizeof(cpu_set_t), &m_processMask ); - ASSERT ( res == 0, "sched_getaffinity failed" ); - } - }; // class AffinityHelper::Initializer - - static Initializer m_initializer; - - public: - static const cpu_set_t& ProcessMask () { return m_processMask; } - }; // class AffinityHelper - - cpu_set_t AffinityHelper::m_processMask; - AffinityHelper::Initializer AffinityHelper::m_initializer; -#endif /* __linux__ */ - - bool PinTheThread ( int cpu_idx, tbb::atomic& nThreads ) { - #if _MSC_VER || __linux__ - cpu_set_t orig_mask, target_mask; - CPU_ZERO( &target_mask ); - CPU_SET( cpu_idx, &target_mask ); - ASSERT ( CPU_ISSET(cpu_idx, &target_mask), "CPU_SET failed" ); - #endif - #if _MSC_VER - orig_mask = SetThreadAffinityMask( GetCurrentThread(), target_mask ); - if ( !orig_mask ) - return false; - #elif __linux__ - CPU_ZERO( &orig_mask ); - int res = sched_getaffinity( gettid(), sizeof(cpu_set_t), &orig_mask ); - ASSERT ( res == 0, "sched_getaffinity failed" ); - res = sched_setaffinity( gettid(), sizeof(cpu_set_t), &target_mask ); - ASSERT ( res == 0, "sched_setaffinity failed" ); - #endif /* _MSC_VER */ - --nThreads; - while ( nThreads ) - __TBB_Yield(); - #if _MSC_VER - SetThreadPriority (GetCurrentThread(), THREAD_PRIORITY_HIGHEST); - #endif - return true; - } - - class AffinitySetterTask : tbb::task { - static bool m_result; - static tbb::atomic m_nThreads; - int m_idx; - - tbb::task* execute () { - //TestAffinityOps(); - m_result = PinTheThread( m_idx, m_nThreads ); - return NULL; - } - - public: - AffinitySetterTask ( int idx ) : m_idx(idx) {} - - friend bool AffinitizeTBB ( int, int /*mode*/ ); - }; - - bool AffinitySetterTask::m_result = true; - tbb::atomic AffinitySetterTask::m_nThreads; - - bool AffinitizeTBB ( int p, int affMode ) { - #if _MSC_VER - SetThreadPriority (GetCurrentThread(), THREAD_PRIORITY_HIGHEST); - SetPriorityClass (GetCurrentProcess(), HIGH_PRIORITY_CLASS); - #endif - AffinitySetterTask::m_result = true; - AffinitySetterTask::m_nThreads = p; - tbb::task_list tl; - for ( int i = 0; i < p; ++i ) { - tbb::task &t = *new( tbb::task::allocate_root() ) AffinitySetterTask( affMode == amSparse ? i * NumCpus / p : i ); - t.set_affinity( tbb::task::affinity_id(i + 1) ); - tl.push_back( t ); - } - tbb::task::spawn_root_and_wait(tl); - return AffinitySetterTask::m_result; - } - - inline - void Affinitize ( int p, int affMode ) { - if ( !AffinitizeTBB (p, affMode) ) - REPORT("Warning: Failed to set affinity for %d TBB threads\n", p); - } - - class TbbWorkersTrapper { - tbb::atomic my_refcount; - tbb::task *my_root; - tbb::task_group_context my_context; - Harness::SpinBarrier my_barrier; - - friend class TrapperTask; - - class TrapperTask : public tbb::task { - TbbWorkersTrapper& my_owner; - - tbb::task* execute () { - my_owner.my_barrier.wait(); - my_owner.my_root->wait_for_all(); - my_owner.my_barrier.wait(); - return NULL; - } - public: - TrapperTask ( TbbWorkersTrapper& owner ) : my_owner(owner) {} - }; - - public: - TbbWorkersTrapper () - : my_context(tbb::task_group_context::bound, - tbb::task_group_context::default_traits | tbb::task_group_context::concurrent_wait) - { - my_root = new ( tbb::task::allocate_root(my_context) ) tbb::empty_task; - my_root->set_ref_count(2); - my_barrier.initialize(NumThreads); - for ( int i = 1; i < NumThreads; ++i ) - tbb::task::spawn( *new(tbb::task::allocate_root()) TrapperTask(*this) ); - my_barrier.wait(); // Wait util all workers are ready - } - - ~TbbWorkersTrapper () { - my_root->decrement_ref_count(); - my_barrier.wait(); // Make sure no tasks are referencing us - tbb::task::destroy(*my_root); - } - }; // TbbWorkersTrapper - - -#if __TBB_STATISTICS - static bool StatisticsMode = true; -#else - static bool StatisticsMode = false; -#endif - -//! Suppresses silly warning -inline bool __TBB_bool( bool b ) { return b; } - -#define START_WORKERS(needScheduler, p, a, setWorkersAffinity, trapWorkers) \ - tbb::task_scheduler_init init(tbb::task_scheduler_init::deferred); \ - TbbWorkersTrapper *trapper = NULL; \ - if ( theSettings.my_opts & UseTaskScheduler \ - && (needScheduler) && ((setWorkersAffinity) || (trapWorkers)) ) \ - { \ - init.initialize( p ); \ - if ( __TBB_bool(setWorkersAffinity) ) \ - Affinitize( p, a ); \ - if ( __TBB_bool(trapWorkers) ) \ - trapper = new TbbWorkersTrapper; \ - } - -#define STOP_WORKERS() \ - if ( theSettings.my_opts & UseTaskScheduler && init.is_active() ) { \ - if ( trapper ) \ - delete trapper; \ - init.terminate(); \ - /* Give asynchronous deinitialization time to complete */ \ - Harness::Sleep(50); \ - } - - typedef void (Test::*RunMemFnPtr)( Test::ThreadInfo& ); - - TimingSeries *TlsTimings; - Harness::SpinBarrier multipleMastersBarrier; - - class TimingFunctor { - Test* my_test; - RunConfig *my_cfg; - RunMemFnPtr my_fnRun; - size_t my_numRuns; - size_t my_numRepeats; - uintptr_t my_availableMethods; - - duration_t TimeSingleRun ( Test::ThreadInfo& ti ) const { - if ( my_availableMethods & idOnStart ) - my_test->OnStart(ti); - // Warming run - (my_test->*my_fnRun)(ti); - multipleMastersBarrier.wait(); - tbb::tick_count t0 = tbb::tick_count::now(); - (my_test->*my_fnRun)(ti); - duration_t t = (tbb::tick_count::now() - t0).seconds(); - if ( my_availableMethods & idOnFinish ) - my_test->OnFinish(ti); - return t; - } - - public: - TimingFunctor ( Test* test, RunConfig *cfg, RunMemFnPtr fnRun, - size_t numRuns, size_t nRepeats, uintptr_t availableMethods ) - : my_test(test), my_cfg(cfg), my_fnRun(fnRun) - , my_numRuns(numRuns), my_numRepeats(nRepeats), my_availableMethods(availableMethods) - {} - - void operator()( int tid ) const { - Test::ThreadInfo ti = { tid, NULL }; - durations_t &d = TlsTimings[tid].my_durations; - bool singleMaster = my_cfg->my_numMasters == 1; - START_WORKERS( (!singleMaster || (singleMaster && StatisticsMode)) && my_fnRun != &Test::RunSerial, - my_cfg->my_numThreads, my_cfg->my_affinityMode, singleMaster, singleMaster ); - for ( uintptr_t k = 0; k < my_numRuns; ++k ) { - if ( my_numRepeats > 1 ) { - d[k] = 0; - if ( my_availableMethods & idPrePostProcess ) { - for ( uintptr_t i = 0; i < my_numRepeats; ++i ) - d[k] += TimeSingleRun(ti); - } - else { - multipleMastersBarrier.wait(); - tbb::tick_count t0 = tbb::tick_count::now(); - for ( uintptr_t i = 0; i < my_numRepeats; ++i ) - (my_test->*my_fnRun)(ti); - d[k] = (tbb::tick_count::now() - t0).seconds(); - } - d[k] /= my_numRepeats; - } - else - d[k] = TimeSingleRun(ti); - } - STOP_WORKERS(); - TlsTimings[tid].CalculateStatistics(); - } - }; // class TimingFunctor - - void DoTiming ( TestResults& tr, RunConfig &cfg, RunMemFnPtr fnRun, size_t nRepeats, TimingSeries& ts ) { - int numThreads = cfg.NumMasters(); - size_t numRuns = ts.my_durations.size() / numThreads; - TimingFunctor body( tr.my_test, &cfg, fnRun, numRuns, nRepeats, tr.my_availableMethods ); - multipleMastersBarrier.initialize(numThreads); - tr.my_test->SetWorkload(cfg.my_workloadID); - if ( numThreads == 1 ) { - TimingSeries *t = TlsTimings; - TlsTimings = &ts; - body(0); - TlsTimings = t; - } - else { - ts.my_durations.resize(numThreads * numRuns); - NativeParallelFor( numThreads, body ); - for ( int i = 0, j = 0; i < numThreads; ++i ) { - durations_t &d = TlsTimings[i].my_durations; - for ( size_t k = 0; k < numRuns; ++k, ++j ) - ts.my_durations[j] = d[k]; - } - ts.CalculateStatistics(); - } - } - - //! Runs the test function, does statistical processing, and, if title is nonzero, prints results. - /** If histogramFileName is a string, the histogram of individual runs is generated and stored - in a file with the given name. If it is NULL then the histogram is printed on the console. - By default no histogram is generated. - The histogram format is: "rate bucket start" "number of tests in this bucket". **/ - void RunTestImpl ( TestResults& tr, RunConfig &cfg, RunMemFnPtr pfnTest, TimingSeries& ts ) { - // nRepeats is a number of repeated calls to the test function made as - // part of the same run. It is determined experimentally by the following - // calibration process so that the total run time was approx. RunDuration. - // This is helpful to increase the measurement precision in case of very - // short tests. - size_t nRepeats = 1; - // A minimal stats is enough when doing calibration - CalibrationTiming.my_durations.resize( (NumRuns < 4 ? NumRuns : 3) * cfg.NumMasters() ); - // There's no need to be too precise when calculating nRepeats. And reasonably - // far extrapolation can speed up the process significantly. - for (;;) { - DoTiming( tr, cfg, pfnTest, nRepeats, CalibrationTiming ); - if ( CalibrationTiming.my_avgTime * nRepeats > 1e-4 ) - break; - nRepeats *= 2; - } - nRepeats *= (uintptr_t)ceil( RunDuration / (CalibrationTiming.my_avgTime * nRepeats) ); - - DoTiming(tr, cfg, pfnTest, nRepeats, ts); - - // No histogram for baseline measurements - if ( pfnTest != &Test::RunSerial && pfnTest != &Test::Baseline ) { - const char* histogramName = theSettings.my_histogramName; - if ( histogramName != NoHistogram && tr.my_test->HistogramName() != DefaultHistogram ) - histogramName = tr.my_test->HistogramName(); - if ( histogramName != NoHistogram ) - TraceHistogram( ts.my_durations, histogramName ); - } - } // RunTestImpl - - typedef void (*TestActionFn) ( TestResults&, int mastersRange, int w, int p, int m, int a, int& numTests ); - - int TestResultIndex ( int mastersRange, int w, int p, int m, int a ) { - return ((w * (MaxThread - MinThread + 1) + (p - MinThread)) * mastersRange + m) * NumActiveAffModes + a; - } - - void RunTest ( TestResults& tr, int mastersRange, int w, int p, int m, int a, int& numTests ) { - size_t r = TestResultIndex(mastersRange, w, p, m, a); - ASSERT( r < tr.my_results.size(), NULL ); - RunConfig &rc = tr.my_results[r].my_config; - rc.my_maxConcurrency = MaxConcurrency; - rc.my_numThreads = p; - rc.my_numMasters = m + tr.my_test->MinNumMasters(); - rc.my_affinityMode = a; - rc.my_workloadID = w; - RunTestImpl( tr, rc, &Test::Run, tr.my_results[r].my_timing ); - printf( "Running tests: %04.1f%%\r", ++numTests * 100. / TotalConfigs ); fflush(stdout); - } - - void WalkTests ( TestActionFn fn, int& numTests, bool setAffinity, bool trapWorkers, bool multipleMasters ) { - for ( int p = MinThread; p <= MaxThread; ++p ) { - NumThreads = p; - MaxConcurrency = p < NumCpus ? p : NumCpus; - for ( int a = 0; a < NumActiveAffModes; ++a ) { - START_WORKERS( multipleMasters || !StatisticsMode, p, a, setAffinity, trapWorkers ); - for ( size_t i = 0; i < theSession.size(); ++i ) { - TestResults &tr = theSession[i]; - Test *t = tr.my_test; - int mastersRange = t->MaxNumMasters() - t->MinNumMasters() + 1; - int numWorkloads = theSettings.my_opts & UseSmallestWorkloadOnly ? 1 : t->NumWorkloads(); - for ( int w = 0; w < numWorkloads; ++w ) { - if ( multipleMasters ) - for ( int m = 1; m < mastersRange; ++m ) - fn( tr, mastersRange, w, p, m, a, numTests ); - else - fn( tr, mastersRange, w, p, 0, a, numTests ); - } - } - STOP_WORKERS(); - } - } - } - - void RunTests () { - int numTests = 0; - WalkTests( &RunTest, numTests, !StatisticsMode, !StatisticsMode, false ); - if ( MaxTbbMasters > 1 ) - WalkTests( &RunTest, numTests, true, false, true ); - } - - void InitTestData ( TestResults& tr, int mastersRange, int w, int p, int m, int a, int& ) { - size_t r = TestResultIndex(mastersRange, w, p, m, a); - ASSERT( r < tr.my_results.size(), NULL ); - tr.my_results[r].my_timing.my_durations.resize( - (theSettings.my_opts & UseTaskScheduler ? tr.my_test->MinNumMasters() + m : p) * NumRuns ); - } - - char WorkloadName[MaxWorkloadNameLen + 1]; - - void PrepareTests () { - printf( "Initializing...\r" ); - NumActiveAffModes = theSettings.my_opts & UseAffinityModes ? NumAffinitizationModes : 1; - TotalConfigs = 0; - TitleFieldLen = strlen( TestNameColumnTitle ); - WorkloadFieldLen = strlen( WorkloadNameColumnTitle ); - int numThreads = MaxThread - MinThread + 1; - int numConfigsBase = numThreads * NumActiveAffModes; - int totalWorkloads = 0; - for ( size_t i = 0; i < theSession.size(); ++i ) { - TestResults &tr = theSession[i]; - Test &t = *tr.my_test; - int numWorkloads = theSettings.my_opts & UseSmallestWorkloadOnly ? 1 : t.NumWorkloads(); - int numConfigs = numConfigsBase * numWorkloads; - if ( t.MaxNumMasters() > 1 ) { - ASSERT( theSettings.my_opts & UseTaskScheduler, "Multiple masters mode is only valid for task scheduler tests" ); - if ( MaxTbbMasters < t.MaxNumMasters() ) - MaxTbbMasters = t.MaxNumMasters(); - numConfigs *= t.MaxNumMasters() - t.MinNumMasters() + 1; - } - totalWorkloads += numWorkloads; - TotalConfigs += numConfigs; - - const char* testName = t.Name(); - if ( testName ) - tr.my_testName = testName; - ASSERT( tr.my_testName, "Neither Test::Name() is implemented, nor RTTI is enabled" ); - TitleFieldLen = max( TitleFieldLen, strlen(tr.my_testName) ); - - tr.my_results.resize( numConfigs ); - tr.my_serialBaselines.resize( numWorkloads ); - tr.my_baselines.resize( numWorkloads ); - tr.my_workloadNames.resize( numWorkloads ); - } - TimingSeries tmpTiming; - TlsTimings = &tmpTiming; // All measurements are serial here - int n = 0; - for ( size_t i = 0; i < theSession.size(); ++i ) { - TestResults &tr = theSession[i]; - Test &t = *tr.my_test; - // Detect which methods are overridden by the test implementation - g_absentMethods = 0; - Test::ThreadInfo ti = { 0 }; - t.SetWorkload(0); - t.OnStart(ti); - t.RunSerial(ti); - t.OnFinish(ti); - if ( theSettings.my_opts & UseSerialBaseline && !(g_absentMethods & idRunSerial) ) - tr.my_availableMethods |= idRunSerial; - if ( !(g_absentMethods & idOnStart) ) - tr.my_availableMethods |= idOnStart; - - RunConfig rc = { 1, 1, 1, 0, 0 }; - int numWorkloads = theSettings.my_opts & UseSmallestWorkloadOnly ? 1 : t.NumWorkloads(); - for ( int w = 0; w < numWorkloads; ++w ) { - WorkloadName[0] = 0; - t.SetWorkload(w); - if ( !WorkloadName[0] ) - sprintf( WorkloadName, "%d", w ); - size_t len = strlen(WorkloadName); - tr.my_workloadNames[w] = new char[len + 1]; - strcpy ( (char*)tr.my_workloadNames[w], WorkloadName ); - WorkloadFieldLen = max( WorkloadFieldLen, len ); - - rc.my_workloadID = w; - if ( theSettings.my_opts & UseBaseline ) - RunTestImpl( tr, rc, &Test::Baseline, tr.my_baselines[w] ); - if ( tr.my_availableMethods & idRunSerial ) - RunTestImpl( tr, rc, &Test::RunSerial, tr.my_serialBaselines[w] ); - printf( "Measuring baselines: %04.1f%%\r", ++n * 100. / totalWorkloads ); fflush(stdout); - } - } - TlsTimings = new TimingSeries[MaxThread + MaxTbbMasters - 1]; - if ( theSettings.my_opts & UseTaskScheduler ? MaxTbbMasters : MaxThread ) - WalkTests( &InitTestData, n, false, false, theSettings.my_opts & UseTaskScheduler ? true : false ); - CalibrationTiming.my_durations.reserve( MaxTbbMasters * 3 ); - printf( " \r"); - } - - FILE* ResFile = NULL; - - void Report ( char const* fmt, ... ) { - va_list args; - if ( ResFile ) { - va_start( args, fmt ); - vfprintf( ResFile, fmt, args ); - va_end( args ); - } - va_start( args, fmt ); - vprintf( fmt, args ); - va_end( args ); - } - - void PrintResults () { - if ( theSettings.my_resFile ) - ResFile = fopen( theSettings.my_resFile, "w" ); - Report( "%-*s %-*s %s", TitleFieldLen, "Test-name", WorkloadFieldLen, "Workload", - MaxTbbMasters > 1 ? "W M " : "T " ); - if ( theSettings.my_opts & UseAffinityModes ) - Report( "Aff " ); - Report( "%-*s SD,%% %-*s %-*s %-*s ", - RateFieldLen, "Avg.time", OvhdFieldLen, "Par.ovhd,%", - RateFieldLen, "Min.time", RateFieldLen, "Max.time" ); - Report( " | Repeats = %lu, CPUs %d\n", (unsigned long)NumRuns, NumCpus ); - for ( size_t i = 0; i < theSession.size(); ++i ) { - TestResults &tr = theSession[i]; - for ( size_t j = 0; j < tr.my_results.size(); ++j ) { - RunResults &rr = tr.my_results[j]; - RunConfig &rc = rr.my_config; - int w = rc.my_workloadID; - TimingSeries &ts = rr.my_timing; - duration_t baselineTime = tr.my_baselines[w].my_avgTime, - cleanTime = ts.my_avgTime - baselineTime; - Report( "%-*s %-*s ", TitleFieldLen, tr.my_testName, WorkloadFieldLen, tr.my_workloadNames[w] ); - if ( MaxTbbMasters > 1 ) - Report( "%-4d %-4d ", rc.my_numThreads - 1, rc.my_numMasters ); - else - Report( "%-4d ", rc.my_numThreads ); - if ( theSettings.my_opts & UseAffinityModes ) - Report( "%%-8s ", AffinitizationModeNames[rc.my_affinityMode] ); - Report( "%-*.2e %-6.1f ", RateFieldLen, cleanTime, ts.my_stdDev); - if ( tr.my_availableMethods & idRunSerial ) { - duration_t serialTime = (tr.my_serialBaselines[w].my_avgTime - baselineTime) / rc.my_maxConcurrency; - Report( "%-*.1f ", OvhdFieldLen, 100*(cleanTime - serialTime)/serialTime ); - } - else - Report( "%*s%*s ", OvhdFieldLen/2, "-", OvhdFieldLen - OvhdFieldLen/2, "" ); - Report( "%-*.2e %-*.2e ", RateFieldLen, ts.my_minTime - baselineTime, RateFieldLen, ts.my_maxTime - baselineTime); - Report( "\n" ); - } - } - delete [] TlsTimings; - if ( ResFile ) - fclose(ResFile); - } - - __TBB_PERF_API void RegisterTest ( Test* t, const char* className, bool takeOwnership ) { - // Just collect test objects at this stage - theSession.push_back( TestResults(t, className, takeOwnership) ); - } - -} // namespace internal - -__TBB_PERF_API void Test::Baseline ( ThreadInfo& ) {} - -__TBB_PERF_API void Test::RunSerial ( ThreadInfo& ) { internal::g_absentMethods |= internal::idRunSerial; } - -__TBB_PERF_API void Test::OnStart ( ThreadInfo& ) { internal::g_absentMethods |= internal::idOnStart; } - -__TBB_PERF_API void Test::OnFinish ( ThreadInfo& ) { internal::g_absentMethods |= internal::idOnFinish; } - -__TBB_PERF_API void WipeCaches () { NativeParallelFor( NumCpus, internal::WiperBody() ); } - -__TBB_PERF_API void EmptyFunc () {} -__TBB_PERF_API void AnchorFunc ( void* ) {} -__TBB_PERF_API void AnchorFunc2 ( void*, void* ) {} - -__TBB_PERF_API void SetWorkloadName( const char* format, ... ) { - internal::WorkloadName[MaxWorkloadNameLen] = 0; - va_list args; - va_start(args, format); - vsnprintf( internal::WorkloadName, MaxWorkloadNameLen, format, args ); - va_end(args); -} - - -__TBB_PERF_API int TestMain( int argc, char* argv[], const SessionSettings* defaultSettings ) { -#if _MSC_VER - HANDLE hMutex = CreateMutex( NULL, FALSE, "Global\\TBB_OMP_PerfSession" ); - WaitForSingleObject( hMutex, INFINITE ); -#endif - MinThread = MaxThread = NumCpus; - if ( defaultSettings ) - theSettings = *defaultSettings; - ParseCommandLine( argc, argv ); // May override data in theSettings - - internal::PrepareTests (); - internal::RunTests (); - internal::PrintResults(); - REPORT("\n"); -#if _MSC_VER - ReleaseMutex( hMutex ); - CloseHandle( hMutex ); -#endif - return 0; -} - -} // namespace Perf diff --git a/src/tbb/src/perf/perf.h b/src/tbb/src/perf/perf.h deleted file mode 100644 index fb5f2ce69..000000000 --- a/src/tbb/src/perf/perf.h +++ /dev/null @@ -1,257 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __tbb_perf_h__ -#define __tbb_perf_h__ - -#ifndef TBB_PERF_TYPEINFO -#define TBB_PERF_TYPEINFO 1 -#endif - -#if TBB_PERF_TYPEINFO - #include - #define __TBB_PERF_TEST_CLASS_NAME(T) typeid(T).name() -#else /* !TBB_PERF_TYPEINFO */ - #define __TBB_PERF_TEST_CLASS_NAME(T) NULL -#endif /* !TBB_PERF_TYPEINFO */ - - -#include "tbb/tick_count.h" - -// TODO: Fix build scripts to provide more reliable build phase identification means -#ifndef __TBB_PERF_API -#if _USRDLL - #if _MSC_VER - #define __TBB_PERF_API __declspec(dllexport) - #else /* !_MSC_VER */ - #define __TBB_PERF_API - #endif /* !_MSC_VER */ -#else /* !_USRDLL */ - #if _MSC_VER - #define __TBB_PERF_API __declspec(dllimport) - #else /* !_MSC_VER */ - #define __TBB_PERF_API - #endif /* !_MSC_VER */ -#endif /* !_USRDLL */ -#endif /* !__TBB_PERF_API */ - -#if _WIN32||_WIN64 - -namespace Perf { - typedef unsigned __int64 tick_t; - #if defined(_M_X64) - inline tick_t rdtsc () { return __rdtsc(); } - #elif _M_IX86 - inline tick_t rdtsc () { __asm { rdtsc } } - #else - #error Unsupported ISA - #endif -} // namespace Perf - -#elif __linux__ || __APPLE__ - -#include - -namespace Perf { - typedef uint64_t tick_t; - #if __x86_64__ || __i386__ || __i386 - inline tick_t rdtsc () { - uint32_t lo, hi; - __asm__ __volatile__ ( "rdtsc" : "=a" (lo), "=d" (hi) ); - return (tick_t)lo | ((tick_t)hi) << 32; - } - #else - #error Unsupported ISA - #endif -} // namespace Perf - -#else - #error Unsupported OS -#endif /* OS */ - -__TBB_PERF_API extern int NumThreads, - MaxConcurrency, - NumCpus; - -// Functions and global variables provided by the benchmarking framework -namespace Perf { - -typedef double duration_t; - -static const int MaxWorkloadNameLen = 64; - -static const char* NoHistogram = (char*)-1; -static const char* DefaultHistogram = (char*)-2; - -__TBB_PERF_API void AnchorFunc ( void* ); -__TBB_PERF_API void AnchorFunc2 ( void*, void* ); - -//! Helper that can be used in the preprocess handler to clean caches -/** Cleaning caches is necessary to obtain reproducible results when a test - accesses significant ranges of memory. **/ -__TBB_PERF_API void WipeCaches (); - -//! Specifies the name to be used to designate the current workload in output -/** Should be used from Test::SetWorkload(). If necessary workload name will be - truncated to MaxWorkloadNameLen characters. **/ -__TBB_PERF_API void SetWorkloadName( const char* format, ... ); - -class __TBB_PERF_API Test { -public: - virtual ~Test () {} - - //! Struct used by tests running in multiple masters mode - struct ThreadInfo { - //! Zero based thread ID - int tid; - //! Pointer to test specific data - /** If used by the test, should be initialized by OnStartLocal(), and - finalized by OnFinishLocal(). **/ - void* data; - }; - - //////////////////////////////////////////////////////////////////////////////// - // Mandatory methods - - //! Returns the number of workloads supported - virtual int NumWorkloads () = 0; - - //! Set workload info for the subsequent calls to Run() and RunSerial() - /** This method can use global helper function Perf::SetWorkloadName() in order - to specify the name of the current workload, which will be used in output - to designate the workload. If SetWorkloadName is not called, workloadIndex - will be used for this purpose. - - When testing task scheduler, make sure that this method does not trigger - its automatic initialization. **/ - virtual void SetWorkload ( int workloadIndex ) = 0; - - //! Test implementation - /** Called by the timing framework several times in a loop to achieve approx. - RunDuration time, and this loop is timed NumRuns times to collect statistics. - Argument ti specifies information about the master thread calling this method. **/ - virtual void Run ( ThreadInfo& ti ) = 0; - - //////////////////////////////////////////////////////////////////////////////// - // Optional methods - - //! Returns short title string to be used in the regular output to identify the test - /** Should uniquely identify the test among other ones in the given benchmark suite. - If not implemented, the test implementation class' RTTI name is used. **/ - virtual const char* Name () { return NULL; }; - - //! Returns minimal number of master threads - /** Used for task scheduler tests only (when UseTbbScheduler option is specified - in session settings). **/ - virtual int MinNumMasters () { return 1; } - - //! Returns maximal number of master threads - /** Used for task scheduler tests only (when UseTbbScheduler option is specified - in session settings). **/ - virtual int MaxNumMasters () { return 1; } - - //! Executes serial workload equivalent to the one processed by Run() - /** Called by the timing framework several times in a loop to collect statistics. **/ - virtual void RunSerial ( ThreadInfo& ti ); - - //! Invoked before each call to Run() - /** Can be used to preinitialize data necessary for the test, clean up - caches (see Perf::WipeCaches), etc. - In multiple masters mode this method is called on each thread. **/ - virtual void OnStart ( ThreadInfo& ti ); - - //! Invoked after each call to Run() - /** Can be used to free resources allocated by OnStart(). - Note that this method must work correctly independently of whether Run(), - RunSerial() or nothing is called between OnStart() and OnFinish(). - In multiple masters mode this method is called on each thread. **/ - virtual void OnFinish ( ThreadInfo& ti ); - - //! Functionality, the cost of which has to be factored out from timing results - /** Applies to both parallel and serial versions. **/ - virtual void Baseline ( ThreadInfo& ); - - //! Returns description string to be used in the benchmark info/summary output - virtual const char* Description () { return NULL; } - - //! Specifies if the histogram of individual run times in a series - /** If the method is not overridden, histogramName argument of TestMain is used. **/ - virtual const char* HistogramName () { return DefaultHistogram; } -}; // class Test - -namespace internal { - __TBB_PERF_API void RegisterTest ( Test*, const char* testClassName, bool takeOwnership ); -} - -template -void RegisterTest() { internal::RegisterTest( new T, __TBB_PERF_TEST_CLASS_NAME(T), true ); } - -template -void RegisterTest( T& t ) { internal::RegisterTest( &t, __TBB_PERF_TEST_CLASS_NAME(T), false ); } - -enum SessionOptions { - //! Use Test::RunSerial if present - UseBaseline = 0x01, - UseSerialBaseline = 0x02, - UseBaselines = UseBaseline | UseSerialBaseline, - UseTaskScheduler = 0x10, - UseAffinityModes = 0x20, - UseSmallestWorkloadOnly = 0x40 -}; - -struct SessionSettings { - //! A combination of SessionOptions flags - uintptr_t my_opts; - - //! Name of a file to store performance results - /** These results are duplicates of what is printed on the console. **/ - const char* my_resFile; - - //! Output destination for the histogram of individual run times in a series - /** If it is a string, the histogram is stored in a file with such name. - If it is NULL, the histogram is printed on the console. By default histograms - are suppressed. - - The histogram is formatted as two column table: - "time bucket start" "number of tests in this bucket" - - When this setting enables histogram generation, an individual test - can override it by implementing HistogramName method. **/ - const char* my_histogramName; - - SessionSettings ( uintptr_t opts = 0, const char* resFile = NULL, const char* histogram = NoHistogram ) - : my_opts(opts) - , my_resFile(resFile) - , my_histogramName(histogram) - {} -}; // struct SessionSettings - -//! Benchmarking session entry point -/** Executes all the individual tests registered previously by means of - RegisterTest **/ -__TBB_PERF_API int TestMain( int argc, char* argv[], - const SessionSettings* defaultSettings = NULL ); - - -} // namespace Perf - -#endif /* __tbb_perf_h__ */ - - diff --git a/src/tbb/src/perf/perf_sched.cpp b/src/tbb/src/perf/perf_sched.cpp deleted file mode 100644 index 321fed68f..000000000 --- a/src/tbb/src/perf/perf_sched.cpp +++ /dev/null @@ -1,456 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#include "perf.h" - -#include - -#include "tbb/blocked_range.h" -#include "tbb/parallel_for.h" -#include "tbb/parallel_reduce.h" - -#define NUM_CHILD_TASKS 2096 -#define NUM_ROOT_TASKS 256 - -#define N 100000000 -#define FINEST_GRAIN 10 -#define FINE_GRAIN 50 -#define MED_GRAIN 200 -#define COARSE_GRAIN 1000 - - -typedef int count_t; - -const count_t N_finest = (count_t)(N/log((double)N)/10); -const count_t N_fine = N_finest * 20; -const count_t N_med = N_fine * (count_t)log((double)N) / 5; - -class StaticTaskHolder { -public: - tbb::task *my_leafTaskPtr; - StaticTaskHolder (); -}; - -static StaticTaskHolder s_tasks; - -static count_t NumIterations; -static count_t NumLeafTasks; -static count_t NumRootTasks; - -class LeafTaskBase : public tbb::task { -public: - count_t my_ID; - - LeafTaskBase () {} - LeafTaskBase ( count_t id ) : my_ID(id) {} -}; - -class SimpleLeafTask : public LeafTaskBase { - task* execute () { - volatile count_t anchor = 0; - for ( count_t i=0; i < NumIterations; ++i ) - anchor += i; - return NULL; - } -public: - SimpleLeafTask ( count_t ) {} -}; - -StaticTaskHolder::StaticTaskHolder () { - static SimpleLeafTask s_t1(0); - my_leafTaskPtr = &s_t1; -} - -class Test_SPMC : public Perf::Test { -protected: - static const int numWorkloads = 4; - static const count_t workloads[numWorkloads]; - - LeafTaskBase* my_leafTaskPtr; - - const char* Name () { return "SPMC"; } - - int NumWorkloads () { return numWorkloads; } - - void SetWorkload ( int idx ) { - NumRootTasks = 1; - NumIterations = workloads[idx]; - NumLeafTasks = NUM_CHILD_TASKS * NUM_ROOT_TASKS / (NumIterations > 1000 ? 32 : 8); - Perf::SetWorkloadName( "%dx%d", NumLeafTasks, NumIterations ); - } - - void Run ( ThreadInfo& ) { - tbb::empty_task &r = *new( tbb::task::allocate_root() ) tbb::empty_task; - r.set_ref_count( NumLeafTasks + 1 ); - for ( count_t i = 0; i < NumLeafTasks; ++i ) - r.spawn( *new(r.allocate_child()) SimpleLeafTask(0) ); - r.wait_for_all(); - tbb::task::destroy(r); - } - - void RunSerial ( ThreadInfo& ) { - const count_t n = NumLeafTasks * NumRootTasks; - for ( count_t i=0; i < n; ++i ) { - my_leafTaskPtr->my_ID = i; - my_leafTaskPtr->execute(); - } - } - -public: - Test_SPMC ( LeafTaskBase* leafTaskPtr = NULL ) { - static SimpleLeafTask t(0); - my_leafTaskPtr = leafTaskPtr ? leafTaskPtr : &t; - } -}; // class Test_SPMC - -const count_t Test_SPMC::workloads[Test_SPMC::numWorkloads] = { 1, 50, 500, 5000 }; - -template -class LeavesLauncherTask : public tbb::task { - count_t my_groupId; - - task* execute () { - count_t base = my_groupId * NumLeafTasks; - set_ref_count(NumLeafTasks + 1); - for ( count_t i = 0; i < NumLeafTasks; ++i ) - spawn( *new(allocate_child()) LeafTask(base + i) ); - wait_for_all(); - return NULL; - } -public: - LeavesLauncherTask ( count_t groupId ) : my_groupId(groupId) {} -}; - -template -void RunShallowTree () { - tbb::empty_task &r = *new( tbb::task::allocate_root() ) tbb::empty_task; - r.set_ref_count( NumRootTasks + 1 ); - for ( count_t i = 0; i < NumRootTasks; ++i ) - r.spawn( *new(r.allocate_child()) LeavesLauncherTask(i) ); - r.wait_for_all(); - tbb::task::destroy(r); -} - -class Test_ShallowTree : public Test_SPMC { - const char* Name () { return "ShallowTree"; } - - void SetWorkload ( int idx ) { - NumRootTasks = NUM_ROOT_TASKS; - NumIterations = workloads[idx]; - NumLeafTasks = NumIterations > 200 ? NUM_CHILD_TASKS / 10 : - (NumIterations > 50 ? NUM_CHILD_TASKS / 2 : NUM_CHILD_TASKS * 2); - Perf::SetWorkloadName( "%dx%d", NumRootTasks * NumLeafTasks, NumIterations ); - } - - void Run ( ThreadInfo& ) { - RunShallowTree(); - } -}; // class Test_ShallowTree - -class LeafTaskSkewed : public LeafTaskBase { - task* execute () { - volatile count_t anchor = 0; - double K = (double)NumRootTasks * NumLeafTasks; - count_t n = count_t(sqrt(double(my_ID)) * double(my_ID) * my_ID / (4 * K * K)); - for ( count_t i = 0; i < n; ++i ) - anchor += i; - return NULL; - } -public: - LeafTaskSkewed ( count_t id ) : LeafTaskBase(id) {} -}; - -class Test_ShallowTree_Skewed : public Test_SPMC { - static LeafTaskSkewed SerialTaskBody; - - const char* Name () { return "ShallowTree_Skewed"; } - - int NumWorkloads () { return 1; } - - void SetWorkload ( int ) { - NumRootTasks = NUM_ROOT_TASKS; - NumLeafTasks = NUM_CHILD_TASKS; - Perf::SetWorkloadName( "%d", NumRootTasks * NumLeafTasks ); - } - - void Run ( ThreadInfo& ) { - RunShallowTree(); - } - -public: - Test_ShallowTree_Skewed () : Test_SPMC(&SerialTaskBody) {} -}; // class Test_ShallowTree_Skewed - -LeafTaskSkewed Test_ShallowTree_Skewed::SerialTaskBody(0); - -typedef tbb::blocked_range range_t; - -static count_t IterRange = N, - IterGrain = 1; - -enum PartitionerType { - SimplePartitioner = 0, - AutoPartitioner = 1 -}; - -class Test_Algs : public Perf::Test { -protected: - static const int numWorkloads = 4; - static const count_t algRanges[numWorkloads]; - static const count_t algGrains[numWorkloads]; - - tbb::simple_partitioner my_simplePartitioner; - tbb::auto_partitioner my_autoPartitioner; - PartitionerType my_partitionerType; - - bool UseAutoPartitioner () const { return my_partitionerType == AutoPartitioner; } - - int NumWorkloads () { return UseAutoPartitioner() ? 3 : numWorkloads; } - - void SetWorkload ( int idx ) { - if ( UseAutoPartitioner() ) { - IterRange = algRanges[idx ? numWorkloads - 1 : 0]; - IterGrain = idx > 1 ? algGrains[numWorkloads - 1] : 1; - } - else { - IterRange = algRanges[idx]; - IterGrain = algGrains[idx]; - } - Perf::SetWorkloadName( "%d/%d", IterRange, IterGrain ); - } -public: - Test_Algs ( PartitionerType pt = SimplePartitioner ) : my_partitionerType(pt) {} -}; // class Test_Algs - -const count_t Test_Algs::algRanges[] = {N_finest, N_fine, N_med, N}; -const count_t Test_Algs::algGrains[] = {1, FINE_GRAIN, MED_GRAIN, COARSE_GRAIN}; - -template -class Test_PFor : public Test_Algs { -protected: - void Run ( ThreadInfo& ) { - if ( UseAutoPartitioner() ) - tbb::parallel_for( range_t(0, IterRange, IterGrain), Body(), my_autoPartitioner ); - else - tbb::parallel_for( range_t(0, IterRange, IterGrain), Body(), my_simplePartitioner ); - } - - void RunSerial ( ThreadInfo& ) { - Body body; - body( range_t(0, IterRange, IterGrain) ); - } -public: - Test_PFor ( PartitionerType pt = SimplePartitioner ) : Test_Algs(pt) {} -}; // class Test_PFor - -class SimpleForBody { -public: - void operator()( const range_t& r ) const { - count_t end = r.end(); - volatile count_t anchor = 0; - for( count_t i = r.begin(); i < end; ++i ) - anchor += i; - } -}; // class SimpleForBody - -class Test_PFor_Simple : public Test_PFor { -protected: - const char* Name () { return UseAutoPartitioner() ? "PFor-AP" : "PFor"; } -public: - Test_PFor_Simple ( PartitionerType pt = SimplePartitioner ) : Test_PFor(pt) {} -}; // class Test_PFor_Simple - -class SkewedForBody { -public: - void operator()( const range_t& r ) const { - count_t end = (r.end() + 1) * (r.end() + 1); - volatile count_t anchor = 0; - for( count_t i = r.begin() * r.begin(); i < end; ++i ) - anchor += i; - } -}; // class SkewedForBody - -class Test_PFor_Skewed : public Test_PFor { - typedef Test_PFor base_type; -protected: - const char* Name () { return UseAutoPartitioner() ? "PFor-Skewed-AP" : "PFor-Skewed"; } - - void SetWorkload ( int idx ) { - base_type::SetWorkload(idx); - IterRange = (count_t)(sqrt((double)IterRange) * sqrt(sqrt((double)N / IterRange))); - Perf::SetWorkloadName( "%d", IterRange ); - } - -public: - Test_PFor_Skewed ( PartitionerType pt = SimplePartitioner ) : base_type(pt) {} -}; // class Test_PFor_Skewed - -PartitionerType gPartitionerType; -count_t NestingRange; -count_t NestingGrain; - -class NestingForBody { - count_t my_depth; - tbb::simple_partitioner my_simplePartitioner; - tbb::auto_partitioner my_autoPartitioner; - - template - void run ( const range_t& r, Partitioner& p ) const { - count_t end = r.end(); - if ( my_depth > 1 ) - for ( count_t i = r.begin(); i < end; ++i ) - tbb::parallel_for( range_t(0, IterRange, IterGrain), NestingForBody(my_depth - 1), p ); - else - for ( count_t i = r.begin(); i < end; ++i ) - tbb::parallel_for( range_t(0, IterRange, IterGrain), SimpleForBody(), p ); - } -public: - void operator()( const range_t& r ) const { - if ( gPartitionerType == AutoPartitioner ) - run( r, my_autoPartitioner ); - else - run( r, my_simplePartitioner ); - } - NestingForBody ( count_t depth = 1 ) : my_depth(depth) {} -}; // class NestingForBody - -enum NestingType { - HollowNesting, - ShallowNesting, - DeepNesting -}; - -class Test_PFor_Nested : public Test_Algs { - typedef Test_Algs base_type; - - NestingType my_nestingType; - count_t my_nestingDepth; - -protected: - const char* Name () { - static const char* names[] = { "PFor-HollowNested", "PFor-HollowNested-AP", - "PFor-ShallowNested", "PFor-ShallowNested-AP", - "PFor-DeeplyNested", "PFor-DeeplyNested-AP" }; - return names[my_nestingType * 2 + my_partitionerType]; - } - - int NumWorkloads () { return my_nestingType == ShallowNesting ? (UseAutoPartitioner() ? 3 : 2) : 1; } - - void SetWorkload ( int idx ) { - gPartitionerType = my_partitionerType; - if ( my_nestingType == DeepNesting ) { - NestingRange = 1024; - IterGrain = NestingGrain = 1; - IterRange = 4; - my_nestingDepth = 4; - } - else if ( my_nestingType == ShallowNesting ) { - int i = idx ? numWorkloads - 1 : 0; - count_t baseRange = algRanges[i]; - count_t baseGrain = !UseAutoPartitioner() || idx > 1 ? algGrains[i] : 1; - NestingRange = IterRange = (count_t)sqrt((double)baseRange); - NestingGrain = IterGrain = (count_t)sqrt((double)baseGrain); - } - else { - NestingRange = N / 100; - NestingGrain = COARSE_GRAIN / 10; - IterRange = 2; - IterGrain = 1; - } - Perf::SetWorkloadName( "%d/%d", NestingRange, NestingGrain ); - } - - void Run ( ThreadInfo& ) { - if ( UseAutoPartitioner() ) - tbb::parallel_for( range_t(0, NestingRange, NestingGrain), NestingForBody(my_nestingDepth), my_autoPartitioner ); - else - tbb::parallel_for( range_t(0, NestingRange, NestingGrain), NestingForBody(my_nestingDepth), my_simplePartitioner ); - } - - void RunSerial ( ThreadInfo& ) { - for ( int i = 0; i < NestingRange; ++i ) { - SimpleForBody body; - body( range_t(0, IterRange, IterGrain) ); - } - } -public: - Test_PFor_Nested ( NestingType nt, PartitionerType pt ) : base_type(pt), my_nestingType(nt), my_nestingDepth(1) {} -}; // class Test_PFor_Nested - -class SimpleReduceBody { -public: - count_t my_sum; - SimpleReduceBody () : my_sum(0) {} - SimpleReduceBody ( SimpleReduceBody&, tbb::split ) : my_sum(0) {} - void join( SimpleReduceBody& rhs ) { my_sum += rhs.my_sum;} - void operator()( const range_t& r ) { - count_t end = r.end(); - volatile count_t anchor = 0; - for( count_t i = r.begin(); i < end; ++i ) - anchor += i; - my_sum = anchor; - } -}; // class SimpleReduceBody - -class Test_PReduce : public Test_Algs { -protected: - const char* Name () { return UseAutoPartitioner() ? "PReduce-AP" : "PReduce"; } - - void Run ( ThreadInfo& ) { - SimpleReduceBody body; - if ( UseAutoPartitioner() ) - tbb::parallel_reduce( range_t(0, IterRange, IterGrain), body, my_autoPartitioner ); - else - tbb::parallel_reduce( range_t(0, IterRange, IterGrain), body, my_simplePartitioner ); - } - - void RunSerial ( ThreadInfo& ) { - SimpleReduceBody body; - body( range_t(0, IterRange, IterGrain) ); - } -public: - Test_PReduce ( PartitionerType pt = SimplePartitioner ) : Test_Algs(pt) {} -}; // class Test_PReduce - -int main( int argc, char* argv[] ) { - Perf::SessionSettings opts (Perf::UseTaskScheduler | Perf::UseSerialBaseline, "perf_sched.txt"); // Perf::UseBaseline, Perf::UseSmallestWorkloadOnly - Perf::RegisterTest(); - Perf::RegisterTest(); - Perf::RegisterTest(); - Test_PFor_Simple pf_sp(SimplePartitioner), pf_ap(AutoPartitioner); - Perf::RegisterTest(pf_sp); - Perf::RegisterTest(pf_ap); - Test_PReduce pr_sp(SimplePartitioner), pr_ap(AutoPartitioner); - Perf::RegisterTest(pr_sp); - Perf::RegisterTest(pr_ap); - Test_PFor_Skewed pf_s_sp(SimplePartitioner), pf_s_ap(AutoPartitioner); - Perf::RegisterTest(pf_s_sp); - Perf::RegisterTest(pf_s_ap); - Test_PFor_Nested pf_hn_sp(HollowNesting, SimplePartitioner), pf_hn_ap(HollowNesting, AutoPartitioner), - pf_sn_sp(ShallowNesting, SimplePartitioner), pf_sn_ap(ShallowNesting, AutoPartitioner), - pf_dn_sp(DeepNesting, SimplePartitioner), pf_dn_ap(DeepNesting, AutoPartitioner); - Perf::RegisterTest(pf_hn_sp); - Perf::RegisterTest(pf_hn_ap); - Perf::RegisterTest(pf_sn_sp); - Perf::RegisterTest(pf_sn_ap); - Perf::RegisterTest(pf_dn_sp); - Perf::RegisterTest(pf_dn_ap); - return Perf::TestMain(argc, argv, &opts); -} diff --git a/src/tbb/src/perf/run_statistics.sh b/src/tbb/src/perf/run_statistics.sh deleted file mode 100644 index ac7ffd7cd..000000000 --- a/src/tbb/src/perf/run_statistics.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash -# -# Copyright (c) 2005-2017 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -# -# - -export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH -#setting output format .csv, 'pivot' - is pivot table mode, ++ means append -export STAT_FORMAT=pivot-csv++ -#check existing files because of apend mode -ls *.csv -rm -i *.csv -#setting a delimiter in txt or csv file -#export STAT_DELIMITER=, -export STAT_RUNINFO1=Host=`hostname -s` -#append a suffix after the filename -#export STAT_SUFFIX=$STAT_RUNINFO1 -for ((i=1;i<=${repeat:=100};++i)); do echo $i of $repeat: && STAT_RUNINFO2=Run=$i $* || break; done diff --git a/src/tbb/src/perf/statistics.cpp b/src/tbb/src/perf/statistics.cpp deleted file mode 100644 index f0c190b4d..000000000 --- a/src/tbb/src/perf/statistics.cpp +++ /dev/null @@ -1,444 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#include "statistics.h" -#include "statistics_xml.h" - -#define COUNT_PARAMETERS 3 - -#ifdef _MSC_VER -#define snprintf _snprintf -#endif - -void GetTime(char* buff,int size_buff) -{ - tm *newtime; - time_t timer; - time(&timer); - newtime=localtime(&timer); - strftime(buff,size_buff,"%H:%M:%S",newtime); -} - -void GetDate(char* buff,int size_buff) -{ - tm *newtime; - time_t timer; - time(&timer); - newtime=localtime(&timer); - strftime(buff,size_buff,"%Y-%m-%d",newtime); -} - - -StatisticsCollector::TestCase StatisticsCollector::SetTestCase(const char *name, const char *mode, int threads) -{ - string KeyName(name); - switch (SortMode) - { - case ByThreads: KeyName += Format("_%02d_%s", threads, mode); break; - default: - case ByAlg: KeyName += Format("_%s_%02d", mode, threads); break; - } - CurrentKey = Statistics[KeyName]; - if(!CurrentKey) { - CurrentKey = new StatisticResults; - CurrentKey->Mode = mode; - CurrentKey->Name = name; - CurrentKey->Threads = threads; - CurrentKey->Results.reserve(RoundTitles.size()); - Statistics[KeyName] = CurrentKey; - } - return TestCase(CurrentKey); -} - -StatisticsCollector::~StatisticsCollector() -{ - for(Statistics_t::iterator i = Statistics.begin(); i != Statistics.end(); i++) - delete i->second; -} - -void StatisticsCollector::ReserveRounds(size_t index) -{ - size_t i = RoundTitles.size(); - if (i > index) return; - char buf[16]; - RoundTitles.resize(index+1); - for(; i <= index; i++) { - snprintf( buf, 15, "%u", unsigned(i+1) ); - RoundTitles[i] = buf; - } - for(Statistics_t::iterator i = Statistics.begin(); i != Statistics.end(); i++) { - if(!i->second) printf("!!!'%s' = NULL\n", i->first.c_str()); - else i->second->Results.reserve(index+1); - } -} - -void StatisticsCollector::AddRoundResult(const TestCase &key, value_t v) -{ - ReserveRounds(key.access->Results.size()); - key.access->Results.push_back(v); -} - -void StatisticsCollector::SetRoundTitle(size_t index, const char *fmt, ...) -{ - vargf2buff(buff, 128, fmt); - ReserveRounds(index); - RoundTitles[index] = buff; -} - -void StatisticsCollector::AddStatisticValue(const TestCase &key, const char *type, const char *fmt, ...) -{ - vargf2buff(buff, 128, fmt); - AnalysisTitles.insert(type); - key.access->Analysis[type] = buff; -} - -void StatisticsCollector::AddStatisticValue(const char *type, const char *fmt, ...) -{ - vargf2buff(buff, 128, fmt); - AnalysisTitles.insert(type); - CurrentKey->Analysis[type] = buff; -} - -void StatisticsCollector::SetRunInfo(const char *title, const char *fmt, ...) -{ - vargf2buff(buff, 256, fmt); - RunInfo.push_back(make_pair(title, buff)); -} - -void StatisticsCollector::SetStatisticFormula(const char *name, const char *formula) -{ - Formulas[name] = formula; -} - -void StatisticsCollector::SetTitle(const char *fmt, ...) -{ - vargf2buff(buff, 256, fmt); - Title = buff; -} - -string ExcelFormula(const string &fmt, size_t place, size_t rounds, bool is_horizontal) -{ - char buff[16]; - if(is_horizontal) - snprintf(buff, 15, "RC[%u]:RC[%u]", unsigned(place), unsigned(place+rounds-1)); - else - snprintf(buff, 15, "R[%u]C:R[%u]C", unsigned(place+1), unsigned(place+rounds)); - string result(fmt); size_t pos = 0; - while ( (pos = result.find("ROUNDS", pos, 6)) != string::npos ) - result.replace(pos, 6, buff); - return result; -} - -void StatisticsCollector::Print(int dataOutput, const char *ModeName) -{ - FILE *OutputFile; - const char *file_suffix = getenv("STAT_SUFFIX"); - if( !file_suffix ) file_suffix = ""; - const char *file_format = getenv("STAT_FORMAT"); - if( file_format ) { - dataOutput = 0; - if( strstr(file_format, "con")||strstr(file_format, "std") ) dataOutput |= StatisticsCollector::Stdout; - if( strstr(file_format, "txt")||strstr(file_format, "csv") ) dataOutput |= StatisticsCollector::TextFile; - if( strstr(file_format, "excel")||strstr(file_format, "xml") ) dataOutput |= StatisticsCollector::ExcelXML; - if( strstr(file_format, "htm") ) dataOutput |= StatisticsCollector::HTMLFile; - if( strstr(file_format, "pivot") ) dataOutput |= StatisticsCollector::PivotMode; - } - for(int i = 1; i < 10; i++) { - string env = Format("STAT_RUNINFO%d", i); - const char *info = getenv(env.c_str()); - if( info ) { - string title(info); - size_t pos = title.find('='); - if( pos != string::npos ) { - env = title.substr(pos+1); - title.resize(pos); - } else env = title; - RunInfo.push_back(make_pair(title, env)); - } - } - - if (dataOutput & StatisticsCollector::Stdout) - { - printf("\n-=# %s #=-\n", Title.c_str()); - if(SortMode == ByThreads) - printf(" Name | # | %s ", ModeName); - else - printf(" Name | %s | # ", ModeName); - for (AnalysisTitles_t::iterator i = AnalysisTitles.begin(); i != AnalysisTitles.end(); i++) - printf("|%s", i->c_str()+1); - - for (Statistics_t::iterator i = Statistics.begin(); i != Statistics.end(); i++) - { - if(SortMode == ByThreads) - printf("\n%12s|% 5d|%6s", i->second->Name.c_str(), i->second->Threads, i->second->Mode.c_str()); - else - printf("\n%12s|%6s|% 5d", i->second->Name.c_str(), i->second->Mode.c_str(), i->second->Threads); - Analysis_t &analisis = i->second->Analysis; - AnalysisTitles_t::iterator t = AnalysisTitles.begin(); - for (Analysis_t::iterator a = analisis.begin(); a != analisis.end(); t++) - { - char fmt[8]; snprintf(fmt, 7, "|%% %us", unsigned(max(size_t(3), t->size()))); - if(*t != a->first) - printf(fmt, ""); - else { - printf(fmt, a->second.c_str()); a++; - } - } - } - printf("\n"); - } - if (dataOutput & StatisticsCollector::TextFile) - { - bool append = false; - const char *file_ext = ".txt"; - if( file_format && strstr(file_format, "++") ) append = true; - if( file_format && strstr(file_format, "csv") ) file_ext = ".csv"; - if ((OutputFile = fopen((Name+file_suffix+file_ext).c_str(), append?"at":"wt")) == NULL) { - printf("Can't open .txt file\n"); - } else { - const char *delim = getenv("STAT_DELIMITER"); - if( !delim || !delim[0] ) { - if( file_format && strstr(file_format, "csv") ) delim = ","; - else delim = "\t"; - } - if( !append || !ftell(OutputFile) ) { // header needed - append = false; - if(SortMode == ByThreads) fprintf(OutputFile, "Name%s#%s%s", delim, delim, ModeName); - else fprintf(OutputFile, "Name%s%s%s#", delim, ModeName, delim); - for( size_t k = 0; k < RunInfo.size(); k++ ) - fprintf(OutputFile, "%s%s", delim, RunInfo[k].first.c_str()); - } - if(dataOutput & StatisticsCollector::PivotMode) { - if( !append) fprintf(OutputFile, "%sColumn%sValue", delim, delim); - for (Statistics_t::iterator i = Statistics.begin(); i != Statistics.end(); i++) - { - string RowHead; - if(SortMode == ByThreads) - RowHead = Format("\n%s%s%d%s%s%s", i->second->Name.c_str(), delim, i->second->Threads, delim, i->second->Mode.c_str(), delim); - else - RowHead = Format("\n%s%s%s%s%d%s", i->second->Name.c_str(), delim, i->second->Mode.c_str(), delim, i->second->Threads, delim); - for( size_t k = 0; k < RunInfo.size(); k++ ) - RowHead.append(RunInfo[k].second + delim); - Analysis_t &analisis = i->second->Analysis; - for (Analysis_t::iterator a = analisis.begin(); a != analisis.end(); ++a) - fprintf(OutputFile, "%s%s%s%s", RowHead.c_str(), a->first.c_str(), delim, a->second.c_str()); - Results_t &r = i->second->Results; - for (size_t k = 0; k < r.size(); k++) { - fprintf(OutputFile, "%s%s%s", RowHead.c_str(), RoundTitles[k].c_str(), delim); - fprintf(OutputFile, ResultsFmt, r[k]); - } - } - } else { - if( !append ) { - for( size_t k = 0; k < RunInfo.size(); k++ ) - fprintf(OutputFile, "%s%s", delim, RunInfo[k].first.c_str()); - for (AnalysisTitles_t::iterator i = AnalysisTitles.begin(); i != AnalysisTitles.end(); i++) - fprintf(OutputFile, "%s%s", delim, i->c_str()+1); - for (size_t i = 0; i < RoundTitles.size(); i++) - fprintf(OutputFile, "%s%s", delim, RoundTitles[i].c_str()); - } - for (Statistics_t::iterator i = Statistics.begin(); i != Statistics.end(); i++) - { - if(SortMode == ByThreads) - fprintf(OutputFile, "\n%s%s%d%s%s", i->second->Name.c_str(), delim, i->second->Threads, delim, i->second->Mode.c_str()); - else - fprintf(OutputFile, "\n%s%s%s%s%d", i->second->Name.c_str(), delim, i->second->Mode.c_str(), delim, i->second->Threads); - for( size_t k = 0; k < RunInfo.size(); k++ ) - fprintf(OutputFile, "%s%s", delim, RunInfo[k].second.c_str()); - Analysis_t &analisis = i->second->Analysis; - AnalysisTitles_t::iterator t = AnalysisTitles.begin(); - for (Analysis_t::iterator a = analisis.begin(); a != analisis.end(); ++t) { - fprintf(OutputFile, "%s", delim); - if(*t == a->first) { - fprintf(OutputFile, "%s", a->second.c_str()); ++a; - } - } - //data - Results_t &r = i->second->Results; - for (size_t k = 0; k < r.size(); k++) - { - fprintf(OutputFile, "%s", delim); - fprintf(OutputFile, ResultsFmt, r[k]); - } - } - } - fprintf(OutputFile, "\n"); - fclose(OutputFile); - } - } - if (dataOutput & StatisticsCollector::HTMLFile) - { - if ((OutputFile = fopen((Name+file_suffix+".html").c_str(), "w+t")) == NULL) { - printf("Can't open .html file\n"); - } else { - char TimerBuff[100], DateBuff[100]; - GetTime(TimerBuff,sizeof(TimerBuff)); - GetDate(DateBuff,sizeof(DateBuff)); - fprintf(OutputFile, "\n%s\n\n", Title.c_str()); - //----------------------- - fprintf(OutputFile, "\n"); - fprintf(OutputFile, "" - "\n", ModeName); - for (AnalysisTitles_t::iterator i = AnalysisTitles.begin(); i != AnalysisTitles.end(); i++) - fprintf(OutputFile, "", i->c_str()+1); - for (size_t i = 0; i < RoundTitles.size(); i++) - fprintf(OutputFile, "", RoundTitles[i].c_str()); - for (Statistics_t::iterator i = Statistics.begin(); i != Statistics.end(); i++) - { - fprintf(OutputFile, "\n", - i->second->Name.c_str(), i->second->Threads, i->second->Mode.c_str()); - //statistics - AnalysisTitles_t::iterator t = AnalysisTitles.begin(); - for (Analysis_t::iterator j = i->second->Analysis.begin(); j != i->second->Analysis.end(); t++) - { - fprintf(OutputFile, "", (*t != j->first)?" ":(i->second->Analysis[j->first]).c_str()); - if(*t == j->first) j++; - } - //data - Results_t &r = i->second->Results; - for (size_t k = 0; k < r.size(); k++) - { - fprintf(OutputFile, ""); - } - } - fprintf(OutputFile, "\n
Flip[H]%s%s%s", - DateBuff, TimerBuff, unsigned(AnalysisTitles.size() + RoundTitles.size()), Title.c_str()); - for( size_t k = 0; k < RunInfo.size(); k++ ) - fprintf(OutputFile, "; %s: %s", RunInfo[k].first.c_str(), RunInfo[k].second.c_str()); - fprintf(OutputFile, "
NameThreads%s%s%s
%s%d%4s%s"); - fprintf(OutputFile, ResultsFmt, r[k]); - fprintf(OutputFile, "
\n"); - ////////////////////////////////////////////////////// - fprintf(OutputFile, "\n"); - fprintf(OutputFile, "\n" - "", - DateBuff, TimerBuff, unsigned(max(Statistics.size()-2,size_t(1))), Title.c_str()); - - fprintf(OutputFile, "\n"); - for (Statistics_t::iterator i = Statistics.begin(); i != Statistics.end(); i++) - fprintf(OutputFile, "", i->second->Name.c_str()); - fprintf(OutputFile, "\n"); - for (Statistics_t::iterator n = Statistics.begin(); n != Statistics.end(); n++) - fprintf(OutputFile, "", n->second->Threads); - fprintf(OutputFile, "\n", ModeName); - for (Statistics_t::iterator m = Statistics.begin(); m != Statistics.end(); m++) - fprintf(OutputFile, "", m->second->Mode.c_str()); - - for (AnalysisTitles_t::iterator t = AnalysisTitles.begin(); t != AnalysisTitles.end(); t++) - { - fprintf(OutputFile, "\n", t->c_str()+1); - for (Statistics_t::iterator i = Statistics.begin(); i != Statistics.end(); i++) - fprintf(OutputFile, "", i->second->Analysis.count(*t)?i->second->Analysis[*t].c_str():" "); - } - - for (size_t r = 0; r < RoundTitles.size(); r++) - { - fprintf(OutputFile, "\n", RoundTitles[r].c_str()); - for (Statistics_t::iterator i = Statistics.begin(); i != Statistics.end(); i++) - { - Results_t &result = i->second->Results; - fprintf(OutputFile, ""); - } - } - fprintf(OutputFile, "\n
Flip[V]%s%s%s
Name%s
Threads%d
%s%s
%s%s
%s"); - if(result.size() > r) - fprintf(OutputFile, ResultsFmt, result[r]); - fprintf(OutputFile, "
\n\n"); - fclose(OutputFile); - } - } - if (dataOutput & StatisticsCollector::ExcelXML) - { - if ((OutputFile = fopen((Name+file_suffix+".xml").c_str(), "w+t")) == NULL) { - printf("Can't open .xml file\n"); - } else { - // TODO:PivotMode - char UserName[100]; - char TimerBuff[100], DateBuff[100]; -#if _WIN32 || _WIN64 - strcpy(UserName,getenv("USERNAME")); -#else - strcpy(UserName,getenv("USER")); -#endif - //-------------------------------- - GetTime(TimerBuff,sizeof(TimerBuff)); - GetDate(DateBuff,sizeof(DateBuff)); - //-------------------------- - fprintf(OutputFile, XMLHead, UserName, TimerBuff); - fprintf(OutputFile, XMLStyles); - fprintf(OutputFile, XMLBeginSheet, "Horizontal"); - fprintf(OutputFile, XMLNames,1,1,1,int(AnalysisTitles.size()+Formulas.size()+COUNT_PARAMETERS)); - fprintf(OutputFile, XMLBeginTable, int(RoundTitles.size()+Formulas.size()+AnalysisTitles.size()+COUNT_PARAMETERS+1/*title*/), int(Statistics.size()+1)); - fprintf(OutputFile, XMLBRow); - fprintf(OutputFile, XMLCellTopName); - fprintf(OutputFile, XMLCellTopThread); - fprintf(OutputFile, XMLCellTopMode, ModeName); - for (AnalysisTitles_t::iterator j = AnalysisTitles.begin(); j != AnalysisTitles.end(); j++) - fprintf(OutputFile, XMLAnalysisTitle, j->c_str()+1); - for (Formulas_t::iterator j = Formulas.begin(); j != Formulas.end(); j++) - fprintf(OutputFile, XMLAnalysisTitle, j->first.c_str()+1); - for (RoundTitles_t::iterator j = RoundTitles.begin(); j != RoundTitles.end(); j++) - fprintf(OutputFile, XMLAnalysisTitle, j->c_str()); - string Info = Title; - for( size_t k = 0; k < RunInfo.size(); k++ ) - Info.append("; " + RunInfo[k].first + "=" + RunInfo[k].second); - fprintf(OutputFile, XMLCellEmptyWhite, Info.c_str()); - fprintf(OutputFile, XMLERow); - //------------------------ - for (Statistics_t::iterator i = Statistics.begin(); i != Statistics.end(); i++) - { - fprintf(OutputFile, XMLBRow); - fprintf(OutputFile, XMLCellName, i->second->Name.c_str()); - fprintf(OutputFile, XMLCellThread,i->second->Threads); - fprintf(OutputFile, XMLCellMode, i->second->Mode.c_str()); - //statistics - AnalysisTitles_t::iterator at = AnalysisTitles.begin(); - for (Analysis_t::iterator j = i->second->Analysis.begin(); j != i->second->Analysis.end(); at++) - { - fprintf(OutputFile, XMLCellAnalysis, (*at != j->first)?"":(i->second->Analysis[j->first]).c_str()); - if(*at == j->first) j++; - } - //formulas - size_t place = 0; - Results_t &v = i->second->Results; - for (Formulas_t::iterator f = Formulas.begin(); f != Formulas.end(); f++, place++) - fprintf(OutputFile, XMLCellFormula, ExcelFormula(f->second, Formulas.size()-place, v.size(), true).c_str()); - //data - for (size_t k = 0; k < v.size(); k++) - { - fprintf(OutputFile, XMLCellData, v[k]); - } - if(v.size() < RoundTitles.size()) - fprintf(OutputFile, XMLMergeRow, int(RoundTitles.size() - v.size())); - fprintf(OutputFile, XMLERow); - } - //------------------------ - fprintf(OutputFile, XMLEndTable); - fprintf(OutputFile, XMLWorkSheetProperties,1,1,3,3,int(RoundTitles.size()+AnalysisTitles.size()+Formulas.size()+COUNT_PARAMETERS)); - fprintf(OutputFile, XMLAutoFilter,1,1,1,int(AnalysisTitles.size()+Formulas.size()+COUNT_PARAMETERS)); - fprintf(OutputFile, XMLEndWorkSheet); - //---------------------------------------- - fprintf(OutputFile, XMLEndWorkbook); - fclose(OutputFile); - } - } -} diff --git a/src/tbb/src/perf/statistics.h b/src/tbb/src/perf/statistics.h deleted file mode 100644 index 7113a7eae..000000000 --- a/src/tbb/src/perf/statistics.h +++ /dev/null @@ -1,191 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -// Internal Intel tool - -#ifndef __STATISTICS_H__ -#define __STATISTICS_H__ - -#define _CRT_SECURE_NO_DEPRECATE 1 - -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace std; -typedef double value_t; - -/* - Statistical collector class. - - Resulting table output: - +---------------------------------------------------------------------------+ - | [Date] ... | - +----------+----v----+--v---+----------------+------------+-..-+------------+ - | TestName | Threads | Mode | Rounds results | Stat_type1 | .. | Stat_typeN | - +----------+---------+------+-+-+-+-..-+-+-+-+------------+-..-+------------+ - | | | | | | | .. | | | | | | | - .. ... ... .................. ...... .. - | | | | | | | .. | | | | | | | - +----------+---------+------+-+-+-+-..-+-+-+-+------------+-..-+------------+ - - Iterating table output: - +---------------------------------------------------------------------------+ - | [Date] <TestName>, Threads: <N>, Mode: <M>; for <Title>... | - +----------+----v----+--v---+----------------+------------+-..-+------------+ - -*/ - -class StatisticsCollector -{ -public: - typedef map<string, string> Analysis_t; - typedef vector<value_t> Results_t; - -protected: - StatisticsCollector(const StatisticsCollector &); - - struct StatisticResults - { - string Name; - string Mode; - int Threads; - Results_t Results; - Analysis_t Analysis; - }; - - // internal members - //bool OpenFile; - StatisticResults *CurrentKey; - string Title; - const char /**Name,*/ *ResultsFmt; - string Name; - //! Data - typedef map<string, StatisticResults*> Statistics_t; - Statistics_t Statistics; - typedef vector<string> RoundTitles_t; - RoundTitles_t RoundTitles; - //TODO: merge those into one structure - typedef map<string, string> Formulas_t; - Formulas_t Formulas; - typedef set<string> AnalysisTitles_t; - AnalysisTitles_t AnalysisTitles; - typedef vector<pair<string, string> > RunInfo_t; - RunInfo_t RunInfo; - -public: - struct TestCase { - StatisticResults *access; - TestCase() : access(0) {} - TestCase(StatisticResults *link) : access(link) {} - const char *getName() const { return access->Name.c_str(); } - const char *getMode() const { return access->Mode.c_str(); } - int getThreads() const { return access->Threads; } - const Results_t &getResults() const { return access->Results; } - const Analysis_t &getAnalysis() const { return access->Analysis; } - }; - - enum Sorting { - ByThreads, ByAlg - }; - - //! Data and output types - enum DataOutput { - // Verbosity level enumeration - Statistic = 1, //< Analytical data - computed after all iterations and rounds passed - Result = 2, //< Testing data - collected after all iterations passed - Iteration = 3, //< Verbose data - collected at each iteration (for each size - in case of containers) - // ExtraVerbose is not applicabe yet :) be happy, but flexibility is always welcome - - // Next constants are bit-fields - Stdout = 1<<8, //< Output to the console - TextFile = 1<<9, //< Output to plain text file "name.txt" (delimiter is TAB by default) - ExcelXML = 1<<10, //< Output to Excel-readable XML-file "name.xml" - HTMLFile = 1<<11, //< Output to HTML file "name.html" - PivotMode= 1<<15 //< Puts all the rounds into one columt to better fit for pivot table in Excel - }; - - //! Constructor. Specify tests set name which used as name of output files - StatisticsCollector(const char *name, Sorting mode = ByThreads, const char *fmt = "%g") - : CurrentKey(NULL), ResultsFmt(fmt), Name(name), SortMode(mode) {} - - ~StatisticsCollector(); - - //! Set tests set title, supporting printf-like arguments - void SetTitle(const char *fmt, ...); - - //! Specify next test key - TestCase SetTestCase(const char *name, const char *mode, int threads); - //! Specify next test key - void SetTestCase(const TestCase &t) { SetTestCase(t.getName(), t.getMode(), t.getThreads()); } - //! Reserve specified number of rounds. Use for efficiency. Used mostly internally - void ReserveRounds(size_t index); - //! Add result of the measure - void AddRoundResult(const TestCase &, value_t v); - //! Add result of the current measure - void AddRoundResult(value_t v) { if(CurrentKey) AddRoundResult(TestCase(CurrentKey), v); } - //! Add title of round - void SetRoundTitle(size_t index, const char *fmt, ...); - //! Add numbered title of round - void SetRoundTitle(size_t index, int num) { SetRoundTitle(index, "%d", num); } - //! Get number of rounds - size_t GetRoundsCount() const { return RoundTitles.size(); } - // Set statistic value for the test - void AddStatisticValue(const TestCase &, const char *type, const char *fmt, ...); - // Set statistic value for the current test - void AddStatisticValue(const char *type, const char *fmt, ...); - //! Add Excel-processing formulas. @arg formula can contain more than one instances of - //! ROUNDS template which transforms into the range of cells with result values - //TODO://! #1 .. #n templates represent data cells from the first to the last - //TODO: merge with Analisis - void SetStatisticFormula(const char *name, const char *formula); - //! Add information about run or compile parameters - void SetRunInfo(const char *title, const char *fmt, ...); - void SetRunInfo(const char *title, int num) { SetRunInfo(title, "%d", num); } - - //! Data output - void Print(int dataOutput, const char *ModeName = "Mode"); - -private: - Sorting SortMode; -}; - -//! using: Func(const char *fmt, ...) { vargf2buff(buff, 128, fmt);... -#define vargf2buff(name, size, fmt) \ - char name[size]; memset(name, 0, size); \ - va_list args; va_start(args, fmt); \ - vsnprintf(name, size-1, fmt, args); \ - va_end(args); - - -inline std::string Format(const char *fmt, ...) { - vargf2buff(buf, 1024, fmt); // from statistics.h - return std::string(buf); -} - -#ifdef STATISTICS_INLINE -#include "statistics.cpp" -#endif -#endif //__STATISTICS_H__ diff --git a/src/tbb/src/perf/statistics_xml.h b/src/tbb/src/perf/statistics_xml.h deleted file mode 100644 index 396241de2..000000000 --- a/src/tbb/src/perf/statistics_xml.h +++ /dev/null @@ -1,200 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -const char XMLBRow[]= -" <Row>\n"; - -const char XMLERow[]= -" </Row>\n"; - -const char XMLHead[]= -"<?xml version=\"1.0\"?>\n" -"<?mso-application progid=\"Excel.Sheet\"?>\n\ -<Workbook xmlns=\"urn:schemas-microsoft-com:office:spreadsheet\"\n\ - xmlns:o=\"urn:schemas-microsoft-com:office:office\"\n\ - xmlns:x=\"urn:schemas-microsoft-com:office:excel\"\n\ - xmlns:ss=\"urn:schemas-microsoft-com:office:spreadsheet\"\n\ - xmlns:html=\"http://www.w3.org/TR/REC-html40\">\n\ - <DocumentProperties xmlns=\"urn:schemas-microsoft-com:office:office\">\n\ - <Author>%s</Author>\n\ - <Created>%s</Created>\n\ - <Company>Intel Corporation</Company>\n\ - </DocumentProperties>\n\ - <ExcelWorkbook xmlns=\"urn:schemas-microsoft-com:office:excel\">\n\ - <RefModeR1C1/>\n\ - </ExcelWorkbook>\n"; - - const char XMLStyles[]= - " <Styles>\n\ - <Style ss:ID=\"Default\" ss:Name=\"Normal\">\n\ - <Alignment ss:Vertical=\"Bottom\" ss:Horizontal=\"Left\" ss:WrapText=\"0\"/>\n\ - </Style>\n\ - <Style ss:ID=\"s26\">\n\ - <Alignment ss:Vertical=\"Top\" ss:Horizontal=\"Left\" ss:WrapText=\"0\"/>\n\ - <Borders>\n\ - <Border ss:Position=\"Bottom\" ss:LineStyle=\"Continuous\" ss:Weight=\"1\"/>\n\ - <Border ss:Position=\"Left\" ss:LineStyle=\"Continuous\" ss:Weight=\"1\"/>\n\ - <Border ss:Position=\"Right\" ss:LineStyle=\"Continuous\" ss:Weight=\"1\"/>\n\ - <Border ss:Position=\"Top\" ss:LineStyle=\"Continuous\" ss:Weight=\"1\"/>\n\ - </Borders>\n\ - <Interior ss:Color=\"#FFFF99\" ss:Pattern=\"Solid\"/>\n\ - </Style>\n\ - <Style ss:ID=\"s25\">\n\ - <Alignment ss:Vertical=\"Top\" ss:Horizontal=\"Left\" ss:WrapText=\"0\"/>\n\ - <Borders>\n\ - <Border ss:Position=\"Bottom\" ss:LineStyle=\"Continuous\" ss:Weight=\"1\"/>\n\ - <Border ss:Position=\"Left\" ss:LineStyle=\"Continuous\" ss:Weight=\"1\"/>\n\ - <Border ss:Position=\"Right\" ss:LineStyle=\"Continuous\" ss:Weight=\"1\"/>\n\ - <Border ss:Position=\"Top\" ss:LineStyle=\"Continuous\" ss:Weight=\"1\"/>\n\ - </Borders>\n\ - <Interior ss:Color=\"#CCFFFF\" ss:Pattern=\"Solid\"/>\n\ - </Style>\n\ - <Style ss:ID=\"s24\">\n\ - <Alignment ss:Vertical=\"Top\" ss:Horizontal=\"Left\" ss:WrapText=\"0\"/>\n\ - <Borders>\n\ - <Border ss:Position=\"Bottom\" ss:LineStyle=\"Continuous\" ss:Weight=\"1\"/>\n\ - <Border ss:Position=\"Left\" ss:LineStyle=\"Continuous\" ss:Weight=\"1\"/>\n\ - <Border ss:Position=\"Right\" ss:LineStyle=\"Continuous\" ss:Weight=\"1\"/>\n\ - <Border ss:Position=\"Top\" ss:LineStyle=\"Continuous\" ss:Weight=\"1\"/>\n\ - </Borders>\n\ - <Interior ss:Color=\"#CCFFCC\" ss:Pattern=\"Solid\"/>\n\ - </Style>\n\ - <Style ss:ID=\"s23\">\n\ - <Alignment ss:Vertical=\"Top\" ss:Horizontal=\"Left\" ss:WrapText=\"0\"/>\n\ - <Borders>\n\ - <Border ss:Position=\"Bottom\" ss:LineStyle=\"Continuous\" ss:Weight=\"1\"/>\n\ - <Border ss:Position=\"Left\" ss:LineStyle=\"Continuous\" ss:Weight=\"1\"/>\n\ - <Border ss:Position=\"Right\" ss:LineStyle=\"Continuous\" ss:Weight=\"1\"/>\n\ - <Border ss:Position=\"Top\" ss:LineStyle=\"Continuous\" ss:Weight=\"1\"/>\n\ - </Borders>\n\ - </Style>\n\ - </Styles>\n"; - -const char XMLBeginSheet[]= -" <Worksheet ss:Name=\"%s\">\n"; - -const char XMLNames[]= -" <Names>\n\ - <NamedRange ss:Name=\"_FilterDatabase\" ss:RefersTo=\"R%dC%d:R%dC%d\" ss:Hidden=\"1\"/>\n\ - </Names>\n"; - -const char XMLBeginTable[]= -" <Table ss:ExpandedColumnCount=\"%d\" ss:ExpandedRowCount=\"%d\" x:FullColumns=\"1\"\n\ - x:FullRows=\"1\">\n"; - -const char XMLColumsHorizontalTable[]= -" <Column ss:Index=\"1\" ss:Width=\"108.75\"/>\n\ - <Column ss:Index=\"%d\" ss:Width=\"77.25\" ss:Span=\"%d\"/>\n"; - -const char XMLColumsVerticalTable[]= -" <Column ss:Index=\"1\" ss:Width=\"77.25\" ss:Span=\"%d\"/>\n"; - -const char XMLNameAndTime[]= -" <Cell><Data ss:Type=\"String\">%s</Data></Cell>\n\ - <Cell><Data ss:Type=\"String\">%s</Data></Cell>\n\ - <Cell><Data ss:Type=\"String\">%s</Data></Cell>\n"; - -const char XMLTableParamAndTitle[]= -" <Cell><Data ss:Type=\"Number\">%d</Data></Cell>\n\ - <Cell><Data ss:Type=\"Number\">%d</Data></Cell>\n\ - <Cell><Data ss:Type=\"Number\">%d</Data></Cell>\n\ - <Cell><Data ss:Type=\"String\">%s</Data></Cell>\n"; - -//-------------- -const char XMLCellTopName[]= -" <Cell ss:StyleID=\"s25\"><Data ss:Type=\"String\">Name</Data></Cell>\n"; -const char XMLCellTopThread[]= -" <Cell ss:StyleID=\"s25\"><Data ss:Type=\"String\">Threads</Data></Cell>\n"; -const char XMLCellTopMode[]= -" <Cell ss:StyleID=\"s25\"><Data ss:Type=\"String\">%s</Data></Cell>\n"; -//--------------------- -const char XMLAnalysisTitle[]= -" <Cell ss:StyleID=\"s25\"><Data ss:Type=\"String\">%s</Data></Cell>\n"; - -const char XMLCellName[]= -" <Cell ss:StyleID=\"s24\"><Data ss:Type=\"String\">%s</Data></Cell>\n"; - -const char XMLCellThread[]= -" <Cell ss:StyleID=\"s24\"><Data ss:Type=\"Number\">%d</Data></Cell>\n"; - -const char XMLCellMode[]= -" <Cell ss:StyleID=\"s24\"><Data ss:Type=\"String\">%s</Data></Cell>\n"; - -const char XMLCellAnalysis[]= -" <Cell ss:StyleID=\"s26\"><Data ss:Type=\"String\">%s</Data></Cell>\n"; - -const char XMLCellFormula[]= -" <Cell ss:StyleID=\"s26\" ss:Formula=\"%s\"><Data ss:Type=\"Number\"></Data></Cell>\n"; - -const char XMLCellData[]= -" <Cell ss:StyleID=\"s23\"><Data ss:Type=\"Number\">%g</Data></Cell>\n"; - -const char XMLMergeRow[]= -" <Cell ss:StyleID=\"s23\" ss:MergeAcross=\"%d\" ><Data ss:Type=\"String\"></Data></Cell>\n"; - -const char XMLCellEmptyWhite[]= -" <Cell><Data ss:Type=\"String\">%s</Data></Cell>\n"; - -const char XMLCellEmptyTitle[]= -" <Cell ss:StyleID=\"s25\"><Data ss:Type=\"String\"></Data></Cell>\n"; - -const char XMLEndTable[]= -" </Table>\n"; - -const char XMLAutoFilter[]= -" <AutoFilter x:Range=\"R%dC%d:R%dC%d\" xmlns=\"urn:schemas-microsoft-com:office:excel\">\n\ - </AutoFilter>\n"; - -const char XMLEndWorkSheet[]= - " </Worksheet>\n"; - -const char XMLWorkSheetProperties[]= -" <WorksheetOptions xmlns=\"urn:schemas-microsoft-com:office:excel\">\n\ - <Unsynced/>\n\ - <Selected/>\n\ - <FreezePanes/>\n\ - <FrozenNoSplit/>\n\ - <SplitHorizontal>%d</SplitHorizontal>\n\ - <TopRowBottomPane>%d</TopRowBottomPane>\n\ - <SplitVertical>%d</SplitVertical>\n\ - <LeftColumnRightPane>%d</LeftColumnRightPane>\n\ - <ActivePane>0</ActivePane>\n\ - <Panes>\n\ - <Pane>\n\ - <Number>3</Number>\n\ - </Pane>\n\ - <Pane>\n\ - <Number>1</Number>\n\ - </Pane>\n\ - <Pane>\n\ - <Number>2</Number>\n\ - </Pane>\n\ - <Pane>\n\ - <Number>0</Number>\n\ - <ActiveRow>0</ActiveRow>\n\ - <ActiveCol>%d</ActiveCol>\n\ - </Pane>\n\ - </Panes>\n\ - <ProtectObjects>False</ProtectObjects>\n\ - <ProtectScenarios>False</ProtectScenarios>\n\ - </WorksheetOptions>\n"; - -const char XMLEndWorkbook[]= - "</Workbook>\n"; diff --git a/src/tbb/src/perf/time_cpq_throughput_test.cpp b/src/tbb/src/perf/time_cpq_throughput_test.cpp deleted file mode 100644 index 02f9bfd36..000000000 --- a/src/tbb/src/perf/time_cpq_throughput_test.cpp +++ /dev/null @@ -1,294 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#define HARNESS_CUSTOM_MAIN 1 -#define HARNESS_NO_PARSE_COMMAND_LINE 1 - -#include <cstdlib> -#include <cmath> -#include <queue> -#include "tbb/tbb_stddef.h" -#include "tbb/spin_mutex.h" -#include "tbb/task_scheduler_init.h" -#include "tbb/tick_count.h" -#include "tbb/cache_aligned_allocator.h" -#include "tbb/concurrent_priority_queue.h" -#include "../test/harness.h" -#include "../examples/common/utility/utility.h" -#if _MSC_VER -#pragma warning(disable: 4996) -#endif - -#define IMPL_SERIAL 0 -#define IMPL_STL 1 -#define IMPL_CPQ 2 - -using namespace tbb; - -// test parameters & defaults -int impl = IMPL_CPQ; // which implementation to test -int contention = 1; // busywork between operations in us -int preload = 0; // # elements to pre-load queue with -double throughput_window = 30.0; // in seconds -int ops_per_iteration = 20; // minimum: 2 (1 push, 1 pop) -const int sample_operations = 1000; // for timing checks - -// global data & types -int pushes_per_iter; -int pops_per_iter; -tbb::atomic<unsigned int> operation_count; -tbb::tick_count start; - -// a non-trivial data element to use in the priority queue -const int padding_size = 15; // change to get cache line size for test machine -class padding_type { -public: - int p[padding_size]; - padding_type& operator=(const padding_type& other) { - if (this != &other) { - for (int i=0; i<padding_size; ++i) { - p[i] = other.p[i]; - } - } - return *this; - } -}; - -class my_data_type { -public: - int priority; - padding_type padding; - my_data_type() : priority(0) {} -}; - -class my_less { -public: - bool operator()(my_data_type d1, my_data_type d2) { - return d1.priority<d2.priority; - } -}; - -// arrays to get/put data from/to to generate non-trivial accesses during busywork -my_data_type *input_data; -my_data_type *output_data; -size_t arrsz; - -// Serial priority queue -std::priority_queue<my_data_type, std::vector<my_data_type>, my_less > *serial_cpq; - -// Coarse-locked priority queue -spin_mutex *my_mutex; -std::priority_queue<my_data_type, std::vector<my_data_type>, my_less > *stl_cpq; - -// TBB concurrent_priority_queue -concurrent_priority_queue<my_data_type, my_less > *agg_cpq; - -// Busy work and calibration helpers -unsigned int one_us_iters = 345; // default value - -// if user wants to calibrate to microseconds on particular machine, call -// this at beginning of program; sets one_us_iters to number of iters to -// busy_wait for approx. 1 us -void calibrate_busy_wait() { - const unsigned niter = 1000000; - tbb::tick_count t0 = tbb::tick_count::now(); - for (volatile unsigned int i=0; i<niter; ++i) continue; - tbb::tick_count t1 = tbb::tick_count::now(); - - one_us_iters = (unsigned int)(niter/(t1-t0).seconds())*1e-6; - printf("one_us_iters: %d\n", one_us_iters); -} - -void busy_wait(int us) -{ - unsigned int iter = us*one_us_iters; - for (volatile unsigned int i=0; i<iter; ++i) continue; -} - -// Push to priority queue, depending on implementation -void do_push(my_data_type elem, int nThr, int impl) { - if (impl == IMPL_SERIAL) { - serial_cpq->push(elem); - } - else if (impl == IMPL_STL) { - tbb::spin_mutex::scoped_lock myLock(*my_mutex); - stl_cpq->push(elem); - } - else if (impl == IMPL_CPQ) { - agg_cpq->push(elem); - } -} - -// Pop from priority queue, depending on implementation -my_data_type do_pop(int nThr, int impl) { - my_data_type elem; - if (impl == IMPL_SERIAL) { - if (!serial_cpq->empty()) { - elem = serial_cpq->top(); - serial_cpq->pop(); - return elem; - } - } - else if (impl == IMPL_STL) { - tbb::spin_mutex::scoped_lock myLock(*my_mutex); - if (!stl_cpq->empty()) { - elem = stl_cpq->top(); - stl_cpq->pop(); - return elem; - } - } - else if (impl == IMPL_CPQ) { - if (agg_cpq->try_pop(elem)) { - return elem; - } - } - return elem; -} - - -struct TestThroughputBody : NoAssign { - int nThread; - int implementation; - - TestThroughputBody(int nThread_, int implementation_) : - nThread(nThread_), implementation(implementation_) {} - - void operator()(const int threadID) const { - tbb::tick_count now; - size_t pos_in = threadID, pos_out = threadID; - my_data_type elem; - while (1) { - for (int i=0; i<sample_operations; i+=ops_per_iteration) { - // do pushes - for (int j=0; j<pushes_per_iter; ++j) { - elem = input_data[pos_in]; - do_push(elem, nThread, implementation); - busy_wait(contention); - pos_in += nThread; - if (pos_in >= arrsz) pos_in = pos_in % arrsz; - } - // do pops - for (int j=0; j<pops_per_iter; ++j) { - output_data[pos_out] = do_pop(nThread, implementation); - busy_wait(contention); - pos_out += nThread; - if (pos_out >= arrsz) pos_out = pos_out % arrsz; - } - } - now = tbb::tick_count::now(); - operation_count += sample_operations; - if ((now-start).seconds() >= throughput_window) break; - } - } -}; - -void TestSerialThroughput() { - tbb::tick_count now; - - serial_cpq = new std::priority_queue<my_data_type, std::vector<my_data_type>, my_less >; - for (int i=0; i<preload; ++i) do_push(input_data[i], 1, IMPL_SERIAL); - - TestThroughputBody my_serial_test(1, IMPL_SERIAL); - start = tbb::tick_count::now(); - NativeParallelFor(1, my_serial_test); - now = tbb::tick_count::now(); - delete serial_cpq; - - printf("SERIAL 1 %10d\n", int(operation_count/(now-start).seconds())); -} - -void TestThroughputCpqOnNThreads(int nThreads) { - tbb::tick_count now; - - if (impl == IMPL_STL) { - stl_cpq = new std::priority_queue<my_data_type, std::vector<my_data_type>, my_less >; - for (int i=0; i<preload; ++i) do_push(input_data[i], nThreads, IMPL_STL); - - TestThroughputBody my_stl_test(nThreads, IMPL_STL); - start = tbb::tick_count::now(); - NativeParallelFor(nThreads, my_stl_test); - now = tbb::tick_count::now(); - delete stl_cpq; - - printf("STL %3d %10d\n", nThreads, int(operation_count/(now-start).seconds())); - } - else if (impl == IMPL_CPQ) { - agg_cpq = new concurrent_priority_queue<my_data_type, my_less >; - for (int i=0; i<preload; ++i) do_push(input_data[i], nThreads, IMPL_CPQ); - - TestThroughputBody my_cpq_test(nThreads, IMPL_CPQ); - start = tbb::tick_count::now(); - NativeParallelFor(nThreads, my_cpq_test); - now = tbb::tick_count::now(); - delete agg_cpq; - - printf("CPQ %3d %10d\n", nThreads, int(operation_count/(now-start).seconds())); - } -} - - -int main(int argc, char *argv[]) { - utility::thread_number_range threads(tbb::task_scheduler_init::default_num_threads); - struct select_impl{ - static bool validate(const int & impl){ - return ((impl == IMPL_SERIAL) || (impl == IMPL_STL) || (impl == IMPL_CPQ)); - } - }; - utility::parse_cli_arguments(argc,argv,utility::cli_argument_pack() - .positional_arg(threads,"n-of-threads",utility::thread_number_range_desc) - .positional_arg(contention,"contention"," busywork between operations, in us") - .positional_arg(impl,"queue_type", "which implementation to test. One of 0(SERIAL), 1(STL), 2(CPQ) ", select_impl::validate) - .positional_arg(preload,"preload","number of elements to pre-load queue with") - .positional_arg(ops_per_iteration, "batch size" ,"minimum: 2 (1 push, 1 pop)") - .positional_arg(throughput_window, "duration", "in seconds") - ); - - std::cout<< "Priority queue performance test "<<impl<<" will run with "<<contention<<"us contention " - "using "<<threads<<" threads, "<<ops_per_iteration<<" batch size, "<<preload<<" pre-loaded elements," - " for "<<throughput_window<<" seconds.\n" - <<std::flush - ; - - srand(42); - arrsz = 100000; - input_data = new my_data_type[arrsz]; - output_data = new my_data_type[arrsz]; - for (size_t i=0; i<arrsz; ++i) { - input_data[i].priority = rand()%100; - } - //calibrate_busy_wait(); - pushes_per_iter = ops_per_iteration/2; - pops_per_iter = ops_per_iteration/2; - operation_count = 0; - - // Initialize mutex for Coarse-locked priority_queue - cache_aligned_allocator<spin_mutex> my_mutex_allocator; - my_mutex = (spin_mutex *)my_mutex_allocator.allocate(1); - - if (impl == IMPL_SERIAL) { - TestSerialThroughput(); - } - else { - for( int p=threads.first; p<=threads.last; p = threads.step(p) ) { - TestThroughputCpqOnNThreads(p); - } - } - return Harness::Done; -} diff --git a/src/tbb/src/perf/time_fibonacci_cutoff.cpp b/src/tbb/src/perf/time_fibonacci_cutoff.cpp deleted file mode 100644 index 2a9e74d00..000000000 --- a/src/tbb/src/perf/time_fibonacci_cutoff.cpp +++ /dev/null @@ -1,130 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#include <cstdio> -#include <cstdlib> - -#include "tbb/task_scheduler_init.h" -#include "tbb/task.h" -#include "tbb/tick_count.h" -#include "fibonacci_impl_tbb.cpp" - -long CutOff = 1; - -long SerialFib( const long n ); - -long ParallelFib( const long n ); - -inline void dump_title() { - printf(" Mode, P, repeat, N, =fib value, cutoff, time, speedup\n"); -} - -inline void output(int P, long n, long c, int T, double serial_elapsed, double elapsed, long result) { - printf("%s,%4d,%7d,%3ld,%11ld,%7ld,%9.3g,%8.3g\n", ( (P == 0) ? " Serial" : "Parallel" ), - P, T, n, result, c, elapsed, serial_elapsed / elapsed); -} - -#define MOVE_BY_FOURTHS 1 -inline long calculate_new_cutoff(const long lo, const long hi) { -#if MOVE_BY_FOURTHS - return lo + (3 + hi - lo ) / 4; -#else - return (hi + lo)/2; -#endif -} - -void find_cutoff(const int P, const long n, const int T, const double serial_elapsed) { - long lo = 1, hi = n; - double elapsed = 0, lo_elapsed = 0, hi_elapsed = 0; - long final_cutoff = -1; - - tbb::task_scheduler_init init(P); - - while(true) { - CutOff = calculate_new_cutoff(lo, hi); - long result = 0; - tbb::tick_count t0; - for (int t = -1; t < T; ++t) { - if (t == 0) t0 = tbb::tick_count::now(); - result += ParallelFib(n); - } - elapsed = (tbb::tick_count::now() - t0).seconds(); - output(P,n,CutOff,T,serial_elapsed,elapsed,result); - - if (serial_elapsed / elapsed >= P/2.0) { - final_cutoff = CutOff; - if (hi == CutOff) { - if (hi == lo) { - // we have had this value at both above and below 50% - lo = 1; lo_elapsed = 0; - } else { - break; - } - } - hi = CutOff; - hi_elapsed = elapsed; - } else { - if (lo == CutOff) break; - lo = CutOff; - lo_elapsed = elapsed; - } - } - - double interpolated_cutoff = lo + ( P/2.0 - serial_elapsed/lo_elapsed ) * ( (hi - lo) / ( serial_elapsed/hi_elapsed - serial_elapsed/lo_elapsed )); - - if (final_cutoff != -1) { - printf("50%% efficiency cutoff is %ld ( linearly interpolated cutoff is %g )\n", final_cutoff, interpolated_cutoff); - } else { - printf("Cannot achieve 50%% efficiency\n"); - } - - return; -} - -int main(int argc, char *argv[]) { - if (argc < 4) { - printf("Usage: %s threads n repetitions\nWhere n make sense in range [25; 45]\n",argv[0]); - return 1; - } - - int P = atoi(argv[1]); - volatile long n = atol(argv[2]); - int T = atoi(argv[3]); - - // warmup parallel engine - ParallelFib(n); - - dump_title(); - - // collect serial time - long serial_result = 0; - tbb::tick_count t0; - for (int t = -1; t < T; ++t) { - if (t == 0) t0 = tbb::tick_count::now(); - serial_result += SerialFib(n); - } - double serial_elapsed = (tbb::tick_count::now() - t0).seconds(); - output(0,n,0,T,serial_elapsed,serial_elapsed,serial_result); - - // perform search - find_cutoff(P,n,T,serial_elapsed); - - return 0; -} diff --git a/src/tbb/src/perf/time_framework.h b/src/tbb/src/perf/time_framework.h deleted file mode 100644 index a749dff69..000000000 --- a/src/tbb/src/perf/time_framework.h +++ /dev/null @@ -1,351 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TIME_FRAMEWORK_H__ -#define __TIME_FRAMEWORK_H__ - -#include <cstdlib> -#include <math.h> -#include <vector> -#include <string> -#include <sstream> -#include "tbb/tbb_stddef.h" -#include "tbb/task_scheduler_init.h" -#include "tbb/tick_count.h" -#define HARNESS_CUSTOM_MAIN 1 -#include "../test/harness.h" -#include "../test/harness_barrier.h" -#define STATISTICS_INLINE -#include "statistics.h" - -#ifndef ARG_TYPE -typedef intptr_t arg_t; -#else -typedef ARG_TYPE arg_t; -#endif - -class Timer { - tbb::tick_count tick; -public: - Timer() { tick = tbb::tick_count::now(); } - double get_time() { return (tbb::tick_count::now() - tick).seconds(); } - double diff_time(const Timer &newer) { return (newer.tick - tick).seconds(); } - double mark_time() { tbb::tick_count t1(tbb::tick_count::now()), t2(tick); tick = t1; return (t1 - t2).seconds(); } - double mark_time(const Timer &newer) { tbb::tick_count t(tick); tick = newer.tick; return (tick - t).seconds(); } -}; - -class TesterBase /*: public tbb::internal::no_copy*/ { -protected: - friend class TestProcessor; - friend class TestRunner; - - //! it is barrier for synchronizing between threads - Harness::SpinBarrier *barrier; - - //! number of tests per this tester - const int tests_count; - - //! number of threads to operate - int threads_count; - - //! some value for tester - arg_t value; - - //! tester name - const char *tester_name; - - // avoid false sharing - char pad[128 - sizeof(arg_t) - sizeof(int)*2 - sizeof(void*)*2 ]; - -public: - //! init tester base. @arg ntests is number of embedded tests in this tester. - TesterBase(int ntests) - : barrier(NULL), tests_count(ntests) - {} - virtual ~TesterBase() {} - - //! internal function - void base_init(arg_t v, int t, Harness::SpinBarrier &b) { - threads_count = t; - barrier = &b; - value = v; - init(); - } - - //! optionally override to init after value and threads count were set. - virtual void init() { } - - //! Override to provide your names - virtual std::string get_name(int testn) { - return Format("test %d", testn); - } - - //! optionally override to init test mode just before execution for a given thread number. - virtual void test_prefix(int testn, int threadn) { } - - //! Override to provide main test's entry function returns a value to record - virtual value_t test(int testn, int threadn) = 0; - - //! Type of aggregation from results of threads - enum result_t { - SUM, AVG, MIN, MAX - }; - - //! Override to change result type for the test. Return postfix for test name or 0 if result type is not needed. - virtual const char *get_result_type(int /*testn*/, result_t type) const { - return type == AVG ? "" : 0; // only average result by default - } -}; - -/***** -a user's tester concept: - -class tester: public TesterBase { -public: - //! init tester with known amount of work - tester() : TesterBase(<user-specified tests count>) { ... } - - //! run a test with sequental number @arg test_number for @arg thread. - / *override* / value_t test(int test_number, int thread); -}; - -******/ - -template<typename Tester, int scale = 1> -class TimeTest : public Tester { - /*override*/ value_t test(int testn, int threadn) { - Timer timer; - Tester::test(testn, threadn); - return timer.get_time() * double(scale); - } -}; - -template<typename Tester> -class NanosecPerValue : public Tester { - /*override*/ value_t test(int testn, int threadn) { - Timer timer; - Tester::test(testn, threadn); - // return time (ns) per value - return timer.get_time()*1e+9/double(Tester::value); - } -}; - -template<typename Tester, int scale = 1> -class ValuePerSecond : public Tester { - /*override*/ value_t test(int testn, int threadn) { - Timer timer; - Tester::test(testn, threadn); - // return value per seconds/scale - return double(Tester::value)/(timer.get_time()*scale); - } -}; - -template<typename Tester, int scale = 1> -class NumberPerSecond : public Tester { - /*override*/ value_t test(int testn, int threadn) { - Timer timer; - Tester::test(testn, threadn); - // return a scale per seconds - return double(scale)/timer.get_time(); - } -}; - -// operate with single tester -class TestRunner { - friend class TestProcessor; - friend struct RunArgsBody; - TestRunner(const TestRunner &); // don't copy - - const char *tester_name; - StatisticsCollector *stat; - std::vector<std::vector<StatisticsCollector::TestCase> > keys; - -public: - TesterBase &tester; - - template<typename Test> - TestRunner(const char *name, Test *test) - : tester_name(name), tester(*static_cast<TesterBase*>(test)) - { - test->tester_name = name; - } - - ~TestRunner() { delete &tester; } - - void init(arg_t value, int threads, Harness::SpinBarrier &barrier, StatisticsCollector *s) { - tester.base_init(value, threads, barrier); - stat = s; - keys.resize(tester.tests_count); - for(int testn = 0; testn < tester.tests_count; testn++) { - keys[testn].resize(threads); - std::string test_name(tester.get_name(testn)); - for(int threadn = 0; threadn < threads; threadn++) - keys[testn][threadn] = stat->SetTestCase(tester_name, test_name.c_str(), threadn); - } - } - - void run_test(int threadn) { - for(int testn = 0; testn < tester.tests_count; testn++) { - tester.test_prefix(testn, threadn); - tester.barrier->wait(); // <<<<<<<<<<<<<<<<< Barrier before running test mode - value_t result = tester.test(testn, threadn); - stat->AddRoundResult(keys[testn][threadn], result); - } - } - - void post_process(StatisticsCollector &report) { - const int threads = tester.threads_count; - for(int testn = 0; testn < tester.tests_count; testn++) { - size_t coln = keys[testn][0].getResults().size()-1; - value_t rsum = keys[testn][0].getResults()[coln]; - value_t rmin = rsum, rmax = rsum; - for(int threadn = 1; threadn < threads; threadn++) { - value_t result = keys[testn][threadn].getResults()[coln]; - rsum += result; // for both SUM or AVG - if(rmin > result) rmin = result; - if(rmax < result) rmax = result; - } - std::string test_name(tester.get_name(testn)); - const char *rname = tester.get_result_type(testn, TesterBase::SUM); - if( rname ) { - report.SetTestCase(tester_name, (test_name+rname).c_str(), threads); - report.AddRoundResult(rsum); - } - rname = tester.get_result_type(testn, TesterBase::MIN); - if( rname ) { - report.SetTestCase(tester_name, (test_name+rname).c_str(), threads); - report.AddRoundResult(rmin); - } - rname = tester.get_result_type(testn, TesterBase::AVG); - if( rname ) { - report.SetTestCase(tester_name, (test_name+rname).c_str(), threads); - report.AddRoundResult(rsum / threads); - } - rname = tester.get_result_type(testn, TesterBase::MAX); - if( rname ) { - report.SetTestCase(tester_name, (test_name+rname).c_str(), threads); - report.AddRoundResult(rmax); - } - } - } -}; - -struct RunArgsBody { - const vector<TestRunner*> &run_list; - RunArgsBody(const vector<TestRunner*> &a) : run_list(a) { } -#ifndef __TBB_parallel_for_H - void operator()(int thread) const { -#else - void operator()(const tbb::blocked_range<int> &r) const { - ASSERT( r.begin() + 1 == r.end(), 0); - int thread = r.begin(); -#endif - for(size_t i = 0; i < run_list.size(); i++) - run_list[i]->run_test(thread); - } -}; - -//! Main test processor. -/** Override or use like this: - class MyTestCollection : public TestProcessor { - void factory(arg_t value, int threads) { - process( value, threads, - run("my1", new tester<my1>() ), - run("my2", new tester<my2>() ), - end ); - if(value == threads) - stat->Print(); - } -}; -*/ - -class TestProcessor { - friend class TesterBase; - - // <threads, collector> - typedef std::map<int, StatisticsCollector *> statistics_collection; - statistics_collection stat_by_threads; - -protected: - // Members - const char *collection_name; - // current stat - StatisticsCollector *stat; - // token - size_t end; - -public: - StatisticsCollector report; - - // token of tests list - template<typename Test> - TestRunner *run(const char *name, Test *test) { - return new TestRunner(name, test); - } - - // iteration processing - void process(arg_t value, int threads, ...) { - // prepare items - stat = stat_by_threads[threads]; - if(!stat) { - stat_by_threads[threads] = stat = new StatisticsCollector((collection_name + Format("@%d", threads)).c_str(), StatisticsCollector::ByAlg); - stat->SetTitle("Detailed log of %s running with %d threads.", collection_name, threads); - } - Harness::SpinBarrier barrier(threads); - // init args - va_list args; va_start(args, threads); - vector<TestRunner*> run_list; run_list.reserve(16); - while(true) { - TestRunner *item = va_arg(args, TestRunner*); - if( !item ) break; - item->init(value, threads, barrier, stat); - run_list.push_back(item); - } - va_end(args); - std::ostringstream buf; - buf << value; - const size_t round_number = stat->GetRoundsCount(); - stat->SetRoundTitle(round_number, buf.str().c_str()); - report.SetRoundTitle(round_number, buf.str().c_str()); - // run them -#ifndef __TBB_parallel_for_H - NativeParallelFor(threads, RunArgsBody(run_list)); -#else - tbb::parallel_for(tbb::blocked_range<int>(0,threads,1), RunArgsBody(run_list)); -#endif - // destroy args - for(size_t i = 0; i < run_list.size(); i++) { - run_list[i]->post_process(report); - delete run_list[i]; - } - } - -public: - TestProcessor(const char *name, StatisticsCollector::Sorting sort_by = StatisticsCollector::ByAlg) - : collection_name(name), stat(NULL), end(0), report(collection_name, sort_by) - { } - - ~TestProcessor() { - for(statistics_collection::iterator i = stat_by_threads.begin(); i != stat_by_threads.end(); i++) - delete i->second; - } -}; - -#endif// __TIME_FRAMEWORK_H__ diff --git a/src/tbb/src/perf/time_hash_map.cpp b/src/tbb/src/perf/time_hash_map.cpp deleted file mode 100644 index 690d092ff..000000000 --- a/src/tbb/src/perf/time_hash_map.cpp +++ /dev/null @@ -1,261 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -// configuration: -#define TBB_USE_THREADING_TOOLS 0 - -//! enable/disable std::map tests -#define STDTABLE 0 - -//! enable/disable old implementation tests (correct include file also) -#define OLDTABLE 0 -#define OLDTABLEHEADER "tbb/concurrent_hash_map-5468.h"//-4329 - -//! enable/disable experimental implementation tests (correct include file also) -#define TESTTABLE 0 -#define TESTTABLEHEADER "tbb/concurrent_unordered_map.h" - -//! avoid erase() -#define TEST_ERASE 1 - -////////////////////////////////////////////////////////////////////////////////// - -#include <cstdlib> -#include <math.h> -#include "tbb/tbb_stddef.h" -#include <vector> -#include <map> -// needed by hash_maps -#include <stdexcept> -#include <iterator> -#include <algorithm> // Need std::swap -#include <utility> // Need std::pair -#include "tbb/cache_aligned_allocator.h" -#include "tbb/tbb_allocator.h" -#include "tbb/spin_rw_mutex.h" -#include "tbb/aligned_space.h" -#include "tbb/atomic.h" -#define __TBB_concurrent_unordered_set_H -#include "tbb/internal/_concurrent_unordered_impl.h" -#undef __TBB_concurrent_unordered_set_H -// for test -#include "tbb/spin_mutex.h" -#include "time_framework.h" - - -using namespace tbb; -using namespace tbb::internal; - -struct IntHashCompare { - size_t operator() ( int x ) const { return x; } - bool operator() ( int x, int y ) const { return x==y; } - static long hash( int x ) { return x; } - bool equal( int x, int y ) const { return x==y; } -}; - -namespace version_current { - namespace tbb { using namespace ::tbb; namespace internal { using namespace ::tbb::internal; } } - namespace tbb { namespace interface5 { using namespace ::tbb::interface5; namespace internal { using namespace ::tbb::interface5::internal; } } } - #include "tbb/concurrent_hash_map.h" -} -typedef version_current::tbb::concurrent_hash_map<int,int> IntTable; - -#if OLDTABLE -#undef __TBB_concurrent_hash_map_H -namespace version_base { - namespace tbb { using namespace ::tbb; namespace internal { using namespace ::tbb::internal; } } - namespace tbb { namespace interface5 { using namespace ::tbb::interface5; namespace internal { using namespace ::tbb::interface5::internal; } } } - #include OLDTABLEHEADER -} -typedef version_base::tbb::concurrent_hash_map<int,int> OldTable; -#endif - -#if TESTTABLE -#undef __TBB_concurrent_hash_map_H -namespace version_new { - namespace tbb { using namespace ::tbb; namespace internal { using namespace ::tbb::internal; } } - namespace tbb { namespace interface5 { using namespace ::tbb::interface5; namespace internal { using namespace ::tbb::interface5::internal; } } } - #include TESTTABLEHEADER -} -typedef version_new::tbb::concurrent_unordered_map<int,int> TestTable; -#define TESTTABLE 1 -#endif - -/////////////////////////////////////// - -static const char *map_testnames[] = { - "1.insert", "2.count1st", "3.count2nd", "4.insert-exists", "5.erase " -}; - -template<typename TableType> -struct TestTBBMap : TesterBase { - TableType Table; - int n_items; - - TestTBBMap() : TesterBase(4+TEST_ERASE), Table(MaxThread*4) {} - void init() { n_items = value/threads_count; } - - std::string get_name(int testn) { - return std::string(map_testnames[testn]); - } - - double test(int test, int t) - { - switch(test) { - case 0: // fill - for(int i = t*n_items, e = (t+1)*n_items; i < e; i++) { - Table.insert( std::make_pair(i,i) ); - } - break; - case 1: // work1 - for(int i = t*n_items, e = (t+1)*n_items; i < e; i++) { - size_t c = Table.count( i ); - ASSERT( c == 1, NULL); - } - break; - case 2: // work2 - for(int i = t*n_items, e = (t+1)*n_items; i < e; i++) { - Table.count( i ); - } - break; - case 3: // work3 - for(int i = t*n_items, e = (t+1)*n_items; i < e; i++) { - Table.insert( std::make_pair(i,i) ); - } - break; -#if TEST_ERASE - case 4: // clean - for(int i = t*n_items, e = (t+1)*n_items; i < e; i++) { - ASSERT( Table.erase( i ), NULL); - } -#endif - } - return 0; - } -}; - -template<typename M> -struct TestSTLMap : TesterBase { - std::map<int, int> Table; - M mutex; - - int n_items; - TestSTLMap() : TesterBase(4+TEST_ERASE) {} - void init() { n_items = value/threads_count; } - - std::string get_name(int testn) { - return std::string(map_testnames[testn]); - } - - double test(int test, int t) - { - switch(test) { - case 0: // fill - for(int i = t*n_items, e = (t+1)*n_items; i < e; i++) { - typename M::scoped_lock with(mutex); - Table[i] = 0; - } - break; - case 1: // work1 - for(int i = t*n_items, e = (t+1)*n_items; i < e; i++) { - typename M::scoped_lock with(mutex); - size_t c = Table.count(i); - ASSERT( c == 1, NULL); - } - break; - case 2: // work2 - for(int i = t*n_items, e = (t+1)*n_items; i < e; i++) { - typename M::scoped_lock with(mutex); - Table.count(i); - } - break; - case 3: // work3 - for(int i = t*n_items, e = (t+1)*n_items; i < e; i++) { - typename M::scoped_lock with(mutex); - Table.insert(std::make_pair(i,i)); - } - break; - case 4: // clean - for(int i = t*n_items, e = (t+1)*n_items; i < e; i++) { - typename M::scoped_lock with(mutex); - Table.erase(i); - } - } - return 0; - } -}; - -class fake_mutex { -public: - class scoped_lock { - fake_mutex *p; - - public: - scoped_lock() {} - scoped_lock( fake_mutex &m ) { p = &m; } - ~scoped_lock() { } - void acquire( fake_mutex &m ) { p = &m; } - void release() { } - }; -}; - -class test_hash_map : public TestProcessor { -public: - test_hash_map() : TestProcessor("time_hash_map") {} - void factory(int value, int threads) { - if(Verbose) printf("Processing with %d threads: %d...\n", threads, value); - process( value, threads, -#if STDTABLE - run("std::map ", new NanosecPerValue<TestSTLMap<spin_mutex> >() ), -#endif -#if OLDTABLE - run("old::hmap", new NanosecPerValue<TestTBBMap<OldTable> >() ), -#endif - run("tbb::hmap", new NanosecPerValue<TestTBBMap<IntTable> >() ), -#if TESTTABLE - run("new::hmap", new NanosecPerValue<TestTBBMap<TestTable> >() ), -#endif - end ); - //stat->Print(StatisticsCollector::Stdout); - //if(value >= 2097152) stat->Print(StatisticsCollector::HTMLFile); - } -}; - -///////////////////////////////////////////////////////////////////////////////////////// - -int main(int argc, char* argv[]) { - if(argc>1) Verbose = true; - //if(argc>2) ExtraVerbose = true; - MinThread = 1; MaxThread = task_scheduler_init::default_num_threads(); - ParseCommandLine( argc, argv ); - - ASSERT(tbb_allocator<int>::allocator_type() == tbb_allocator<int>::scalable, "expecting scalable allocator library to be loaded. Please build it by:\n\t\tmake tbbmalloc"); - - { - test_hash_map the_test; - for( int t=MinThread; t <= MaxThread; t++) - for( int o=/*2048*/(1<<8)*8; o<2200000; o*=2 ) - the_test.factory(o, t); - the_test.report.SetTitle("Nanoseconds per operation of (Mode) for N items in container (Name)"); - the_test.report.SetStatisticFormula("1AVG per size", "=AVERAGE(ROUNDS)"); - the_test.report.Print(StatisticsCollector::HTMLFile|StatisticsCollector::ExcelXML); - } - return 0; -} diff --git a/src/tbb/src/perf/time_hash_map_fill.cpp b/src/tbb/src/perf/time_hash_map_fill.cpp deleted file mode 100644 index f0919375b..000000000 --- a/src/tbb/src/perf/time_hash_map_fill.cpp +++ /dev/null @@ -1,167 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -// configuration: - -// Size of final table (must be multiple of STEP_*) -int MAX_TABLE_SIZE = 2000000; - -// Specify list of unique percents (5-30,100) to test against. Max 10 values -#define UNIQUE_PERCENTS PERCENT(5); PERCENT(10); PERCENT(20); PERCENT(30); PERCENT(100) - -#define SECONDS_RATIO 1000000 // microseconds - -// enable/disable tests for: -#define BOX1 "CHMap" -#define BOX1TEST ValuePerSecond<Uniques<tbb::concurrent_hash_map<int,int> >, SECONDS_RATIO> -#define BOX1HEADER "tbb/concurrent_hash_map.h" - -// enable/disable tests for: -#define BOX2 "CUMap" -#define BOX2TEST ValuePerSecond<Uniques<tbb::concurrent_unordered_map<int,int> >, SECONDS_RATIO> -#define BOX2HEADER "tbb/concurrent_unordered_map.h" - -// enable/disable tests for: -//#define BOX3 "OLD" -#define BOX3TEST ValuePerSecond<Uniques<tbb::concurrent_hash_map<int,int> >, SECONDS_RATIO> -#define BOX3HEADER "tbb/concurrent_hash_map-5468.h" - -#define TBB_USE_THREADING_TOOLS 0 -////////////////////////////////////////////////////////////////////////////////// - -#include <cstdlib> -#include <math.h> -#include "tbb/tbb_stddef.h" -#include <vector> -#include <map> -// needed by hash_maps -#include <stdexcept> -#include <iterator> -#include <algorithm> // Need std::swap -#include <utility> // Need std::pair -#include <cstring> // Need std::memset -#include <typeinfo> -#include "tbb/cache_aligned_allocator.h" -#include "tbb/tbb_allocator.h" -#include "tbb/spin_rw_mutex.h" -#include "tbb/aligned_space.h" -#include "tbb/atomic.h" -#define __TBB_concurrent_unordered_set_H -#include "tbb/internal/_concurrent_unordered_impl.h" -#undef __TBB_concurrent_unordered_set_H -// for test -#include "tbb/spin_mutex.h" -#include "time_framework.h" - - -using namespace tbb; -using namespace tbb::internal; - -///////////////////////////////////////////////////////////////////////////////////////// -// Input data built for test -int *Data; - -// Main test class used to run the timing tests. All overridden methods are called by the framework -template<typename TableType> -struct Uniques : TesterBase { - TableType Table; - int n_items; - - // Initializes base class with number of test modes - Uniques() : TesterBase(2), Table(MaxThread*16) { - //Table->max_load_factor(1); // add stub into hash_map to uncomment it - } - ~Uniques() {} - - // Returns name of test mode specified by number - /*override*/ std::string get_name(int testn) { - if(testn == 1) return "find"; - return "insert"; - } - - // Informs the class that value and threads number become known - /*override*/ void init() { - n_items = value/threads_count; // operations - } - - // Informs the class that the test mode for specified thread is about to start - /*override*/ void test_prefix(int testn, int t) { - barrier->wait(); - if(Verbose && !t && testn) printf("%s: inserted %u, %g%% of operations\n", tester_name, unsigned(Table.size()), 100.0*Table.size()/(value*testn)); - } - - // Executes test mode for a given thread. Return value is ignored when used with timing wrappers. - /*override*/ double test(int testn, int t) - { - if( testn == 0 ) { // do insertions - for(int i = t*n_items, e = (t+1)*n_items; i < e; i++) { - Table.insert( std::make_pair(Data[i],t) ); - } - } else { // do last finds - for(int i = t*n_items, e = (t+1)*n_items; i < e; i++) { - size_t c = - Table.count( Data[i] ); - ASSERT( c == 1, NULL ); // must exist - } - } - return 0; - } -}; - -///////////////////////////////////////////////////////////////////////////////////////// -#undef max -#include <limits> - -// Using BOX declarations from configuration -#include "time_sandbox.h" - -int rounds = 0; -// Prepares the input data for given unique percent -void execute_percent(test_sandbox &the_test, int p) { - int input_size = MAX_TABLE_SIZE*100/p; - Data = new int[input_size]; - int uniques = p==100?std::numeric_limits<int>::max() : MAX_TABLE_SIZE; - ASSERT(p==100 || p <= 30, "Function is broken for %% > 30 except for 100%%"); - for(int i = 0; i < input_size; i++) - Data[i] = (rand()*rand())%uniques; - for(int t = MinThread; t <= MaxThread; t++) - the_test.factory(input_size, t); // executes the tests specified in BOX-es for given 'value' and threads - the_test.report.SetRoundTitle(rounds++, "%d%%", p); -} -#define PERCENT(x) execute_percent(the_test, x) - -int main(int argc, char* argv[]) { - if(argc>1) Verbose = true; - //if(argc>2) ExtraVerbose = true; - MinThread = 1; MaxThread = task_scheduler_init::default_num_threads(); - ParseCommandLine( argc, argv ); - if(getenv("TABLE_SIZE")) - MAX_TABLE_SIZE = atoi(getenv("TABLE_SIZE")); - - ASSERT(tbb_allocator<int>::allocator_type() == tbb_allocator<int>::scalable, "expecting scalable allocator library to be loaded. Please build it by:\n\t\tmake tbbmalloc"); - // Declares test processor - test_sandbox the_test("time_hash_map_fill"/*, StatisticsCollector::ByThreads*/); - srand(10101); - UNIQUE_PERCENTS; // test the percents - the_test.report.SetTitle("Operations per microsecond"); - the_test.report.SetRunInfo("Items", MAX_TABLE_SIZE); - the_test.report.Print(StatisticsCollector::HTMLFile|StatisticsCollector::ExcelXML); // Write files - return 0; -} diff --git a/src/tbb/src/perf/time_hash_map_fill.html b/src/tbb/src/perf/time_hash_map_fill.html deleted file mode 100644 index 766e85e18..000000000 --- a/src/tbb/src/perf/time_hash_map_fill.html +++ /dev/null @@ -1,122 +0,0 @@ -<HTML><BODY> -<H2>time_hash_map_fill</H2> -<P><a href=time_hash_map_fill.cpp>time_hash_map_fill.cpp</a> is a micro-benchmark specifically designed to highlight aspects of concurrent resizing algorithm of the hash tables. -It was derived from the Count Strings example that counts the number of unique words. But to exclude synchronization on the counters from the picture, -it was simplified to build just a set of unique numbers from an input array. The array is filled evenly by using a pseudo-random number generator from the standard C library for various proportions of unique numbers. -For example, for 5% of unique numbers, the same number is repeated 20 times on average. Together, it gives 5% of actual insertions and 95% are just lookups. However, in the beginning, there are more new keys occur than in the end. -In addition, a size of the source array correlates with input rates in order to produce the same number of unique keys at the end, and so exclude cache effects from the equation. -</P> -<H2>Diagram</H2><img src="time_hash_map_fill.gif"/> -<H3>Prepare results</H3> -<P>This benchmark outputs results in Excel* and html file formats by default. To generate text (CSV) file instead, specify STAT_FORMAT=pivot-csv evironment variable. To change the default table size, set TABLE_SIZE. -<code><b><pre>src$ make time_hash_map_fill args=-v STAT_FORMAT=pivot-csv TABLE_SIZE=250000</pre></b></code>Or to get statistics from different runs: -<code><b><pre>src$ make time_hash_map_fill TABLE_SIZE=50000 run_cmd="bash ../../src/perf/<a href=run_statistics.sh>run_statistics.sh</a>"</pre></b></code> -</P> -<H3>Build diagram</H3>You can use <a href="http://ploticus.sourceforge.net/">Ploticus</a> to build diagram from the prepared data using this html file as a script. But first, the input data file should be sorted to join lines from different runs together, e.g.: -<code><b><pre>src$ sort -t , -k 1dr,2 -k 3n,4 -k 7n,7 ../build/<i>{scrambled_path}</i>/time_hash_map_fill.csv -o perf/time_hash_map_fill.csv</pre></b></code>Here, field 7 is "Column" field that contains input rates because run_statistics.sh adds hostname and number of the run as 5 and 6 fields. Now, to build gif diagram, run: -<code><b><pre>perf$ pl -maxrows 200000 -maxfields 1500000 -maxvector 1200000 -gif -scale 1.8 time_hash_map_fill.html</pre></b></code> -<H3>Script body</H3> -<hr/><pre> - -#setifnotgiven NAMES = $makelist("1.CHMap 2.CUMap 3.OLD") -#setifnotgiven LABLESIZE = 0.06 - -#proc settings - encodenames: yes - units: cm - -#proc getdata - file: time_hash_map_fill.csv - fieldnameheader: yes - delim: comma - showdata: no - select: @@Mode = insert - pf_fieldnames: Name Mode Threads Value - filter: - ##print @@Name,"@@Items on @@Column",@@3,@@Value - -#endproc - -#proc page - pagesize: 70 50 - tightcrop: yes -#endproc - -#proc processdata - action: summary - fields: Name Mode Threads - valfield: Value - fieldnames: Name Mode Threads Average sd sem n_obs Min Max - showdata: no - -#proc categories - axis: x - datafield: Mode - -#proc areadef - title: Throughput on Insert operation - titledetails: size=14 align=C - areaname: slide - xscaletype: categories - xautorange: datafield=Mode - xaxis.stubs: usecategories - xaxis.label: Threads across table sizes and % of input rates -// yrange: 0 70 - yautorange: datafield=Max,Min - yaxis.stubs: inc - yaxis.label: ops/ns -// yaxis.stubformat: %3.1f - autowidth: 1.1 - autoheight: 0.07 - frame: yes - -#for LABEL in @NAMES -#set NLABEL = $arithl(@NLABEL+1) -#set COLOR = $icolor( @NLABEL ) -#proc legendentry - label: @LABEL - sampletype: color - details: @COLOR - -#procdef catlines - select: @Name = @LABEL - catfield: Mode - subcatfield: Threads - subcats: auto - plotwidth: 0.8 - #saveas C - -#proc catlines - #clone C - dpsymbol: shape=square radius=@LABLESIZE style=solid color=@COLOR - valfield: Average - errfield: sd - -#proc catlines - #clone C - valfield: Max - dpsymbol: shape=triangle radius=@LABLESIZE style=solid color=@COLOR - -#proc catlines - #clone C - valfield: Min - dpsymbol: shape=downtriangle radius=@LABLESIZE style=solid color=@COLOR - -#endloop - -#proc legend - location: 3.2 max - seglen: 0.2 -#endproc -</pre> -<HR/> -<A HREF="../index.html">Up to parent directory</A> -<p></p> -Copyright © 2005-2017 Intel Corporation. All Rights Reserved. -<P></P> -Intel is a registered trademark or trademark of Intel Corporation -or its subsidiaries in the United States and other countries. -<p></p> -* Other names and brands may be claimed as the property of others. -</BODY> -</HTML> diff --git a/src/tbb/src/perf/time_locked_work.cpp b/src/tbb/src/perf/time_locked_work.cpp deleted file mode 100644 index 3a966bbb3..000000000 --- a/src/tbb/src/perf/time_locked_work.cpp +++ /dev/null @@ -1,166 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -////// Test configuration //////////////////////////////////////////////////// -#define SECONDS_RATIO 1000000 // microseconds - -#ifndef REPEAT_K -#define REPEAT_K 50 // repeat coefficient -#endif - -int outer_work[] = {/*256,*/ 64, 16, 4, 0}; -int inner_work[] = {32, 8, 0 }; - -// keep it to calibrate the time of work without synchronization -#define BOX1 "baseline" -#define BOX1TEST TimeTest< TBB_Mutex<tbb::null_mutex>, SECONDS_RATIO > - -// enable/disable tests for: -#define BOX2 "spin_mutex" -#define BOX2TEST TimeTest< TBB_Mutex<tbb::spin_mutex>, SECONDS_RATIO > - -// enable/disable tests for: -#define BOX3 "spin_rw_mutex" -#define BOX3TEST TimeTest< TBB_Mutex<tbb::spin_rw_mutex>, SECONDS_RATIO > - -// enable/disable tests for: -#define BOX4 "queuing_mutex" -#define BOX4TEST TimeTest< TBB_Mutex<tbb::queuing_mutex>, SECONDS_RATIO > - -// enable/disable tests for: -//#define BOX5 "queuing_rw_mutex" -#define BOX5TEST TimeTest< TBB_Mutex<tbb::queuing_rw_mutex>, SECONDS_RATIO > - -////////////////////////////////////////////////////////////////////////////// - -#include <cstdlib> -#include <math.h> -#include <algorithm> // Need std::swap -#include <utility> // Need std::pair -#include <sstream> -#include "tbb/tbb_stddef.h" -#include "tbb/null_mutex.h" -#include "tbb/spin_rw_mutex.h" -#include "tbb/spin_mutex.h" -#include "tbb/queuing_mutex.h" -#include "tbb/queuing_rw_mutex.h" -#include "tbb/mutex.h" - -#if INTEL_TRIAL==2 -#include "tbb/parallel_for.h" // enable threading by TBB scheduler -#include "tbb/task_scheduler_init.h" -#include "tbb/blocked_range.h" -#endif -// for test -#include "time_framework.h" - -using namespace tbb; -using namespace tbb::internal; - -///////////////////////////////////////////////////////////////////////////////////////// - -//! base class for tests family -struct TestLocks : TesterBase { - // Inherits "value", "threads_count", and other variables - TestLocks() : TesterBase(/*number of modes*/sizeof(outer_work)/sizeof(int)) {} - //! returns name of test part/mode - /*override*/std::string get_name(int testn) { - std::ostringstream buf; - buf.width(4); buf.fill('0'); - buf << outer_work[testn]; // mode number - return buf.str(); - } - //! enables results types and returns theirs suffixes - /*override*/const char *get_result_type(int, result_t type) const { - switch(type) { - case MIN: return " min"; - case MAX: return " max"; - default: return 0; - } - } - //! repeats count - int repeat_until(int /*test_n*/) const { - return REPEAT_K*100;//TODO: suggest better? - } - //! fake work - void do_work(int work) volatile { - for(int i = 0; i < work; i++) { - volatile int x = i; - __TBB_Pause(0); // just to call inline assembler - x *= work/threads_count; - } - } -}; - -//! template test unit for any of TBB mutexes -template<typename M> -struct TBB_Mutex : TestLocks { - M mutex; - - double test(int testn, int /*threadn*/) - { - for(int r = 0; r < repeat_until(testn); ++r) { - do_work(outer_work[testn]); - { - typename M::scoped_lock with(mutex); - do_work(/*inner work*/value); - } - } - return 0; - } -}; - -///////////////////////////////////////////////////////////////////////////////////////// - -//Using BOX declarations -#include "time_sandbox.h" - -// run tests for each of inner work value -void RunLoops(test_sandbox &the_test, int thread) { - for( unsigned i=0; i<sizeof(inner_work)/sizeof(int); ++i ) - the_test.factory(inner_work[i], thread); -} - -int main(int argc, char* argv[]) { - if(argc>1) Verbose = true; - int DefThread = task_scheduler_init::default_num_threads(); - MinThread = 1; MaxThread = DefThread+1; - ParseCommandLine( argc, argv ); - ASSERT(MinThread <= MaxThread, 0); -#if INTEL_TRIAL && defined(__TBB_parallel_for_H) - task_scheduler_init me(MaxThread); -#endif - { - test_sandbox the_test("time_locked_work", StatisticsCollector::ByThreads); - //TODO: refactor this out as RunThreads(test&) - for( int t = MinThread; t < DefThread && t <= MaxThread; t *= 2) - RunLoops( the_test, t ); // execute undersubscribed threads - if( DefThread > MinThread && DefThread <= MaxThread ) - RunLoops( the_test, DefThread ); // execute on all hw threads - if( DefThread < MaxThread) - RunLoops( the_test, MaxThread ); // execute requested oversubscribed threads - - the_test.report.SetTitle("Time of lock/unlock for mutex Name with Outer and Inner work"); - //the_test.report.SetStatisticFormula("1AVG per size", "=AVERAGE(ROUNDS)"); - the_test.report.Print(StatisticsCollector::HTMLFile|StatisticsCollector::ExcelXML, /*ModeName*/ "Outer work"); - } - return 0; -} - diff --git a/src/tbb/src/perf/time_lru_cache_throughput.cpp b/src/tbb/src/perf/time_lru_cache_throughput.cpp deleted file mode 100644 index 151ef061a..000000000 --- a/src/tbb/src/perf/time_lru_cache_throughput.cpp +++ /dev/null @@ -1,217 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#include "../examples/common/utility/utility.h" -#include "tbb/tick_count.h" -//#include <tbb/parallel_for.h> -#include "tbb/task_scheduler_init.h" //for number of threads -#include <functional> - -#include "coarse_grained_raii_lru_cache.h" -#define TBB_PREVIEW_CONCURRENT_LRU_CACHE 1 -#include "tbb/concurrent_lru_cache.h" - -#define HARNESS_CUSTOM_MAIN 1 -#define HARNESS_NO_PARSE_COMMAND_LINE 1 - -#include "../src/test/harness.h" -#include "../src/test/harness_barrier.h" - -#include <vector> -#include <algorithm> -#include "tbb/mutex.h" - -//TODO: probably move this to separate header utlity file -namespace micro_benchmarking{ -namespace utils{ - template <typename type> - void disable_elimination(type const& v){ - volatile type dummy = v; - (void) dummy; - } - //Busy work and calibration helpers - unsigned int one_us_iters = 345; // default value - - //TODO: add a CLI parameter for calibration run - // if user wants to calibrate to microseconds on particular machine, call - // this at beginning of program; sets one_us_iters to number of iters to - // busy_wait for approx. 1 us - void calibrate_busy_wait() { - const unsigned niter = 1000000; - tbb::tick_count t0 = tbb::tick_count::now(); - for (volatile unsigned int i=0; i<niter; ++i) continue; - tbb::tick_count t1 = tbb::tick_count::now(); - - one_us_iters = (unsigned int)(niter/(t1-t0).seconds())*1e-6; - } - - void busy_wait(int us) - { - unsigned int iter = us*one_us_iters; - for (volatile unsigned int i=0; i<iter; ++i) continue; - } -} -} - -struct parameter_pack{ - size_t time_window_sec; - size_t time_check_granularity_ops; - size_t cache_lru_history_size; - size_t time_of_item_use_usec; - size_t cache_miss_percent; - int threads_number; - size_t weight_of_initiation_call_usec; - bool use_serial_initiation_function; - parameter_pack( - size_t a_time_window_sec - ,size_t a_time_check_granularity_ops - ,size_t a_cache_lru_history_size - ,size_t a_time_of_item_use_usec, size_t a_cache_miss_percent - , int a_threads_number ,size_t a_weight_of_initiation_call_usec - , bool a_use_serial_initiation_function - ) : - time_window_sec(a_time_window_sec) - ,time_check_granularity_ops(a_time_check_granularity_ops) - ,cache_lru_history_size(a_cache_lru_history_size) - ,time_of_item_use_usec(a_time_of_item_use_usec) - ,cache_miss_percent(a_cache_miss_percent) - ,threads_number(a_threads_number) - ,weight_of_initiation_call_usec(a_weight_of_initiation_call_usec) - ,use_serial_initiation_function(a_use_serial_initiation_function) - {} -}; - -struct return_size_t { - size_t m_weight_of_initiation_call_usec; - bool use_serial_initiation_function; - return_size_t(size_t a_weight_of_initiation_call_usec, bool a_use_serial_initiation_function) - :m_weight_of_initiation_call_usec(a_weight_of_initiation_call_usec), use_serial_initiation_function(a_use_serial_initiation_function) - {} - size_t operator()(size_t key){ - static tbb::mutex mtx; - if (use_serial_initiation_function){ - mtx.lock(); - } - micro_benchmarking::utils::busy_wait(m_weight_of_initiation_call_usec); - if (use_serial_initiation_function){ - mtx.unlock(); - } - - return key; - } -}; - -template< typename a_cache_type> -struct throughput { - typedef throughput self_type; - typedef a_cache_type cache_type; - - parameter_pack m_parameter_pack; - - - const size_t per_thread_sample_size ; - typedef std::vector<size_t> access_sequence_type; - access_sequence_type m_access_sequence; - cache_type m_cache; - Harness::SpinBarrier m_barrier; - tbb::atomic<size_t> loops_count; - - throughput(parameter_pack a_parameter_pack) - :m_parameter_pack(a_parameter_pack) - ,per_thread_sample_size(m_parameter_pack.cache_lru_history_size *(1 + m_parameter_pack.cache_miss_percent/100)) - ,m_access_sequence(m_parameter_pack.threads_number * per_thread_sample_size ) - ,m_cache(return_size_t(m_parameter_pack.weight_of_initiation_call_usec,m_parameter_pack.use_serial_initiation_function),m_parameter_pack.cache_lru_history_size) - - { - loops_count=0; - //TODO: check if changing from generating longer sequence to generating indexes in a specified range (i.e. making per_thread_sample_size fixed) give any change - std::generate(m_access_sequence.begin(),m_access_sequence.end(),std::rand); - } - - size_t operator()(){ - struct _{ static void retrieve_from_cache(self_type* _this, size_t thread_index){ - parameter_pack& p = _this->m_parameter_pack; - access_sequence_type::iterator const begin_it =_this->m_access_sequence.begin()+ thread_index * _this->per_thread_sample_size; - access_sequence_type::iterator const end_it = begin_it + _this->per_thread_sample_size; - - _this->m_barrier.wait(); - tbb::tick_count start = tbb::tick_count::now(); - - size_t local_loops_count =0; - do { - size_t part_of_the_sample_so_far = (local_loops_count * p.time_check_granularity_ops) % _this->per_thread_sample_size; - access_sequence_type::iterator const iteration_begin_it = begin_it + part_of_the_sample_so_far; - access_sequence_type::iterator const iteration_end_it = iteration_begin_it + - (std::min)(p.time_check_granularity_ops, _this->per_thread_sample_size - part_of_the_sample_so_far); - - for (access_sequence_type::iterator it = iteration_begin_it; it < iteration_end_it; ++it){ - typename cache_type::handle h = _this->m_cache[*it]; - micro_benchmarking::utils::busy_wait(p.time_of_item_use_usec); - micro_benchmarking::utils::disable_elimination(h.value()); - } - ++local_loops_count; - }while((tbb::tick_count::now()-start).seconds() < p.time_window_sec); - _this->loops_count+=local_loops_count; - }}; - m_barrier.initialize(m_parameter_pack.threads_number); - - NativeParallelFor(m_parameter_pack.threads_number,std::bind1st(std::ptr_fun(&_::retrieve_from_cache),this)); - - return loops_count * m_parameter_pack.time_check_granularity_ops; - } -}; - -int main(int argc,const char** args ){ - - size_t time_window_sec = 10; - size_t cache_lru_history_size = 1000; - size_t time_check_granularity_ops = 200; - size_t time_of_item_use_usec = 100; - size_t cache_miss_percent = 5; - int threads_number =tbb::task_scheduler_init::default_num_threads(); - size_t weight_of_initiation_call_usec =1000; - bool use_serial_initiation_function = false; - bool use_coarse_grained_locked_cache = false; - - parameter_pack p(time_window_sec, time_check_granularity_ops, cache_lru_history_size,time_of_item_use_usec,cache_miss_percent,threads_number,weight_of_initiation_call_usec,use_serial_initiation_function); - - utility::parse_cli_arguments(argc,args,utility::cli_argument_pack() - .arg(p.cache_lru_history_size,"cache-lru-history-size","") - .arg(p.time_window_sec,"time-window","time frame for measuring, in seconds") - .arg(p.threads_number,"n-of-threads","number of threads to run on") - .arg(p.time_of_item_use_usec,"time-of-item-use","time between consequent requests to the cache, in microseconds") - .arg(p.cache_miss_percent,"cache-miss-percent","cache miss percent ") - .arg(p.weight_of_initiation_call_usec,"initiation-call-weight","time occupied by a single call to initiation function, in microseconds") - .arg(p.use_serial_initiation_function,"use-serial-initiation-function","limit lock-based serial initiation function") - .arg(use_coarse_grained_locked_cache,"use-locked-version","use stl coarse grained lock based version") - ); - - typedef tbb::concurrent_lru_cache<size_t,size_t,return_size_t> tbb_cache; - typedef coarse_grained_raii_lru_cache<size_t,size_t,return_size_t> coarse_grained_locked_cache; - - size_t operations =0; - if (!use_coarse_grained_locked_cache){ - operations = throughput<tbb_cache>(p)(); - }else{ - operations = throughput<coarse_grained_locked_cache>(p)(); - } - std::cout<<"operations: "<<operations<<std::endl; - return 0; -} diff --git a/src/tbb/src/perf/time_parallel_for_each.cpp b/src/tbb/src/perf/time_parallel_for_each.cpp deleted file mode 100644 index 10e99c0d0..000000000 --- a/src/tbb/src/perf/time_parallel_for_each.cpp +++ /dev/null @@ -1,70 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#include <vector> -#include <list> -#include <iostream> -#include <cstdlib> -#include <algorithm> -#include <string> - -#include "tbb/parallel_for_each.h" -#include "tbb/tick_count.h" - -template <typename Type> -void foo( Type &f ) { - f += 1.0f; -} - -template <typename Container> -void test( std::string testName, const int N, const int numRepeats ) { - typedef typename Container::value_type Type; - Container v; - - for ( int i = 0; i < N; ++i ) { - v.push_back( static_cast<Type>(std::rand()) ); - } - - std::vector<double> times; - times.reserve( numRepeats ); - - for ( int i = 0; i < numRepeats; ++i ) { - tbb::tick_count t0 = tbb::tick_count::now(); - tbb::parallel_for_each( v.begin(), v.end(), foo<Type> ); - tbb::tick_count t1 = tbb::tick_count::now(); - times.push_back( (t1 - t0).seconds()*1e+3 ); - } - - std::sort( times.begin(), times.end() ); - std::cout << "Test " << testName << std::endl - << "min " << times[times.size() / 20] << " ms " << std::endl - << "med " << times[times.size() / 2] << " ms " << std::endl - << "max " << times[times.size() - times.size() / 20 - 1] << " ms " << std::endl; -} - -int main( int argc, char* argv[] ) { - const int N = argc > 1 ? std::atoi( argv[1] ) : 10 * 1000; - const int numRepeats = argc > 2 ? std::atoi( argv[2] ) : 10; - - test< std::vector<float> >( "std::vector<float>", N, numRepeats ); - test< std::list<float> >( "std::list<float>", N / 100, numRepeats ); - - return 0; -} diff --git a/src/tbb/src/perf/time_sandbox.h b/src/tbb/src/perf/time_sandbox.h deleted file mode 100644 index 120b1b502..000000000 --- a/src/tbb/src/perf/time_sandbox.h +++ /dev/null @@ -1,171 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __TIME_FRAMEWORK_H__ -#error time_framework.h must be included -#endif - -#define INJECT_TBB namespace tbb { using namespace ::tbb; namespace internal { using namespace ::tbb::internal; } } -#define INJECT_TBB5 namespace tbb { namespace interface5 { using namespace ::tbb::interface5; namespace internal { using namespace ::tbb::interface5::internal; } } } - -#ifndef INJECT_BOX_NAMES -#if defined(__TBB_task_H) || defined(__TBB_concurrent_unordered_internal_H) || defined(__TBB_reader_writer_lock_H) || defined(__TBB__concurrent_unordered_impl_H) -#define INJECT_BOX_NAMES INJECT_TBB INJECT_TBB5 -#else -#define INJECT_BOX_NAMES INJECT_TBB -#endif -#endif - -#ifdef BOX1 -namespace sandbox1 { - INJECT_BOX_NAMES -# ifdef BOX1HEADER -# include BOX1HEADER -# endif - typedef ::BOX1TEST testbox; -} -#endif -#ifdef BOX2 -namespace sandbox2 { - INJECT_BOX_NAMES -# ifdef BOX2HEADER -# include BOX2HEADER -# endif - typedef ::BOX2TEST testbox; -} -#endif -#ifdef BOX3 -namespace sandbox3 { - INJECT_BOX_NAMES -# ifdef BOX3HEADER -# include BOX3HEADER -# endif - typedef ::BOX3TEST testbox; -} -#endif -#ifdef BOX4 -namespace sandbox4 { - INJECT_BOX_NAMES -# ifdef BOX4HEADER -# include BOX4HEADER -# endif - typedef ::BOX4TEST testbox; -} -#endif -#ifdef BOX5 -namespace sandbox5 { - INJECT_BOX_NAMES -# ifdef BOX5HEADER -# include BOX5HEADER -# endif - typedef ::BOX5TEST testbox; -} -#endif -#ifdef BOX6 -namespace sandbox6 { - INJECT_BOX_NAMES -# ifdef BOX6HEADER -# include BOX6HEADER -# endif - typedef ::BOX6TEST testbox; -} -#endif -#ifdef BOX7 -namespace sandbox7 { - INJECT_BOX_NAMES -# ifdef BOX7HEADER -# include BOX7HEADER -# endif - typedef ::BOX7TEST testbox; -} -#endif -#ifdef BOX8 -namespace sandbox8 { - INJECT_BOX_NAMES -# ifdef BOX8HEADER -# include BOX8HEADER -# endif - typedef ::BOX8TEST testbox; -} -#endif -#ifdef BOX9 -namespace sandbox9 { - INJECT_BOX_NAMES -# ifdef BOX9HEADER -# include BOX9HEADER -# endif - typedef ::BOX9TEST testbox; -} -#endif - -//if harness.h included -#if defined(ASSERT) && !HARNESS_NO_PARSE_COMMAND_LINE -#ifndef TEST_PREFIX -#define TEST_PREFIX if(Verbose) printf("Processing with %d threads: %ld...\n", threads, long(value)); -#endif -#endif//harness included - -#ifndef TEST_PROCESSOR_NAME -#define TEST_PROCESSOR_NAME test_sandbox -#endif - -class TEST_PROCESSOR_NAME : public TestProcessor { -public: - TEST_PROCESSOR_NAME(const char *name, StatisticsCollector::Sorting sort_by = StatisticsCollector::ByAlg) - : TestProcessor(name, sort_by) {} - void factory(arg_t value, int threads) { -#ifdef TEST_PREFIX - TEST_PREFIX -#endif - process( value, threads, -#define RUNBOX(n) run(#n"."BOX##n, new sandbox##n::testbox() ) -#ifdef BOX1 - RUNBOX(1), -#endif -#ifdef BOX2 - RUNBOX(2), -#endif -#ifdef BOX3 - RUNBOX(3), -#endif -#ifdef BOX4 - RUNBOX(4), -#endif -#ifdef BOX5 - RUNBOX(5), -#endif -#ifdef BOX6 - RUNBOX(6), -#endif -#ifdef BOX7 - RUNBOX(7), -#endif -#ifdef BOX8 - RUNBOX(8), -#endif -#ifdef BOX9 - RUNBOX(9), -#endif - end ); -#ifdef TEST_POSTFIX - TEST_POSTFIX -#endif - } -}; diff --git a/src/tbb/src/perf/time_split_node.cpp b/src/tbb/src/perf/time_split_node.cpp deleted file mode 100644 index dab9ac42d..000000000 --- a/src/tbb/src/perf/time_split_node.cpp +++ /dev/null @@ -1,120 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#include <iostream> -#include "tbb/flow_graph.h" -#include "tbb/tick_count.h" -#include "tbb/task_scheduler_init.h" - -static double bm_split_node(tbb::flow::graph& g, int nIter); -static double bm_broadcast_node(tbb::flow::graph& g, int nIter); -static double bm_queue_node(tbb::flow::graph& g, int nIter); - -typedef int my_type; -//typedef std::vector<int> my_type; - -const int nIter = 1 << 24; //16M -const int nSize = 100000000; - -int main() -{ - //set up one thread to eliminate scheduler overheads - tbb::task_scheduler_init tsi(1); - - tbb::flow::graph g; - - //1. queue_node benchmark; calculate queue_node time + plus threads creation time (if we have multi-threading) - std::cout << "queue benchmark: number of calls of putting element:" << nIter; - const double tQueue = bm_queue_node(g, nIter); - std::cout << "; time:" << tQueue << std::endl << std::endl; - - //2. split_node benchmark - std::cout << "split_node benchmark: number of calls:" << nIter; - const double tSplitNode = bm_split_node(g, nIter); - //output split_node benchmark result - std::cout << "; time:" << tSplitNode << std::endl; - std::cout << "exclusive split_node time:" << tSplitNode - tQueue << std::endl << std::endl; - - //3. broadcast_node benchmark - std::cout << "broadcast_node benchmark: number of calls:" << nIter; - const double tBNode = bm_broadcast_node(g, nIter); - //output broadcast_node benchmark result - std::cout << "; time:" << tBNode << std::endl; - std::cout << "exclusive broadcast_node time:" << tBNode - tQueue << std::endl; - - return 0; -} - -//! Dummy executing split_node, "nIter" calls; Returns time in seconds. -double bm_split_node(tbb::flow::graph& g, int nIter) -{ - my_type v1(nSize); - - tbb::flow::queue_node<my_type> my_queue1(g); - tbb::flow::tuple<my_type> my_tuple(1); - - tbb::flow::split_node< tbb::flow::tuple<my_type> > my_split_node(g); - make_edge(tbb::flow::get<0>(my_split_node.output_ports()), my_queue1); - - const tbb::tick_count t0 = tbb::tick_count::now(); - - //using split_node - for (int i = 0; i < nIter; ++i) - my_split_node.try_put(my_tuple); - - //barrier sync - g.wait_for_all(); - - return (tbb::tick_count::now() - t0).seconds(); -} - -//! Dummy executing broadcast_node; "nIter" calls; Returns time in seconds. -double bm_broadcast_node(tbb::flow::graph& g, int nIter) -{ - tbb::flow::queue_node<my_type> my_queue(g); - tbb::flow::broadcast_node<my_type> my_broadcast_node(g); - make_edge(my_broadcast_node, my_queue); - - my_type v(nSize); - - const tbb::tick_count t0 = tbb::tick_count::now(); - - //using broadcast_node - for (int i = 0; i < nIter; ++i) - my_broadcast_node.try_put(v); - //barrier sync - g.wait_for_all(); - - return (tbb::tick_count::now() - t0).seconds(); -} - -double bm_queue_node(tbb::flow::graph& g, int nIter) -{ - tbb::flow::queue_node<my_type> first_queue(g); - - my_type v(nSize); - - tbb::tick_count t0 = tbb::tick_count::now(); - //using queue_node - for (int i = 0; i < nIter; ++i) - first_queue.try_put(v); - g.wait_for_all(); - return (tbb::tick_count::now() - t0).seconds(); -} diff --git a/src/tbb/src/perf/time_vector.cpp b/src/tbb/src/perf/time_vector.cpp deleted file mode 100644 index 02ae84c79..000000000 --- a/src/tbb/src/perf/time_vector.cpp +++ /dev/null @@ -1,249 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -//#define DO_SCALABLEALLOC - -#include <cstdlib> -#include <cmath> -#include <vector> -#include <algorithm> -#include <functional> -#include <numeric> -#include "tbb/tbb_stddef.h" -#include "tbb/spin_mutex.h" -#ifdef DO_SCALABLEALLOC -#include "tbb/scalable_allocator.h" -#endif -#include "tbb/concurrent_vector.h" -#include "tbb/tbb_allocator.h" -#include "tbb/cache_aligned_allocator.h" -#include "tbb/task_scheduler_init.h" -#include "tbb/parallel_for.h" -#include "tbb/tick_count.h" -#include "tbb/blocked_range.h" -#define HARNESS_CUSTOM_MAIN 1 -#include "../test/harness.h" -//#include "harness_barrier.h" -#include "../test/harness_allocator.h" -#define STATISTICS_INLINE -#include "statistics.h" - -using namespace tbb; -bool ExtraVerbose = false; - -class Timer { - tbb::tick_count tick; -public: - Timer() { tick = tbb::tick_count::now(); } - double get_time() { return (tbb::tick_count::now() - tick).seconds(); } - double diff_time(const Timer &newer) { return (newer.tick - tick).seconds(); } - double mark_time() { tick_count t1(tbb::tick_count::now()), t2(tick); tick = t1; return (t1 - t2).seconds(); } - double mark_time(const Timer &newer) { tick_count t(tick); tick = newer.tick; return (tick - t).seconds(); } -}; - -/************************************************************************/ -/* TEST1 */ -/************************************************************************/ -#define mk_vector_test1(v, a) vector_test1<v<Timer, static_counting_allocator<a<Timer> > >, v<double, static_counting_allocator<a<double> > > > -template<class timers_vector_t, class values_vector_t> -class vector_test1 { - const char *mode; - StatisticsCollector &stat; - StatisticsCollector::TestCase key[16]; - -public: - vector_test1(const char *m, StatisticsCollector &s) : mode(m), stat(s) {} - - vector_test1 &operator()(size_t len) { - if(Verbose) printf("test1<%s>(%u): collecting timing statistics\n", mode, unsigned(len)); - __TBB_ASSERT(sizeof(Timer) == sizeof(double), NULL); - static const char *test_names[] = { - "b)creation wholly", - "a)creation by push", - "c)operation time per item", - 0 }; - for(int i = 0; test_names[i]; ++i) key[i] = stat.SetTestCase(test_names[i], mode, len); - - Timer timer0; timers_vector_t::allocator_type::init_counters(); - timers_vector_t tv(len); - Timer timer1; values_vector_t::allocator_type::init_counters(); - values_vector_t dv; - for (size_t i = 0; i < len; ++i) - dv.push_back( i ); - Timer timer2; - for (size_t i = 0; i < len; ++i) - { - dv[len-i-1] = timer0.diff_time(tv[i]); - tv[i].mark_time(); - } - stat.AddStatisticValue( key[2], "1total, ms", "%.3f", timer2.get_time()*1e+3 ); - stat.AddStatisticValue( key[1], "1total, ms", "%.3f", timer1.diff_time(timer2)*1e+3 ); - stat.AddStatisticValue( key[0], "1total, ms", "%.3f", timer0.diff_time(timer1)*1e+3 ); - //allocator statistics - stat.AddStatisticValue( key[0], "2total allocations", "%d", int(timers_vector_t::allocator_type::allocations) ); - stat.AddStatisticValue( key[1], "2total allocations", "%d", int(values_vector_t::allocator_type::allocations) ); - stat.AddStatisticValue( key[2], "2total allocations", "%d", 0); - stat.AddStatisticValue( key[0], "3total alloc#items", "%d", int(timers_vector_t::allocator_type::items_allocated) ); - stat.AddStatisticValue( key[1], "3total alloc#items", "%d", int(values_vector_t::allocator_type::items_allocated) ); - stat.AddStatisticValue( key[2], "3total alloc#items", "%d", 0); - //remarks - stat.AddStatisticValue( key[0], "9note", "segment creation time, us:"); - stat.AddStatisticValue( key[2], "9note", "average op-time per item, us:"); - Timer last_timer(timer2); double last_value = 0; - for (size_t j = 0, i = 2; i < len; i *= 2, j++) { - stat.AddRoundResult( key[0], (dv[len-i-1]-last_value)*1e+6 ); - last_value = dv[len-i-1]; - stat.AddRoundResult( key[2], last_timer.diff_time(tv[i])/double(i)*1e+6 ); - last_timer = tv[i]; - stat.SetRoundTitle(j, i); - } - tv.clear(); dv.clear(); - //__TBB_ASSERT(timers_vector_t::allocator_type::items_allocated == timers_vector_t::allocator_type::items_freed, NULL); - //__TBB_ASSERT(values_vector_t::allocator_type::items_allocated == values_vector_t::allocator_type::items_freed, NULL); - return *this; - } -}; - -/************************************************************************/ -/* TEST2 */ -/************************************************************************/ -#define mk_vector_test2(v, a) vector_test2<v<size_t, a<size_t> > > -template<class vector_t> -class vector_test2 { - const char *mode; - static const int ntrial = 10; - StatisticsCollector &stat; - -public: - vector_test2(const char *m, StatisticsCollector &s) : mode(m), stat(s) {} - - vector_test2 &operator()(size_t len) { - if(Verbose) printf("test2<%s>(%u): performing standard transformation sequence on vector\n", mode, unsigned(len)); - StatisticsCollector::TestCase init_key = stat.SetTestCase("allocate", mode, len); - StatisticsCollector::TestCase fill_key = stat.SetTestCase("fill", mode, len); - StatisticsCollector::TestCase proc_key = stat.SetTestCase("process", mode, len); - StatisticsCollector::TestCase full_key = stat.SetTestCase("total time", mode, len); - for (int i = 0; i < ntrial; i++) { - Timer timer0; - vector_t v1(len); - vector_t v2(len); - Timer timer1; - std::generate(v1.begin(), v1.end(), values(0)); - std::generate(v2.begin(), v2.end(), values(size_t(-len))); - Timer timer2; - std::reverse(v1.rbegin(), v1.rend()); - std::inner_product(v1.begin(), v1.end(), v2.rbegin(), 1); - std::sort(v1.rbegin(), v1.rend()); - std::sort(v2.rbegin(), v2.rend()); - std::set_intersection(v1.begin(), v1.end(), v2.rbegin(), v2.rend(), v1.begin()); - Timer timer3; - stat.AddRoundResult( proc_key, timer2.diff_time(timer3)*1e+3 ); - stat.AddRoundResult( fill_key, timer1.diff_time(timer2)*1e+3 ); - stat.AddRoundResult( init_key, timer0.diff_time(timer1)*1e+3 ); - stat.AddRoundResult( full_key, timer0.diff_time(timer3)*1e+3 ); - } - stat.SetStatisticFormula("1Average", "=AVERAGE(ROUNDS)"); - stat.SetStatisticFormula("2+/-", "=(MAX(ROUNDS)-MIN(ROUNDS))/2"); - return *this; - } - - class values - { - size_t value; - public: - values(size_t i) : value(i) {} - size_t operator()() { - return value++%(1|(value^55)); - } - }; -}; - -/************************************************************************/ -/* TEST3 */ -/************************************************************************/ -#define mk_vector_test3(v, a) vector_test3<v<char, local_counting_allocator<a<char>, size_t > > > -template<class vector_t> -class vector_test3 { - const char *mode; - StatisticsCollector &stat; - -public: - vector_test3(const char *m, StatisticsCollector &s) : mode(m), stat(s) {} - - vector_test3 &operator()(size_t len) { - if(Verbose) printf("test3<%s>(%u): collecting allocator statistics\n", mode, unsigned(len)); - static const size_t sz = 1024; - vector_t V[sz]; - StatisticsCollector::TestCase vinst_key = stat.SetTestCase("instances number", mode, len); - StatisticsCollector::TestCase count_key = stat.SetTestCase("allocations count", mode, len); - StatisticsCollector::TestCase items_key = stat.SetTestCase("allocated items", mode, len); - //stat.ReserveRounds(sz-1); - for (size_t c = 0, i = 0, s = sz/2; s >= 1 && i < sz; s /= 2, c++) - { - const size_t count = c? 1<<(c-1) : 0; - for (size_t e = i+s; i < e; i++) { - //if(count >= 16) V[i].reserve(count); - for (size_t j = 0; j < count; j++) - V[i].push_back(j); - } - stat.SetRoundTitle ( c, count ); - stat.AddRoundResult( vinst_key, s ); - stat.AddRoundResult( count_key, V[i-1].get_allocator().allocations ); - stat.AddRoundResult( items_key, V[i-1].get_allocator().items_allocated ); - } - return *this; - } -}; - -/************************************************************************/ -/* TYPES SET FOR TESTS */ -/************************************************************************/ -#define types_set(n, title, op) { StatisticsCollector Collector("time_vector"#n); Collector.SetTitle title; \ - {mk_vector_test##n(tbb::concurrent_vector, tbb::cache_aligned_allocator) ("TBB:NFS", Collector)op;} \ - {mk_vector_test##n(tbb::concurrent_vector, tbb::tbb_allocator) ("TBB:TBB", Collector)op;} \ - {mk_vector_test##n(tbb::concurrent_vector, std::allocator) ("TBB:STD", Collector)op;} \ - {mk_vector_test##n(std::vector, tbb::cache_aligned_allocator) ("STL:NFS", Collector)op;} \ - {mk_vector_test##n(std::vector, tbb::tbb_allocator) ("STL:TBB", Collector)op;} \ - {mk_vector_test##n(std::vector, std::allocator) ("STL:STD", Collector)op;} \ - Collector.Print(StatisticsCollector::Stdout|StatisticsCollector::HTMLFile|StatisticsCollector::ExcelXML); } - - -/************************************************************************/ -/* MAIN DRIVER */ -/************************************************************************/ -int main(int argc, char* argv[]) { - if(argc>1) Verbose = true; - if(argc>2) ExtraVerbose = true; - MinThread = 0; MaxThread = 500000; // use in another meaning - test#:problem size - ParseCommandLine( argc, argv ); - - ASSERT(tbb_allocator<int>::allocator_type() == tbb_allocator<int>::scalable, "expecting scalable allocator library to be loaded"); - - if(!MinThread || MinThread == 1) - types_set(1, ("Vectors performance test #1 for %d", MaxThread), (MaxThread) ) - if(!MinThread || MinThread == 2) - types_set(2, ("Vectors performance test #2 for %d", MaxThread), (MaxThread) ) - if(!MinThread || MinThread == 3) - types_set(3, ("Vectors performance test #3 for %d", MaxThread), (MaxThread) ) - - if(!Verbose) printf("done\n"); - return 0; -} - diff --git a/src/tbb/src/rml/client/index.html b/src/tbb/src/rml/client/index.html deleted file mode 100644 index 53152ca9f..000000000 --- a/src/tbb/src/rml/client/index.html +++ /dev/null @@ -1,42 +0,0 @@ -<HTML> -<BODY> -<H2>Overview</H2> - -This directory has source code that must be statically linked into an RML client. - -<H2>Files</H2> - -<DL> -<DT><A HREF="rml_factory.h">rml_factory.h</A> -<DD>Text shared by <A HREF="rml_omp.cpp">rml_omp.cpp</A> and <A HREF="rml_tbb.cpp">rml_tbb.cpp</A>. - This is not an ordinary include file, so it does not have an #ifndef guard.</DD></DT> -</DL> - -<H3> Specific to client=OpenMP</H3> -<DL> -<DT><A HREF="rml_omp.cpp">rml_omp.cpp</A> -<DD>Source file for OpenMP client.</DD></DT> -<DT><A HREF="omp_dynamic_link.h">omp_dynamic_link.h</A></DT> -<DT><A HREF="omp_dynamic_link.cpp">omp_dynamic_link.cpp</A> -<DD>Source files for dynamic linking support. - The code is the code from the TBB source directory, but adjusted so that it - appears in namespace <TT>__kmp</TT> instead of namespace <TT>tbb::internal</TT>.</DD></DT> -</DL> -<H3> Specific to client=TBB</H3> -<DL> -<DT><A HREF="rml_tbb.cpp">rml_tbb.cpp</A> -<DD>Source file for TBB client. It uses the dynamic linking support from the TBB source directory.</DD></DT> -</DL> - -<HR/> -<A HREF="../index.html">Up to parent directory</A> -<p></p> -Copyright © 2005-2017 Intel Corporation. All Rights Reserved. -<P></P> -Intel is a registered trademark or trademark of Intel Corporation -or its subsidiaries in the United States and other countries. -<p></p> -* Other names and brands may be claimed as the property of others. -</BODY> -</HTML> - diff --git a/src/tbb/src/rml/client/library_assert.h b/src/tbb/src/rml/client/library_assert.h deleted file mode 100644 index df3fcdbdf..000000000 --- a/src/tbb/src/rml/client/library_assert.h +++ /dev/null @@ -1,34 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef LIBRARY_ASSERT_H -#define LIBRARY_ASSERT_H - -#ifndef LIBRARY_ASSERT -#ifdef KMP_ASSERT2 -#define LIBRARY_ASSERT(x,y) KMP_ASSERT2((x),(y)) -#else -#include <assert.h> -#define LIBRARY_ASSERT(x,y) assert(x) -#define __TBB_DYNAMIC_LOAD_ENABLED 1 -#endif -#endif /* LIBRARY_ASSERT */ - -#endif /* LIBRARY_ASSERT_H */ diff --git a/src/tbb/src/rml/client/omp_dynamic_link.h b/src/tbb/src/rml/client/omp_dynamic_link.h deleted file mode 100644 index 302029315..000000000 --- a/src/tbb/src/rml/client/omp_dynamic_link.h +++ /dev/null @@ -1,30 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __KMP_omp_dynamic_link_H -#define __KMP_omp_dynamic_link_H - -#define OPEN_INTERNAL_NAMESPACE namespace __kmp { -#define CLOSE_INTERNAL_NAMESPACE } - -#include "library_assert.h" -#include "tbb/dynamic_link.h" // Refers to src/tbb, not include/tbb - -#endif /* __KMP_omp_dynamic_link_H */ diff --git a/src/tbb/src/rml/client/rml_omp.cpp b/src/tbb/src/rml/client/rml_omp.cpp deleted file mode 100644 index 86935ed23..000000000 --- a/src/tbb/src/rml/client/rml_omp.cpp +++ /dev/null @@ -1,46 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#include "rml_omp.h" -#include "omp_dynamic_link.h" -#include <assert.h> - -namespace __kmp { -namespace rml { - -#define MAKE_SERVER(x) DLD(__KMP_make_rml_server,x) -#define GET_INFO(x) DLD(__KMP_call_with_my_server_info,x) -#define SERVER omp_server -#define CLIENT omp_client -#define FACTORY omp_factory - -#if __TBB_WEAK_SYMBOLS_PRESENT - #pragma weak __KMP_make_rml_server - #pragma weak __KMP_call_with_my_server_info - extern "C" { - omp_factory::status_type __KMP_make_rml_server( omp_factory& f, omp_server*& server, omp_client& client ); - void __KMP_call_with_my_server_info( ::rml::server_info_callback_t cb, void* arg ); - } -#endif /* __TBB_WEAK_SYMBOLS_PRESENT */ - -#include "rml_factory.h" - -} // rml -} // __kmp diff --git a/src/tbb/src/rml/client/rml_tbb.cpp b/src/tbb/src/rml/client/rml_tbb.cpp deleted file mode 100644 index 5842d1ed3..000000000 --- a/src/tbb/src/rml/client/rml_tbb.cpp +++ /dev/null @@ -1,48 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#include "../include/rml_tbb.h" -#include "tbb/dynamic_link.h" -#include <assert.h> - -namespace tbb { -namespace internal { -namespace rml { - -#define MAKE_SERVER(x) DLD(__TBB_make_rml_server,x) -#define GET_INFO(x) DLD(__TBB_call_with_my_server_info,x) -#define SERVER tbb_server -#define CLIENT tbb_client -#define FACTORY tbb_factory - -#if __TBB_WEAK_SYMBOLS_PRESENT - #pragma weak __TBB_make_rml_server - #pragma weak __TBB_call_with_my_server_info - extern "C" { - ::rml::factory::status_type __TBB_make_rml_server( tbb::internal::rml::tbb_factory& f, tbb::internal::rml::tbb_server*& server, tbb::internal::rml::tbb_client& client ); - void __TBB_call_with_my_server_info( ::rml::server_info_callback_t cb, void* arg ); - } -#endif /* __TBB_WEAK_SYMBOLS_PRESENT */ - -#include "rml_factory.h" - -} // rml -} // internal -} // tbb diff --git a/src/tbb/src/rml/include/index.html b/src/tbb/src/rml/include/index.html deleted file mode 100644 index 46c8b0a78..000000000 --- a/src/tbb/src/rml/include/index.html +++ /dev/null @@ -1,29 +0,0 @@ -<HTML> -<BODY> -<H2>Overview</H2> - -This directory has the include files for the Resource Management Layer (RML). - -<H2>Files</H2> - -<DL> -<DT><P><A HREF="rml_base.h">rml_base.h</A> -<DD>Interfaces shared by TBB and OpenMP.</P> -<DT><P><A HREF="rml_omp.h">rml_omp.h</A> -<DD>Interface exclusive to OpenMP.</P> -<DT><P><A HREF="rml_tbb.h">rml_tbb.h</A> -<DD>Interface exclusive to TBB.</P> -</DL> - -<HR> -<A HREF="../index.html">Up to parent directory</A> -<p></p> -Copyright © 2005-2017 Intel Corporation. All Rights Reserved. -<P></P> -Intel is a registered trademark or trademark of Intel Corporation -or its subsidiaries in the United States and other countries. -<p></p> -* Other names and brands may be claimed as the property of others. -</BODY> -</HTML> - diff --git a/src/tbb/src/rml/include/rml_omp.h b/src/tbb/src/rml/include/rml_omp.h deleted file mode 100644 index 103936bb1..000000000 --- a/src/tbb/src/rml/include/rml_omp.h +++ /dev/null @@ -1,130 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -// Header guard and namespace names follow OpenMP runtime conventions. - -#ifndef KMP_RML_OMP_H -#define KMP_RML_OMP_H - -#include "rml_base.h" - -namespace __kmp { -namespace rml { - -class omp_client; - -//------------------------------------------------------------------------ -// Classes instantiated by the server -//------------------------------------------------------------------------ - -//! Represents a set of omp worker threads provided by the server. -class omp_server: public ::rml::server { -public: - //! A number of coins (i.e., threads) - typedef unsigned size_type; - - //! Return the number of coins in the bank. (negative if machine is oversubscribed). - virtual int current_balance() const = 0; - - //! Request n coins. Returns number of coins granted. Oversubscription amount if negative. - /** Always granted if is_strict is true. - - Positive or zero result indicates that the number of coins was taken from the bank. - - Negative result indicates that no coins were taken, and that the bank has deficit - by that amount and the caller (if being a good citizen) should return that many coins. - */ - virtual int try_increase_load( size_type /*n*/, bool /*strict*/ ) = 0; - - //! Return n coins into the bank. - virtual void decrease_load( size_type /*n*/ ) = 0; - - //! Convert n coins into n threads. - /** When a thread returns, it is converted back into a coin and the coin is returned to the bank. */ - virtual void get_threads( size_type /*m*/, void* /*cookie*/, job* /*array*/[] ) = 0; - - /** Putting a thread to sleep - convert a thread into a coin - Waking up a thread - convert a coin into a thread - - Note: conversion between a coin and a thread does not affect the accounting. - */ -#if _WIN32||_WIN64 - //! Inform server of a tbb master thread. - virtual void register_master( execution_resource_t& /*v*/ ) = 0; - - //! Inform server that the tbb master thread is done with its work. - virtual void unregister_master( execution_resource_t /*v*/ ) = 0; - - //! deactivate - /** give control to ConcRT RM */ - virtual void deactivate( job* ) = 0; - - //! reactivate - virtual void reactivate( job* ) = 0; -#endif /* _WIN32||_WIN64 */ -}; - - -//------------------------------------------------------------------------ -// Classes (or base classes thereof) instantiated by the client -//------------------------------------------------------------------------ - -class omp_client: public ::rml::client { -public: - //! Called by server thread when it delivers a thread to client - /** The index argument is a 0-origin index of the job for this thread within the array - returned by method get_threads. Server decreases the load by 1 (i.e., returning the coin - back to the bank) after this method returns. */ - virtual void process( job&, void* /*cookie*/, size_type /*index*/ ) RML_PURE(void) -}; - -/** Client must ensure that instance is zero-inited, typically by being a file-scope object. */ -class omp_factory: public ::rml::factory { - - //! Pointer to routine that creates an RML server. - status_type (*my_make_server_routine)( omp_factory&, omp_server*&, omp_client& ); - - //! Pointer to routine that calls callback function with server version info. - void (*my_call_with_server_info_routine)( ::rml::server_info_callback_t cb, void* arg ); - -public: - typedef ::rml::versioned_object::version_type version_type; - typedef omp_client client_type; - typedef omp_server server_type; - - //! Open factory. - /** Dynamically links against RML library. - Returns st_success, st_incompatible, or st_not_found. */ - status_type open(); - - //! Factory method to be called by client to create a server object. - /** Factory must be open. - Returns st_success or st_incompatible . */ - status_type make_server( server_type*&, client_type& ); - - //! Close factory. - void close(); - - //! Call the callback with the server build info. - void call_with_server_info( ::rml::server_info_callback_t cb, void* arg ) const; -}; - -} // namespace rml -} // namespace __kmp - -#endif /* KMP_RML_OMP_H */ diff --git a/src/tbb/src/rml/index.html b/src/tbb/src/rml/index.html deleted file mode 100644 index 7c8e7ea2e..000000000 --- a/src/tbb/src/rml/index.html +++ /dev/null @@ -1,31 +0,0 @@ -<HTML> -<BODY> -<H2>Overview</H2> - -The subdirectories pertain to the Resource Management Layer (RML). - -<H2>Directories</H2> - -<DL> -<DT><P><A HREF="include/index.html">include/</A> -<DD>Include files used by clients of RML.</P> -<DT><P><A HREF="client/index.html">client/</A> -<DD>Source files for code that must be statically linked with a client.</P> -<DT><P><A HREF="server/index.html">server/</A> -<DD>Source files for the RML server.</P> -<DT><P><A HREF="test">test/</A> -<DD>Unit tests for RML server and its components.</P> -</DL> - -<HR> -<A HREF="../index.html">Up to parent directory</A> -<p></p> -Copyright © 2005-2017 Intel Corporation. All Rights Reserved. -<P></P> -Intel is a registered trademark or trademark of Intel Corporation -or its subsidiaries in the United States and other countries. -<p></p> -* Other names and brands may be claimed as the property of others. -</BODY> -</HTML> - diff --git a/src/tbb/src/rml/perfor/omp_nested.cpp b/src/tbb/src/rml/perfor/omp_nested.cpp deleted file mode 100644 index f0f675fe9..000000000 --- a/src/tbb/src/rml/perfor/omp_nested.cpp +++ /dev/null @@ -1,144 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#include <cstddef> -#include <cstdlib> -#include <cstdio> -#include <float.h> -#include <math.h> -#include <time.h> - -#include <omp.h> -#include <assert.h> - -#include "thread_level.h" - -#if _WIN32||_WIN64 -#include <Windows.h> /* Need Sleep */ -#else -#include <unistd.h> /* Need usleep */ -#endif - -void MilliSleep( unsigned milliseconds ) { -#if _WIN32||_WIN64 - Sleep( milliseconds ); -#else - usleep( milliseconds*1000 ); -#endif /* _WIN32||_WIN64 */ -} - -// Algorithm parameters -const int Max_OMP_Outer_Threads = 8; - -// Global variables -int max_outer_threads = Max_OMP_Outer_Threads; - -// Print help on command-line arguments -void help_message(char *prog_name) { - fprintf(stderr, "\n%s usage:\n", prog_name); - fprintf(stderr, - " Parameters:\n" - " -o<num> : max # of threads OMP should use at outer level\n" - "\n Help:\n" - " -h : print this help message\n"); -} - -// Process command-line arguments -void process_args(int argc, char *argv[], int *max_outer_t) { - (*max_outer_t) = omp_get_max_threads(); - for (int i=1; i<argc; ++i) { - if (argv[i][0] == '-') { - switch (argv[i][1]) { - case 'o': // set max_outer_threads - if (sscanf(&argv[i][2], "%d", max_outer_t) != 1 || *max_outer_t < 1) { - fprintf(stderr, "%s Warning: argument of -o option unacceptable: %s\n", argv[0], &argv[i][2]); - help_message(argv[0]); - } - break; - case 'h': // print help message - help_message(argv[0]); - exit(0); - break; - default: - fprintf(stderr, "%s: Warning: command-line option ignored: %s\n", argv[0], argv[i]); - help_message(argv[0]); - break; - } - } else { - fprintf(stderr, "%s: Warning: command-line option ignored: %s\n", argv[0], argv[i]); - help_message(argv[0]); - } - } -} - -int main(int argc, char *argv[]) { - process_args(argc, argv, &max_outer_threads); -#ifdef LOG_THREADS - TotalThreadLevel.init(); -#endif - - double start, end; - start = omp_get_wtime( ); - -#pragma omp parallel num_threads(max_outer_threads) - { - int omp_thread = omp_get_thread_num(); -#ifdef LOG_THREADS - if (omp_thread == 0) - TotalThreadLevel.change_level(omp_get_num_threads(), omp_outer); -#endif - if (omp_thread == 0) { - MilliSleep(3000); -#ifdef LOG_THREADS - TotalThreadLevel.change_level(-1, omp_outer); -#endif -#pragma omp parallel - { - int my_omp_thread = omp_get_thread_num(); -#ifdef LOG_THREADS - if (my_omp_thread == 0) - TotalThreadLevel.change_level(omp_get_num_threads(), omp_inner); -#endif - printf("Inner thread %d nested inside outer thread %d\n", my_omp_thread, omp_thread); -#ifdef LOG_THREADS - if (my_omp_thread == 0) - TotalThreadLevel.change_level(-omp_get_num_threads(), omp_inner); -#endif - } -#ifdef LOG_THREADS - TotalThreadLevel.change_level(1, omp_outer); -#endif - } - else { - MilliSleep(6000); - } -#ifdef LOG_THREADS - if (omp_thread == 0) - TotalThreadLevel.change_level(-omp_get_num_threads(), omp_outer); -#endif - } - end = omp_get_wtime( ); - printf("Simple test of nested OMP (%d outer threads max) took: %6.6f\n", - max_outer_threads, end-start); -#ifdef LOG_THREADS - TotalThreadLevel.dump(); -#endif - return 0; -} diff --git a/src/tbb/src/rml/perfor/omp_simple.cpp b/src/tbb/src/rml/perfor/omp_simple.cpp deleted file mode 100644 index 334b5717e..000000000 --- a/src/tbb/src/rml/perfor/omp_simple.cpp +++ /dev/null @@ -1,160 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#include <cstddef> -#include <cstdlib> -#include <cstdio> -#include <float.h> -#include <math.h> -#include <time.h> - -#include <omp.h> -#include <assert.h> - -#include "thread_level.h" - -#include "tbb/task.h" -#include "tbb/task_scheduler_init.h" -#include "tbb/parallel_for.h" -#include "tbb/blocked_range.h" - -#if _WIN32||_WIN64 -#include <Windows.h> /* Need Sleep */ -#else -#include <unistd.h> /* Need usleep */ -#endif - -void MilliSleep( unsigned milliseconds ) { -#if _WIN32||_WIN64 - Sleep( milliseconds ); -#else - usleep( milliseconds*1000 ); -#endif /* _WIN32||_WIN64 */ -} - -using namespace std; -using namespace tbb; - -// Algorithm parameters -const int Max_TBB_Threads = 16; -const int Max_OMP_Threads = 16; - -// Global variables -int max_tbb_threads = Max_TBB_Threads; -int max_omp_threads = Max_OMP_Threads; - -// Print help on command-line arguments -void help_message(char *prog_name) { - fprintf(stderr, "\n%s usage:\n", prog_name); - fprintf(stderr, - " Parameters:\n" - " -t<num> : max # of threads TBB should use\n" - " -o<num> : max # of threads OMP should use\n" - "\n Help:\n" - " -h : print this help message\n"); -} - -// Process command-line arguments -void process_args(int argc, char *argv[], int *max_tbb_t, int *max_omp_t) { - for (int i=1; i<argc; ++i) { - if (argv[i][0] == '-') { - switch (argv[i][1]) { - case 't': // set max_tbb_threads - if (sscanf(&argv[i][2], "%d", max_tbb_t) != 1 || *max_tbb_t < 1) { - fprintf(stderr, "%s Warning: argument of -t option unacceptable: %s\n", argv[0], &argv[i][2]); - help_message(argv[0]); - } - break; - case 'o': // set max_omp_threads - if (sscanf(&argv[i][2], "%d", max_omp_t) != 1 || *max_omp_t < 1) { - fprintf(stderr, "%s Warning: argument of -o option unacceptable: %s\n", argv[0], &argv[i][2]); - help_message(argv[0]); - } - break; - case 'h': // print help message - help_message(argv[0]); - exit(0); - break; - default: - fprintf(stderr, "%s: Warning: command-line option ignored: %s\n", argv[0], argv[i]); - help_message(argv[0]); - break; - } - } else { - fprintf(stderr, "%s: Warning: command-line option ignored: %s\n", argv[0], argv[i]); - help_message(argv[0]); - } - } -} - -int main(int argc, char *argv[]) { - process_args(argc, argv, &max_tbb_threads, &max_omp_threads); - TotalThreadLevel.init(); - - double start, end; - start = omp_get_wtime(); - -#pragma omp parallel num_threads(max_omp_threads) - { - int omp_thread = omp_get_thread_num(); -#ifdef LOG_THREADS - if (omp_thread == 0) - TotalThreadLevel.change_level(omp_get_num_threads(), omp_outer); -#endif - task_scheduler_init phase(max_tbb_threads); - if (omp_thread == 0) { - MilliSleep(3000); -#ifdef LOG_THREADS - TotalThreadLevel.change_level(-1, omp_outer); -#endif - parallel_for(blocked_range<size_t>(0, 1000), - [=](const blocked_range<size_t>& range) { -#ifdef LOG_THREADS - TotalThreadLevel.change_level(1, tbb_inner); -#endif -#pragma ivdep - for (size_t i=range.begin(); i!=range.end(); ++i) { - if (i==range.begin()) - printf("TBB range starting at %d on OMP thread %d\n", (int)i, omp_thread); - } -#ifdef LOG_THREADS - TotalThreadLevel.change_level(-1, tbb_inner); -#endif - }, auto_partitioner()); -#ifdef LOG_THREADS - TotalThreadLevel.change_level(1, omp_outer); -#endif - } - else { - MilliSleep(6000); - } -#ifdef LOG_THREADS - if (omp_thread == 0) - TotalThreadLevel.change_level(-omp_get_num_threads(), omp_outer); -#endif - } - end = omp_get_wtime(); - printf("Simple test of OMP (%d threads max) with TBB (%d threads max) inside took: %6.6f\n", - max_omp_threads, max_tbb_threads, end-start); -#ifdef LOG_THREADS - TotalThreadLevel.dump(); -#endif - return 0; -} diff --git a/src/tbb/src/rml/perfor/tbb_multi_omp.cpp b/src/tbb/src/rml/perfor/tbb_multi_omp.cpp deleted file mode 100644 index b324d13ff..000000000 --- a/src/tbb/src/rml/perfor/tbb_multi_omp.cpp +++ /dev/null @@ -1,186 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#include <cstddef> -#include <cstdlib> -#include <cstdio> -#include <float.h> -#include <math.h> -#include <time.h> - -#include <omp.h> -#include <assert.h> - -#include "thread_level.h" - -#include "tbb/task.h" -#include "tbb/tick_count.h" -#include "tbb/task_scheduler_init.h" -#include "tbb/scalable_allocator.h" - -#if _WIN32||_WIN64 -#include <Windows.h> /* Need Sleep */ -#else -#include <unistd.h> /* Need usleep */ -#endif - -void MilliSleep( unsigned milliseconds ) { -#if _WIN32||_WIN64 - Sleep( milliseconds ); -#else - usleep( milliseconds*1000 ); -#endif /* _WIN32||_WIN64 */ -} - -using namespace std; -using namespace tbb; - -// Algorithm parameters -const int Max_TBB_Threads = 16; -const int Max_OMP_Threads = 16; - -// Global variables -int max_tbb_threads = Max_TBB_Threads; -int max_omp_threads = Max_OMP_Threads; - -// Print help on command-line arguments -void help_message(char *prog_name) { - fprintf(stderr, "\n%s usage:\n", prog_name); - fprintf(stderr, - " Parameters:\n" - " -t<num> : max # of threads TBB should use\n" - " -o<num> : max # of threads OMP should use\n" - "\n Help:\n" - " -h : print this help message\n"); -} - -// Process command-line arguments -void process_args(int argc, char *argv[], int *max_tbb_t, int *max_omp_t) { - for (int i=1; i<argc; ++i) { - if (argv[i][0] == '-') { - switch (argv[i][1]) { - case 't': // set max_tbb_threads - if (sscanf(&argv[i][2], "%d", max_tbb_t) != 1 || *max_tbb_t < 1) { - fprintf(stderr, "%s Warning: argument of -t option unacceptable: %s\n", argv[0], &argv[i][2]); - help_message(argv[0]); - } - break; - case 'o': // set max_omp_threads - if (sscanf(&argv[i][2], "%d", max_omp_t) != 1 || *max_omp_t < 1) { - fprintf(stderr, "%s Warning: argument of -o option unacceptable: %s\n", argv[0], &argv[i][2]); - help_message(argv[0]); - } - break; - case 'h': // print help message - help_message(argv[0]); - exit(0); - break; - default: - fprintf(stderr, "%s: Warning: command-line option ignored: %s\n", argv[0], argv[i]); - help_message(argv[0]); - break; - } - } else { - fprintf(stderr, "%s: Warning: command-line option ignored: %s\n", argv[0], argv[i]); - help_message(argv[0]); - } - } -} - -class SimpleTask : public task { - bool isLeaf; - int myId; -public: - SimpleTask(bool isLeaf_, int myId_) : isLeaf(isLeaf_), myId(myId_) {} - task* execute() { -#ifdef LOG_THREADS - TotalThreadLevel.change_level(1, tbb_outer); -#endif - omp_set_num_threads(max_omp_threads); - if (!isLeaf) { - set_ref_count(65); - for (int i=0; i<64; ++i) { - SimpleTask& st = *new(allocate_child()) SimpleTask(true, i); - spawn(st); - } -#ifdef LOG_THREADS - TotalThreadLevel.change_level(-1, tbb_outer); -#endif - wait_for_all(); -#ifdef LOG_THREADS - TotalThreadLevel.change_level(1, tbb_outer); -#endif - } - else { - if (myId%2 == 0) { - MilliSleep(3000); -#pragma omp parallel - { -#ifdef LOG_THREADS - if (omp_get_thread_num() == 0) - TotalThreadLevel.change_level(omp_get_num_threads()-1, omp_inner); -#endif - //printf("In OMP parallel region on TBB task with myId=0: thread %d of %d\n", omp_get_thread_num(), omp_get_num_threads()); -#ifdef LOG_THREADS - if (omp_get_thread_num() == 0) - TotalThreadLevel.change_level(-(omp_get_num_threads()-1), omp_inner); -#endif - } - } - else { - MilliSleep(6000); - } - } -#ifdef LOG_THREADS - TotalThreadLevel.change_level(-1, tbb_outer); -#endif - return NULL; - } -}; - - -int main(int argc, char *argv[]) { -#ifdef LOG_THREADS - TotalThreadLevel.init(); - TotalThreadLevel.change_level(1, tbb_outer); -#endif - process_args(argc, argv, &max_tbb_threads, &max_omp_threads); - - task_scheduler_init phase(max_tbb_threads); - tick_count start, end; - start = tick_count::now(); - SimpleTask& st = *new(task::allocate_root()) SimpleTask(false, -1); -#ifdef LOG_THREADS - TotalThreadLevel.change_level(-1, tbb_outer); -#endif - task::spawn_root_and_wait(st); -#ifdef LOG_THREADS - TotalThreadLevel.change_level(1, tbb_outer); -#endif - end = tick_count::now(); - printf("Simple Test of TBB (%d threads max) with OMP (%d threads max) inside took: %6.6f\n", - max_tbb_threads, max_omp_threads, (end-start).seconds()); - -#ifdef LOG_THREADS - TotalThreadLevel.change_level(-1, tbb_outer); - TotalThreadLevel.dump(); -#endif - return 0; -} diff --git a/src/tbb/src/rml/perfor/tbb_simple.cpp b/src/tbb/src/rml/perfor/tbb_simple.cpp deleted file mode 100644 index b00bad93e..000000000 --- a/src/tbb/src/rml/perfor/tbb_simple.cpp +++ /dev/null @@ -1,191 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#include <cstddef> -#include <cstdlib> -#include <cstdio> -#include <float.h> -#include <math.h> -#include <time.h> - -#include <omp.h> -#include <assert.h> - -#include "thread_level.h" - -#include "tbb/task.h" -#include "tbb/tick_count.h" -#include "tbb/task_scheduler_init.h" - -#if _WIN32||_WIN64 -#include <Windows.h> /* Need Sleep */ -#else -#include <unistd.h> /* Need usleep */ -#endif - -void MilliSleep( unsigned milliseconds ) { -#if _WIN32||_WIN64 - Sleep( milliseconds ); -#else - usleep( milliseconds*1000 ); -#endif /* _WIN32||_WIN64 */ -} - -using namespace std; -using namespace tbb; - -// Algorithm parameters -const int Max_TBB_Threads = 16; -const int Max_OMP_Threads = 16; - -// Global variables -int max_tbb_threads = Max_TBB_Threads; -int max_omp_threads = Max_OMP_Threads; - -// Print help on command-line arguments -void help_message(char *prog_name) { - fprintf(stderr, "\n%s usage:\n", prog_name); - fprintf(stderr, - " Parameters:\n" - " -t<num> : max # of threads TBB should use\n" - " -o<num> : max # of threads OMP should use\n" - "\n Help:\n" - " -h : print this help message\n"); -} - -// Process command-line arguments -void process_args(int argc, char *argv[], int *max_tbb_t, int *max_omp_t) { - for (int i=1; i<argc; ++i) { - if (argv[i][0] == '-') { - switch (argv[i][1]) { - case 't': // set max_tbb_threads - if (sscanf(&argv[i][2], "%d", max_tbb_t) != 1 || *max_tbb_t < 1) { - fprintf(stderr, "%s Warning: argument of -t option unacceptable: %s\n", argv[0], &argv[i][2]); - help_message(argv[0]); - } - break; - case 'o': // set max_omp_threads - if (sscanf(&argv[i][2], "%d", max_omp_t) != 1 || *max_omp_t < 1) { - fprintf(stderr, "%s Warning: argument of -o option unacceptable: %s\n", argv[0], &argv[i][2]); - help_message(argv[0]); - } - break; - case 'h': // print help message - help_message(argv[0]); - exit(0); - break; - default: - fprintf(stderr, "%s: Warning: command-line option ignored: %s\n", argv[0], argv[i]); - help_message(argv[0]); - break; - } - } else { - fprintf(stderr, "%s: Warning: command-line option ignored: %s\n", argv[0], argv[i]); - help_message(argv[0]); - } - } -} - -class SimpleTask : public task { - bool isLeaf; - int myId; -public: - SimpleTask(bool isLeaf_, int myId_) : isLeaf(isLeaf_), myId(myId_) {} - task* execute() { -#ifdef LOG_THREADS - TotalThreadLevel.change_level(1, tbb_outer); -#endif - omp_set_num_threads(max_omp_threads); - if (!isLeaf) { - set_ref_count(17); - for (int i=0; i<16; ++i) { - SimpleTask& st = *new(allocate_child()) SimpleTask(true, i); - spawn(st); - } -#ifdef LOG_THREADS - TotalThreadLevel.change_level(-1, tbb_outer); -#endif - wait_for_all(); -#ifdef LOG_THREADS - TotalThreadLevel.change_level(1, tbb_outer); -#endif - } - else { - if (myId == 0) { - MilliSleep(3000); -#ifdef LOG_THREADS - TotalThreadLevel.change_level(-1, tbb_outer); -#endif -#pragma omp parallel - { -#ifdef LOG_THREADS - if (omp_get_thread_num() == 0) - TotalThreadLevel.change_level(omp_get_num_threads(), omp_inner); -#endif - printf("In OMP parallel region on TBB task with myId=0: thread %d of %d\n", - omp_get_thread_num(), omp_get_num_threads()); -#ifdef LOG_THREADS - if (omp_get_thread_num() == 0) - TotalThreadLevel.change_level(-omp_get_num_threads(), omp_inner); -#endif - } -#ifdef LOG_THREADS - TotalThreadLevel.change_level(1, tbb_outer); -#endif - } - else { - MilliSleep(6000); - } - } -#ifdef LOG_THREADS - TotalThreadLevel.change_level(-1, tbb_outer); -#endif - return NULL; - } -}; - - -int main(int argc, char *argv[]) { -#ifdef LOG_THREADS - TotalThreadLevel.init(); - TotalThreadLevel.change_level(1, tbb_outer); -#endif - process_args(argc, argv, &max_tbb_threads, &max_omp_threads); - - task_scheduler_init phase(max_tbb_threads); - tick_count start, end; - start = tick_count::now(); - SimpleTask& st = *new(task::allocate_root()) SimpleTask(false, -1); -#ifdef LOG_THREADS - TotalThreadLevel.change_level(-1, tbb_outer); -#endif - task::spawn_root_and_wait(st); -#ifdef LOG_THREADS - TotalThreadLevel.change_level(1, tbb_outer); -#endif - end = tick_count::now(); - printf("Simple Test of TBB (%d threads max) with OMP (%d threads max) inside took: %6.6f\n", - max_tbb_threads, max_omp_threads, (end-start).seconds()); -#ifdef LOG_THREADS - TotalThreadLevel.change_level(-1, tbb_outer); - TotalThreadLevel.dump(); -#endif - return 0; -} diff --git a/src/tbb/src/rml/perfor/thread_level.h b/src/tbb/src/rml/perfor/thread_level.h deleted file mode 100644 index 6a2912ee2..000000000 --- a/src/tbb/src/rml/perfor/thread_level.h +++ /dev/null @@ -1,134 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -// Thread level recorder -#ifndef __THREAD_LEVEL_H -#define __THREAD_LEVEL_H -#include <cstdio> -#include <omp.h> -#include <assert.h> -#include "tbb/atomic.h" -#include "tbb/tick_count.h" - -//#define LOG_THREADS // use this to ifdef out calls to this class -//#define NO_BAIL_OUT // continue execution after detecting oversubscription - -using namespace tbb; - -typedef enum {tbb_outer, tbb_inner, omp_outer, omp_inner} client_t; - -class ThreadLevelRecorder { - tbb::atomic<int> tbb_outer_level; - tbb::atomic<int> tbb_inner_level; - tbb::atomic<int> omp_outer_level; - tbb::atomic<int> omp_inner_level; - struct record { - tbb::tick_count time; - int n_tbb_outer_thread; - int n_tbb_inner_thread; - int n_omp_outer_thread; - int n_omp_inner_thread; - }; - tbb::atomic<unsigned> next; - /** Must be power of two */ - static const unsigned max_record_count = 1<<20; - record array[max_record_count]; - int max_threads; - bool fail; - public: - void change_level(int delta, client_t whichClient); - void dump(); - void init(); -}; - -void ThreadLevelRecorder::change_level(int delta, client_t whichClient) { - int tox=tbb_outer_level, tix=tbb_inner_level, oox=omp_outer_level, oix=omp_inner_level; - if (whichClient == tbb_outer) { - tox = tbb_outer_level+=delta; - } else if (whichClient == tbb_inner) { - tix = tbb_inner_level+=delta; - } else if (whichClient == omp_outer) { - oox = omp_outer_level+=delta; - } else if (whichClient == omp_inner) { - oix = omp_inner_level+=delta; - } else { - printf("WARNING: Bad client type; ignoring.\n"); - return; - } - // log non-negative entries - tbb::tick_count t = tbb::tick_count::now(); - unsigned k = next++; - if (k<max_record_count) { - record& r = array[k]; - r.time = t; - r.n_tbb_outer_thread = tox>=0?tox:0; - r.n_omp_outer_thread = oox>=0?oox:0; - r.n_tbb_inner_thread = tix>=0?tix:0; - r.n_omp_inner_thread = oix>=0?oix:0; - } - char errStr[100]; - int tot_threads; - tot_threads = tox+tix+oox+oix; - sprintf(errStr, "ERROR: Number of threads (%d+%d+%d+%d=%d) in use exceeds maximum (%d).\n", - tox, tix, oox, oix, tot_threads, max_threads); - if (tot_threads > max_threads) { -#ifdef NO_BAIL_OUT - if (!fail) { - printf("%sContinuing...\n", errStr); - fail = true; - } -#else - dump(); - printf("%s\n", errStr); - assert(tot_threads <= max_threads); -#endif - } -} - -void ThreadLevelRecorder::dump() { - FILE* f = fopen("time.txt","w"); - if (!f) { - perror("fopen(time.txt)\n"); - exit(1); - } - unsigned limit = next; - if (limit>max_record_count) { // Clip - limit = max_record_count; - } - for (unsigned i=0; i<limit; ++i) { - fprintf(f,"%f\t%d\t%d\t%d\t%d\n",(array[i].time-array[0].time).seconds(), array[i].n_tbb_outer_thread, - array[i].n_tbb_inner_thread, array[i].n_omp_outer_thread, array[i].n_omp_inner_thread); - } - fclose(f); - int tox=tbb_outer_level, tix=tbb_inner_level, oox=omp_outer_level, oix=omp_inner_level; - int tot_threads; - tot_threads = tox+tix+oox+oix; - if (!fail) printf("INFO: Passed.\n"); - else printf("INFO: Failed.\n"); -} - -void ThreadLevelRecorder::init() { - fail = false; - max_threads = omp_get_max_threads(); - printf("INFO: Getting maximum hardware threads... %d.\n", max_threads); -} - -ThreadLevelRecorder TotalThreadLevel; -#endif diff --git a/src/tbb/src/rml/server/index.html b/src/tbb/src/rml/server/index.html deleted file mode 100644 index dc2487465..000000000 --- a/src/tbb/src/rml/server/index.html +++ /dev/null @@ -1,18 +0,0 @@ -<HTML> -<BODY> -<H2>Overview</H2> - -This directory has source code internal to the server. - -<HR> -<A HREF="../index.html">Up to parent directory</A> -<p></p> -Copyright © 2005-2017 Intel Corporation. All Rights Reserved. -<P></P> -Intel is a registered trademark or trademark of Intel Corporation -or its subsidiaries in the United States and other countries. -<p></p> -* Other names and brands may be claimed as the property of others. -</BODY> -</HTML> - diff --git a/src/tbb/src/rml/server/job_automaton.h b/src/tbb/src/rml/server/job_automaton.h deleted file mode 100644 index 8fe81c967..000000000 --- a/src/tbb/src/rml/server/job_automaton.h +++ /dev/null @@ -1,145 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __RML_job_automaton_H -#define __RML_job_automaton_H - -#include "rml_base.h" -#include "tbb/atomic.h" - -#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) - // Workaround for overzealous compiler warnings - #pragma warning (push) - #pragma warning (disable: 4244) -#endif - -namespace rml { - -namespace internal { - -//! Finite state machine. -/** /--------------\ - / V - 0 --> 1--> ptr --> -1 - ^ - | - | - V - ptr|1 - -"owner" = corresponding server_thread. -Odd states (except -1) indicate that someone is executing code on the job. -Most transitions driven only by owner. -Transition 0-->-1 is driven by non-owner. -Transition ptr->-1 is driven by owner or non-owner. -*/ -class job_automaton: no_copy { -private: - tbb::atomic<intptr_t> my_job; -public: - /** Created by non-owner */ - job_automaton() { - my_job = 0; - } - - ~job_automaton() { - __TBB_ASSERT( my_job==-1, "must plug before destroying" ); - } - - //! Try to transition 0-->1 or ptr-->ptr|1. - /** Should only be called by owner. */ - bool try_acquire() { - intptr_t snapshot = my_job; - if( snapshot==-1 ) { - return false; - } else { - __TBB_ASSERT( (snapshot&1)==0, "already marked that way" ); - intptr_t old = my_job.compare_and_swap( snapshot|1, snapshot ); - __TBB_ASSERT( old==snapshot || old==-1, "unexpected interference" ); - return old==snapshot; - } - } - //! Transition ptr|1-->ptr - /** Should only be called by owner. */ - void release() { - intptr_t snapshot = my_job; - __TBB_ASSERT( snapshot&1, NULL ); - // Atomic store suffices here. - my_job = snapshot&~1; - } - - //! Transition 1-->ptr - /** Should only be called by owner. */ - void set_and_release( rml::job* job ) { - intptr_t value = reinterpret_cast<intptr_t>(job); - __TBB_ASSERT( (value&1)==0, "job misaligned" ); - __TBB_ASSERT( value!=0, "null job" ); - __TBB_ASSERT( my_job==1, "already set, or not marked busy?" ); - // Atomic store suffices here. - my_job = value; - } - - //! Transition 0-->-1 - /** If successful, return true. called by non-owner (for TBB and the likes) */ - bool try_plug_null() { - return my_job.compare_and_swap( -1, 0 )==0; - } - - //! Try to transition to -1. If successful, set j to contents and return true. - /** Called by owner or non-owner. (for OpenMP and the likes) */ - bool try_plug( rml::job*&j ) { - for(;;) { - intptr_t snapshot = my_job; - if( snapshot&1 ) { - j = NULL; - return false; - } - // Not busy - if( my_job.compare_and_swap( -1, snapshot )==snapshot ) { - j = reinterpret_cast<rml::job*>(snapshot); - return true; - } - // Need to retry, because current thread may be non-owner that read a 0, and owner might have - // caused transition 0->1->ptr after we took our snapshot. - } - } - - /** Called by non-owner to wait for transition to ptr. */ - rml::job* wait_for_job() const { - intptr_t snapshot; - for(;;) { - snapshot = my_job; - if( snapshot&~1 ) break; - __TBB_Yield(); - } - __TBB_ASSERT( snapshot!=-1, "wait on plugged job_automaton" ); - return reinterpret_cast<rml::job*>(snapshot&~1); - } -}; - -} // namespace internal -} // namespace rml - - -#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) - #pragma warning (pop) -#endif // warning 4244 are back - -#endif /* __RML_job_automaton_H */ diff --git a/src/tbb/src/rml/server/rml_server.cpp b/src/tbb/src/rml/server/rml_server.cpp deleted file mode 100644 index b25974164..000000000 --- a/src/tbb/src/rml/server/rml_server.cpp +++ /dev/null @@ -1,3309 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#include "rml_tbb.h" -#define private public /* Sleazy trick to avoid publishing internal names in public header. */ -#include "rml_omp.h" -#undef private - -#include "tbb/tbb_allocator.h" -#include "tbb/cache_aligned_allocator.h" -#include "tbb/aligned_space.h" -#include "tbb/atomic.h" -#include "tbb/spin_mutex.h" -#include "tbb/tbb_misc.h" // Get AvailableHwConcurrency() from here. -#if _MSC_VER==1500 && !defined(__INTEL_COMPILER) -// VS2008/VC9 seems to have an issue; -#pragma warning( push ) -#pragma warning( disable: 4985 ) -#endif -#include "tbb/concurrent_vector.h" -#if _MSC_VER==1500 && !defined(__INTEL_COMPILER) -#pragma warning( pop ) -#endif -#if _MSC_VER && defined(_Wp64) -// Workaround for overzealous compiler warnings -#pragma warning (push) -#pragma warning (disable: 4244) -#endif - -#include "job_automaton.h" -#include "wait_counter.h" -#include "thread_monitor.h" - -#if RML_USE_WCRM -#include <concrt.h> -#include <concrtrm.h> -using namespace Concurrency; -#include <vector> -#include <hash_map> -#define __RML_REMOVE_VIRTUAL_PROCESSORS_DISABLED 0 -#endif /* RML_USE_WCRM */ - -#define STRINGIFY(x) #x -#define TOSTRING(x) STRINGIFY(x) - -namespace rml { -namespace internal { - -using tbb::internal::rml::tbb_client; -using tbb::internal::rml::tbb_server; - -using __kmp::rml::omp_client; -using __kmp::rml::omp_server; - -typedef versioned_object::version_type version_type; - -#define SERVER_VERSION 2 -#define EARLIEST_COMPATIBLE_CLIENT_VERSION 2 - -static const size_t cache_line_size = tbb::internal::NFS_MaxLineSize; - -template<typename Server, typename Client> class generic_connection; -class tbb_connection_v2; -class omp_connection_v2; - -#if RML_USE_WCRM -//! State of a server_thread -/** Below are diagrams of legal state transitions. - - ts_busy - ^ ^ - / \ - / V - ts_done <----- ts_asleep <------> ts_idle -*/ - -enum thread_state_t { - ts_idle, - ts_asleep, - ts_busy, - ts_done -}; - -//! Extra state of an omp server thread -enum thread_extra_state_t { - ts_none, - ts_removed, - ts_lent -}; - -//! Results from try_grab_for() -enum thread_grab_t { - wk_failed, - wk_from_asleep, - wk_from_idle -}; - -#else /* !RML_USE_WCRM */ - -//! State of a server_thread -/** Below are diagrams of legal state transitions. - - OMP - ts_omp_busy - ^ ^ - / \ - / V - ts_asleep <-----------> ts_idle - - - ts_deactivated - ^ ^ - / \ - V \ - ts_none <--------------> ts_reactivated - - TBB - ts_tbb_busy - ^ ^ - / \ - / V - ts_asleep <-----------> ts_idle --> ts_done - - For TBB only. Extra state transition. - - ts_created -> ts_started -> ts_visited - */ -enum thread_state_t { - //! Thread not doing anything useful, but running and looking for work. - ts_idle, - //! Thread not doing anything useful and is asleep */ - ts_asleep, - //! Thread is enlisted into OpenMP team - ts_omp_busy, - //! Thread is busy doing TBB work. - ts_tbb_busy, - //! For tbb threads only - ts_done, - ts_created, - ts_started, - ts_visited, - //! For omp threads only - ts_none, - ts_deactivated, - ts_reactivated -}; -#endif /* RML_USE_WCRM */ - -#if TBB_USE_ASSERT -#define PRODUCE_ARG(x) ,x -#else -#define PRODUCE_ARG(x) -#endif /* TBB_USE_ASSERT */ - -//! Synchronizes dispatch of OpenMP work. -class omp_dispatch_type { - typedef ::rml::job job_type; - omp_client* client; - void* cookie; - omp_client::size_type index; - tbb::atomic<job_type*> job; -#if TBB_USE_ASSERT - omp_connection_v2* server; -#endif /* TBB_USE_ASSERT */ -public: - omp_dispatch_type() {job=NULL;} - void consume(); - void produce( omp_client& c, job_type* j, void* cookie_, omp_client::size_type index_ PRODUCE_ARG( omp_connection_v2& s )) { - __TBB_ASSERT( j, NULL ); - __TBB_ASSERT( !job, "job already set" ); - client = &c; -#if TBB_USE_ASSERT - server = &s; -#endif /* TBB_USE_ASSERT */ - cookie = cookie_; - index = index_; - // Must be last - job = j; - } -}; - -//! A reference count. -/** No default constructor, because users of ref_count must be very careful about whether the - initial reference count is 0 or 1. */ -class ref_count: no_copy { - friend class thread_map; - tbb::atomic<int> my_ref_count; -public: - ref_count(int k ) {my_ref_count=k;} - ~ref_count() {__TBB_ASSERT( !my_ref_count, "premature destruction of refcounted object" );} - //! Add one and return new value. - int add_ref() { - int k = ++my_ref_count; - __TBB_ASSERT(k>=1,"reference count underflowed before add_ref"); - return k; - } - //! Subtract one and return new value. - int remove_ref() { - int k = --my_ref_count; - __TBB_ASSERT(k>=0,"reference count underflow"); - return k; - } -}; - -#if RML_USE_WCRM - -#if USE_UMS_THREAD -#define RML_THREAD_KIND UmsThreadDefault -#define RML_THREAD_KIND_STRING "UmsThread" -#else -#define RML_THREAD_KIND ThreadScheduler -#define RML_THREAD_KIND_STRING "WinThread" -#endif - -// Forward declaration -class thread_map; - -static const IExecutionResource* c_remove_prepare = (IExecutionResource*)0; -static const IExecutionResource* c_remove_returned = (IExecutionResource*)1; - -//! Server thread representation -class server_thread_rep : no_copy { - friend class thread_map; - friend class omp_connection_v2; - friend class server_thread; - friend class tbb_server_thread; - friend class omp_server_thread; - template<typename Connection> friend void make_job( Connection& c, typename Connection::server_thread_type& t ); - typedef int thread_state_rep_t; -public: - //! Ctor - server_thread_rep( bool assigned, IScheduler* s, IExecutionResource* r, thread_map& map, rml::client& cl ) : - uid( GetExecutionContextId() ), my_scheduler(s), my_proxy(NULL), - my_thread_map(map), my_client(cl), my_job(NULL) - { - my_state = assigned ? ts_busy : ts_idle; - my_extra_state = ts_none; - terminate = false; - my_execution_resource = r; - } - //! Dtor - ~server_thread_rep() {} - - //! Synchronization routine - inline rml::job* wait_for_job() { - if( !my_job ) my_job = my_job_automaton.wait_for_job(); - return my_job; - } - - // Getters and setters - inline thread_state_t read_state() const { thread_state_rep_t s = my_state; return static_cast<thread_state_t>(s); } - inline void set_state( thread_state_t to ) {my_state = to;} - inline void set_removed() { __TBB_ASSERT( my_extra_state==ts_none, NULL ); my_extra_state = ts_removed; } - inline bool is_removed() const { return my_extra_state==ts_removed; } - inline bool is_lent() const {return my_extra_state==ts_lent;} - inline void set_lent() { my_extra_state=ts_lent; } - inline void set_returned() { my_extra_state=ts_none; } - inline IExecutionResource* get_execution_resource() { return my_execution_resource; } - inline IVirtualProcessorRoot* get_virtual_processor() { return (IVirtualProcessorRoot*)get_execution_resource(); } - - //! Enlist the thread for work - inline bool wakeup( thread_state_t to, thread_state_t from ) { - __TBB_ASSERT( from==ts_asleep && (to==ts_idle||to==ts_busy||to==ts_done), NULL ); - return my_state.compare_and_swap( to, from )==from; - } - - //! Enlist the thread for. - thread_grab_t try_grab_for(); - - //! Destroy the client job associated with the thread - template<typename Connection> bool destroy_job( Connection* c ); - - //! Try to re-use the thread - void revive( IScheduler* s, IExecutionResource* r, rml::client& c ) { - // the variables may not have been set before a thread was told to quit - __TBB_ASSERT( my_scheduler==s, "my_scheduler has been altered?\n" ); - my_scheduler = s; - __TBB_ASSERT( &my_client==&c, "my_client has been altered?\n" ); - if( r ) my_execution_resource = r; - my_client = c; - my_state = ts_idle; - __TBB_ASSERT( my_extra_state==ts_removed, NULL ); - my_extra_state = ts_none; - } - -protected: - const int uid; - IScheduler* my_scheduler; - IThreadProxy* my_proxy; - tbb::atomic<IExecutionResource*> my_execution_resource; /* for non-masters, it is IVirtualProcessorRoot */ - thread_map& my_thread_map; - rml::client& my_client; - job* my_job; - job_automaton my_job_automaton; - tbb::atomic<bool> terminate; - tbb::atomic<thread_state_rep_t> my_state; - tbb::atomic<thread_extra_state_t> my_extra_state; -}; - -//! Class that implements IExecutionContext -class server_thread : public IExecutionContext, public server_thread_rep { - friend class tbb_connection_v2; - friend class omp_connection_v2; - friend class tbb_server_thread; - friend class omp_server_thread; - friend class thread_map; - template<typename Connection> friend void make_job( Connection& c, typename Connection::server_thread_type& t ); -protected: - server_thread( bool is_tbb, bool assigned, IScheduler* s, IExecutionResource* r, thread_map& map, rml::client& cl ) : server_thread_rep(assigned,s,r,map,cl), tbb_thread(is_tbb) {} - ~server_thread() {} - unsigned int GetId() const __TBB_override { return uid; } - IScheduler* GetScheduler() __TBB_override { return my_scheduler; } - IThreadProxy* GetProxy() __TBB_override { return my_proxy; } - void SetProxy( IThreadProxy* thr_proxy ) __TBB_override { my_proxy = thr_proxy; } - -private: - bool tbb_thread; -}; - -// Forward declaration -class tbb_connection_v2; -class omp_connection_v2; - -//! TBB server thread -class tbb_server_thread : public server_thread { - friend class tbb_connection_v2; -public: - tbb_server_thread( bool assigned, IScheduler* s, IExecutionResource* r, tbb_connection_v2* con, thread_map& map, rml::client& cl ) : server_thread(true,assigned,s,r,map,cl), my_conn(con) { - activation_count = 0; - } - ~tbb_server_thread() {} - void Dispatch( DispatchState* ) __TBB_override; - inline bool initiate_termination(); - bool sleep_perhaps(); - //! Switch out this thread - bool switch_out(); -private: - tbb_connection_v2* my_conn; -public: - tbb::atomic<int> activation_count; -}; - -//! OMP server thread -class omp_server_thread : public server_thread { - friend class omp_connection_v2; -public: - omp_server_thread( bool assigned, IScheduler* s, IExecutionResource* r, omp_connection_v2* con, thread_map& map, rml::client& cl ) : - server_thread(false,assigned,s,r,map,cl), my_conn(con), my_cookie(NULL), my_index(UINT_MAX) {} - ~omp_server_thread() {} - void Dispatch( DispatchState* ) __TBB_override; - inline void* get_cookie() {return my_cookie;} - inline ::__kmp::rml::omp_client::size_type get_index() {return my_index;} - - inline IExecutionResource* get_execution_resource() { return get_execution_resource(); } - inline bool initiate_termination() { return destroy_job( (omp_connection_v2*) my_conn ); } - void sleep_perhaps(); -private: - omp_connection_v2* my_conn; - void* my_cookie; - ::__kmp::rml::omp_client::size_type my_index; - omp_dispatch_type omp_data; -}; - -//! Class that implements IScheduler -template<typename Connection> -class scheduler : no_copy, public IScheduler { -public: - unsigned int GetId() const __TBB_override {return uid;} - void Statistics( unsigned int* /*pTaskCompletionRate*/, unsigned int* /*pTaskArrivalRate*/, unsigned int* /*pNumberOfTaskEnqueued*/) __TBB_override {} - SchedulerPolicy GetPolicy() const __TBB_override { __TBB_ASSERT(my_policy,NULL); return *my_policy; } - void AddVirtualProcessors( IVirtualProcessorRoot** vproots, unsigned int count ) __TBB_override { if( !my_conn.is_closing() ) my_conn.add_virtual_processors( vproots, count); } - void RemoveVirtualProcessors( IVirtualProcessorRoot** vproots, unsigned int count ) __TBB_override; - void NotifyResourcesExternallyIdle( IVirtualProcessorRoot** vproots, unsigned int count ) __TBB_override { __TBB_ASSERT( false, "This call is not allowed for TBB" ); } - void NotifyResourcesExternallyBusy( IVirtualProcessorRoot** vproots, unsigned int count ) __TBB_override { __TBB_ASSERT( false, "This call is not allowed for TBB" ); } -protected: - scheduler( Connection& conn ); - virtual ~scheduler() { __TBB_ASSERT( my_policy, NULL ); delete my_policy; } - -public: - static scheduler* create( Connection& conn ) {return new scheduler( conn );} - -private: - const int uid; - Connection& my_conn; - SchedulerPolicy* my_policy; -}; - - -/* - * --> ts_busy --> ts_done - */ -class thread_scavenger_thread : public IExecutionContext, no_copy { -public: - thread_scavenger_thread( IScheduler* s, IVirtualProcessorRoot* r, thread_map& map ) : - uid( GetExecutionContextId() ), my_scheduler(s), my_virtual_processor_root(r), my_proxy(NULL), my_thread_map(map) - { - my_state = ts_busy; -#if TBB_USE_ASSERT - activation_count = 0; -#endif - } - ~thread_scavenger_thread() {} - unsigned int GetId() const __TBB_override { return uid; } - IScheduler* GetScheduler() __TBB_override { return my_scheduler; } - IThreadProxy* GetProxy() __TBB_override { return my_proxy; } - void SetProxy( IThreadProxy* thr_proxy ) __TBB_override { my_proxy = thr_proxy; } - void Dispatch( DispatchState* ) __TBB_override; - inline thread_state_t read_state() { return my_state; } - inline void set_state( thread_state_t s ) { my_state = s; } - inline IVirtualProcessorRoot* get_virtual_processor() { return my_virtual_processor_root; } -private: - const int uid; - IScheduler* my_scheduler; - IVirtualProcessorRoot* my_virtual_processor_root; - IThreadProxy* my_proxy; - thread_map& my_thread_map; - tbb::atomic<thread_state_t> my_state; -#if TBB_USE_ASSERT -public: - tbb::atomic<int> activation_count; -#endif -}; - -static const thread_scavenger_thread* c_claimed = reinterpret_cast<thread_scavenger_thread*>(1); - -struct garbage_connection_queue { - tbb::atomic<uintptr_t> head; - tbb::atomic<uintptr_t> tail; - static const uintptr_t empty = 0; // connection scavenger thread empty list - static const uintptr_t plugged = 1; // end of use of the list - static const uintptr_t plugged_acked = 2; // connection scavenger saw the plugged flag, and it freed all connections -}; - -//! Connection scavenger -/** It collects closed connection objects, wait for worker threads belonging to the connection to return to ConcRT RM - * then return the object to the memory manager. - */ -class connection_scavenger_thread { - friend void assist_cleanup_connections(); - /* - * connection_scavenger_thread's state - * ts_busy <----> ts_asleep <-- - */ - tbb::atomic<thread_state_t> state; - - /* We steal two bits from a connection pointer to encode - * whether the connection is for TBB or for OMP. - * - * ---------------------------------- - * | | | | - * ---------------------------------- - * ^ ^ - * / | - * 1 : tbb, 0 : omp | - * if set, terminate - */ - // FIXME: pad these? - thread_monitor monitor; - HANDLE thr_handle; -#if TBB_USE_ASSERT - tbb::atomic<int> n_scavenger_threads; -#endif - -public: - connection_scavenger_thread() : thr_handle(NULL) { - state = ts_asleep; -#if TBB_USE_ASSERT - n_scavenger_threads = 0; -#endif - } - - ~connection_scavenger_thread() {} - - void wakeup() { - if( state.compare_and_swap( ts_busy, ts_asleep )==ts_asleep ) - monitor.notify(); - } - - void sleep_perhaps(); - - void process_requests( uintptr_t conn_ex ); - - static __RML_DECL_THREAD_ROUTINE thread_routine( void* arg ); - - void launch() { - thread_monitor::launch( connection_scavenger_thread::thread_routine, this, NULL ); - } - - template<typename Server, typename Client> - void add_request( generic_connection<Server,Client>* conn_to_close ); - - template<typename Server, typename Client> - uintptr_t grab_and_prepend( generic_connection<Server,Client>* last_conn_to_close ); -}; - -void free_all_connections( uintptr_t ); - -#endif /* RML_USE_WCRM */ - -#if !RML_USE_WCRM -class server_thread; - -//! thread_map_base; we need to make the iterator type available to server_thread -struct thread_map_base { - //! A value in the map - class value_type { - public: - server_thread& thread() { - __TBB_ASSERT( my_thread, "thread_map::value_type::thread() called when !my_thread" ); - return *my_thread; - } - rml::job& job() { - __TBB_ASSERT( my_job, "thread_map::value_type::job() called when !my_job" ); - return *my_job; - } - value_type() : my_thread(NULL), my_job(NULL) {} - server_thread& wait_for_thread() const { - for(;;) { - server_thread* ptr=const_cast<server_thread*volatile&>(my_thread); - if( ptr ) - return *ptr; - __TBB_Yield(); - } - } - /** Shortly after when a connection is established, it is possible for the server - to grab a server_thread that has not yet created a job object for that server. */ - rml::job* wait_for_job() const { - if( !my_job ) { - my_job = my_automaton.wait_for_job(); - } - return my_job; - } - private: - server_thread* my_thread; - /** Marked mutable because though it is physically modified, conceptually it is a duplicate of - the job held by job_automaton. */ - mutable rml::job* my_job; - job_automaton my_automaton; - // FIXME - pad out to cache line, because my_automaton is hit hard by thread() - friend class thread_map; - }; - typedef tbb::concurrent_vector<value_type,tbb::zero_allocator<value_type,tbb::cache_aligned_allocator> > array_type; -}; -#endif /* !RML_USE_WCRM */ - -#if _MSC_VER && !defined(__INTEL_COMPILER) - // Suppress overzealous compiler warnings about uninstantiable class - #pragma warning(push) - #pragma warning(disable:4510 4610) -#endif - -template<typename T> -class padded: public T { - char pad[cache_line_size - sizeof(T)%cache_line_size]; -}; - -#if _MSC_VER && !defined(__INTEL_COMPILER) - #pragma warning(pop) -#endif - -// FIXME - should we pad out memory to avoid false sharing of our global variables? -static unsigned the_default_concurrency; -static tbb::atomic<int> the_balance; -static tbb::atomic<tbb::internal::do_once_state> rml_module_state; - -#if !RML_USE_WCRM -//! Per thread information -/** ref_count holds number of clients that are using this, - plus 1 if a host thread owns this instance. */ -class server_thread: public ref_count { - friend class thread_map; - template<typename Server, typename Client> friend class generic_connection; - friend class tbb_connection_v2; - friend class omp_connection_v2; - //! Integral type that can hold a thread_state_t - typedef int thread_state_rep_t; - tbb::atomic<thread_state_rep_t> state; -public: - thread_monitor monitor; -private: - bool is_omp_thread; - tbb::atomic<thread_state_rep_t> my_extra_state; - server_thread* link; - thread_map_base::array_type::iterator my_map_pos; - rml::server *my_conn; - rml::job* my_job; - job_automaton* my_ja; - size_t my_index; - tbb::atomic<bool> terminate; - omp_dispatch_type omp_dispatch; - -#if TBB_USE_ASSERT - //! Flag used to check if thread is still using *this. - bool has_active_thread; -#endif /* TBB_USE_ASSERT */ - - //! Volunteer to sleep. - void sleep_perhaps( thread_state_t asleep ); - - //! Destroy job corresponding to given client - /** Return true if thread must quit. */ - template<typename Connection> - bool destroy_job( Connection& c ); - - //! Do terminate the thread - /** Return true if thread must quit. */ - bool do_termination(); - - void loop(); - static __RML_DECL_THREAD_ROUTINE thread_routine( void* arg ); - -public: - server_thread(); - - ~server_thread(); - - //! Read the thread state - thread_state_t read_state() const { - thread_state_rep_t s = state; - __TBB_ASSERT( unsigned(s)<=unsigned(ts_done), "corrupted server thread?" ); - return thread_state_t(s); - } - - //! Read the tbb-specific extra thread state - thread_state_t read_extra_state() const { - thread_state_rep_t s = my_extra_state; - return thread_state_t(s); - } - - //! Launch a thread that is bound to *this. - void launch( size_t stack_size ); - - //! Attempt to wakeup a thread - /** The value "to" is the new state for the thread, if it was woken up. - Returns true if thread was woken up, false otherwise. */ - bool wakeup( thread_state_t to, thread_state_t from ); - - //! Attempt to enslave a thread for OpenMP/TBB. - /** Returns true if state is successfully changed. 's' takes either ts_omp_busy or ts_tbb_busy */ - bool try_grab_for( thread_state_t s ); - -#if _WIN32||_WIN64 - //! Send the worker thread to sleep temporarily - void deactivate(); - - //! Wake the worker thread up - void reactivate(); -#endif /* _WIN32||_WIN64 */ -}; - -//! Bag of threads that are private to a client. -class private_thread_bag { - struct list_thread: server_thread { - list_thread* next; - }; - //! Root of atomic linked list of list_thread - /** ABA problem is avoided because items are only atomically pushed, never popped. */ - tbb::atomic<list_thread*> my_root; - tbb::cache_aligned_allocator<padded<list_thread> > my_allocator; -public: - //! Construct empty bag - private_thread_bag() {my_root=NULL;} - - //! Create a fresh server_thread object. - server_thread& add_one_thread() { - list_thread* t = my_allocator.allocate(1); - new( t ) list_thread; - // Atomically add to list - list_thread* old_root; - do { - old_root = my_root; - t->next = old_root; - } while( my_root.compare_and_swap( t, old_root )!=old_root ); - return *t; - } - - //! Destroy the bag and threads in it. - ~private_thread_bag() { - while( my_root ) { - // Unlink thread from list. - list_thread* t = my_root; - my_root = t->next; - // Destroy and deallocate the thread. - t->~list_thread(); - my_allocator.deallocate(static_cast<padded<list_thread>*>(t),1); - } - } -}; - -//! Forward declaration -void wakeup_some_tbb_threads(); - -//! Type-independent part of class generic_connection. -/** One to one map from server threads to jobs, and associated reference counting. */ -class thread_map : public thread_map_base { -public: - typedef rml::client::size_type size_type; - //! ctor - thread_map( wait_counter& fc, ::rml::client& client ) : - all_visited_at_least_once(false), my_min_stack_size(0), my_server_ref_count(1), - my_client_ref_count(1), my_client(client), my_factory_counter(fc) - { my_unrealized_threads = 0; } - //! dtor - ~thread_map() {} - typedef array_type::iterator iterator; - iterator begin() {return my_array.begin();} - iterator end() {return my_array.end();} - void bind(); - void unbind(); - void assist_cleanup( bool assist_null_only ); - - /** Returns number of unrealized threads to create. */ - size_type wakeup_tbb_threads( size_type n ); - bool wakeup_next_thread( iterator i, tbb_connection_v2& conn ); - void release_tbb_threads( server_thread* t ); - void adjust_balance( int delta ); - - //! Add a server_thread object to the map, but do not bind it. - /** Return NULL if out of unrealized threads. */ - value_type* add_one_thread( bool is_omp_thread_ ); - - void bind_one_thread( rml::server& server, value_type& x ); - - void remove_client_ref(); - int add_server_ref() {return my_server_ref_count.add_ref();} - int remove_server_ref() {return my_server_ref_count.remove_ref();} - - ::rml::client& client() const {return my_client;} - - size_type get_unrealized_threads() { return my_unrealized_threads; } - -private: - private_thread_bag my_private_threads; - bool all_visited_at_least_once; - array_type my_array; - size_t my_min_stack_size; - tbb::atomic<size_type> my_unrealized_threads; - - //! Number of threads referencing *this, plus one extra. - /** When it becomes zero, the containing server object can be safely deleted. */ - ref_count my_server_ref_count; - - //! Number of jobs that need cleanup, plus one extra. - /** When it becomes zero, acknowledge_close_connection is called. */ - ref_count my_client_ref_count; - - ::rml::client& my_client; - //! Counter owned by factory that produced this thread_map. - wait_counter& my_factory_counter; -}; - -void thread_map::bind_one_thread( rml::server& server, value_type& x ) { - // Add one to account for the thread referencing this map hereforth. - server_thread& t = x.thread(); - my_server_ref_count.add_ref(); - my_client_ref_count.add_ref(); -#if TBB_USE_ASSERT - __TBB_ASSERT( t.add_ref()==1, NULL ); -#else - t.add_ref(); -#endif - // Have responsibility to start the thread. - t.my_conn = &server; - t.my_ja = &x.my_automaton; - t.launch( my_min_stack_size ); - /* Must wake thread up so it can fill in its "my_job" field in *this. - Otherwise deadlock can occur where wait_for_job spins on thread that is sleeping. */ - __TBB_ASSERT( t.state!=ts_tbb_busy, NULL ); - t.wakeup( ts_idle, ts_asleep ); -} - -thread_map::value_type* thread_map::add_one_thread( bool is_omp_thread_ ) { - size_type u; - do { - u = my_unrealized_threads; - if( !u ) return NULL; - } while( my_unrealized_threads.compare_and_swap(u-1,u)!=u ); - server_thread& t = my_private_threads.add_one_thread(); - t.is_omp_thread = is_omp_thread_; - __TBB_ASSERT( u>=1, NULL ); - t.my_index = u - 1; - __TBB_ASSERT( t.state!=ts_tbb_busy, NULL ); - t.my_extra_state = t.is_omp_thread ? ts_none : ts_created; - - iterator i = t.my_map_pos = my_array.grow_by(1); - value_type& v = *i; - v.my_thread = &t; - return &v; -} - -void thread_map::bind() { - ++my_factory_counter; - my_min_stack_size = my_client.min_stack_size(); - __TBB_ASSERT( my_unrealized_threads==0, "already called bind?" ); - my_unrealized_threads = my_client.max_job_count(); -} - -void thread_map::unbind() { - // Ask each server_thread to cleanup its job for this server. - for( iterator i=begin(); i!=end(); ++i ) { - server_thread& t = i->thread(); - t.terminate = true; - t.wakeup( ts_idle, ts_asleep ); - } - // Remove extra ref to client. - remove_client_ref(); -} - -void thread_map::assist_cleanup( bool assist_null_only ) { - // To avoid deadlock, the current thread *must* help out with cleanups that have not started, - // because the thread that created the job may be busy for a long time. - for( iterator i = begin(); i!=end(); ++i ) { - rml::job* j=0; - job_automaton& ja = i->my_automaton; - if( assist_null_only ? ja.try_plug_null() : ja.try_plug(j) ) { - if( j ) { - my_client.cleanup(*j); - } else { - // server thread did not get a chance to create a job. - } - remove_client_ref(); - } - } -} - -thread_map::size_type thread_map::wakeup_tbb_threads( size_type n ) { - __TBB_ASSERT(n>0,"must specify positive number of threads to wake up"); - iterator e = end(); - for( iterator k=begin(); k!=e; ++k ) { - // If another thread added *k, there is a tiny timing window where thread() is invalid. - server_thread& t = k->wait_for_thread(); - thread_state_t thr_s = t.read_state(); - if( t.read_extra_state()==ts_created || thr_s==ts_tbb_busy || thr_s==ts_done ) - continue; - if( --the_balance>=0 ) { // try to withdraw a coin from the deposit - while( !t.try_grab_for( ts_tbb_busy ) ) { - thr_s = t.read_state(); - if( thr_s==ts_tbb_busy || thr_s==ts_done ) { - // we lost; move on to the next. - ++the_balance; - goto skip; - } - } - if( --n==0 ) - return 0; - } else { - // overdraft. - ++the_balance; - break; - } -skip: - ; - } - return n<my_unrealized_threads ? n : size_type(my_unrealized_threads); -} -#else /* RML_USE_WCRM */ - -class thread_map : no_copy { - friend class omp_connection_v2; - typedef ::std::hash_map<uintptr_t,server_thread*> hash_map_type; - size_t my_min_stack_size; - size_t my_unrealized_threads; - ::rml::client& my_client; - //! Counter owned by factory that produced this thread_map. - wait_counter& my_factory_counter; - //! Ref counters - ref_count my_server_ref_count; - ref_count my_client_ref_count; - // FIXME: pad this? - hash_map_type my_map; - bool shutdown_in_progress; - std::vector<IExecutionResource*> original_exec_resources; - tbb::cache_aligned_allocator<padded<tbb_server_thread> > my_tbb_allocator; - tbb::cache_aligned_allocator<padded<omp_server_thread> > my_omp_allocator; - tbb::cache_aligned_allocator<padded<thread_scavenger_thread> > my_scavenger_allocator; - IResourceManager* my_concrt_resource_manager; - IScheduler* my_scheduler; - ISchedulerProxy* my_scheduler_proxy; - tbb::atomic<thread_scavenger_thread*> my_thread_scavenger_thread; -#if TBB_USE_ASSERT - tbb::atomic<int> n_add_vp_requests; - tbb::atomic<int> n_thread_scavengers_created; -#endif -public: - thread_map( wait_counter& fc, ::rml::client& client ) : - my_min_stack_size(0), my_client(client), my_factory_counter(fc), - my_server_ref_count(1), my_client_ref_count(1), shutdown_in_progress(false), - my_concrt_resource_manager(NULL), my_scheduler(NULL), my_scheduler_proxy(NULL) - { - my_thread_scavenger_thread = NULL; -#if TBB_USE_ASSERT - n_add_vp_requests = 0; - n_thread_scavengers_created; -#endif - } - - ~thread_map() { - __TBB_ASSERT( n_thread_scavengers_created<=1, "too many scavenger thread created" ); - // if thread_scavenger_thread is launched, wait for it to complete - if( my_thread_scavenger_thread ) { - __TBB_ASSERT( my_thread_scavenger_thread!=c_claimed, NULL ); - while( my_thread_scavenger_thread->read_state()==ts_busy ) - __TBB_Yield(); - thread_scavenger_thread* tst = my_thread_scavenger_thread; - my_scavenger_allocator.deallocate(static_cast<padded<thread_scavenger_thread>*>(tst),1); - } - // deallocate thread contexts - for( hash_map_type::const_iterator hi=my_map.begin(); hi!=my_map.end(); ++hi ) { - server_thread* thr = hi->second; - if( thr->tbb_thread ) { - while( ((tbb_server_thread*)thr)->activation_count>1 ) - __TBB_Yield(); - ((tbb_server_thread*)thr)->~tbb_server_thread(); - my_tbb_allocator.deallocate(static_cast<padded<tbb_server_thread>*>(thr),1); - } else { - ((omp_server_thread*)thr)->~omp_server_thread(); - my_omp_allocator.deallocate(static_cast<padded<omp_server_thread>*>(thr),1); - } - } - if( my_scheduler_proxy ) { - my_scheduler_proxy->Shutdown(); - my_concrt_resource_manager->Release(); - __TBB_ASSERT( my_scheduler, NULL ); - delete my_scheduler; - } else { - __TBB_ASSERT( !my_scheduler, NULL ); - } - } - typedef hash_map_type::key_type key_type; - typedef hash_map_type::value_type value_type; - typedef hash_map_type::iterator iterator; - iterator begin() {return my_map.begin();} - iterator end() {return my_map.end();} - iterator find( key_type k ) {return my_map.find( k );} - iterator insert( key_type k, server_thread* v ) { - std::pair<iterator,bool> res = my_map.insert( value_type(k,v) ); - return res.first; - } - void bind( IScheduler* s ) { - ++my_factory_counter; - if( s ) { - my_unrealized_threads = s->GetPolicy().GetPolicyValue( MaxConcurrency ); - __TBB_ASSERT( my_unrealized_threads>0, NULL ); - my_scheduler = s; - my_concrt_resource_manager = CreateResourceManager(); // reference count==3 when first created. - my_scheduler_proxy = my_concrt_resource_manager->RegisterScheduler( s, CONCRT_RM_VERSION_1 ); - my_scheduler_proxy->RequestInitialVirtualProcessors( false ); - } - } - bool is_closing() { return shutdown_in_progress; } - void unbind( rml::server& server, ::tbb::spin_mutex& mtx ); - void add_client_ref() { my_server_ref_count.add_ref(); } - void remove_client_ref(); - void add_server_ref() {my_server_ref_count.add_ref();} - int remove_server_ref() {return my_server_ref_count.remove_ref();} - int get_server_ref_count() { int k = my_server_ref_count.my_ref_count; return k; } - void assist_cleanup( bool assist_null_only ); - void adjust_balance( int delta ); - int current_balance() const {int k = the_balance; return k;} - ::rml::client& client() const {return my_client;} - void register_as_master( server::execution_resource_t& v ) const { (IExecutionResource*&)v = my_scheduler_proxy ? my_scheduler_proxy->SubscribeCurrentThread() : NULL; } - // Remove() should be called from the same thread that subscribed the current h/w thread (i.e., the one that - // called register_as_master() ). - void unregister( server::execution_resource_t v ) const {if( v ) ((IExecutionResource*)v)->Remove( my_scheduler );} - void add_virtual_processors( IVirtualProcessorRoot** vprocs, unsigned int count, tbb_connection_v2& conn, ::tbb::spin_mutex& mtx ); - void add_virtual_processors( IVirtualProcessorRoot** vprocs, unsigned int count, omp_connection_v2& conn, ::tbb::spin_mutex& mtx ); - void remove_virtual_processors( IVirtualProcessorRoot** vproots, unsigned count, ::tbb::spin_mutex& mtx ); - void mark_virtual_processors_as_lent( IVirtualProcessorRoot** vproots, unsigned count, ::tbb::spin_mutex& mtx ); - void create_oversubscribers( unsigned n, std::vector<server_thread*>& thr_vec, omp_connection_v2& conn, ::tbb::spin_mutex& mtx ); - void wakeup_tbb_threads( int c, ::tbb::spin_mutex& mtx ); - void mark_virtual_processors_as_returned( IVirtualProcessorRoot** vprocs, unsigned int count, tbb::spin_mutex& mtx ); - inline void addto_original_exec_resources( IExecutionResource* r, ::tbb::spin_mutex& mtx ) { - ::tbb::spin_mutex::scoped_lock lck(mtx); - __TBB_ASSERT( !is_closing(), "trying to register master while connection is being shutdown?" ); - original_exec_resources.push_back( r ); - } -#if !__RML_REMOVE_VIRTUAL_PROCESSORS_DISABLED - void allocate_thread_scavenger( IExecutionResource* v ); -#endif - inline thread_scavenger_thread* get_thread_scavenger() { return my_thread_scavenger_thread; } -}; - -garbage_connection_queue connections_to_reclaim; -connection_scavenger_thread connection_scavenger; - -#endif /* !RML_USE_WCRM */ - -//------------------------------------------------------------------------ -// generic_connection -//------------------------------------------------------------------------ - -template<typename Server, typename Client> -struct connection_traits {}; - -// head of the active tbb connections -static tbb::atomic<uintptr_t> active_tbb_connections; -static tbb::atomic<int> current_tbb_conn_readers; -static size_t current_tbb_conn_reader_epoch; -static tbb::atomic<size_t> close_tbb_connection_event_count; - -#if RML_USE_WCRM -template<typename Connection> -void make_job( Connection& c, server_thread& t ); -#endif - -template<typename Server, typename Client> -class generic_connection: public Server, no_copy { - version_type version() const __TBB_override {return SERVER_VERSION;} - void yield() __TBB_override {thread_monitor::yield();} - void independent_thread_number_changed( int delta ) __TBB_override { my_thread_map.adjust_balance( -delta ); } - unsigned default_concurrency() const __TBB_override { return the_default_concurrency; } - friend void wakeup_some_tbb_threads(); - friend class connection_scavenger_thread; - -protected: - thread_map my_thread_map; - generic_connection* next_conn; - size_t my_ec; -#if RML_USE_WCRM - // FIXME: pad it? - tbb::spin_mutex map_mtx; - IScheduler* my_scheduler; - void do_open( IScheduler* s ) { - my_scheduler = s; - my_thread_map.bind( s ); - } - bool is_closing() { return my_thread_map.is_closing(); } - void request_close_connection( bool existing ); -#else - void do_open() {my_thread_map.bind();} - void request_close_connection( bool ); -#endif /* RML_USE_WCRM */ - //! Make destructor virtual - virtual ~generic_connection() {} -#if !RML_USE_WCRM - generic_connection( wait_counter& fc, Client& c ) : my_thread_map(fc,c), next_conn(NULL), my_ec(0) {} -#else - generic_connection( wait_counter& fc, Client& c ) : - my_thread_map(fc,c), next_conn(NULL), my_ec(0), map_mtx(), my_scheduler(NULL) {} - void add_virtual_processors( IVirtualProcessorRoot** vprocs, unsigned int count ); - void remove_virtual_processors( IVirtualProcessorRoot** vprocs, unsigned int count ); - void notify_resources_externally_busy( IVirtualProcessorRoot** vprocs, unsigned int count ) { my_thread_map.mark_virtual_processors_as_lent( vprocs, count, map_mtx ); } - void notify_resources_externally_idle( IVirtualProcessorRoot** vprocs, unsigned int count ) { - my_thread_map.mark_virtual_processors_as_returned( vprocs, count, map_mtx ); - } -#endif /* !RML_USE_WCRM */ - -public: - typedef Server server_type; - typedef Client client_type; - Client& client() const {return static_cast<Client&>(my_thread_map.client());} - void set_scratch_ptr( job& j, void* ptr ) { ::rml::server::scratch_ptr(j) = ptr; } -#if RML_USE_WCRM - template<typename Connection> - friend void make_job( Connection& c, server_thread& t ); - void add_server_ref () {my_thread_map.add_server_ref();} - void remove_server_ref() {if( my_thread_map.remove_server_ref()==0 ) delete this;} - void add_client_ref () {my_thread_map.add_client_ref();} - void remove_client_ref() {my_thread_map.remove_client_ref();} -#else /* !RML_USE_WCRM */ - int add_server_ref () {return my_thread_map.add_server_ref();} - void remove_server_ref() {if( my_thread_map.remove_server_ref()==0 ) delete this;} - void remove_client_ref() {my_thread_map.remove_client_ref();} - void make_job( server_thread& t, job_automaton& ja ); -#endif /* RML_USE_WCRM */ - static generic_connection* get_addr( uintptr_t addr_ex ) { - return reinterpret_cast<generic_connection*>( addr_ex&~(uintptr_t)3 ); - } -}; - -//------------------------------------------------------------------------ -// TBB server -//------------------------------------------------------------------------ - -template<> -struct connection_traits<tbb_server,tbb_client> { - static const bool assist_null_only = true; - static const bool is_tbb = true; -}; - -//! Represents a server/client binding. -/** The internal representation uses inheritance for the server part and a pointer for the client part. */ -class tbb_connection_v2: public generic_connection<tbb_server,tbb_client> { - void adjust_job_count_estimate( int delta ) __TBB_override; -#if !RML_USE_WCRM -#if _WIN32||_WIN64 - void register_master ( rml::server::execution_resource_t& /*v*/ ) __TBB_override {} - void unregister_master ( rml::server::execution_resource_t /*v*/ ) __TBB_override {} -#endif -#else - void register_master ( rml::server::execution_resource_t& v ) __TBB_override { - my_thread_map.register_as_master(v); - if( v ) ++nesting; - } - void unregister_master ( rml::server::execution_resource_t v ) __TBB_override { - if( v ) { - __TBB_ASSERT( nesting>0, NULL ); - if( --nesting==0 ) { -#if !__RML_REMOVE_VIRTUAL_PROCESSORS_DISABLED - my_thread_map.allocate_thread_scavenger( (IExecutionResource*)v ); -#endif - } - } - my_thread_map.unregister(v); - } - IScheduler* create_scheduler() {return( scheduler<tbb_connection_v2>::create( *this ) );} - friend void free_all_connections( uintptr_t ); - friend class scheduler<tbb_connection_v2>; - friend class execution_context; - friend class connection_scavenger_thread; -#endif /* RML_USE_WCRM */ - friend void wakeup_some_tbb_threads(); - //! Estimate on number of jobs without threads working on them. - tbb::atomic<int> my_slack; - friend class dummy_class_to_shut_up_gratuitous_warning_from_gcc_3_2_3; -#if TBB_USE_ASSERT - tbb::atomic<int> my_job_count_estimate; -#endif /* TBB_USE_ASSERT */ - - tbb::atomic<int> n_adjust_job_count_requests; -#if RML_USE_WCRM - tbb::atomic<int> nesting; -#endif - - // dtor - ~tbb_connection_v2(); - -public: -#if RML_USE_WCRM - typedef tbb_server_thread server_thread_type; -#endif - //! True if there is slack that try_process can use. - bool has_slack() const {return my_slack>0;} - -#if RML_USE_WCRM - bool try_process( job& job ) -#else - bool try_process( server_thread& t, job& job ) -#endif - { - bool visited = false; - // No check for my_slack>0 here because caller is expected to do that check. - int k = --my_slack; - if( k>=0 ) { -#if !RML_USE_WCRM - t.my_extra_state = ts_visited; // remember the thread paid a trip to process() at least once -#endif - client().process(job); - visited = true; - } - ++my_slack; - return visited; - } - - tbb_connection_v2( wait_counter& fc, tbb_client& client ) : generic_connection<tbb_server,tbb_client>(fc,client) - { - my_slack = 0; -#if RML_USE_WCRM - nesting = 0; -#endif -#if TBB_USE_ASSERT - my_job_count_estimate = 0; -#endif /* TBB_USE_ASSERT */ - __TBB_ASSERT( !my_slack, NULL ); - -#if RML_USE_WCRM - do_open( client.max_job_count()>0 ? create_scheduler() : NULL ); -#else - do_open(); -#endif /* !RML_USE_WCRM */ - n_adjust_job_count_requests = 0; - - // Acquire head of active_tbb_connections & push the connection into the list - uintptr_t conn; - do { - for( ; (conn=active_tbb_connections)&1; ) - __TBB_Yield(); - } while( active_tbb_connections.compare_and_swap( conn|1, conn )!=conn ); - - this->next_conn = generic_connection<tbb_server,tbb_client>::get_addr(conn); - // Update and release head of active_tbb_connections - active_tbb_connections = (uintptr_t) this; // set and release - } - inline void wakeup_tbb_threads( unsigned n ) { - my_thread_map.wakeup_tbb_threads( n -#if RML_USE_WCRM - , map_mtx -#endif - ); - } -#if RML_USE_WCRM - inline int get_nesting_level() { return nesting; } -#else - inline bool wakeup_next_thread( thread_map::iterator i ) {return my_thread_map.wakeup_next_thread( i, *this );} - inline thread_map::size_type get_unrealized_threads () {return my_thread_map.get_unrealized_threads();} -#endif /* !RML_USE_WCRM */ -}; - -//------------------------------------------------------------------------ -// OpenMP server -//------------------------------------------------------------------------ - -template<> -struct connection_traits<omp_server,omp_client> { - static const bool assist_null_only = false; - static const bool is_tbb = false; -}; - -class omp_connection_v2: public generic_connection<omp_server,omp_client> { -#if !RML_USE_WCRM - int current_balance() const __TBB_override {return the_balance;} -#else - friend void free_all_connections( uintptr_t ); - friend class scheduler<omp_connection_v2>; - int current_balance() const __TBB_override {return my_thread_map.current_balance();} -#endif /* !RML_USE_WCRM */ - int try_increase_load( size_type n, bool strict ) __TBB_override; - void decrease_load( size_type n ) __TBB_override; - void get_threads( size_type request_size, void* cookie, job* array[] ) __TBB_override; -#if !RML_USE_WCRM -#if _WIN32||_WIN64 - void register_master ( rml::server::execution_resource_t& /*v*/ ) __TBB_override {} - void unregister_master ( rml::server::execution_resource_t /*v*/ ) __TBB_override {} -#endif -#else - void register_master ( rml::server::execution_resource_t& v ) __TBB_override { - my_thread_map.register_as_master( v ); - my_thread_map.addto_original_exec_resources( (IExecutionResource*)v, map_mtx ); - } - void unregister_master ( rml::server::execution_resource_t v ) __TBB_override { my_thread_map.unregister(v); } -#endif /* !RML_USE_WCRM */ -#if _WIN32||_WIN64 - void deactivate( rml::job* j ) __TBB_override; - void reactivate( rml::job* j ) __TBB_override; -#endif /* _WIN32||_WIN64 */ -#if RML_USE_WCRM -public: - typedef omp_server_thread server_thread_type; -private: - IScheduler* create_scheduler() {return( scheduler<omp_connection_v2>::create( *this ) );} -#endif /* RML_USE_WCRM */ -public: -#if TBB_USE_ASSERT - //! Net change in delta caused by this connection. - /** Should be zero when connection is broken */ - tbb::atomic<int> net_delta; -#endif /* TBB_USE_ASSERT */ - - omp_connection_v2( wait_counter& fc, omp_client& client ) : generic_connection<omp_server,omp_client>(fc,client) { -#if TBB_USE_ASSERT - net_delta = 0; -#endif /* TBB_USE_ASSERT */ -#if RML_USE_WCRM - do_open( create_scheduler() ); -#else - do_open(); -#endif /* RML_USE_WCRM */ - } - ~omp_connection_v2() {__TBB_ASSERT( net_delta==0, "net increase/decrease of load is nonzero" );} -}; - -#if !RML_USE_WCRM -/* to deal with cases where the machine is oversubscribed; we want each thread to trip to try_process() at least once */ -/* this should not involve computing the_balance */ -bool thread_map::wakeup_next_thread( thread_map::iterator this_thr, tbb_connection_v2& conn ) { - if( all_visited_at_least_once ) - return false; - - iterator e = end(); -retry: - bool exist = false; - iterator k=this_thr; - for( ++k; k!=e; ++k ) { - // If another thread added *k, there is a tiny timing window where thread() is invalid. - server_thread& t = k->wait_for_thread(); - if( t.my_extra_state!=ts_visited ) - exist = true; - if( t.read_state()!=ts_tbb_busy && t.my_extra_state==ts_started ) - if( t.try_grab_for( ts_tbb_busy ) ) - return true; - } - for( k=begin(); k!=this_thr; ++k ) { - server_thread& t = k->wait_for_thread(); - if( t.my_extra_state!=ts_visited ) - exist = true; - if( t.read_state()!=ts_tbb_busy && t.my_extra_state==ts_started ) - if( t.try_grab_for( ts_tbb_busy ) ) - return true; - } - - if( exist ) - if( conn.has_slack() ) - goto retry; - else - all_visited_at_least_once = true; - return false; -} - -void thread_map::release_tbb_threads( server_thread* t ) { - for( ; t; t = t->link ) { - while( t->read_state()!=ts_asleep ) - __TBB_Yield(); - t->my_extra_state = ts_started; - } -} -#endif /* !RML_USE_WCRM */ - -void thread_map::adjust_balance( int delta ) { - int new_balance = the_balance += delta; - if( new_balance>0 && 0>=new_balance-delta /*== old the_balance*/ ) - wakeup_some_tbb_threads(); -} - -void thread_map::remove_client_ref() { - int k = my_client_ref_count.remove_ref(); - if( k==0 ) { - // Notify factory that thread has crossed back into RML. - --my_factory_counter; - // Notify client that RML is done with the client object. - my_client.acknowledge_close_connection(); - } -} - -#if RML_USE_WCRM -/** Not a member of generic_connection because we need Connection to be the derived class. */ -template<typename Connection> -void make_job( Connection& c, typename Connection::server_thread_type& t ) { - if( t.my_job_automaton.try_acquire() ) { - rml::job* j = t.my_client.create_one_job(); - __TBB_ASSERT( j!=NULL, "client:::create_one_job returned NULL" ); - __TBB_ASSERT( (intptr_t(j)&1)==0, "client::create_one_job returned misaligned job" ); - t.my_job_automaton.set_and_release( j ); - c.set_scratch_ptr( *j, (void*) &t ); - } -} -#endif /* RML_USE_WCRM */ - -#if _MSC_VER && !defined(__INTEL_COMPILER) -// Suppress "conditional expression is constant" warning. -#pragma warning( push ) -#pragma warning( disable: 4127 ) -#endif -#if RML_USE_WCRM -template<typename Server, typename Client> -void generic_connection<Server,Client>::request_close_connection( bool exiting ) { - // for TBB connections, exiting should always be false - if( connection_traits<Server,Client>::is_tbb ) - __TBB_ASSERT( !exiting, NULL); -#if TBB_USE_ASSERT - else if( exiting ) - reinterpret_cast<omp_connection_v2*>(this)->net_delta = 0; -#endif - if( exiting ) { - uintptr_t tail = connections_to_reclaim.tail; - while( connections_to_reclaim.tail.compare_and_swap( garbage_connection_queue::plugged, tail )!=tail ) - __TBB_Yield(); - my_thread_map.unbind( *this, map_mtx ); - my_thread_map.assist_cleanup( connection_traits<Server,Client>::assist_null_only ); - // It is assumed that the client waits for all other threads to terminate before - // calling request_close_connection with true. Thus, it is safe to return all - // outstanding connection objects that are reachable. It is possible that there may - // be some unreachable connection objects lying somewhere. - free_all_connections( connection_scavenger.grab_and_prepend( this ) ); - return; - } -#else /* !RML_USE_WCRM */ -template<typename Server, typename Client> -void generic_connection<Server,Client>::request_close_connection( bool ) { -#endif /* RML_USE_WCRM */ - if( connection_traits<Server,Client>::is_tbb ) { - // acquire the head of active tbb connections - uintptr_t conn; - do { - for( ; (conn=active_tbb_connections)&1; ) - __TBB_Yield(); - } while( active_tbb_connections.compare_and_swap( conn|1, conn )!=conn ); - - // Locate the current connection - generic_connection* pred_conn = NULL; - generic_connection* curr_conn = (generic_connection*) conn; - for( ; curr_conn && curr_conn!=this; curr_conn=curr_conn->next_conn ) - pred_conn = curr_conn; - __TBB_ASSERT( curr_conn==this, "the current connection is not in the list?" ); - - // Remove this from the list - if( pred_conn ) { - pred_conn->next_conn = curr_conn->next_conn; - active_tbb_connections = reinterpret_cast<uintptr_t>(generic_connection<tbb_server,tbb_client>::get_addr(active_tbb_connections)); // release it - } else - active_tbb_connections = (uintptr_t) curr_conn->next_conn; // update & release it - curr_conn->next_conn = NULL; - // Increment the tbb connection close event count - my_ec = ++close_tbb_connection_event_count; - // Wait happens in tbb_connection_v2::~tbb_connection_v2() - } -#if RML_USE_WCRM - my_thread_map.unbind( *this, map_mtx ); - my_thread_map.assist_cleanup( connection_traits<Server,Client>::assist_null_only ); - connection_scavenger.add_request( this ); -#else - my_thread_map.unbind(); - my_thread_map.assist_cleanup( connection_traits<Server,Client>::assist_null_only ); - // Remove extra reference - remove_server_ref(); -#endif -} -#if _MSC_VER && !defined(__INTEL_COMPILER) -#pragma warning( pop ) -#endif - -#if RML_USE_WCRM - -template<typename Server, typename Client> -void generic_connection<Server,Client>::add_virtual_processors( IVirtualProcessorRoot** vproots, unsigned int count ) -{} - -template<> -void generic_connection<tbb_server,tbb_client>::add_virtual_processors( IVirtualProcessorRoot** vproots, unsigned int count ) -{ - my_thread_map.add_virtual_processors( vproots, count, (tbb_connection_v2&)*this, map_mtx ); -} -template<> -void generic_connection<omp_server,omp_client>::add_virtual_processors( IVirtualProcessorRoot** vproots, unsigned int count ) -{ - // For OMP, since it uses ScheudlerPolicy of MinThreads==MaxThreads, this is called once when - // RequestInitialVirtualProcessors() is called. - my_thread_map.add_virtual_processors( vproots, count, (omp_connection_v2&)*this, map_mtx ); -} - -template<typename Server, typename Client> -void generic_connection<Server,Client>::remove_virtual_processors( IVirtualProcessorRoot** vproots, unsigned int count ) -{ - __TBB_ASSERT( false, "should not be called" ); -} -/* For OMP, RemoveVirtualProcessors() will never be called. */ - -template<> -void generic_connection<tbb_server,tbb_client>::remove_virtual_processors( IVirtualProcessorRoot** vproots, unsigned int count ) -{ - my_thread_map.remove_virtual_processors( vproots, count, map_mtx ); -} - -void tbb_connection_v2::adjust_job_count_estimate( int delta ) { -#if TBB_USE_ASSERT - my_job_count_estimate += delta; -#endif /* TBB_USE_ASSERT */ - // Atomically update slack. - int c = my_slack+=delta; - if( c>0 ) { - ++n_adjust_job_count_requests; - my_thread_map.wakeup_tbb_threads( c, map_mtx ); - --n_adjust_job_count_requests; - } -} -#endif /* RML_USE_WCRM */ - -tbb_connection_v2::~tbb_connection_v2() { -#if TBB_USE_ASSERT - if( my_job_count_estimate!=0 ) { - fprintf(stderr, "TBB client tried to disconnect with non-zero net job count estimate of %d\n", int(my_job_count_estimate )); - abort(); - } - __TBB_ASSERT( !my_slack, "attempt to destroy tbb_server with nonzero slack" ); - __TBB_ASSERT( this!=static_cast<tbb_connection_v2*>(generic_connection<tbb_server,tbb_client >::get_addr(active_tbb_connections)), "request_close_connection() must be called" ); -#endif /* TBB_USE_ASSERT */ -#if !RML_USE_WCRM - // If there are other threads ready for work, give them coins - if( the_balance>0 ) - wakeup_some_tbb_threads(); -#endif - // Someone might be accessing my data members - while( current_tbb_conn_readers>0 && (ptrdiff_t)(my_ec-current_tbb_conn_reader_epoch)>0 ) - __TBB_Yield(); -} - -#if !RML_USE_WCRM -template<typename Server, typename Client> -void generic_connection<Server,Client>::make_job( server_thread& t, job_automaton& ja ) { - if( ja.try_acquire() ) { - rml::job* j = client().create_one_job(); - __TBB_ASSERT( j!=NULL, "client:::create_one_job returned NULL" ); - __TBB_ASSERT( (intptr_t(j)&1)==0, "client::create_one_job returned misaligned job" ); - ja.set_and_release( j ); - __TBB_ASSERT( t.my_conn && t.my_ja && t.my_job==NULL, NULL ); - t.my_job = j; - set_scratch_ptr( *j, (void*) &t ); - } -} - -void tbb_connection_v2::adjust_job_count_estimate( int delta ) { -#if TBB_USE_ASSERT - my_job_count_estimate += delta; -#endif /* TBB_USE_ASSERT */ - // Atomically update slack. - int c = my_slack+=delta; - if( c>0 ) { - ++n_adjust_job_count_requests; - // The client has work to do and there are threads available - thread_map::size_type n = my_thread_map.wakeup_tbb_threads(c); - - server_thread* new_threads_anchor = NULL; - thread_map::size_type i; - { - tbb::internal::affinity_helper fpa; - for( i=0; i<n; ++i ) { - // Obtain unrealized threads - thread_map::value_type* k = my_thread_map.add_one_thread( false ); - if( !k ) - // No unrealized threads left. - break; - // Eagerly start the thread off. - fpa.protect_affinity_mask( /*restore_process_mask=*/true ); - my_thread_map.bind_one_thread( *this, *k ); - server_thread& t = k->thread(); - __TBB_ASSERT( !t.link, NULL ); - t.link = new_threads_anchor; - new_threads_anchor = &t; - } - // Implicit destruction of fpa resets original affinity mask. - } - - thread_map::size_type j=0; - for( ; the_balance>0 && j<i; ++j ) { - if( --the_balance>=0 ) { - // Withdraw a coin from the bank - __TBB_ASSERT( new_threads_anchor, NULL ); - - server_thread* t = new_threads_anchor; - new_threads_anchor = t->link; - while( !t->try_grab_for( ts_tbb_busy ) ) - __TBB_Yield(); - t->my_extra_state = ts_started; - } else { - // Overdraft. return it to the bank - ++the_balance; - break; - } - } - __TBB_ASSERT( i-j!=0||new_threads_anchor==NULL, NULL ); - // Mark the ones that did not get started as eligible for being snatched. - if( new_threads_anchor ) - my_thread_map.release_tbb_threads( new_threads_anchor ); - - --n_adjust_job_count_requests; - } -} -#endif /* RML_USE_WCRM */ - -#if RML_USE_WCRM -int omp_connection_v2::try_increase_load( size_type n, bool strict ) { - __TBB_ASSERT(int(n)>=0,NULL); - if( strict ) { - the_balance -= int(n); - } else { - int avail, old; - do { - avail = the_balance; - if( avail<=0 ) { - // No atomic read-write-modify operation necessary. - return avail; - } - // Don't read the_system_balance; if it changes, compare_and_swap will fail anyway. - old = the_balance.compare_and_swap( int(n)<avail ? avail-n : 0, avail ); - } while( old!=avail ); - if( int(n)>avail ) - n=avail; - } -#if TBB_USE_ASSERT - net_delta += n; -#endif /* TBB_USE_ASSERT */ - return n; -} - -void omp_connection_v2::decrease_load( size_type /*n*/ ) {} - -void omp_connection_v2::get_threads( size_type request_size, void* cookie, job* array[] ) { - unsigned index = 0; - std::vector<omp_server_thread*> enlisted(request_size); - std::vector<thread_grab_t> to_activate(request_size); - - if( request_size==0 ) return; - - { - tbb::spin_mutex::scoped_lock lock(map_mtx); - - __TBB_ASSERT( !is_closing(), "try to get threads while connection is being shutdown?" ); - - for( int scan=0; scan<2; ++scan ) { - for( thread_map::iterator i=my_thread_map.begin(); i!=my_thread_map.end(); ++i ) { - omp_server_thread* thr = (omp_server_thread*) (*i).second; - // in the first scan, skip VPs that are lent - if( scan==0 && thr->is_lent() ) continue; - thread_grab_t res = thr->try_grab_for(); - if( res!=wk_failed ) {// && if is not busy by some other scheduler - to_activate[index] = res; - enlisted[index] = thr; - if( ++index==request_size ) - goto activate_threads; - } - } - } - } - -activate_threads: - - for( unsigned i=0; i<index; ++i ) { - omp_server_thread* thr = enlisted[i]; - if( to_activate[i]==wk_from_asleep ) - thr->get_virtual_processor()->Activate( thr ); - job* j = thr->wait_for_job(); - array[i] = j; - thr->omp_data.produce( client(), j, cookie, i PRODUCE_ARG(*this) ); - } - - if( index==request_size ) - return; - - // If we come to this point, it must be because dynamic==false - // Create Oversubscribers.. - - // Note that our policy is such that MinConcurrency==MaxConcurrency. - // RM will deliver MaxConcurrency of VirtualProcessors and no more. - __TBB_ASSERT( request_size>index, NULL ); - unsigned n = request_size - index; - std::vector<server_thread*> thr_vec(n); - typedef std::vector<server_thread*>::iterator iterator_thr; - my_thread_map.create_oversubscribers( n, thr_vec, *this, map_mtx ); - for( iterator_thr ti=thr_vec.begin(); ti!=thr_vec.end(); ++ti ) { - omp_server_thread* thr = (omp_server_thread*) *ti; - __TBB_ASSERT( thr, "thread not created?" ); - // Thread is already grabbed; since it is newly created, we need to activate it. - thr->get_virtual_processor()->Activate( thr ); - job* j = thr->wait_for_job(); - array[index] = j; - thr->omp_data.produce( client(), j, cookie, index PRODUCE_ARG(*this) ); - ++index; - } -} - -#if _WIN32||_WIN64 -void omp_connection_v2::deactivate( rml::job* j ) -{ - my_thread_map.adjust_balance(1); -#if TBB_USE_ASSERT - net_delta -= 1; -#endif - omp_server_thread* thr = (omp_server_thread*) scratch_ptr( *j ); - (thr->get_virtual_processor())->Deactivate( thr ); -} - -void omp_connection_v2::reactivate( rml::job* j ) -{ - // Should not adjust the_balance because OMP client is supposed to - // do try_increase_load() to reserve the threads to use. - omp_server_thread* thr = (omp_server_thread*) scratch_ptr( *j ); - (thr->get_virtual_processor())->Activate( thr ); -} -#endif /* !_WIN32||_WIN64 */ - -#endif /* RML_USE_WCRM */ - -//! Wake up some available tbb threads -void wakeup_some_tbb_threads() -{ - /* First, atomically grab the connection, then increase the server ref count to keep - it from being released prematurely. Second, check if the balance is available for TBB - and the tbb conneciton has slack to exploit. If the answer is true, go ahead and - try to wake some up. */ - if( generic_connection<tbb_server,tbb_client >::get_addr(active_tbb_connections)==0 ) - // the next connection will see the change; return. - return; - -start_it_over: - int n_curr_readers = ++current_tbb_conn_readers; - if( n_curr_readers>1 ) // I lost - return; - // if n_curr_readers==1, i am the first one, so I will take responsibility for waking tbb threads up. - - // update the current epoch - current_tbb_conn_reader_epoch = close_tbb_connection_event_count; - - // read and clear - // Newly added connection will not invalidate the pointer, and it will - // compete with the current one to claim coins. - // One that is about to close the connection increments the event count - // after it removes the connection from the list. But it will keep around - // the connection until all readers including this one catch up. So, reading - // the head and clearing the lock bit should be o.k. - generic_connection<tbb_server,tbb_client>* next_conn_wake_up = generic_connection<tbb_server,tbb_client>::get_addr( active_tbb_connections ); - - for( ; next_conn_wake_up; ) { - /* some threads are creating tbb server threads; they may not see my changes made to the_balance */ - /* When a thread is in adjust_job_count_estimate() to increase the slack - RML tries to activate worker threads on behalf of the requesting thread - by repeatedly drawing a coin from the bank optimistically and grabbing a - thread. If it finds the bank overdrafted, it returns the coin back to - the bank and returns the control to the thread (return from the method). - There lies a tiny timing hole. - - When the overdraft occurs (note that multiple masters may be in - adjust_job_count_estimate() so the_balance can be any negative value) and - a worker returns from the TBB work at that moment, its returning the coin - does not bump up the_balance over 0, so it happily returns from - wakeup_some_tbb_threads() without attempting to give coins to worker threads - that are ready. - */ - while( ((tbb_connection_v2*)next_conn_wake_up)->n_adjust_job_count_requests>0 ) - __TBB_Yield(); - - int bal = the_balance; - n_curr_readers = current_tbb_conn_readers; // get the snapshot - if( bal<=0 ) break; - // if the connection is deleted, the following will immediately return because its slack would be 0 or less. - - tbb_connection_v2* tbb_conn = (tbb_connection_v2*)next_conn_wake_up; - int my_slack = tbb_conn->my_slack; - if( my_slack>0 ) tbb_conn->wakeup_tbb_threads( my_slack ); - next_conn_wake_up = next_conn_wake_up->next_conn; - } - - int delta = current_tbb_conn_readers -= n_curr_readers; - //if delta>0, more threads entered the routine since this one took the snapshot - if( delta>0 ) { - current_tbb_conn_readers = 0; - if( the_balance>0 && generic_connection<tbb_server,tbb_client >::get_addr(active_tbb_connections)!=0 ) - goto start_it_over; - } - - // Signal any connection that is waiting for me to complete my access that I am done. - current_tbb_conn_reader_epoch = close_tbb_connection_event_count; -} - -#if !RML_USE_WCRM -int omp_connection_v2::try_increase_load( size_type n, bool strict ) { - __TBB_ASSERT(int(n)>=0,NULL); - if( strict ) { - the_balance -= int(n); - } else { - int avail, old; - do { - avail = the_balance; - if( avail<=0 ) { - // No atomic read-write-modify operation necessary. - return avail; - } - // don't read the_balance; if it changes, compare_and_swap will fail anyway. - old = the_balance.compare_and_swap( int(n)<avail ? avail-n : 0, avail ); - } while( old!=avail ); - if( int(n)>avail ) - n=avail; - } -#if TBB_USE_ASSERT - net_delta += n; -#endif /* TBB_USE_ASSERT */ - return n; -} - -void omp_connection_v2::decrease_load( size_type n ) { - __TBB_ASSERT(int(n)>=0,NULL); - my_thread_map.adjust_balance(int(n)); -#if TBB_USE_ASSERT - net_delta -= n; -#endif /* TBB_USE_ASSERT */ -} - -void omp_connection_v2::get_threads( size_type request_size, void* cookie, job* array[] ) { - - if( !request_size ) - return; - - unsigned index = 0; - for(;;) { // don't return until all request_size threads are grabbed. - // Need to grab some threads - thread_map::iterator k_end=my_thread_map.end(); - for( thread_map::iterator k=my_thread_map.begin(); k!=k_end; ++k ) { - // If another thread added *k, there is a tiny timing window where thread() is invalid. - server_thread& t = k->wait_for_thread(); - if( t.try_grab_for( ts_omp_busy ) ) { - // The preincrement instead of post-increment of index is deliberate. - job* j = k->wait_for_job(); - array[index] = j; - t.omp_dispatch.produce( client(), j, cookie, index PRODUCE_ARG(*this) ); - if( ++index==request_size ) - return; - } - } - // Need to allocate more threads - for( unsigned i=index; i<request_size; ++i ) { - __TBB_ASSERT( index<request_size, NULL ); - thread_map::value_type* k = my_thread_map.add_one_thread( true ); -#if TBB_USE_ASSERT - if( !k ) { - // Client erred - __TBB_ASSERT(false, "server::get_threads: exceeded job_count\n"); - } -#endif - my_thread_map.bind_one_thread( *this, *k ); - server_thread& t = k->thread(); - if( t.try_grab_for( ts_omp_busy ) ) { - job* j = k->wait_for_job(); - array[index] = j; - // The preincrement instead of post-increment of index is deliberate. - t.omp_dispatch.produce( client(), j, cookie, index PRODUCE_ARG(*this) ); - if( ++index==request_size ) - return; - } // else someone else snatched it. - } - } -} -#endif /* !RML_USE_WCRM */ - -//------------------------------------------------------------------------ -// Methods of omp_dispatch_type -//------------------------------------------------------------------------ -void omp_dispatch_type::consume() { - // Wait for short window between when master sets state of this thread to ts_omp_busy - // and master thread calls produce. - job_type* j; - tbb::internal::atomic_backoff backoff; - while( (j = job)==NULL ) backoff.pause(); - job = static_cast<job_type*>(NULL); - client->process(*j,cookie,index); -#if TBB_USE_ASSERT - // Return of method process implies "decrease_load" from client's viewpoint, even though - // the actual adjustment of the_balance only happens when this thread really goes to sleep. - --server->net_delta; -#endif /* TBB_USE_ASSERT */ -} - -#if !RML_USE_WCRM -#if _WIN32||_WIN64 -void omp_connection_v2::deactivate( rml::job* j ) -{ -#if TBB_USE_ASSERT - net_delta -= 1; -#endif - __TBB_ASSERT( j, NULL ); - server_thread* thr = (server_thread*) scratch_ptr( *j ); - thr->deactivate(); -} - -void omp_connection_v2::reactivate( rml::job* j ) -{ - // Should not adjust the_balance because OMP client is supposed to - // do try_increase_load() to reserve the threads to use. - __TBB_ASSERT( j, NULL ); - server_thread* thr = (server_thread*) scratch_ptr( *j ); - thr->reactivate(); -} -#endif /* _WIN32||_WIN64 */ - -//------------------------------------------------------------------------ -// Methods of server_thread -//------------------------------------------------------------------------ - -server_thread::server_thread() : - ref_count(0), - link(NULL), - my_map_pos(), - my_conn(NULL), my_job(NULL), my_ja(NULL) -{ - state = ts_idle; - terminate = false; -#if TBB_USE_ASSERT - has_active_thread = false; -#endif /* TBB_USE_ASSERT */ -} - -server_thread::~server_thread() { - __TBB_ASSERT( !has_active_thread, NULL ); -} - -#if _MSC_VER && !defined(__INTEL_COMPILER) - // Suppress overzealous compiler warnings about an initialized variable 'sink_for_alloca' not referenced - #pragma warning(push) - #pragma warning(disable:4189) -#endif -__RML_DECL_THREAD_ROUTINE server_thread::thread_routine( void* arg ) { - server_thread* self = static_cast<server_thread*>(arg); - AVOID_64K_ALIASING( self->my_index ); -#if TBB_USE_ASSERT - __TBB_ASSERT( !self->has_active_thread, NULL ); - self->has_active_thread = true; -#endif /* TBB_USE_ASSERT */ - self->loop(); - return 0; -} -#if _MSC_VER && !defined(__INTEL_COMPILER) - #pragma warning(pop) -#endif - -void server_thread::launch( size_t stack_size ) { -#if USE_WINTHREAD - thread_monitor::launch( thread_routine, this, stack_size, &this->my_index ); -#else - thread_monitor::launch( thread_routine, this, stack_size ); -#endif /* USE_PTHREAD */ -} - -void server_thread::sleep_perhaps( thread_state_t asleep ) { - if( terminate ) return; - __TBB_ASSERT( asleep==ts_asleep, NULL ); - thread_monitor::cookie c; - monitor.prepare_wait(c); - if( state.compare_and_swap( asleep, ts_idle )==ts_idle ) { - if( !terminate ) { - monitor.commit_wait(c); - // Someone else woke me up. The compare_and_swap further below deals with spurious wakeups. - } else { - monitor.cancel_wait(); - } - thread_state_t s = read_state(); - if( s==ts_asleep ) { - state.compare_and_swap( ts_idle, ts_asleep ); - // I woke myself up, either because I cancelled the wait or suffered a spurious wakeup. - } else { - // Someone else woke me up; there the_balance is decremented by 1. -- tbb only - if( !is_omp_thread ) { - __TBB_ASSERT( s==ts_tbb_busy||s==ts_idle, NULL ); - } - } - } else { - // someone else made it busy ; see try_grab_for when state==ts_idle. - __TBB_ASSERT( state==ts_omp_busy||state==ts_tbb_busy, NULL ); - monitor.cancel_wait(); - } - __TBB_ASSERT( read_state()!=asleep, "a thread can only put itself to sleep" ); -} - -bool server_thread::wakeup( thread_state_t to, thread_state_t from ) { - bool success = false; - __TBB_ASSERT( from==ts_asleep && (to==ts_idle||to==ts_omp_busy||to==ts_tbb_busy), NULL ); - if( state.compare_and_swap( to, from )==from ) { - if( !is_omp_thread ) __TBB_ASSERT( to==ts_idle||to==ts_tbb_busy, NULL ); - // There is a small timing window that permits balance to become negative, - // but such occurrences are probably rare enough to not worry about, since - // at worst the result is slight temporary oversubscription. - monitor.notify(); - success = true; - } - return success; -} - -//! Attempt to change a thread's state to ts_omp_busy, and waking it up if necessary. -bool server_thread::try_grab_for( thread_state_t target_state ) { - bool success = false; - switch( read_state() ) { - case ts_asleep: - success = wakeup( target_state, ts_asleep ); - break; - case ts_idle: - success = state.compare_and_swap( target_state, ts_idle )==ts_idle; - break; - default: - // Thread is not available to be part of an OpenMP thread team. - break; - } - return success; -} - -#if _WIN32||_WIN64 -void server_thread::deactivate() { - thread_state_t es = (thread_state_t) my_extra_state.fetch_and_store( ts_deactivated ); - __TBB_ASSERT( my_extra_state==ts_deactivated, "someone else tampered with my_extra_state?" ); - if( es==ts_none ) - state = ts_idle; - else - __TBB_ASSERT( es==ts_reactivated, "Cannot call deactivate() while in ts_deactivated" ); - // only the thread can transition itself from ts_deactivted to ts_none - __TBB_ASSERT( my_extra_state==ts_deactivated, "someone else tampered with my_extra_state?" ); - my_extra_state = ts_none; // release the critical section - int bal = ++the_balance; - if( bal>0 ) - wakeup_some_tbb_threads(); - if( es==ts_none ) - sleep_perhaps( ts_asleep ); -} - -void server_thread::reactivate() { - thread_state_t es; - do { - while( (es=read_extra_state())==ts_deactivated ) - __TBB_Yield(); - if( es==ts_reactivated ) { - __TBB_ASSERT( false, "two Reactivate() calls in a row. Should not happen" ); - return; - } - __TBB_ASSERT( es==ts_none, NULL ); - } while( (thread_state_t)my_extra_state.compare_and_swap( ts_reactivated, ts_none )!=ts_none ); - if( state!=ts_omp_busy ) { - my_extra_state = ts_none; - while( !try_grab_for( ts_omp_busy ) ) - __TBB_Yield(); - } -} -#endif /* _WIN32||_WIN64 */ - - -template<typename Connection> -bool server_thread::destroy_job( Connection& c ) { - __TBB_ASSERT( !is_omp_thread||(state==ts_idle||state==ts_omp_busy), NULL ); - __TBB_ASSERT( is_omp_thread||(state==ts_idle||state==ts_tbb_busy), NULL ); - if( !is_omp_thread ) { - __TBB_ASSERT( state==ts_idle||state==ts_tbb_busy, NULL ); - if( state==ts_idle ) - state.compare_and_swap( ts_done, ts_idle ); - // 'state' may be set to ts_tbb_busy by another thread. - - if( state==ts_tbb_busy ) { // return the coin to the deposit - // need to deposit first to let the next connection see the change - ++the_balance; - state = ts_done; // no other thread changes the state when it is ts_*_busy - } - } - if( job_automaton* ja = my_ja ) { - rml::job* j; - if( ja->try_plug(j) ) { - __TBB_ASSERT( j, NULL ); - c.client().cleanup(*j); - c.remove_client_ref(); - } else { - // Some other thread took responsibility for cleaning up the job. - } - } - // Must do remove client reference first, because execution of - // c.remove_ref() can cause *this to be destroyed. - int k = remove_ref(); - __TBB_ASSERT_EX( k==0, "more than one references?" ); -#if TBB_USE_ASSERT - has_active_thread = false; -#endif /* TBB_USE_ASSERT */ - c.remove_server_ref(); - return true; -} - -bool server_thread::do_termination() { - if( is_omp_thread ) - return destroy_job( *static_cast<omp_connection_v2*>(my_conn) ); - else - return destroy_job( *static_cast<tbb_connection_v2*>(my_conn) ); -} - -//! Loop that each thread executes -void server_thread::loop() { - if( is_omp_thread ) - static_cast<omp_connection_v2*>(my_conn)->make_job( *this, *my_ja ); - else - static_cast<tbb_connection_v2*>(my_conn)->make_job( *this, *my_ja ); - for(;;) { - __TBB_Yield(); - if( state==ts_idle ) - sleep_perhaps( ts_asleep ); - - // Check whether I should quit. - if( terminate ) - if( do_termination() ) - return; - - // read the state - thread_state_t s = read_state(); - __TBB_ASSERT( s==ts_idle||s==ts_omp_busy||s==ts_tbb_busy, NULL ); - - if( s==ts_omp_busy ) { - // Enslaved by OpenMP team. - omp_dispatch.consume(); - /* here wake tbb threads up if feasible */ - if( ++the_balance>0 ) - wakeup_some_tbb_threads(); - state = ts_idle; - } else if( s==ts_tbb_busy ) { - // do some TBB work. - __TBB_ASSERT( my_conn && my_job, NULL ); - tbb_connection_v2& conn = *static_cast<tbb_connection_v2*>(my_conn); - // give openmp higher priority - bool has_coin = true; - if( conn.has_slack() ) { - // it has the coin, it should trip to the scheduler at least once as long as its slack is positive - do { - if( conn.try_process( *this, *my_job ) ) - if( conn.has_slack() && the_balance>=0 ) - has_coin = !conn.wakeup_next_thread( my_map_pos ); - } while( has_coin && conn.has_slack() && the_balance>=0 ); - } - state = ts_idle; - if( has_coin ) { - ++the_balance; // return the coin back to the deposit - if( conn.has_slack() ) { // a new adjust_job_request_estimate() is in progress - // it may have missed my changes to state and/or the_balance - if( --the_balance>=0 ) { // try to grab the coin back - // I got the coin - if( state.compare_and_swap( ts_tbb_busy, ts_idle )!=ts_idle ) - ++the_balance; // someone else enlisted me. - } else { - // overdraft. return the coin - ++the_balance; - } - } // else the new request will see my changes to state & the_balance. - } - /* here wake tbb threads up if feasible */ - if( the_balance>0 ) - wakeup_some_tbb_threads(); - } - } -} -#endif /* !RML_USE_WCRM */ - -#if RML_USE_WCRM - -class tbb_connection_v2; -class omp_connection_v2; - -#define CREATE_SCHEDULER_POLICY(policy,min_thrs,max_thrs,stack_size) \ - try { \ - policy = new SchedulerPolicy (7, \ - SchedulerKind, RML_THREAD_KIND, /*defined in _rml_serer_msrt.h*/ \ - MinConcurrency, min_thrs, \ - MaxConcurrency, max_thrs, \ - TargetOversubscriptionFactor, 1, \ - ContextStackSize, stack_size/1000, /*ConcRT:kB, iRML:bytes*/ \ - ContextPriority, THREAD_PRIORITY_NORMAL, \ - DynamicProgressFeedback, ProgressFeedbackDisabled ); \ - } catch ( invalid_scheduler_policy_key & ) { \ - __TBB_ASSERT( false, "invalid scheduler policy key exception caught" );\ - } catch ( invalid_scheduler_policy_value & ) { \ - __TBB_ASSERT( false, "invalid scheduler policy value exception caught" );\ - } - -static unsigned int core_count; -static tbb::atomic<int> core_count_inited; - - -static unsigned int get_processor_count() -{ - if( core_count_inited!=2 ) { - if( core_count_inited.compare_and_swap( 1, 0 )==0 ) { - core_count = GetProcessorCount(); - core_count_inited = 2; - } else { - tbb::internal::spin_wait_until_eq( core_count_inited, 2 ); - } - } - return core_count; -} - -template<typename Connection> -scheduler<Connection>::scheduler( Connection& conn ) : uid(GetSchedulerId()), my_conn(conn) {} - -template<> -scheduler<tbb_connection_v2>::scheduler( tbb_connection_v2& conn ) : uid(GetSchedulerId()), my_conn(conn) -{ - rml::client& cl = my_conn.client(); - unsigned max_job_count = cl.max_job_count(); - unsigned count = get_processor_count(); - __TBB_ASSERT( max_job_count>0, "max job count must be positive" ); - __TBB_ASSERT( count>1, "The processor count must be greater than 1" ); - if( max_job_count>count-1) max_job_count = count-1; - CREATE_SCHEDULER_POLICY( my_policy, 0, max_job_count, cl.min_stack_size() ); -} - -#if __RML_REMOVE_VIRTUAL_PROCESSORS_DISABLED -template<> -void scheduler<tbb_connection_v2>::RemoveVirtualProcessors( IVirtualProcessorRoot**, unsigned int) -{ -} -#else -template<> -void scheduler<tbb_connection_v2>::RemoveVirtualProcessors( IVirtualProcessorRoot** vproots, unsigned int count ) -{ - if( !my_conn.is_closing() ) - my_conn.remove_virtual_processors( vproots, count ); -} -#endif - -template<> -void scheduler<tbb_connection_v2>::NotifyResourcesExternallyIdle( IVirtualProcessorRoot** /*vproots*/, unsigned int /*count*/) -{ - __TBB_ASSERT( false, "NotifyResourcesExternallyIdle() is not allowed for TBB" ); -} - -template<> -void scheduler<tbb_connection_v2>::NotifyResourcesExternallyBusy( IVirtualProcessorRoot** /*vproots*/, unsigned int /*count*/ ) -{ - __TBB_ASSERT( false, "NotifyResourcesExternallyBusy() is not allowed for TBB" ); -} - -template<> -scheduler<omp_connection_v2>::scheduler( omp_connection_v2& conn ) : uid(GetSchedulerId()), my_conn(conn) -{ - unsigned count = get_processor_count(); - rml::client& cl = my_conn.client(); - __TBB_ASSERT( count>1, "The processor count must be greater than 1" ); - CREATE_SCHEDULER_POLICY( my_policy, count-1, count-1, cl.min_stack_size() ); -} - -template<> -void scheduler<omp_connection_v2>::RemoveVirtualProcessors( IVirtualProcessorRoot** /*vproots*/, unsigned int /*count*/ ) { - __TBB_ASSERT( false, "RemoveVirtualProcessors() is not allowed for OMP" ); -} - -template<> -void scheduler<omp_connection_v2>::NotifyResourcesExternallyIdle( IVirtualProcessorRoot** vproots, unsigned int count ){ - if( !my_conn.is_closing() ) - my_conn.notify_resources_externally_idle( vproots, count ); -} - -template<> -void scheduler<omp_connection_v2>::NotifyResourcesExternallyBusy( IVirtualProcessorRoot** vproots, unsigned int count ){ - if( !my_conn.is_closing() ) - my_conn.notify_resources_externally_busy( vproots, count ); -} - -/* ts_idle, ts_asleep, ts_busy */ -void tbb_server_thread::Dispatch( DispatchState* ) { - // Activate() will resume a thread right after Deactivate() as if it returns from the call - tbb_connection_v2* tbb_conn = static_cast<tbb_connection_v2*>(my_conn); - make_job( *tbb_conn, *this ); - - for( ;; ) { - // Try to wake some tbb threads if the balance is positive. - // When a thread is added by ConcRT and enter here for the first time, - // the thread may wake itself up (i.e., atomically change its state to ts_busy. - if( the_balance>0 ) - wakeup_some_tbb_threads(); - if( read_state()!=ts_busy ) - if( sleep_perhaps() ) - return; - if( terminate ) - if( initiate_termination() ) - return; - if( read_state()==ts_busy ) { - // this thread has a coin (i.e., state=ts_busy; it should trip to the scheduler at least once - if ( tbb_conn->has_slack() ) { - do { - tbb_conn->try_process( *wait_for_job() ); - } while( tbb_conn->has_slack() && the_balance>=0 && !is_removed() ); - } - __TBB_ASSERT( read_state()==ts_busy, "thread is not in busy state after returning from process()" ); - // see remove_virtual_processors() - if( my_state.compare_and_swap( ts_idle, ts_busy )==ts_busy ) { - int bal = ++the_balance; - if( tbb_conn->has_slack() ) { - // slack is positive, volunteer to help - bal = --the_balance; // try to grab the coin back - if( bal>=0 ) { // got the coin back - if( my_state.compare_and_swap( ts_busy, ts_idle )!=ts_idle ) - ++the_balance; // someone else enlisted me. - // else my_state is ts_busy, I will come back to tbb_conn->try_process(). - } else { - // overdraft. return the coin - ++the_balance; - } - } // else the new request will see my changes to state & the_balance. - } else { - __TBB_ASSERT( false, "someone tampered with my state" ); - } - } // someone else might set the state to something other than ts_idle - } -} - -void omp_server_thread::Dispatch( DispatchState* ) { - // Activate() will resume a thread right after Deactivate() as if it returns from the call - make_job( *static_cast<omp_connection_v2*>(my_conn), *this ); - - for( ;; ) { - if( read_state()!=ts_busy ) - sleep_perhaps(); - if( terminate ) { - if( initiate_termination() ) - return; - } - if( read_state()==ts_busy ) { - omp_data.consume(); - __TBB_ASSERT( read_state()==ts_busy, "thread is not in busy state after returning from process()" ); - my_thread_map.adjust_balance( 1 ); - set_state( ts_idle ); - } - // someone else might set the state to something other than ts_idle - } -} - -//! Attempt to change a thread's state to ts_omp_busy, and waking it up if necessary. -thread_grab_t server_thread_rep::try_grab_for() { - thread_grab_t res = wk_failed; - thread_state_t s = read_state(); - switch( s ) { - case ts_asleep: - if( wakeup( ts_busy, ts_asleep ) ) - res = wk_from_asleep; - __TBB_ASSERT( res==wk_failed||read_state()==ts_busy, NULL ); - break; - case ts_idle: - if( my_state.compare_and_swap( ts_busy, ts_idle )==ts_idle ) - res = wk_from_idle; - // At this point a thread is grabbed (i.e., its state has changed to ts_busy. - // It is possible that the thread 1) processes the job, returns from process() and - // sets its state ts_idle again. In some cases, it even sets its state to ts_asleep. - break; - default: - break; - } - return res; -} - -bool tbb_server_thread::switch_out() { - thread_state_t s = read_state(); - __TBB_ASSERT( s==ts_asleep||s==ts_busy, NULL ); - // This thread comes back from the TBB scheduler, and changed its state to ts_asleep successfully. - // The master enlisted it and woke it up by Activate()'ing it; now it is emerging from Deactivated(). - // ConcRT requested for removal of the vp associated with the thread, and RML marks it removed. - // Now, it has ts_busy, and removed. -- we should remove it. - IExecutionResource* old_vp = my_execution_resource; - if( s==ts_busy ) { - ++the_balance; - my_state = ts_asleep; - } - IThreadProxy* proxy = my_proxy; - __TBB_ASSERT( proxy, NULL ); - my_execution_resource = (IExecutionResource*) c_remove_prepare; - old_vp->Remove( my_scheduler ); - my_execution_resource = (IExecutionResource*) c_remove_returned; - int cnt = --activation_count; - __TBB_ASSERT_EX( cnt==0||cnt==1, "too many activations?" ); - proxy->SwitchOut(); - if( terminate ) { - bool activated = activation_count==1; -#if TBB_USE_ASSERT - /* In a rare sequence of events, a thread comes out of SwitchOut with activation_count==1. - * 1) The thread is SwitchOut'ed. - * 2) AddVirtualProcessors() arrived and the thread is Activated. - * 3) The thread is coming out of SwitchOut(). - * 4) request_close_connection arrives and inform the thread that it is time to terminate. - * 5) The thread hits the check and falls into the path with 'activated==true'. - * In that case, do the clean-up but do not switch to the thread scavenger; rather simply return to RM. - */ - if( activated ) { - // thread is 'revived' in add_virtual_processors after being Activated(). - // so, if the thread extra state is still marked 'removed', it will shortly change to 'none' - // i.e., !is_remove(). The thread state is changed to ts_idle before the extra state, so - // the thread's state should be either ts_idle or ts_done. - while( is_removed() ) - __TBB_Yield(); - thread_state_t s = read_state(); - __TBB_ASSERT( s==ts_idle || s==ts_done, NULL ); - } -#endif - __TBB_ASSERT( my_state==ts_asleep||my_state==ts_idle, NULL ); - // it is possible that in make_job() the thread may not have a chance to create a job. - // my_job may not be set if the thread did not get a chance to process client's job (i.e., call try_process()) - rml::job* j; - if( my_job_automaton.try_plug(j) ) { - __TBB_ASSERT( j, NULL ); - my_client.cleanup(*j); - my_conn->remove_client_ref(); - } - // Must do remove client reference first, because execution of - // c.remove_ref() can cause *this to be destroyed. - if( !activated ) - proxy->SwitchTo( my_thread_map.get_thread_scavenger(), Idle ); - my_conn->remove_server_ref(); - return true; - } - // We revive a thread in add_virtual_processors() after we Activate the thread on a new virtual processor. - // So briefly wait until the thread's my_execution_resource gets set. - while( get_virtual_processor()==c_remove_returned ) - __TBB_Yield(); - return false; -} - -bool tbb_server_thread::sleep_perhaps () { - if( terminate ) return false; - thread_state_t s = read_state(); - if( s==ts_idle ) { - if( my_state.compare_and_swap( ts_asleep, ts_idle )==ts_idle ) { - // If a thread is between read_state() and compare_and_swap(), and the master tries to terminate, - // the master's compare_and_swap() will fail because the thread's state is ts_idle. - // We need to check if terminate is true or not before letting the thread go to sleep, - // otherwise we will miss the terminate signal. - if( !terminate ) { - if( !is_removed() ) { - --activation_count; - get_virtual_processor()->Deactivate( this ); - } - if( is_removed() ) { - if( switch_out() ) - return true; - __TBB_ASSERT( my_execution_resource>c_remove_returned, NULL ); - } - // in add_virtual_processors(), when we revive a thread, we change its state after Activate the thread - // in that case the state may be ts_asleep for a short period - while( read_state()==ts_asleep ) - __TBB_Yield(); - } else { - if( my_state.compare_and_swap( ts_done, ts_asleep )!=ts_asleep ) { - --activation_count; - // unbind() changed my state. It will call Activate(). So issue a matching Deactivate() - get_virtual_processor()->Deactivate( this ); - } - } - } - } else { - __TBB_ASSERT( s==ts_busy, NULL ); - } - return false; -} - -void omp_server_thread::sleep_perhaps () { - if( terminate ) return; - thread_state_t s = read_state(); - if( s==ts_idle ) { - if( my_state.compare_and_swap( ts_asleep, ts_idle )==ts_idle ) { - // If a thread is between read_state() and compare_and_swap(), and the master tries to terminate, - // the master's compare_and_swap() will fail because the thread's state is ts_idle. - // We need to check if terminate is true or not before letting the thread go to sleep, - // otherwise we will miss the terminate signal. - if( !terminate ) { - get_virtual_processor()->Deactivate( this ); - __TBB_ASSERT( !is_removed(), "OMP threads should not be deprived of a virtual processor" ); - __TBB_ASSERT( read_state()!=ts_asleep, NULL ); - } else { - if( my_state.compare_and_swap( ts_done, ts_asleep )!=ts_asleep ) - // unbind() changed my state. It will call Activate(). So issue a matching Deactivate() - get_virtual_processor()->Deactivate( this ); - } - } - } else { - __TBB_ASSERT( s==ts_busy, NULL ); - } -} - -bool tbb_server_thread::initiate_termination() { - if( read_state()==ts_busy ) { - int bal = ++the_balance; - if( bal>0 ) wakeup_some_tbb_threads(); - } - return destroy_job( (tbb_connection_v2*) my_conn ); -} - -template<typename Connection> -bool server_thread_rep::destroy_job( Connection* c ) { - __TBB_ASSERT( my_state!=ts_asleep, NULL ); - rml::job* j; - if( my_job_automaton.try_plug(j) ) { - __TBB_ASSERT( j, NULL ); - my_client.cleanup(*j); - c->remove_client_ref(); - } - // Must do remove client reference first, because execution of - // c.remove_ref() can cause *this to be destroyed. - c->remove_server_ref(); - return true; -} - -void thread_map::assist_cleanup( bool assist_null_only ) { - // To avoid deadlock, the current thread *must* help out with cleanups that have not started, - // because the thread that created the job may be busy for a long time. - for( iterator i = begin(); i!=end(); ++i ) { - rml::job* j=0; - server_thread* thr = (*i).second; - job_automaton& ja = thr->my_job_automaton; - if( assist_null_only ? ja.try_plug_null() : ja.try_plug(j) ) { - if( j ) { - my_client.cleanup(*j); - } else { - // server thread did not get a chance to create a job. - } - remove_client_ref(); - } - } -} - -void thread_map::add_virtual_processors( IVirtualProcessorRoot** vproots, unsigned int count, tbb_connection_v2& conn, ::tbb::spin_mutex& mtx ) -{ -#if TBB_USE_ASSERT - int req_cnt = ++n_add_vp_requests; - __TBB_ASSERT( req_cnt==1, NULL ); -#endif - std::vector<thread_map::iterator> vec(count); - std::vector<tbb_server_thread*> tvec(count); - iterator end; - - { - tbb::spin_mutex::scoped_lock lck( mtx ); - __TBB_ASSERT( my_map.size()==0||count==1, NULL ); - end = my_map.end(); //remember 'end' at the time of 'find' - // find entries in the map for those VPs that were previously added and then removed. - for( size_t i=0; i<count; ++i ) { - vec[i] = my_map.find( (key_type) vproots[i] ); -#if TBB_USE_DEBUG - if( vec[i]!=end ) { - tbb_server_thread* t = (tbb_server_thread*) (*vec[i]).second; - IVirtualProcessorRoot* v = t->get_virtual_processor(); - __TBB_ASSERT( v==c_remove_prepare||v==c_remove_returned, NULL ); - } -#endif - } - - iterator nxt = my_map.begin(); - for( size_t i=0; i<count; ++i ) { - if( vec[i]!=end ) { -#if TBB_USE_ASSERT - tbb_server_thread* t = (tbb_server_thread*) (*vec[i]).second; - __TBB_ASSERT( t->read_state()==ts_asleep, NULL ); - IVirtualProcessorRoot* r = t->get_virtual_processor(); - __TBB_ASSERT( r==c_remove_prepare||r==c_remove_returned, NULL ); -#endif - continue; - } - - if( my_unrealized_threads>0 ) { - --my_unrealized_threads; - } else { - __TBB_ASSERT( nxt!=end, "nxt should not be thread_map::iterator::end" ); - // find a removed thread context for i - for( ; nxt!=end; ++nxt ) { - tbb_server_thread* t = (tbb_server_thread*) (*nxt).second; - if( t->is_removed() && t->read_state()==ts_asleep && t->get_virtual_processor()==c_remove_returned ) { - vec[i] = nxt++; - break; - } - } - // break target - if( vec[i]==end ) // ignore excessive VP. - vproots[i] = NULL; - } - } - } - - for( size_t i=0; i<count; ++i ) { - __TBB_ASSERT( !tvec[i], NULL ); - if( vec[i]==end ) { - if( vproots[i] ) { - tvec[i] = my_tbb_allocator.allocate(1); - new ( tvec[i] ) tbb_server_thread( false, my_scheduler, (IExecutionResource*)vproots[i], &conn, *this, my_client ); - } -#if TBB_USE_ASSERT - } else { - tbb_server_thread* t = (tbb_server_thread*) (*vec[i]).second; - __TBB_ASSERT( t->GetProxy(), "Proxy is cleared?" ); -#endif - } - } - - { - tbb::spin_mutex::scoped_lock lck( mtx ); - - bool closing = is_closing(); - - for( size_t i=0; i<count; ++i ) { - if( vec[i]==end ) { - if( vproots[i] ) { - thread_map::key_type key = (thread_map::key_type) vproots[i]; - vec[i] = insert( key, (server_thread*) tvec[i] ); - my_client_ref_count.add_ref(); - my_server_ref_count.add_ref(); - } - } else if( !closing ) { - tbb_server_thread* t = (tbb_server_thread*) (*vec[i]).second; - - if( (*vec[i]).first!=(thread_map::key_type)vproots[i] ) { - my_map.erase( vec[i] ); - thread_map::key_type key = (thread_map::key_type) vproots[i]; - __TBB_ASSERT( key, NULL ); - vec[i] = insert( key, t ); - } - __TBB_ASSERT( t->read_state()==ts_asleep, NULL ); - // We did not decrement server/client ref count when a thread is removed. - // So, don't increment server/client ref count here. - } - } - - // we could check is_closing() earlier. That requires marking the newly allocated server_thread objects - // that are not inserted into the thread_map, and deallocate them. Doing so seems more cumbersome - // than simply adding these to the thread_map and let thread_map's destructor take care of reclamation. - __TBB_ASSERT( closing==is_closing(), NULL ); - if( closing ) return; - } - - for( size_t i=0; i<count; ++i ) { - if( vproots[i] ) { - tbb_server_thread* t = (tbb_server_thread*) (*vec[i]).second; - __TBB_ASSERT( tvec[i]!=NULL||t->GetProxy(), "Proxy is cleared?" ); - if( t->is_removed() ) - __TBB_ASSERT( t->get_virtual_processor()==c_remove_returned, NULL ); - int cnt = ++t->activation_count; - __TBB_ASSERT_EX( cnt==0||cnt==1, NULL ); - vproots[i]->Activate( t ); - if( t->is_removed() ) - t->revive( my_scheduler, vproots[i], my_client ); - } - } -#if TBB_USE_ASSERT - req_cnt = --n_add_vp_requests; - __TBB_ASSERT( req_cnt==0, NULL ); -#endif -} - -void thread_map::remove_virtual_processors( IVirtualProcessorRoot** vproots, unsigned count, ::tbb::spin_mutex& mtx ) { - if( my_map.size()==0 ) - return; - tbb::spin_mutex::scoped_lock lck( mtx ); - - if( is_closing() ) return; - - for( unsigned int c=0; c<count; ++c ) { - iterator i = my_map.find( (key_type) vproots[c] ); - if( i==my_map.end() ) { - thread_scavenger_thread* tst = my_thread_scavenger_thread; - if( !tst ) { - // Remove unknown vp from my scheduler; - vproots[c]->Remove( my_scheduler ); - } else { - while( (tst=my_thread_scavenger_thread)==c_claimed ) - __TBB_Yield(); - if( vproots[c]!=tst->get_virtual_processor() ) - vproots[c]->Remove( my_scheduler ); - } - continue; - } - tbb_server_thread* thr = (tbb_server_thread*) (*i).second; - __TBB_ASSERT( thr->tbb_thread, "incorrect type of server_thread" ); - thr->set_removed(); - if( thr->read_state()==ts_asleep ) { - while( thr->activation_count>0 ) { - if( thr->get_virtual_processor()<=c_remove_returned ) - break; - __TBB_Yield(); - } - if( thr->get_virtual_processor()>c_remove_returned ) { - // the thread is in Deactivated state - ++thr->activation_count; - // wake the thread up so that it Switches Out itself. - thr->get_virtual_processor()->Activate( thr ); - } // else, it is Switched Out - } // else the thread will see that it is removed and proceed to switch itself out without Deactivation - } -} - -void thread_map::add_virtual_processors( IVirtualProcessorRoot** vproots, unsigned int count, omp_connection_v2& conn, ::tbb::spin_mutex& mtx ) -{ - std::vector<thread_map::iterator> vec(count); - std::vector<server_thread*> tvec(count); - iterator end; - - { - tbb::spin_mutex::scoped_lock lck( mtx ); - // read the map - end = my_map.end(); //remember 'end' at the time of 'find' - for( size_t i=0; i<count; ++i ) - vec[i] = my_map.find( (key_type) vproots[i] ); - } - - for( size_t i=0; i<count; ++i ) { - __TBB_ASSERT( !tvec[i], NULL ); - if( vec[i]==end ) { - tvec[i] = my_omp_allocator.allocate(1); - new ( tvec[i] ) omp_server_thread( false, my_scheduler, (IExecutionResource*)vproots[i], &conn, *this, my_client ); - } - } - - { - tbb::spin_mutex::scoped_lock lck( mtx ); - - for( size_t i=0; i<count; ++i ) { - if( vec[i]==my_map.end() ) { - thread_map::key_type key = (thread_map::key_type) vproots[i]; - vec[i] = insert( key, tvec[i] ); - my_client_ref_count.add_ref(); - my_server_ref_count.add_ref(); - } - } - - // we could check is_closing() earlier. That requires marking the newly allocated server_thread objects - // that are not inserted into the thread_map, and deallocate them. Doing so seems more cumbersome - // than simply adding these to the thread_map and let thread_map's destructor take care of reclamation. - if( is_closing() ) return; - } - - for( size_t i=0; i<count; ++i ) - vproots[i]->Activate( (*vec[i]).second ); - - { - tbb::spin_mutex::scoped_lock lck( mtx ); - for( size_t i=0; i<count; ++i ) - original_exec_resources.push_back( vproots[i] ); - } -} - -void thread_map::mark_virtual_processors_as_lent( IVirtualProcessorRoot** vproots, unsigned count, ::tbb::spin_mutex& mtx ) { - tbb::spin_mutex::scoped_lock lck( mtx ); - - if( is_closing() ) return; - - iterator end = my_map.end(); - for( unsigned int c=0; c<count; ++c ) { - iterator i = my_map.find( (key_type) vproots[c] ); - if( i==end ) { - // The vproc has not been added to the map in create_oversubscribers() - my_map.insert( hash_map_type::value_type( (key_type) vproots[c], (server_thread*)1 ) ); - } else { - server_thread* thr = (*i).second; - if( ((uintptr_t)thr)&~(uintptr_t)1 ) { - __TBB_ASSERT( !thr->is_removed(), "incorrectly removed" ); - ((omp_server_thread*)thr)->set_lent(); - } - } - } -} - -void thread_map::create_oversubscribers( unsigned n, std::vector<server_thread*>& thr_vec, omp_connection_v2& conn, ::tbb::spin_mutex& mtx ) { - std::vector<IExecutionResource*> curr_exec_rsc; - { - tbb::spin_mutex::scoped_lock lck( mtx ); - curr_exec_rsc = original_exec_resources; // copy construct - } - typedef std::vector<IExecutionResource*>::iterator iterator_er; - typedef ::std::vector<std::pair<hash_map_type::key_type, hash_map_type::mapped_type> > hash_val_vector_t; - hash_val_vector_t v_vec(n); - iterator_er begin = curr_exec_rsc.begin(); - iterator_er end = curr_exec_rsc.end(); - iterator_er i = begin; - for( unsigned c=0; c<n; ++c ) { - IVirtualProcessorRoot* vpr = my_scheduler_proxy->CreateOversubscriber( *i ); - omp_server_thread* t = new ( my_omp_allocator.allocate(1) ) omp_server_thread( true, my_scheduler, (IExecutionResource*)vpr, &conn, *this, my_client ); - thr_vec[c] = t; - v_vec[c] = hash_map_type::value_type( (key_type) vpr, t ); - if( ++i==end ) i = begin; - } - - { - tbb::spin_mutex::scoped_lock lck( mtx ); - - if( is_closing() ) return; - - iterator end = my_map.end(); - unsigned c = 0; - for( hash_val_vector_t::iterator vi=v_vec.begin(); vi!=v_vec.end(); ++vi, ++c ) { - iterator i = my_map.find( (key_type) (*vi).first ); - if( i==end ) { - my_map.insert( *vi ); - } else { - // the vproc has not been added to the map in mark_virtual_processors_as_returned(); - uintptr_t lent = (uintptr_t) (*i).second; - __TBB_ASSERT( lent<=1, "vproc map entry added incorrectly?"); - (*i).second = thr_vec[c]; - if( lent ) - ((omp_server_thread*)thr_vec[c])->set_lent(); - else - ((omp_server_thread*)thr_vec[c])->set_returned(); - } - my_client_ref_count.add_ref(); - my_server_ref_count.add_ref(); - } - } -} - -void thread_map::wakeup_tbb_threads( int c, ::tbb::spin_mutex& mtx ) { - std::vector<tbb_server_thread*> vec(c); - - size_t idx = 0; - { - tbb::spin_mutex::scoped_lock lck( mtx ); - - if( is_closing() ) return; - // only one RML thread is in here to wake worker threads up. - - int bal = the_balance; - int cnt = c<bal ? c : bal; - - if( cnt<=0 ) { return; } - - for( iterator i=begin(); i!=end(); ++i ) { - tbb_server_thread* thr = (tbb_server_thread*) (*i).second; - // ConcRT RM should take threads away from TBB scheduler instead of lending them to another scheduler - if( thr->is_removed() ) - continue; - - if( --the_balance>=0 ) { - thread_grab_t res; - while( (res=thr->try_grab_for())!=wk_from_idle ) { - if( res==wk_from_asleep ) { - vec[idx++] = thr; - break; - } else { - thread_state_t s = thr->read_state(); - if( s==ts_busy ) {// failed because already assigned. move on. - ++the_balance; - goto skip; - } - } - } - thread_state_t s = thr->read_state(); - __TBB_ASSERT_EX( s==ts_busy, "should have set the state to ts_busy" ); - if( --cnt==0 ) - break; - } else { - // overdraft - ++the_balance; - break; - } -skip: - ; - } - } - - for( size_t i=0; i<idx; ++i ) { - tbb_server_thread* thr = vec[i]; - __TBB_ASSERT( thr, NULL ); - thread_state_t s = thr->read_state(); - __TBB_ASSERT_EX( s==ts_busy, "should have set the state to ts_busy" ); - ++thr->activation_count; - thr->get_virtual_processor()->Activate( thr ); - } - -} - -void thread_map::mark_virtual_processors_as_returned( IVirtualProcessorRoot** vprocs, unsigned int count, tbb::spin_mutex& mtx ) { - { - tbb::spin_mutex::scoped_lock lck( mtx ); - - if( is_closing() ) return; - - iterator end = my_map.end(); - for(unsigned c=0; c<count; ++c ) { - iterator i = my_map.find( (key_type) vprocs[c] ); - if( i==end ) { - // the vproc has not been added to the map in create_oversubscribers() - my_map.insert( hash_map_type::value_type( (key_type) vprocs[c], static_cast<server_thread*>(0) ) ); - } else { - omp_server_thread* thr = (omp_server_thread*) (*i).second; - if( ((uintptr_t)thr)&~(uintptr_t)1 ) { - __TBB_ASSERT( !thr->is_removed(), "incorrectly removed" ); - // we should not make any assumption on the initial state of an added vproc. - thr->set_returned(); - } - } - } - } -} - - -void thread_map::unbind( rml::server& /*server*/, tbb::spin_mutex& mtx ) { - { - tbb::spin_mutex::scoped_lock lck( mtx ); - shutdown_in_progress = true; // ignore any callbacks from ConcRT RM - - // Ask each server_thread to cleanup its job for this server. - for( iterator i = begin(); i!=end(); ++i ) { - server_thread* t = (*i).second; - t->terminate = true; - if( t->is_removed() ) { - // This is for TBB only as ConcRT RM does not request OMP schedulers to remove virtual processors - if( t->read_state()==ts_asleep ) { - __TBB_ASSERT( my_thread_scavenger_thread, "this is TBB connection; thread_scavenger_thread must be allocated" ); - // thread is on its way to switch_out; see remove_virtual_processors() where - // the thread is Activated() to bring it back from 'Deactivated' in sleep_perhaps() - // now assume that the thread will go to SwitchOut() -#if TBB_USE_ASSERT - while( t->get_virtual_processor()>c_remove_returned ) - __TBB_Yield(); -#endif - // A removed thread is supposed to proceed to SwithcOut. - // There, we remove client&server references. - } - } else { - if( t->wakeup( ts_done, ts_asleep ) ) { - if( t->tbb_thread ) - ++((tbb_server_thread*)t)->activation_count; - t->get_virtual_processor()->Activate( t ); - // We mark in the thread_map such that when termination sequence started, we ignore - // all notification from ConcRT RM. - } - } - } - } - // Remove extra ref to client. - remove_client_ref(); - - if( my_thread_scavenger_thread ) { - thread_scavenger_thread* tst; - while( (tst=my_thread_scavenger_thread)==c_claimed ) - __TBB_Yield(); -#if TBB_USE_ASSERT - ++my_thread_scavenger_thread->activation_count; -#endif - tst->get_virtual_processor()->Activate( tst ); - } -} - -#if !__RML_REMOVE_VIRTUAL_PROCESSORS_DISABLED -void thread_map::allocate_thread_scavenger( IExecutionResource* v ) -{ - if( my_thread_scavenger_thread>c_claimed ) return; - thread_scavenger_thread* c = my_thread_scavenger_thread.fetch_and_store((thread_scavenger_thread*)c_claimed); - if( c==NULL ) { // successfully claimed - add_server_ref(); -#if TBB_USE_ASSERT - ++n_thread_scavengers_created; -#endif - __TBB_ASSERT( v, NULL ); - IVirtualProcessorRoot* vpr = my_scheduler_proxy->CreateOversubscriber( v ); - my_thread_scavenger_thread = c = new ( my_scavenger_allocator.allocate(1) ) thread_scavenger_thread( my_scheduler, vpr, *this ); -#if TBB_USE_ASSERT - ++c->activation_count; -#endif - vpr->Activate( c ); - } else if( c>c_claimed ) { - my_thread_scavenger_thread = c; - } -} -#endif - -void thread_scavenger_thread::Dispatch( DispatchState* ) -{ - __TBB_ASSERT( my_proxy, NULL ); -#if TBB_USE_ASSERT - --activation_count; -#endif - get_virtual_processor()->Deactivate( this ); - for( thread_map::iterator i=my_thread_map.begin(); i!=my_thread_map.end(); ++i ) { - tbb_server_thread* t = (tbb_server_thread*) (*i).second; - if( t->read_state()==ts_asleep && t->is_removed() ) { - while( t->get_execution_resource()!=c_remove_returned ) - __TBB_Yield(); - my_proxy->SwitchTo( t, Blocking ); - } - } - get_virtual_processor()->Remove( my_scheduler ); - my_thread_map.remove_server_ref(); - // signal to the connection scavenger that i am done with the map. - __TBB_ASSERT( activation_count==1, NULL ); - set_state( ts_done ); -} - -//! Windows "DllMain" that handles startup and shutdown of dynamic library. -extern "C" bool WINAPI DllMain( HINSTANCE /*hinstDLL*/, DWORD fwdReason, LPVOID lpvReserved ) { - void assist_cleanup_connections(); - if( fwdReason==DLL_PROCESS_DETACH ) { - // dll is being unloaded - if( !lpvReserved ) // if FreeLibrary has been called - assist_cleanup_connections(); - } - return true; -} - -void free_all_connections( uintptr_t conn_ex ) { - while( conn_ex ) { - bool is_tbb = (conn_ex&2)>0; - //clear extra bits - uintptr_t curr_conn = conn_ex & ~(uintptr_t)3; - __TBB_ASSERT( curr_conn, NULL ); - - // Wait for worker threads to return - if( is_tbb ) { - tbb_connection_v2* tbb_conn = reinterpret_cast<tbb_connection_v2*>(curr_conn); - conn_ex = reinterpret_cast<uintptr_t>(tbb_conn->next_conn); - while( tbb_conn->my_thread_map.remove_server_ref()>0 ) - __TBB_Yield(); - delete tbb_conn; - } else { - omp_connection_v2* omp_conn = reinterpret_cast<omp_connection_v2*>(curr_conn); - conn_ex = reinterpret_cast<uintptr_t>(omp_conn->next_conn); - while( omp_conn->my_thread_map.remove_server_ref()>0 ) - __TBB_Yield(); - delete omp_conn; - } - } -} - -void assist_cleanup_connections() -{ - //signal to connection_scavenger_thread to terminate - uintptr_t tail = connections_to_reclaim.tail; - while( connections_to_reclaim.tail.compare_and_swap( garbage_connection_queue::plugged, tail )!=tail ) { - __TBB_Yield(); - tail = connections_to_reclaim.tail; - } - - __TBB_ASSERT( connection_scavenger.state==ts_busy || connection_scavenger.state==ts_asleep, NULL ); - // Scavenger thread may be busy freeing connections - DWORD thr_exit_code = STILL_ACTIVE; - while( connection_scavenger.state==ts_busy ) { - if( GetExitCodeThread( connection_scavenger.thr_handle, &thr_exit_code )>0 ) - if( thr_exit_code!=STILL_ACTIVE ) - break; - __TBB_Yield(); - thr_exit_code = STILL_ACTIVE; - } - if( connection_scavenger.state==ts_asleep && thr_exit_code==STILL_ACTIVE ) - connection_scavenger.wakeup(); // wake the connection scavenger thread up - - // it is possible that the connection scavenger thread already exited. Take over its responsibility. - if( tail && connections_to_reclaim.tail!=garbage_connection_queue::plugged_acked ) { - // atomically claim the head of the list. - uintptr_t head = connections_to_reclaim.head.fetch_and_store( garbage_connection_queue::empty ); - if( head==garbage_connection_queue::empty ) - head = tail; - connection_scavenger.process_requests( head ); - } - __TBB_ASSERT( connections_to_reclaim.tail==garbage_connection_queue::plugged||connections_to_reclaim.tail==garbage_connection_queue::plugged_acked, "someone else added a request after termination has initiated" ); - __TBB_ASSERT( (unsigned)the_balance==the_default_concurrency, NULL ); -} - -void connection_scavenger_thread::sleep_perhaps() { - uintptr_t tail = connections_to_reclaim.tail; - // connections_to_reclaim.tail==garbage_connection_queue::plugged --> terminate, - // connections_to_reclaim.tail>garbage_connection_queue::plugged : we got work to do - if( tail>=garbage_connection_queue::plugged ) return; - __TBB_ASSERT( !tail, NULL ); - thread_monitor::cookie c; - monitor.prepare_wait(c); - if( state.compare_and_swap( ts_asleep, ts_busy )==ts_busy ) { - if( connections_to_reclaim.tail!=garbage_connection_queue::plugged ) { - monitor.commit_wait(c); - // Someone else woke me up. The compare_and_swap further below deals with spurious wakeups. - } else { - monitor.cancel_wait(); - } - thread_state_t s = state; - if( s==ts_asleep ) // if spurious wakeup. - state.compare_and_swap( ts_busy, ts_asleep ); - // I woke myself up, either because I cancelled the wait or suffered a spurious wakeup. - } else { - __TBB_ASSERT( false, "someone else tampered with my state" ); - } - __TBB_ASSERT( state==ts_busy, "a thread can only put itself to sleep" ); -} - -void connection_scavenger_thread::process_requests( uintptr_t conn_ex ) -{ - __TBB_ASSERT( conn_ex>1, NULL ); - __TBB_ASSERT( n_scavenger_threads==1||connections_to_reclaim.tail==garbage_connection_queue::plugged, "more than one connection_scavenger_thread being active?" ); - - bool done = false; - while( !done ) { - bool is_tbb = (conn_ex&2)>0; - //clear extra bits - uintptr_t curr_conn = conn_ex & ~(uintptr_t)3; - - // no contention. there is only one connection_scavenger_thread!! - uintptr_t next_conn; - tbb_connection_v2* tbb_conn = NULL; - omp_connection_v2* omp_conn = NULL; - // Wait for worker threads to return - if( is_tbb ) { - tbb_conn = reinterpret_cast<tbb_connection_v2*>(curr_conn); - next_conn = reinterpret_cast<uintptr_t>(tbb_conn->next_conn); - while( tbb_conn->my_thread_map.get_server_ref_count()>1 ) - __TBB_Yield(); - } else { - omp_conn = reinterpret_cast<omp_connection_v2*>(curr_conn); - next_conn = reinterpret_cast<uintptr_t>(omp_conn->next_conn); - while( omp_conn->my_thread_map.get_server_ref_count()>1 ) - __TBB_Yield(); - } - - //someone else may try to write into this connection object. - //So access next_conn field first before remove the extra server ref count. - - if( next_conn==0 ) { - uintptr_t tail = connections_to_reclaim.tail; - if( tail==garbage_connection_queue::plugged ) { - tail = garbage_connection_queue::plugged_acked; // connection scavenger saw the flag, and it freed all connections. - done = true; - } else if( tail==conn_ex ) { - if( connections_to_reclaim.tail.compare_and_swap( garbage_connection_queue::empty, tail )==tail ) { - __TBB_ASSERT( !connections_to_reclaim.head, NULL ); - done = true; - } - } - - if( !done ) { - // A new connection to close is added to connections_to_reclaim.tail; - // Wait for curr_conn->next_conn to be set. - if( is_tbb ) { - while( !tbb_conn->next_conn ) - __TBB_Yield(); - conn_ex = reinterpret_cast<uintptr_t>(tbb_conn->next_conn); - } else { - while( !omp_conn->next_conn ) - __TBB_Yield(); - conn_ex = reinterpret_cast<uintptr_t>(omp_conn->next_conn); - } - } - } else { - conn_ex = next_conn; - } - __TBB_ASSERT( conn_ex, NULL ); - if( is_tbb ) - // remove extra server ref count; this will trigger Shutdown/Release of ConcRT RM - tbb_conn->remove_server_ref(); - else - // remove extra server ref count; this will trigger Shutdown/Release of ConcRT RM - omp_conn->remove_server_ref(); - } -} - -__RML_DECL_THREAD_ROUTINE connection_scavenger_thread::thread_routine( void* arg ) { - connection_scavenger_thread* thr = (connection_scavenger_thread*) arg; - thr->state = ts_busy; - thr->thr_handle = GetCurrentThread(); -#if TBB_USE_ASSERT - ++thr->n_scavenger_threads; -#endif - for(;;) { - __TBB_Yield(); - thr->sleep_perhaps(); - if( connections_to_reclaim.tail==garbage_connection_queue::plugged || connections_to_reclaim.tail==garbage_connection_queue::plugged_acked ) { - thr->state = ts_asleep; - return 0; - } - - __TBB_ASSERT( connections_to_reclaim.tail!=garbage_connection_queue::plugged_acked, NULL ); - __TBB_ASSERT( connections_to_reclaim.tail>garbage_connection_queue::plugged && (connections_to_reclaim.tail&garbage_connection_queue::plugged)==0 , NULL ); - while( connections_to_reclaim.head==garbage_connection_queue::empty ) - __TBB_Yield(); - uintptr_t head = connections_to_reclaim.head.fetch_and_store( garbage_connection_queue::empty ); - thr->process_requests( head ); - wakeup_some_tbb_threads(); - } -} - -template<typename Server, typename Client> -void connection_scavenger_thread::add_request( generic_connection<Server,Client>* conn_to_close ) -{ - uintptr_t conn_ex = (uintptr_t)conn_to_close | (connection_traits<Server,Client>::is_tbb<<1); - __TBB_ASSERT( !conn_to_close->next_conn, NULL ); - const uintptr_t old_tail_ex = connections_to_reclaim.tail.fetch_and_store(conn_ex); - __TBB_ASSERT( old_tail_ex==0||old_tail_ex>garbage_connection_queue::plugged_acked, "Unloading DLL called while this connection is being closed?" ); - - if( old_tail_ex==garbage_connection_queue::empty ) - connections_to_reclaim.head = conn_ex; - else { - bool is_tbb = (old_tail_ex&2)>0; - uintptr_t old_tail = old_tail_ex & ~(uintptr_t)3; - if( is_tbb ) - reinterpret_cast<tbb_connection_v2*>(old_tail)->next_conn = reinterpret_cast<tbb_connection_v2*>(conn_ex); - else - reinterpret_cast<omp_connection_v2*>(old_tail)->next_conn = reinterpret_cast<omp_connection_v2*>(conn_ex); - } - - if( state==ts_asleep ) - wakeup(); -} - -template<> -uintptr_t connection_scavenger_thread::grab_and_prepend( generic_connection<tbb_server,tbb_client>* /*last_conn_to_close*/ ) { return 0;} - -template<> -uintptr_t connection_scavenger_thread::grab_and_prepend( generic_connection<omp_server,omp_client>* last_conn_to_close ) -{ - uintptr_t conn_ex = (uintptr_t)last_conn_to_close; - uintptr_t head = connections_to_reclaim.head.fetch_and_store( garbage_connection_queue::empty ); - reinterpret_cast<omp_connection_v2*>(last_conn_to_close)->next_conn = reinterpret_cast<omp_connection_v2*>(head); - return conn_ex; -} - -extern "C" ULONGLONG NTAPI VerSetConditionMask( ULONGLONG, DWORD, BYTE); - -bool is_windows7_or_later () -{ - try { - return GetOSVersion()>=IResourceManager::Win7OrLater; - } catch( ... ) { - return false; - } -} - -#endif /* RML_USE_WCRM */ - -template<typename Connection, typename Server, typename Client> -static factory::status_type connect( factory& f, Server*& server, Client& client ) { - server = new Connection(*static_cast<wait_counter*>(f.scratch_ptr),client); - return factory::st_success; -} - -void init_rml_module () { - the_balance = the_default_concurrency = tbb::internal::AvailableHwConcurrency() - 1; -#if RML_USE_WCRM - connection_scavenger.launch(); -#endif -} - -extern "C" factory::status_type __RML_open_factory( factory& f, version_type& server_version, version_type client_version ) { - // Hack to keep this library from being closed by causing the first client's dlopen to not have a corresponding dlclose. - // This code will be removed once we figure out how to do shutdown of the RML perfectly. - static tbb::atomic<bool> one_time_flag; - if( one_time_flag.compare_and_swap(true,false)==false) { - __TBB_ASSERT( (size_t)f.library_handle!=factory::c_dont_unload, NULL ); -#if _WIN32||_WIN64 - f.library_handle = reinterpret_cast<HMODULE>(factory::c_dont_unload); -#else - f.library_handle = reinterpret_cast<void*>(factory::c_dont_unload); -#endif - } - // End of hack - - // Initialize the_balance only once - tbb::internal::atomic_do_once ( &init_rml_module, rml_module_state ); - - server_version = SERVER_VERSION; - f.scratch_ptr = 0; - if( client_version==0 ) { - return factory::st_incompatible; -#if RML_USE_WCRM - } else if ( !is_windows7_or_later() ) { -#if TBB_USE_DEBUG - fprintf(stderr, "This version of the RML library requires Windows 7 to run on.\nConnection request denied.\n"); -#endif - return factory::st_incompatible; -#endif - } else { -#if TBB_USE_DEBUG - if( client_version<EARLIEST_COMPATIBLE_CLIENT_VERSION ) - fprintf(stderr, "This client library is too old for the current RML server.\nThe connection request is granted but oversubscription/undersubscription may occur.\n"); -#endif - f.scratch_ptr = new wait_counter; - return factory::st_success; - } -} - -extern "C" void __RML_close_factory( factory& f ) { - if( wait_counter* fc = static_cast<wait_counter*>(f.scratch_ptr) ) { - f.scratch_ptr = 0; - fc->wait(); - size_t bal = the_balance; - f.scratch_ptr = (void*)bal; - delete fc; - } -} - -void call_with_build_date_str( ::rml::server_info_callback_t cb, void* arg ); - -}} // rml::internal - -namespace tbb { -namespace internal { -namespace rml { - -extern "C" tbb_factory::status_type __TBB_make_rml_server( tbb_factory& f, tbb_server*& server, tbb_client& client ) { - return ::rml::internal::connect< ::rml::internal::tbb_connection_v2>(f,server,client); -} - -extern "C" void __TBB_call_with_my_server_info( ::rml::server_info_callback_t cb, void* arg ) { - return ::rml::internal::call_with_build_date_str( cb, arg ); -} - -}}} - -namespace __kmp { -namespace rml { - -extern "C" omp_factory::status_type __KMP_make_rml_server( omp_factory& f, omp_server*& server, omp_client& client ) { - return ::rml::internal::connect< ::rml::internal::omp_connection_v2>(f,server,client); -} - -extern "C" void __KMP_call_with_my_server_info( ::rml::server_info_callback_t cb, void* arg ) { - return ::rml::internal::call_with_build_date_str( cb, arg ); -} - -}} - -/* - * RML server info - */ -#include "version_string.ver" - -#ifndef __TBB_VERSION_STRINGS -#pragma message("Warning: version_string.ver isn't generated properly by version_info.sh script!") -#endif - -// We use the build time as the RML server info. TBB is required to build RML, so we make it the same as the TBB build time. -#ifndef __TBB_DATETIME -#define __TBB_DATETIME __DATE__ " " __TIME__ -#endif - -#if !RML_USE_WCRM -#define RML_SERVER_BUILD_TIME "Intel(R) RML library built: " __TBB_DATETIME -#define RML_SERVER_VERSION_ST "Intel(R) RML library version: v" TOSTRING(SERVER_VERSION) -#else -#define RML_SERVER_BUILD_TIME "Intel(R) RML library built: " __TBB_DATETIME -#define RML_SERVER_VERSION_ST "Intel(R) RML library version: v" TOSTRING(SERVER_VERSION) " on ConcRT RM with " RML_THREAD_KIND_STRING -#endif - -namespace rml { -namespace internal { - -void call_with_build_date_str( ::rml::server_info_callback_t cb, void* arg ) -{ - (*cb)( arg, RML_SERVER_BUILD_TIME ); - (*cb)( arg, RML_SERVER_VERSION_ST ); -} -}} // rml::internal diff --git a/src/tbb/src/rml/server/wait_counter.h b/src/tbb/src/rml/server/wait_counter.h deleted file mode 100644 index 3ddd8e036..000000000 --- a/src/tbb/src/rml/server/wait_counter.h +++ /dev/null @@ -1,73 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef __RML_wait_counter_H -#define __RML_wait_counter_H - -#include "thread_monitor.h" -#include "tbb/atomic.h" - -namespace rml { -namespace internal { - -class wait_counter { - thread_monitor my_monitor; - tbb::atomic<int> my_count; - tbb::atomic<int> n_transients; -public: - wait_counter() { - // The "1" here is subtracted by the call to "wait". - my_count=1; - n_transients=0; - } - - //! Wait for number of operator-- invocations to match number of operator++ invocations. - /** Exactly one thread should call this method. */ - void wait() { - int k = --my_count; - __TBB_ASSERT( k>=0, "counter underflow" ); - if( k>0 ) { - thread_monitor::cookie c; - my_monitor.prepare_wait(c); - if( my_count ) - my_monitor.commit_wait(c); - else - my_monitor.cancel_wait(); - } - while( n_transients>0 ) - __TBB_Yield(); - } - void operator++() { - ++my_count; - } - void operator--() { - ++n_transients; - int k = --my_count; - __TBB_ASSERT( k>=0, "counter underflow" ); - if( k==0 ) - my_monitor.notify(); - --n_transients; - } -}; - -} // namespace internal -} // namespace rml - -#endif /* __RML_wait_counter_H */ diff --git a/src/tbb/src/rml/server/win32-rml-export.def b/src/tbb/src/rml/server/win32-rml-export.def deleted file mode 100644 index 2a550d0b5..000000000 --- a/src/tbb/src/rml/server/win32-rml-export.def +++ /dev/null @@ -1,27 +0,0 @@ -; Copyright (c) 2005-2017 Intel Corporation -; -; Licensed under the Apache License, Version 2.0 (the "License"); -; you may not use this file except in compliance with the License. -; You may obtain a copy of the License at -; -; http://www.apache.org/licenses/LICENSE-2.0 -; -; Unless required by applicable law or agreed to in writing, software -; distributed under the License is distributed on an "AS IS" BASIS, -; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -; See the License for the specific language governing permissions and -; limitations under the License. -; -; -; -; - -EXPORTS - -__RML_open_factory -__RML_close_factory -__TBB_make_rml_server -__KMP_make_rml_server -__TBB_call_with_my_server_info -__KMP_call_with_my_server_info - diff --git a/src/tbb/src/rml/test/rml_omp_stub.cpp b/src/tbb/src/rml/test/rml_omp_stub.cpp deleted file mode 100644 index eb701b171..000000000 --- a/src/tbb/src/rml/test/rml_omp_stub.cpp +++ /dev/null @@ -1,72 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -// This file is compiled with C++, but linked with a program written in C. -// The intent is to find dependencies on the C++ run-time. - -#include <stdlib.h> -#include "../../../include/tbb/tbb_stddef.h" // __TBB_override -#include "harness_defs.h" -#define RML_PURE_VIRTUAL_HANDLER abort - -#if _MSC_VER==1500 && !defined(__INTEL_COMPILER) -// VS2008/VC9 seems to have an issue; -#pragma warning( push ) -#pragma warning( disable: 4100 ) -#elif __TBB_MSVC_UNREACHABLE_CODE_IGNORED -// VS2012-2013 issues "warning C4702: unreachable code" for the code which really -// shouldn't be reached according to the test logic: rml::client has the -// implementation for the "pure" virtual methods to be aborted if they are -// called. -#pragma warning( push ) -#pragma warning( disable: 4702 ) -#endif -#include "rml_omp.h" -#if ( _MSC_VER==1500 && !defined(__INTEL_COMPILER)) || __TBB_MSVC_UNREACHABLE_CODE_IGNORED -#pragma warning( pop ) -#endif - -rml::versioned_object::version_type Version; - -class MyClient: public __kmp::rml::omp_client { -public: - rml::versioned_object::version_type version() const __TBB_override {return 0;} - size_type max_job_count() const __TBB_override {return 1024;} - size_t min_stack_size() const __TBB_override {return 1<<20;} - rml::job* create_one_job() __TBB_override {return NULL;} - void acknowledge_close_connection() __TBB_override {} - void cleanup(job&) __TBB_override {} - policy_type policy() const __TBB_override {return throughput;} - void process( job&, void*, __kmp::rml::omp_client::size_type ) __TBB_override {} - -}; - -//! Never actually set, because point of test is to find linkage issues. -__kmp::rml::omp_server* MyServerPtr; - -#define HARNESS_NO_PARSE_COMMAND_LINE 1 -#define HARNESS_CUSTOM_MAIN 1 -#include "harness.h" - -extern "C" void Cplusplus() { - MyClient client; - Version = client.version(); - REPORT("done\n"); -} diff --git a/src/tbb/src/rml/test/test_job_automaton.cpp b/src/tbb/src/rml/test/test_job_automaton.cpp deleted file mode 100644 index f416e58b1..000000000 --- a/src/tbb/src/rml/test/test_job_automaton.cpp +++ /dev/null @@ -1,152 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#include "harness.h" -#if __TBB_MIC_OFFLOAD -int TestMain () { - return Harness::Skipped; -} -#else -#include "job_automaton.h" -#define HARNESS_NO_PARSE_COMMAND_LINE 1 -#include "harness_barrier.h" - -class State { - Harness::SpinBarrier barrier; - rml::internal::job_automaton ja; - rml::job job; - tbb::atomic<int> job_created; - tbb::atomic<int> job_destroyed; - tbb::atomic<bool> job_received; -public: - State() : barrier(2) { - job_created = 0; - job_destroyed = 0; - job_received = false; - } - void exercise( bool is_owner ); - ~State() { - ASSERT( job_created==job_destroyed, "accounting error" ); - ASSERT( job_destroyed<=1, "destroyed job twice" ); - } -}; - -int DelayMask; -const int N = 14; -tbb::atomic<int> Coverage[N]; - -//! Mark kth interval as covered and insert delay if kth bit of DelayMask is set. -/** An interval is the code between two operations on the job_automaton that we are testing. */ -void Cover( int k ) { - ASSERT( k<N, NULL ); - ++Coverage[k]; - if( DelayMask>>k&1 ) { - // Introduce delay (and possibly a thread context switch) - __TBB_Yield(); - } -} - -void State::exercise( bool is_owner ) { - barrier.wait(); - if( is_owner ) { - Cover(0); - if( ja.try_acquire() ) { - Cover(1); - ++job_created; - ja.set_and_release(&job); - Cover(2); - if( ja.try_acquire() ) { - Cover(3); - ja.release(); - Cover(4); - if( ja.try_acquire() ) { - Cover(5); - ja.release(); - } - } - Cover(6); - } else { - Cover(7); - } - if( DelayMask&1<<N ) { - while( !job_received ) - __TBB_Yield(); - } - } else { - // Using extra bit of DelayMask for choosing whether to run wait_for_job or not. - if( DelayMask&1<<N ) { - rml::job* j= ja.wait_for_job(); - if( j!=&job ) REPORT("%p\n",j); - ASSERT( j==&job, NULL ); - job_received = true; - } - Cover(8); - } - rml::job* j; - if( ja.try_plug(j) ) { - ASSERT( j==&job || !j, NULL ); - if( j ) { - Cover(9+is_owner); - ++job_destroyed; - } else { - __TBB_ASSERT( !is_owner, "owner failed to create job but plugged self" ); - Cover(11); - } - } else { - Cover(12+is_owner); - } -} - -class Loop: NoAssign { - State& s; -public: - Loop(State& s_) : s(s_) {} - void operator()( int i ) const {s.exercise(i==0);} -}; - -/** Return true if coverage is acceptable. - If report==true, issue message if it is unacceptable. */ -bool CheckCoverage( bool report ) { - bool okay = true; - for( int i=0; i<N; ++i ) { - const int min_coverage = 4; - if( Coverage[i]<min_coverage ) { - okay = false; - if( report ) - REPORT("Warning: Coverage[%d]=%d is less than acceptable minimum of %d\n", i, int(Coverage[i]),min_coverage); - } - } - return okay; -} - -int TestMain () { - for( DelayMask=0; DelayMask<8<<N; ++DelayMask ) { - State s; - NativeParallelFor( 2, Loop(s) ); - if( CheckCoverage(false) ) { - // Reached acceptable code coverage level - break; - } - } - CheckCoverage(true); - return Harness::Done; -} - -#endif /* __TBB_MIC_OFFLOAD */ diff --git a/src/tbb/src/rml/test/test_rml_mixed.cpp b/src/tbb/src/rml/test/test_rml_mixed.cpp deleted file mode 100644 index 0575df503..000000000 --- a/src/tbb/src/rml/test/test_rml_mixed.cpp +++ /dev/null @@ -1,318 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#include <tbb/tbb_config.h> -#if __TBB_WIN8UI_SUPPORT || __TBB_MIC_OFFLOAD -#include "harness.h" -int TestMain () { - return Harness::Skipped; -} -#else -#include "rml_tbb.h" -#include "rml_omp.h" -#include "tbb/atomic.h" -#include "tbb/tick_count.h" - -#define HARNESS_DEFAULT_MIN_THREADS 4 -#include "harness.h" - -// dynamic_link initializes its data structures in a static constructor. But -// the initialization order of static constructors in different modules is -// non-deterministic. Thus dynamic_link fails on some systems when the -// application changes its current directory after the library (TBB/OpenMP/...) -// is loaded but before the static constructors in the library are executed. -#define CHDIR_SUPPORT_BROKEN ( __TBB_GCC_VERSION >= 40600 || (__linux__ && __TBB_CLANG_VERSION >= 30500) ) - -const int OMP_ParallelRegionSize = 16; -int TBB_MaxThread = 4; // Includes master -int OMP_MaxThread = int(~0u>>1); // Includes master - -template<typename Client> -class ClientBase: public Client { -protected: - typedef typename Client::version_type version_type; - typedef typename Client::job job; - typedef typename Client::policy_type policy_type; - -private: - version_type version() const __TBB_override { - return 0; - } - size_t min_stack_size() const __TBB_override { - return 1<<20; - } - job* create_one_job() __TBB_override { - return new rml::job; - } - policy_type policy() const __TBB_override { - return Client::turnaround; - } - void acknowledge_close_connection() __TBB_override { - delete this; - } - void cleanup( job& j ) __TBB_override {delete &j;} - -public: - virtual ~ClientBase() {} -}; - -#if _WIN32 -#include <direct.h> -#define PATH_LEN MAX_PATH+1 -#define SLASH '\\' -#define ROOT_DIR "\\" -// ROOT_DIR_REST means how many symbols before first slash in the path -#define ROOT_DIR_REST 2 -#else -#include <unistd.h> -#include <limits.h> -#define PATH_LEN PATH_MAX+1 -#define SLASH '/' -#define ROOT_DIR "/" -// ROOT_DIR_REST means how many symbols before first slash in the path -#define ROOT_DIR_REST 0 -#define _getcwd getcwd -#define _chdir chdir -#endif - -#if !CHDIR_SUPPORT_BROKEN -class ChangeCurrentDir { - char dir[PATH_LEN+1]; - char *last_slash; -public: - ChangeCurrentDir() { - if ( !_getcwd( dir, PATH_LEN ) ) { - REPORT_FATAL_ERROR("ERROR: Couldn't get current working directory\n"); - } - - last_slash = strrchr( dir, SLASH ); - ASSERT( last_slash, "The current directory doesn't contain slashes" ); - *last_slash = 0; - - if ( _chdir( last_slash-dir == ROOT_DIR_REST ? ROOT_DIR : dir ) ) { - REPORT_FATAL_ERROR("ERROR: Couldn't change current working directory (%s)\n", dir ); - } - } - - // Restore current dir - ~ChangeCurrentDir() { - *last_slash = SLASH; - if ( _chdir(dir) ) { - REPORT_FATAL_ERROR("ERROR: Couldn't change current working directory\n"); - } - } -}; -#endif - -//! Represents a TBB or OpenMP run-time that uses RML. -template<typename Factory, typename Client> -class RunTime { -public: - //! Factory that run-time uses to make servers. - Factory factory; - Client* client; - typename Factory::server_type* server; -#if _WIN32||_WIN64 - ::rml::server::execution_resource_t me; -#endif - RunTime() { - factory.open(); - } - ~RunTime() { - factory.close(); - } - //! Create server for this run-time - void create_connection(); - - //! Destroy server for this run-time - void destroy_connection(); -}; - -class ThreadLevelRecorder { - tbb::atomic<int> level; - struct record { - tbb::tick_count time; - int nthread; - }; - tbb::atomic<unsigned> next; - /** Must be power of two */ - static const unsigned max_record_count = 1<<20; - record array[max_record_count]; -public: - void change_level( int delta ); - void dump(); -}; - -void ThreadLevelRecorder::change_level( int delta ) { - int x = level+=delta; - tbb::tick_count t = tbb::tick_count::now(); - unsigned k = next++; - if( k<max_record_count ) { - record& r = array[k]; - r.time = t; - r.nthread = x; - } -} - -void ThreadLevelRecorder::dump() { - FILE* f = fopen("time.txt","w"); - if( !f ) { - perror("fopen(time.txt)\n"); - exit(1); - } - unsigned limit = next; - if( limit>max_record_count ) { - // Clip - limit = next; - } - for( unsigned i=0; i<limit; ++i ) { - fprintf(f,"%f\t%d\n",(array[i].time-array[0].time).seconds(),array[i].nthread); - } - fclose(f); -} - -class TBB_Client: public ClientBase<tbb::internal::rml::tbb_client> { - void process( job& j ) __TBB_override; - size_type max_job_count() const __TBB_override { - return TBB_MaxThread-1; - } -}; - -class OMP_Client: public ClientBase<__kmp::rml::omp_client> { - void process( job&, void* cookie, omp_client::size_type ) __TBB_override; - size_type max_job_count() const __TBB_override { - return OMP_MaxThread-1; - } -}; - -#if !CHDIR_SUPPORT_BROKEN -// A global instance of ChangeCurrentDir should be declared before TBB_RunTime and OMP_RunTime -// since we want to change current directory before opening factory -ChangeCurrentDir Changer; -#endif -RunTime<tbb::internal::rml::tbb_factory, TBB_Client> TBB_RunTime; -RunTime<__kmp::rml::omp_factory, OMP_Client> OMP_RunTime; -ThreadLevelRecorder TotalThreadLevel; - -template<typename Factory, typename Client> -void RunTime<Factory,Client>::create_connection() { - client = new Client; - typename Factory::status_type status = factory.make_server( server, *client ); - ASSERT( status==Factory::st_success, NULL ); -#if _WIN32||_WIN64 - server->register_master( me ); -#endif /* _WIN32||_WIN64 */ -} - -template<typename Factory, typename Client> -void RunTime<Factory,Client>::destroy_connection() { -#if _WIN32||_WIN64 - server->unregister_master( me ); -#endif /* _WIN32||_WIN64 */ - server->request_close_connection(); - server = NULL; -} - -class OMP_Team { -public: - OMP_Team( __kmp::rml::omp_server& ) {} - tbb::atomic<unsigned> barrier; -}; - -tbb::atomic<int> AvailWork; -tbb::atomic<int> CompletionCount; - -void OMPWork() { - tbb::atomic<int> x; - for( x=0; x<2000000; ++x ) { - continue; - } -} - -void TBBWork() { - if( AvailWork>=0 ) { - int k = --AvailWork; - if( k==-1 ) { - TBB_RunTime.server->adjust_job_count_estimate(-(TBB_MaxThread-1)); - ++CompletionCount; - } else if( k>=0 ) { - for( int j=0; j<4; ++j ) { - OMP_Team team( *OMP_RunTime.server ); - int n = OMP_RunTime.server->try_increase_load( OMP_ParallelRegionSize-1, /*strict=*/false ); - team.barrier = 0; - ::rml::job* array[OMP_ParallelRegionSize-1]; - if( n>0) - OMP_RunTime.server->get_threads( n, &team, array ); - // Master does work inside parallel region too. - OMPWork(); - // Master waits for workers to finish - if( n>0 ) - while( team.barrier!=unsigned(n) ) { - __TBB_Yield(); - } - } - ++CompletionCount; - } - } -} - -void TBB_Client::process( job& ) { - TotalThreadLevel.change_level(1); - TBBWork(); - TotalThreadLevel.change_level(-1); -} - -void OMP_Client::process( job& /* j */, void* cookie, omp_client::size_type ) { - TotalThreadLevel.change_level(1); - ASSERT( OMP_RunTime.server, NULL ); - OMPWork(); - ASSERT( OMP_RunTime.server, NULL ); - static_cast<OMP_Team*>(cookie)->barrier+=1; - TotalThreadLevel.change_level(-1); -} - -void TBBOutSideOpenMPInside() { - TotalThreadLevel.change_level(1); - CompletionCount = 0; - int tbbtasks = 32; - AvailWork = tbbtasks; - TBB_RunTime.server->adjust_job_count_estimate(TBB_MaxThread-1); - while( CompletionCount!=tbbtasks+1 ) { - TBBWork(); - } - TotalThreadLevel.change_level(-1); -} - -int TestMain () { -#if CHDIR_SUPPORT_BROKEN - REPORT("Known issue: dynamic_link does not support current directory changing before its initialization.\n"); -#endif - for( TBB_MaxThread=MinThread; TBB_MaxThread<=MaxThread; ++TBB_MaxThread ) { - REMARK("Testing with TBB_MaxThread=%d\n", TBB_MaxThread); - TBB_RunTime.create_connection(); - OMP_RunTime.create_connection(); - TBBOutSideOpenMPInside(); - OMP_RunTime.destroy_connection(); - TBB_RunTime.destroy_connection(); - } - TotalThreadLevel.dump(); - return Harness::Done; -} -#endif /* __TBB_WIN8UI_SUPPORT || __TBB_MIC_OFFLOAD */ diff --git a/src/tbb/src/rml/test/test_rml_omp.cpp b/src/tbb/src/rml/test/test_rml_omp.cpp deleted file mode 100644 index 89096de52..000000000 --- a/src/tbb/src/rml/test/test_rml_omp.cpp +++ /dev/null @@ -1,203 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#include <tbb/tbb_config.h> -#if __TBB_WIN8UI_SUPPORT || __TBB_MIC_OFFLOAD -#include "harness.h" -int TestMain () { - return Harness::Skipped; -} -#else -#include "rml_omp.h" - -typedef __kmp::rml::omp_server MyServer; -typedef __kmp::rml::omp_factory MyFactory; - -// Forward declaration for the function used in test_server.h -void DoClientSpecificVerification( MyServer& , int ); - -#define HARNESS_DEFAULT_MIN_THREADS 0 -#include "test_server.h" -#include "tbb/tbb_misc.h" - -static bool StrictTeam; - -class MyTeam { - MyTeam& operator=( const MyTeam& ) ; -public: - struct info_type { - rml::job* job; - bool ran; - info_type() : job(NULL), ran(false) {} - }; - MyTeam( MyServer& /* server */, size_t max_thread_ ) : - max_thread(max_thread_) - { - self_ptr = this; - info = new info_type[max_thread]; - } - ~MyTeam() { - delete[] info; - } - const size_t max_thread; - size_t n_thread; - tbb::atomic<int> barrier; - /** Indexed with 1-origin index */ - info_type* info; - int iteration; - MyTeam* self_ptr; -}; - -class MyClient: public ClientBase<__kmp::rml::omp_client> { -public: - MyServer* server; - void process( job& j, void* cookie, size_type index ) __TBB_override { - MyTeam& t = *static_cast<MyTeam*>(cookie); - ASSERT( t.self_ptr==&t, "trashed cookie" ); - ASSERT( index<t.max_thread, NULL ); - ASSERT( !t.info[index].ran, "duplicate index?" ); - t.info[index].job = &j; - t.info[index].ran = true; - do_process(&j); - if( index==1 && nesting.level<nesting.limit ) { - DoOneConnection<MyFactory,MyClient> doc(MaxThread,Nesting(nesting.level+1,nesting.limit),0,false); - doc(0); - } -#if _WIN32||_WIN64 - // test activate/deactivate - if( t.n_thread>1 && t.n_thread%2==0 ) { - if( nesting.level==0 ) { - if( index&1 ) { - size_type target = index-1; - ASSERT( target<t.max_thread, NULL ); - // wait until t.info[target].job is defined - tbb::internal::spin_wait_until_eq( t.info[target].ran, true ); - server->try_increase_load( 1, true ); - server->reactivate( t.info[target].job ); - } else { - server->deactivate( &j ); - } - } - } -#endif /* _WIN32||_WIN64 */ - ++t.barrier; - } - static const bool is_omp = true; - bool is_strict() const {return StrictTeam;} -}; - -void FireUpJobs( MyServer& server, MyClient& client, int max_thread, int n_extra, Checker* checker ) { - ASSERT( max_thread>=0, NULL ); -#if _WIN32||_WIN64 - ::rml::server::execution_resource_t me; - server.register_master( me ); -#endif /* _WIN32||_WIN64 */ - client.server = &server; - MyTeam team(server,size_t(max_thread)); - MyServer::size_type n_thread = 0; - for( int iteration=0; iteration<4; ++iteration ) { - for( size_t i=0; i<team.max_thread; ++i ) - team.info[i].ran = false; - switch( iteration ) { - default: - n_thread = int(max_thread); - break; - case 1: - // No change in number of threads - break; - case 2: - // Decrease number of threads. - n_thread = int(max_thread)/2; - break; - // Case 3 is same code as the default, but has effect of increasing the number of threads. - } - team.barrier = 0; - REMARK("client %d: server.run with n_thread=%d\n", client.client_id(), int(n_thread) ); - server.independent_thread_number_changed( n_extra ); - if( checker ) { - // Give RML time to respond to change in number of threads. - Harness::Sleep(1); - } - int n_delivered = server.try_increase_load( n_thread, StrictTeam ); - ASSERT( !StrictTeam || n_delivered==int(n_thread), "server failed to satisfy strict request" ); - if( n_delivered<0 ) { - REMARK( "client %d: oversubscription occurred (by %d)\n", client.client_id(), -n_delivered ); - server.independent_thread_number_changed( -n_extra ); - n_delivered = 0; - } else { - team.n_thread = n_delivered; - ::rml::job* job_array[JobArraySize]; - job_array[n_delivered] = (::rml::job*)intptr_t(-1); - server.get_threads( n_delivered, &team, job_array ); - __TBB_ASSERT( job_array[n_delivered]== (::rml::job*)intptr_t(-1), NULL ); - for( int i=0; i<n_delivered; ++i ) { - MyJob* j = static_cast<MyJob*>(job_array[i]); - int s = j->state; - ASSERT( s==MyJob::idle||s==MyJob::busy, NULL ); - } - server.independent_thread_number_changed( -n_extra ); - REMARK("client %d: team size is %d\n", client.client_id(), n_delivered); - if( checker ) { - checker->check_number_of_threads_delivered( n_delivered, n_thread, n_extra ); - } - // Protocol requires that master wait until workers have called "done_processing" - while( team.barrier!=n_delivered ) { - ASSERT( team.barrier>=0, NULL ); - ASSERT( team.barrier<=n_delivered, NULL ); - __TBB_Yield(); - } - REMARK("client %d: team completed\n", client.client_id() ); - for( int i=0; i<n_delivered; ++i ) { - ASSERT( team.info[i].ran, "thread on team allegedly delivered, but did not run?" ); - } - } - for( MyServer::size_type i=n_delivered; i<MyServer::size_type(max_thread); ++i ) { - ASSERT( !team.info[i].ran, "thread on team ran with illegal index" ); - } - } -#if _WIN32||_WIN64 - server.unregister_master( me ); -#endif -} - -void DoClientSpecificVerification( MyServer& server, int /*n_thread*/ ) -{ - ASSERT( server.current_balance()==int(tbb::internal::AvailableHwConcurrency())-1, NULL ); -} - -int TestMain () { -#if _MSC_VER == 1600 && RML_USE_WCRM - REPORT("Known issue: RML resets the process mask when Concurrency Runtime is used.\n"); - // AvailableHwConcurrency reads process mask when the first call. That's why it should - // be called before RML initialization. - tbb::internal::AvailableHwConcurrency(); -#endif - - StrictTeam = true; - VerifyInitialization<MyFactory,MyClient>( MaxThread ); - SimpleTest<MyFactory,MyClient>(); - - StrictTeam = false; - VerifyInitialization<MyFactory,MyClient>( MaxThread ); - SimpleTest<MyFactory,MyClient>(); - - return Harness::Done; -} -#endif /* __TBB_WIN8UI_SUPPORT || __TBB_MIC_OFFLOAD */ diff --git a/src/tbb/src/rml/test/test_rml_tbb.cpp b/src/tbb/src/rml/test/test_rml_tbb.cpp deleted file mode 100644 index f679baeb6..000000000 --- a/src/tbb/src/rml/test/test_rml_tbb.cpp +++ /dev/null @@ -1,205 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#include <tbb/tbb_config.h> -#if __TBB_WIN8UI_SUPPORT || __TBB_MIC_OFFLOAD -#include "harness.h" -int TestMain () { - return Harness::Skipped; -} -#else -#include "rml_tbb.h" - -typedef tbb::internal::rml::tbb_server MyServer; -typedef tbb::internal::rml::tbb_factory MyFactory; - -// Forward declaration of the function used in test_server.h -void DoClientSpecificVerification( MyServer&, int ); - -#define HARNESS_DEFAULT_MIN_THREADS 0 -#include "test_server.h" - -tbb::atomic<int> n_available_hw_threads; - -class MyClient: public ClientBase<tbb::internal::rml::tbb_client> { - tbb::atomic<int> counter; - tbb::atomic<int> gate; - void process( job& j ) __TBB_override { - do_process(&j); - //wait until the gate is open. - while( gate==0 ) - Harness::Sleep(1); - - __TBB_ASSERT( nesting.limit<=2, NULL ); - if( nesting.level>=nesting.limit ) - return; - - size_type max_outstanding_connections = max_job_count(); // if nesting.level==0 - if( nesting.level==1 ) - max_outstanding_connections *= (1+max_outstanding_connections); - - if( default_concurrency()<=max_outstanding_connections+2 ) - // i.e., if it is not guaranteed that at least two connections may be made without depleting the_balance - return; - - // at this point, ( nesting.level<nesting.limit ) && ( my_server->default_concurrency()-max_outstanding_connections>2 ) - for( ;; ) { - while( n_available_hw_threads<=1 ) - Harness::Sleep(1); - - int n = --n_available_hw_threads; - if( n>0 ) break; - // else I lost - ++n_available_hw_threads; - } - - DoOneConnection<MyFactory,MyClient> doc(max_job_count(),Nesting(nesting.level+1,nesting.limit),0,false); - doc(0); - - ++n_available_hw_threads; - } -public: - MyClient() {counter=1;} - static const bool is_omp = false; - bool is_strict() const {return false;} - void open_the_gate() { gate = 1; } - void close_the_gate() { gate = 0; } -}; - -void FireUpJobs( MyServer& server, MyClient& client, int n_thread, int n_extra, Checker* checker ) { - REMARK("client %d: calling adjust_job_count_estimate(%d)\n", client.client_id(),n_thread); - // Exercise independent_thread_number_changed, even for zero values. - server.independent_thread_number_changed( n_extra ); -#if _WIN32||_WIN64 - ::rml::server::execution_resource_t me; - server.register_master( me ); -#endif /* _WIN32||_WIN64 */ - // Experiments indicate that when oversubscribing, the main thread should wait a little - // while for the RML worker threads to do some work. - if( checker ) { - // Give RML time to respond to change in number of threads. - Harness::Sleep(1); - for( int k=0; k<n_thread; ++k ) - client.job_array[k].processing_count = 0; - } - //close the gate to keep worker threads from returning to RML until a snapshot is taken - client.close_the_gate(); - server.adjust_job_count_estimate( n_thread ); - int n_used = 0; - if( checker ) { - Harness::Sleep(100); - for( int k=0; k<n_thread; ++k ) - if( client.job_array[k].processing_count ) - ++n_used; - } - // open the gate - client.open_the_gate(); - // Logic further below presumes that jobs never starve, so undo previous call - // to independent_thread_number_changed before waiting on those jobs. - server.independent_thread_number_changed( -n_extra ); - REMARK("client %d: wait for each job to be processed at least once\n",client.client_id()); - // Calculate the number of jobs that are expected to get threads. - int expected = n_thread; - // Wait for expected number of jobs to be processed. -#if RML_USE_WCRM - int default_concurrency = server.default_concurrency(); - if( N_TestConnections>0 ) { - if( default_concurrency+1>=8 && n_thread<=3 && N_TestConnections<=3 && (default_concurrency/int(N_TestConnections)-1)>=n_thread ) { -#endif /* RML_USE_WCRM */ - for(;;) { - int n = 0; - for( int k=0; k<n_thread; ++k ) - if( client.job_array[k].processing_count!=0 ) - ++n; - if( n>=expected ) break; - server.yield(); - } -#if RML_USE_WCRM - } else if( n_thread>0 ) { - for( int m=0; m<20; ++m ) { - int n = 0; - for( int k=0; k<n_thread; ++k ) - if( client.job_array[k].processing_count!=0 ) - ++n; - if( n>=expected ) break; - Harness::Sleep(1); - } - } - } -#endif /* RML_USE_WCRM */ - server.adjust_job_count_estimate(-n_thread); -#if _WIN32||_WIN64 - server.unregister_master( me ); -#endif - // Give RML some time to respond - if( checker ) { - Harness::Sleep(1); - checker->check_number_of_threads_delivered( n_used, n_thread, n_extra ); - } -} - -void DoClientSpecificVerification( MyServer&, int n_thread ) -{ - MyClient* client = new MyClient; - client->initialize( n_thread, Nesting(), ClientStackSize[0] ); - MyFactory factory; - memset( &factory, 0, sizeof(factory) ); - MyFactory::status_type status = factory.open(); - ASSERT( status!=MyFactory::st_not_found, "could not find RML library" ); - ASSERT( status!=MyFactory::st_incompatible, NULL ); - ASSERT( status==MyFactory::st_success, NULL ); - MyFactory::server_type* server; - status = factory.make_server( server, *client ); - ASSERT( status==MyFactory::st_success, NULL ); - client->set_server( server ); - client->expect_close_connection = true; - server->request_close_connection(); - // Client deletes itself when it sees call to acknowledge_close_connection from server. - factory.close(); -} - -void Initialize() -{ - MyClient* client = new MyClient; - client->initialize( 1, Nesting(), ClientStackSize[0] ); - MyFactory factory; - memset( &factory, 0, sizeof(factory) ); - factory.open(); - MyFactory::server_type* server; - factory.make_server( server, *client ); - client->set_server( server ); - n_available_hw_threads = server->default_concurrency(); - client->expect_close_connection = true; - server->request_close_connection(); - // Client deletes itself when it sees call to acknowledge_close_connection from server. - factory.close(); -} - -int TestMain () { - VerifyInitialization<MyFactory,MyClient>( MaxThread ); - if ( server_concurrency<1 ) { - REPORT("The test is not intended to run on 1 thread\n"); - return Harness::Skipped; - } - Initialize(); - SimpleTest<MyFactory,MyClient>(); - return Harness::Done; -} -#endif /* __TBB_WIN8UI_SUPPORT || __TBB_MIC_OFFLOAD */ diff --git a/src/tbb/src/rml/test/test_server.h b/src/tbb/src/rml/test/test_server.h deleted file mode 100644 index a72142fa1..000000000 --- a/src/tbb/src/rml/test/test_server.h +++ /dev/null @@ -1,433 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -/* This header contains code shared by test_omp_server.cpp and test_tbb_server.cpp - There is no ifndef guard - test is supposed to include this file exactly once. - The test is also exected to have #include of rml_omp.h or rml_tbb.h before - including this header. - - This header should not use any parts of TBB that require linking in the TBB run-time. - It uses a few instances of tbb::atomic<T>, all of which are completely inlined. */ - -#include "tbb/atomic.h" -#include "tbb/tbb_thread.h" -#include "harness.h" -#include "harness_memory.h" -#include "harness_concurrency_tracker.h" - -//! Define TRIVIAL as 1 to test only a single client, no nesting, no extra threads. -#define TRIVIAL 0 - -//! Maximum number of clients -#if TRIVIAL -const size_t MaxClient = 1; -#else -const size_t MaxClient = 4; -#endif - -const size_t ClientStackSize[MaxClient] = { - 1000000 -#if !TRIVIAL - ,2000000 - ,1000000 - ,4000000 -#endif /* TRIVIAL */ -}; - -const size_t OverheadStackSize = 500000; - -const size_t JobArraySize = 1000; - -static bool TestSingleConnection; - -static size_t N_TestConnections; - -static int server_concurrency; - -class MyJob: public ::rml::job { -public: - //! Enumeration for tracking states of a job. - enum state_t { - //! Job has not yet been allocated. - unallocated, - //! Is idle. - idle, - //! Has a thread working on it. - busy, - //! After call to client::cleanup - clean - }; - tbb::atomic<int> state; - tbb::atomic<int> processing_count; - void update( state_t new_state, state_t old_state ) { - int o = state.compare_and_swap(new_state,old_state); - ASSERT( o==old_state, "illegal transition" ); - } - void update_from_either( state_t new_state, state_t old_state1, state_t old_state2 ) { - int snapshot; - do { - snapshot = state; - ASSERT( snapshot==old_state1||snapshot==old_state2, "illegal transition" ); - } while( state.compare_and_swap(new_state,snapshot)!=snapshot ); - } - MyJob() { - state=unallocated; - processing_count=0; - } - ~MyJob() { - // Overwrite so that accidental use after destruction can be detected. - memset(this,-1,sizeof(*this)); - } -}; - -static tbb::atomic<int> ClientConstructions; -static tbb::atomic<int> ClientDestructions; - -struct Nesting { - int level; - int limit; - Nesting() : level(0), limit(0) {} - Nesting( int level_, int limit_ ) : level(level_), limit(limit_) {} -}; - -template<typename Client> -class ClientBase: public Client { -protected: - typedef typename Client::size_type size_type; - typedef typename Client::version_type version_type; - typedef typename Client::policy_type policy_type; - typedef typename Client::job job; -private: - size_type my_max_job_count; - size_t my_stack_size; - tbb::atomic<size_t> next_job_index; - int my_client_id; - rml::server* my_server; - -public: - enum state_t { - //! Treat *this as constructed. - live=0x1234, - //! Treat *this as destroyed. - destroyed=0xDEAD - }; - - tbb::atomic<int> state; - void update( state_t new_state, state_t old_state ) { - int o = state.compare_and_swap(new_state,old_state); - ASSERT( o==old_state, NULL ); - } - - tbb::atomic<bool> expect_close_connection; - - MyJob *job_array; - - version_type version() const __TBB_override { - ASSERT( state==live, NULL ); - return 1; - } - - size_type max_job_count() const __TBB_override { - ASSERT( state==live, NULL ); - return my_max_job_count; - } - - size_t min_stack_size() const __TBB_override { - ASSERT( state==live, NULL ); - return my_stack_size; - } - - policy_type policy() const __TBB_override {return Client::throughput;} - - void acknowledge_close_connection() __TBB_override { - ASSERT( expect_close_connection, NULL ); - for( size_t k=next_job_index; k>0; ) { - --k; - ASSERT( job_array[k].state==MyJob::clean, NULL ); - } - delete[] job_array; - job_array = NULL; - ASSERT( my_server, NULL ); - update( destroyed, live ); - delete this; - } - - void cleanup( job& j_ ) __TBB_override { - REMARK("client %d: cleanup(%p) called\n",client_id(),&j_); - ASSERT( state==live, NULL ); - MyJob& j = static_cast<MyJob&>(j_); - while( j.state==MyJob::busy ) - my_server->yield(); - j.update(MyJob::clean,MyJob::idle); - REMARK("client %d: cleanup(%p) returns\n",client_id(),&j_); - } - - job* create_one_job(); - -protected: - void do_process( job* j_ ) { - ASSERT( state==live, NULL ); - MyJob& j = static_cast<MyJob&>(*j_); - ASSERT( j_, NULL ); - j.update(MyJob::busy,MyJob::idle); - // use of the plain addition (not the atomic increment) is intentonial - j.processing_count = j.processing_count + 1; - ASSERT( my_stack_size>OverheadStackSize, NULL ); -#ifdef __ia64__ - // Half of the stack is reserved for RSE, so test only remaining half. - UseStackSpace( (my_stack_size-OverheadStackSize)/2 ); -#else - UseStackSpace( my_stack_size-OverheadStackSize ); -#endif - j.update(MyJob::idle,MyJob::busy); - my_server->yield(); - } -public: - ClientBase() : my_server(NULL) { - my_client_id = ClientConstructions++; - next_job_index = 0; - } - int client_id() const {return my_client_id;} - - Nesting nesting; - - void initialize( size_type max_job_count, Nesting nesting_, size_t stack_size ) { - ASSERT( stack_size>0, NULL ); - my_max_job_count = max_job_count; - nesting = nesting_; - my_stack_size = stack_size; - job_array = new MyJob[JobArraySize]; - expect_close_connection = false; - state = live; - } - - void set_server( rml::server* s ) {my_server=s;} - - unsigned default_concurrency() const { ASSERT( my_server, NULL); return my_server->default_concurrency(); } - - virtual ~ClientBase() { - ASSERT( state==destroyed, NULL ); - ++ClientDestructions; - } -}; - -template<typename Client> -typename Client::job* ClientBase<Client>::create_one_job() { - REMARK("client %d: create_one_job() called\n",client_id()); - size_t k = next_job_index++; - ASSERT( state==live, NULL ); - // Following assertion depends on assumption that implementation does not destroy jobs until - // the connection is closed. If the implementation is changed to destroy jobs sooner, the - // test logic in this header will have to be reworked. - ASSERT( k<my_max_job_count, "RML allocated more than max_job_count jobs simultaneously" ); - ASSERT( k<JobArraySize, "JobArraySize not big enough (problem is in test, not RML)" ); - MyJob& j = job_array[k]; - j.update(MyJob::idle,MyJob::unallocated); - REMARK("client %d: create_one_job() for k=%d returns %p\n",client_id(),int(k),&j); - return &j; -} - -struct warning_tracker { - tbb::atomic<int> n_more_than_available; - tbb::atomic<int> n_too_many_threads; - tbb::atomic<int> n_system_overload; - warning_tracker() { - n_more_than_available = 0; - n_too_many_threads = 0; - n_system_overload = 0; - } - bool all_set() { return n_more_than_available>0 && n_too_many_threads>0 && n_system_overload>0; } -} tracker; - -class Checker { -public: - int default_concurrency; - void check_number_of_threads_delivered( int n_delivered, int n_requested, int n_extra ) const; - Checker( rml::server& server ) : default_concurrency(int(server.default_concurrency())) {} -}; - -void Checker::check_number_of_threads_delivered( int n_delivered, int n_requested, int n_extra ) const { - ASSERT( default_concurrency>=0, NULL ); - if( tracker.all_set() ) return; - // Check that number of threads delivered is reasonable. - int n_avail = default_concurrency; - if( n_extra>0 ) - n_avail-=n_extra; - if( n_avail<0 ) - n_avail=0; - if( n_requested>default_concurrency ) - n_avail += n_requested-default_concurrency; - int n_expected = n_requested; - if( n_expected>n_avail ) - n_expected=n_avail; - const char* msg = NULL; - if( n_delivered>n_avail ) { - if( ++tracker.n_more_than_available>1 ) - return; - msg = "server delivered more threads than were theoretically available"; - } else if( n_delivered>n_expected ) { - if( ++tracker.n_too_many_threads>1 ) - return; - msg = "server delivered more threads than expected"; - } else if( n_delivered<n_expected ) { - if( ++tracker.n_system_overload>1 ) - return; - msg = "server delivered fewer threads than ideal; or, the system is overloaded?"; - } - if( msg ) { - REPORT("Warning: %s (n_delivered=%d n_avail=%d n_requested=%d n_extra=%d default_concurrency=%d)\n", - msg, n_delivered, n_avail, n_requested, n_extra, default_concurrency ); - } -} - -template<typename Factory,typename Client> -class DoOneConnection: NoAssign { - //! Number of threads to request - const int n_thread; - //! Nesting - const Nesting nesting; - //! Number of extra threads to pretend having outside the RML - const int n_extra; - //! If true, check number of threads actually delivered. - const bool check_delivered; -public: - DoOneConnection( int n_thread_, Nesting nesting_, int n_extra_, bool check_delivered_ ) : - n_thread(n_thread_), - nesting(nesting_), - n_extra(n_extra_), - check_delivered(check_delivered_) - { - } - - //! Test ith connection - void operator()( size_t i ) const; -}; - -template<typename Factory,typename Client> -void DoOneConnection<Factory,Client>::operator()( size_t i ) const { - ASSERT( i<MaxClient, NULL ); - Client* client = new Client; - client->initialize( Client::is_omp ? JobArraySize : n_thread, nesting, ClientStackSize[i] ); - Factory factory; - memset( &factory, 0, sizeof(factory) ); - typename Factory::status_type status = factory.open(); - ASSERT( status==Factory::st_success, NULL ); - - typename Factory::server_type* server; - status = factory.make_server( server, *client ); - ASSERT( status==Factory::st_success, NULL ); - Harness::ConcurrencyTracker ct; - REMARK("client %d: opened server n_thread=%d nesting=(%d,%d)\n", - client->client_id(), n_thread, nesting.level, nesting.limit); - client->set_server( server ); - Checker checker( *server ); - - FireUpJobs( *server, *client, n_thread, n_extra, check_delivered && !client->is_strict() ? &checker : NULL ); - - // Close the connection - client->expect_close_connection = true; - REMARK("client %d: calling request_close_connection\n", client->client_id()); -#if !RML_USE_WCRM - int default_concurrency = server->default_concurrency(); -#endif - server->request_close_connection(); - // Client deletes itself when it sees call to acknowledge_close_connection from server. - factory.close(); -#if !RML_USE_WCRM - if( TestSingleConnection ) - __TBB_ASSERT_EX( uintptr_t(factory.scratch_ptr)==uintptr_t(default_concurrency), "under/over subscription?" ); -#endif -} - -//! Test with n_threads threads and n_client clients. -template<typename Factory, typename Client> -void SimpleTest() { - Harness::ConcurrencyTracker::Reset(); - TestSingleConnection = true; - N_TestConnections = 1; - for( int n_thread=MinThread; n_thread<=MaxThread; ++n_thread ) { - // Test a single connection, no nesting, no extra threads - DoOneConnection<Factory,Client> doc(n_thread,Nesting(0,0),0,false); - doc(0); - } -#if !TRIVIAL - TestSingleConnection = false; - for( int n_thread=MinThread; n_thread<=MaxThread; ++n_thread ) { - // Test parallel connections - for( int n_client=1; n_client<=int(MaxClient); ++n_client ) { - N_TestConnections = n_client; - REMARK("SimpleTest: n_thread=%d n_client=%d\n",n_thread,n_client); - NativeParallelFor( n_client, DoOneConnection<Factory,Client>(n_thread,Nesting(0,0),0,false) ); - } - // Test server::independent_thread_number_changed - N_TestConnections = 1; - for( int n_extra=-4; n_extra<=32; n_extra=n_extra+1+n_extra/5 ) { - DoOneConnection<Factory,Client> doc(n_thread,Nesting(0,0),n_extra,true); - doc(0); - } -#if !RML_USE_WCRM - // Test nested connections - DoOneConnection<Factory,Client> doc(n_thread,Nesting(0,2),0,false); - doc(0); -#endif - } - ASSERT( Harness::ConcurrencyTracker::PeakParallelism()>1 || server_concurrency==0, "No multiple connections exercised?" ); -#endif /* !TRIVIAL */ - // Let RML catch up. - while( ClientConstructions!=ClientDestructions ) - Harness::Sleep(1); -} - -static void check_server_info( void* arg, const char* server_info ) -{ - ASSERT( strstr(server_info, (char*)arg), NULL ); -} - -template<typename Factory, typename Client> -void VerifyInitialization( int n_thread ) { - Client* client = new Client; - client->initialize( Client::is_omp ? JobArraySize : n_thread, Nesting(), ClientStackSize[0] ); - Factory factory; - memset( &factory, 0, sizeof(factory) ); - typename Factory::status_type status = factory.open(); - ASSERT( status!=Factory::st_not_found, "could not find RML library" ); - ASSERT( status!=Factory::st_incompatible, NULL ); - ASSERT( status==Factory::st_success, NULL ); - factory.call_with_server_info( check_server_info, (void*)"Intel(R) RML library" ); - typename Factory::server_type* server; - status = factory.make_server( server, *client ); - ASSERT( status!=Factory::st_incompatible, NULL ); - ASSERT( status!=Factory::st_not_found, NULL ); - ASSERT( status==Factory::st_success, NULL ); - REMARK("client %d: opened server n_thread=%d nesting=(%d,%d)\n", - client->client_id(), n_thread, 0, 0); - ASSERT( server, NULL ); - client->set_server( server ); - server_concurrency = server->default_concurrency(); - - DoClientSpecificVerification( *server, n_thread ); - - // Close the connection - client->expect_close_connection = true; - REMARK("client %d: calling request_close_connection\n", client->client_id()); - server->request_close_connection(); - // Client deletes itself when it sees call to acknowledge_close_connection from server. - factory.close(); -} diff --git a/src/tbb/src/rml/test/test_thread_monitor.cpp b/src/tbb/src/rml/test/test_thread_monitor.cpp deleted file mode 100644 index ca6325936..000000000 --- a/src/tbb/src/rml/test/test_thread_monitor.cpp +++ /dev/null @@ -1,117 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#include "harness.h" -#if __TBB_MIC_OFFLOAD -int TestMain () { - return Harness::Skipped; -} -#else -#include "thread_monitor.h" -#include "harness_memory.h" -#include "tbb/semaphore.cpp" - -class ThreadState { - void loop(); -public: - static __RML_DECL_THREAD_ROUTINE routine( void* arg ) { - static_cast<ThreadState*>(arg)->loop(); - return 0; - } - typedef rml::internal::thread_monitor thread_monitor; - thread_monitor monitor; - volatile int request; - volatile int ack; - volatile unsigned clock; - volatile unsigned stamp; - ThreadState() : request(-1), ack(-1), clock(0) {} -}; - -void ThreadState::loop() { - for(;;) { - ++clock; - if( ack==request ) { - thread_monitor::cookie c; - monitor.prepare_wait(c); - if( ack==request ) { - REMARK("%p: request=%d ack=%d\n", this, request, ack ); - monitor.commit_wait(c); - } else - monitor.cancel_wait(); - } else { - // Throw in delay occasionally - switch( request%8 ) { - case 0: - case 1: - case 5: - rml::internal::thread_monitor::yield(); - } - int r = request; - ack = request; - if( !r ) return; - } - } -} - -// Linux on IA-64 architecture seems to require at least 1<<18 bytes per stack. -const size_t MinStackSize = 1<<18; -const size_t MaxStackSize = 1<<22; - -int TestMain () { - for( int p=MinThread; p<=MaxThread; ++p ) { - ThreadState* t = new ThreadState[p]; - for( size_t stack_size = MinStackSize; stack_size<=MaxStackSize; stack_size*=2 ) { - REMARK("launching %d threads\n",p); - for( int i=0; i<p; ++i ) - rml::internal::thread_monitor::launch( ThreadState::routine, t+i, stack_size ); - for( int k=1000; k>=0; --k ) { - if( k%8==0 ) { - // Wait for threads to wait. - for( int i=0; i<p; ++i ) { - unsigned count = 0; - do { - t[i].stamp = t[i].clock; - rml::internal::thread_monitor::yield(); - if( ++count>=1000 ) { - REPORT("Warning: thread %d not waiting\n",i); - break; - } - } while( t[i].stamp!=t[i].clock ); - } - } - REMARK("notifying threads\n"); - for( int i=0; i<p; ++i ) { - // Change state visible to launched thread - t[i].request = k; - t[i].monitor.notify(); - } - REMARK("waiting for threads to respond\n"); - for( int i=0; i<p; ++i ) - // Wait for thread to respond - while( t[i].ack!=k ) - rml::internal::thread_monitor::yield(); - } - } - delete[] t; - } - - return Harness::Done; -} -#endif /* __TBB_MIC_OFFLOAD */ diff --git a/src/tbb/src/tbb/CMakeLists.txt b/src/tbb/src/tbb/CMakeLists.txt new file mode 100644 index 000000000..63fd4c73f --- /dev/null +++ b/src/tbb/src/tbb/CMakeLists.txt @@ -0,0 +1,219 @@ +# Copyright (c) 2020-2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_library(tbb + address_waiter.cpp + allocator.cpp + arena.cpp + arena_slot.cpp + concurrent_bounded_queue.cpp + dynamic_link.cpp + exception.cpp + governor.cpp + global_control.cpp + itt_notify.cpp + main.cpp + market.cpp + tcm_adaptor.cpp + misc.cpp + misc_ex.cpp + observer_proxy.cpp + parallel_pipeline.cpp + private_server.cpp + profiling.cpp + rml_tbb.cpp + rtm_mutex.cpp + rtm_rw_mutex.cpp + semaphore.cpp + small_object_pool.cpp + task.cpp + task_dispatcher.cpp + task_group_context.cpp + thread_dispatcher.cpp + thread_request_serializer.cpp + threading_control.cpp + version.cpp + queuing_rw_mutex.cpp) + +add_library(TBB::tbb ALIAS tbb) + +if (WIN32) + target_sources(tbb PRIVATE tbb.rc) + set_target_properties(tbb PROPERTIES OUTPUT_NAME "tbb${TBB_BINARY_VERSION}") +endif() + +# TODO: Add statistics.cpp + +target_compile_definitions(tbb + PUBLIC + $<$<CONFIG:DEBUG>:TBB_USE_DEBUG> + PRIVATE + __TBB_BUILD + ${TBB_RESUMABLE_TASKS_USE_THREADS} + $<$<NOT:$<BOOL:${BUILD_SHARED_LIBS}>>:__TBB_DYNAMIC_LOAD_ENABLED=0> + $<$<NOT:$<BOOL:${BUILD_SHARED_LIBS}>>:__TBB_SOURCE_DIRECTLY_INCLUDED=1>) + +if (NOT ("${CMAKE_SYSTEM_PROCESSOR}" MATCHES "(armv7-a|aarch64|mips|arm64|riscv)" OR + "${CMAKE_OSX_ARCHITECTURES}" MATCHES "arm64" OR + WINDOWS_STORE OR + TBB_WINDOWS_DRIVER)) + target_compile_definitions(tbb PRIVATE __TBB_USE_ITT_NOTIFY) +endif() + +target_include_directories(tbb + PUBLIC + $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../../include> + $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>) + +target_compile_options(tbb + PRIVATE + ${TBB_CXX_STD_FLAG} # TODO: consider making it PUBLIC. + ${TBB_MMD_FLAG} + ${TBB_DSE_FLAG} + ${TBB_WARNING_LEVEL} + ${TBB_WARNING_SUPPRESS} + ${TBB_LIB_COMPILE_FLAGS} + ${TBB_COMMON_COMPILE_FLAGS} +) + +# Avoid use of target_link_libraries here as it changes /DEF option to \DEF on Windows. +set_target_properties(tbb PROPERTIES + DEFINE_SYMBOL "" +) + +tbb_handle_ipo(tbb) + +if (TBB_DEF_FILE_PREFIX) # If there's no prefix, assume we're using export directives + set_target_properties(tbb PROPERTIES + LINK_FLAGS "${TBB_LINK_DEF_FILE_FLAG}\"${CMAKE_CURRENT_SOURCE_DIR}/def/${TBB_DEF_FILE_PREFIX}-tbb.def\"" + LINK_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/def/${TBB_DEF_FILE_PREFIX}-tbb.def" + ) +endif() + +# Prefer using target_link_options instead of target_link_libraries to specify link options because +# target_link_libraries may incorrectly handle some options (on Windows, for example). +if (COMMAND target_link_options) + target_link_options(tbb + PRIVATE + ${TBB_LIB_LINK_FLAGS} + ${TBB_COMMON_LINK_FLAGS} + ) +else() + target_link_libraries(tbb + PRIVATE + ${TBB_LIB_LINK_FLAGS} + ${TBB_COMMON_LINK_FLAGS} + ) +endif() + +target_link_libraries(tbb + PRIVATE + Threads::Threads + ${TBB_LIB_LINK_LIBS} + ${TBB_COMMON_LINK_LIBS} +) + +# Strip debug symbols into a separate .dbg file +if(TBB_LINUX_SEPARATE_DBG) + if(NOT CMAKE_BUILD_TYPE STREQUAL "release") + find_program(OBJCOPY_COMMAND objcopy) + if(NOT OBJCOPY_COMMAND) + message(WARNING "objcopy command not found in the system") + else() + add_custom_command(TARGET tbb POST_BUILD + COMMAND objcopy --only-keep-debug $<TARGET_FILE:tbb> $<TARGET_FILE:tbb>.dbg + COMMAND objcopy --strip-debug $<TARGET_FILE:tbb> + COMMAND objcopy --add-gnu-debuglink=$<TARGET_FILE:tbb>.dbg $<TARGET_FILE:tbb> + COMMENT "Creating and associating .dbg file with tbb" + ) + endif() + else() + message(WARNING " TBB_LINUX_SEPARATE_DBG flag is not used on release config") + endif() +endif() + +if(TBB_BUILD_APPLE_FRAMEWORKS) + set_target_properties(tbb PROPERTIES + FRAMEWORK TRUE + FRAMEWORK_VERSION ${TBB_BINARY_VERSION}.${TBB_BINARY_MINOR_VERSION} + XCODE_ATTRIBUTE_PRODUCT_BUNDLE_IDENTIFIER com.intel.tbb + MACOSX_FRAMEWORK_IDENTIFIER com.intel.tbb + MACOSX_FRAMEWORK_BUNDLE_VERSION ${TBB_BINARY_VERSION}.${TBB_BINARY_MINOR_VERSION} + MACOSX_FRAMEWORK_SHORT_VERSION_STRING ${TBB_BINARY_VERSION}) +endif() + +tbb_install_target(tbb) + +if (TBB_INSTALL) + if (MSVC) + # Create a copy of target linker file (tbb<ver>[_debug].lib) with legacy name (tbb[_debug].lib) + # to support previous user experience for linkage. + install(FILES + $<TARGET_LINKER_FILE:tbb> + DESTINATION lib + CONFIGURATIONS RelWithDebInfo Release MinSizeRel + RENAME tbb.lib + COMPONENT devel + ) + + install(FILES + $<TARGET_LINKER_FILE:tbb> + DESTINATION lib + CONFIGURATIONS Debug + RENAME tbb_debug.lib + COMPONENT devel + ) + endif() + if(TBB_LINUX_SEPARATE_DBG) + install(FILES + $<TARGET_FILE:tbb>.dbg + DESTINATION lib + COMPONENT devel + ) + endif() + set(_tbb_pc_lib_name tbb) + + if (WIN32) + set(_tbb_pc_lib_name ${_tbb_pc_lib_name}${TBB_BINARY_VERSION}) + endif() + + if (CMAKE_SIZEOF_VOID_P EQUAL 8) + set(TBB_PC_NAME tbb) + else() + set(TBB_PC_NAME tbb32) + endif() + + set(_prefix_for_pc_file "${CMAKE_INSTALL_PREFIX}") + + if (IS_ABSOLUTE "${CMAKE_INSTALL_LIBDIR}") + set(_libdir_for_pc_file "${CMAKE_INSTALL_LIBDIR}") + else() + set(_libdir_for_pc_file "\${prefix}/${CMAKE_INSTALL_LIBDIR}") + endif() + + if (IS_ABSOLUTE "${CMAKE_INSTALL_INCLUDEDIR}") + set(_includedir_for_pc_file "${CMAKE_INSTALL_INCLUDEDIR}") + else() + set(_includedir_for_pc_file "\${prefix}/${CMAKE_INSTALL_INCLUDEDIR}") + endif() + + configure_file(${PROJECT_SOURCE_DIR}/integration/pkg-config/tbb.pc.in ${CMAKE_CURRENT_BINARY_DIR}/${TBB_PC_NAME}.pc @ONLY) + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/${TBB_PC_NAME}.pc + DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig/ + COMPONENT devel) +endif() + +if (COMMAND tbb_gen_vars) + tbb_gen_vars(tbb) +endif() diff --git a/src/tbb/src/tbb/address_waiter.cpp b/src/tbb/src/tbb/address_waiter.cpp new file mode 100644 index 000000000..1fd3dea8e --- /dev/null +++ b/src/tbb/src/tbb/address_waiter.cpp @@ -0,0 +1,106 @@ +/* + Copyright (c) 2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "oneapi/tbb/detail/_utils.h" +#include "governor.h" +#include "concurrent_monitor.h" +#include "oneapi/tbb/detail/_waitable_atomic.h" + +#include <type_traits> + +namespace tbb { +namespace detail { +namespace r1 { + +struct address_context { + address_context() = default; + + address_context(void* address, std::uintptr_t context) : + my_address(address), my_context(context) + {} + + void* my_address{nullptr}; + std::uintptr_t my_context{0}; +}; + +class address_waiter : public concurrent_monitor_base<address_context> { + using base_type = concurrent_monitor_base<address_context>; +public: + using base_type::base_type; + /** per-thread descriptor for concurrent_monitor */ + using thread_context = sleep_node<address_context>; +}; + +// 1024 is a rough estimate based on two assumptions: +// 1) there are no more than 1000 threads in the application; +// 2) the mutexes are optimized for short critical sections less than a couple of microseconds, +// which is less than 1/1000 of a time slice. +// In the worst case, we have single mutex that is locked and its thread is preempted. +// Therefore, the probability of a collision while taking unrelated mutex is about 1/size of a table. +static constexpr std::size_t num_address_waiters = 2 << 10; +static_assert(std::is_standard_layout<address_waiter>::value, + "address_waiter must be with standard layout"); +static address_waiter address_waiter_table[num_address_waiters]; + +void clear_address_waiter_table() { + for (std::size_t i = 0; i < num_address_waiters; ++i) { + address_waiter_table[i].destroy(); + } +} + +static address_waiter& get_address_waiter(void* address) { + std::uintptr_t tag = std::uintptr_t(address); + return address_waiter_table[((tag >> 5) ^ tag) % num_address_waiters]; +} + +void wait_on_address(void* address, d1::delegate_base& predicate, std::uintptr_t context) { + address_waiter& waiter = get_address_waiter(address); + waiter.wait<address_waiter::thread_context>(predicate, address_context{address, context}); +} + +void notify_by_address(void* address, std::uintptr_t target_context) { + address_waiter& waiter = get_address_waiter(address); + + auto predicate = [address, target_context] (address_context ctx) { + return ctx.my_address == address && ctx.my_context == target_context; + }; + + waiter.notify_relaxed(predicate); +} + +void notify_by_address_one(void* address) { + address_waiter& waiter = get_address_waiter(address); + + auto predicate = [address] (address_context ctx) { + return ctx.my_address == address; + }; + + waiter.notify_one_relaxed(predicate); +} + +void notify_by_address_all(void* address) { + address_waiter& waiter = get_address_waiter(address); + + auto predicate = [address] (address_context ctx) { + return ctx.my_address == address; + }; + + waiter.notify_relaxed(predicate); +} + +} // namespace r1 +} // namespace detail +} // namespace tbb diff --git a/src/tbb/src/tbb/allocator.cpp b/src/tbb/src/tbb/allocator.cpp new file mode 100644 index 000000000..689c51255 --- /dev/null +++ b/src/tbb/src/tbb/allocator.cpp @@ -0,0 +1,288 @@ +/* + Copyright (c) 2005-2024 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "oneapi/tbb/version.h" + +#include "oneapi/tbb/detail/_exception.h" +#include "oneapi/tbb/detail/_assert.h" +#include "oneapi/tbb/detail/_utils.h" +#include "oneapi/tbb/tbb_allocator.h" // Is this OK? +#include "oneapi/tbb/cache_aligned_allocator.h" + +#include "dynamic_link.h" +#include "misc.h" + +#include <cstdlib> + +#ifdef _WIN32 +#include <windows.h> +#else +#include <dlfcn.h> +#endif + +#if (!defined(_WIN32) && !defined(_WIN64)) || defined(__CYGWIN__) +#include <stdlib.h> // posix_memalign, free +// With glibc, uClibc and musl on Linux and bionic on Android it is safe to use memalign(), as the allocated memory +// can be freed with free(). It is also better to use memalign() since posix_memalign() is just a wrapper on top of +// memalign() and it offers nothing but overhead due to inconvenient interface. This is likely the case with other +// standard libraries as well, and more libraries can be added to the preprocessor check below. Unfortunately, we +// can't detect musl, so we simply enable memalign() on Linux and Android in general. +#if defined(linux) || defined(__linux) || defined(__linux__) || defined(__ANDROID__) +#include <malloc.h> // memalign +#define __TBB_USE_MEMALIGN +#else +#define __TBB_USE_POSIX_MEMALIGN +#endif +#elif defined(_MSC_VER) || defined(__MINGW32__) +#include <malloc.h> // _aligned_malloc, _aligned_free +#define __TBB_USE_MSVC_ALIGNED_MALLOC +#endif + +#if __TBB_WEAK_SYMBOLS_PRESENT + +#pragma weak scalable_malloc +#pragma weak scalable_free +#pragma weak scalable_aligned_malloc +#pragma weak scalable_aligned_free + +extern "C" { + void* scalable_malloc(std::size_t); + void scalable_free(void*); + void* scalable_aligned_malloc(std::size_t, std::size_t); + void scalable_aligned_free(void*); +} + +#endif /* __TBB_WEAK_SYMBOLS_PRESENT */ + +namespace tbb { +namespace detail { +namespace r1 { + +//! Initialization routine used for first indirect call via allocate_handler. +static void* initialize_allocate_handler(std::size_t size); + +//! Handler for memory allocation +using allocate_handler_type = void* (*)(std::size_t size); +static std::atomic<allocate_handler_type> allocate_handler{ &initialize_allocate_handler }; +allocate_handler_type allocate_handler_unsafe = nullptr; + +//! Handler for memory deallocation +static void (*deallocate_handler)(void* pointer) = nullptr; + +//! Initialization routine used for first indirect call via cache_aligned_allocate_handler. +static void* initialize_cache_aligned_allocate_handler(std::size_t n, std::size_t alignment); + +//! Allocates overaligned memory using standard memory allocator. It is used when scalable_allocator is not available. +static void* std_cache_aligned_allocate(std::size_t n, std::size_t alignment); + +//! Deallocates overaligned memory using standard memory allocator. It is used when scalable_allocator is not available. +static void std_cache_aligned_deallocate(void* p); + +//! Handler for padded memory allocation +using cache_aligned_allocate_handler_type = void* (*)(std::size_t n, std::size_t alignment); +static std::atomic<cache_aligned_allocate_handler_type> cache_aligned_allocate_handler{ &initialize_cache_aligned_allocate_handler }; +cache_aligned_allocate_handler_type cache_aligned_allocate_handler_unsafe = nullptr; + +//! Handler for padded memory deallocation +static void (*cache_aligned_deallocate_handler)(void* p) = nullptr; + +//! Table describing how to link the handlers. +static const dynamic_link_descriptor MallocLinkTable[] = { + DLD(scalable_malloc, allocate_handler_unsafe), + DLD(scalable_free, deallocate_handler), + DLD(scalable_aligned_malloc, cache_aligned_allocate_handler_unsafe), + DLD(scalable_aligned_free, cache_aligned_deallocate_handler), +}; + + +#if TBB_USE_DEBUG +#define DEBUG_SUFFIX "_debug" +#else +#define DEBUG_SUFFIX +#endif /* TBB_USE_DEBUG */ + +// MALLOCLIB_NAME is the name of the oneTBB memory allocator library. +#if _WIN32||_WIN64 +#define MALLOCLIB_NAME "tbbmalloc" DEBUG_SUFFIX ".dll" +#elif __APPLE__ +#define MALLOCLIB_NAME "libtbbmalloc" DEBUG_SUFFIX ".2.dylib" +#elif __FreeBSD__ || __NetBSD__ || __OpenBSD__ || __sun || _AIX || __ANDROID__ +#define MALLOCLIB_NAME "libtbbmalloc" DEBUG_SUFFIX ".so" +#elif __unix__ // Note that order of these #elif's is important! +#define MALLOCLIB_NAME "libtbbmalloc" DEBUG_SUFFIX ".so.2" +#else +#error Unknown OS +#endif + +//! Initialize the allocation/free handler pointers. +/** Caller is responsible for ensuring this routine is called exactly once. + The routine attempts to dynamically link with the TBB memory allocator. + If that allocator is not found, it links to malloc and free. */ +void initialize_handler_pointers() { + __TBB_ASSERT(allocate_handler == &initialize_allocate_handler, nullptr); + bool success = dynamic_link(MALLOCLIB_NAME, MallocLinkTable, 4); + if(!success) { + // If unsuccessful, set the handlers to the default routines. + // This must be done now, and not before FillDynamicLinks runs, because if other + // threads call the handlers, we want them to go through the DoOneTimeInitializations logic, + // which forces them to wait. + allocate_handler_unsafe = &std::malloc; + deallocate_handler = &std::free; + cache_aligned_allocate_handler_unsafe = &std_cache_aligned_allocate; + cache_aligned_deallocate_handler = &std_cache_aligned_deallocate; + } + + allocate_handler.store(allocate_handler_unsafe, std::memory_order_release); + cache_aligned_allocate_handler.store(cache_aligned_allocate_handler_unsafe, std::memory_order_release); + + PrintExtraVersionInfo( "ALLOCATOR", success?"scalable_malloc":"malloc" ); +} + +static std::once_flag initialization_state; +void initialize_cache_aligned_allocator() { + std::call_once(initialization_state, &initialize_handler_pointers); +} + +//! Executed on very first call through allocate_handler +/** Only one of initialize_allocate_handler() and initialize_cache_aligned_allocate_handler() + is called, since each one of them also initializes the other. + + In the current implementation of oneTBB library initialization, cache_aligned_allocate() is + used, which in turn calls initialize_cache_aligned_allocate_handler(). As mentioned above, + that also initializes the regular allocate_handler. + + Therefore, initialize_allocate_handler() is not called in the current library implementation. */ +static void* initialize_allocate_handler(std::size_t size) { + initialize_cache_aligned_allocator(); + __TBB_ASSERT(allocate_handler != &initialize_allocate_handler, nullptr); + return (*allocate_handler)(size); +} + +//! Executed on very first call through cache_aligned_allocate_handler +static void* initialize_cache_aligned_allocate_handler(std::size_t bytes, std::size_t alignment) { + initialize_cache_aligned_allocator(); + __TBB_ASSERT(cache_aligned_allocate_handler != &initialize_cache_aligned_allocate_handler, nullptr); + return (*cache_aligned_allocate_handler)(bytes, alignment); +} + +// TODO: use CPUID to find actual line size, though consider backward compatibility +// nfs - no false sharing +static constexpr std::size_t nfs_size = 128; + +std::size_t __TBB_EXPORTED_FUNC cache_line_size() { + return nfs_size; +} + +void* __TBB_EXPORTED_FUNC cache_aligned_allocate(std::size_t size) { + const std::size_t cache_line_size = nfs_size; + __TBB_ASSERT(is_power_of_two(cache_line_size), "must be power of two"); + + // Check for overflow + if (size + cache_line_size < size) { + throw_exception(exception_id::bad_alloc); + } + // scalable_aligned_malloc considers zero size request an error, and returns nullptr + if (size == 0) size = 1; + + void* result = cache_aligned_allocate_handler.load(std::memory_order_acquire)(size, cache_line_size); + if (!result) { + throw_exception(exception_id::bad_alloc); + } + __TBB_ASSERT(is_aligned(result, cache_line_size), "The returned address isn't aligned"); + return result; +} + +void __TBB_EXPORTED_FUNC cache_aligned_deallocate(void* p) { + __TBB_ASSERT(cache_aligned_deallocate_handler, "Initialization has not been yet."); + (*cache_aligned_deallocate_handler)(p); +} + +static void* std_cache_aligned_allocate(std::size_t bytes, std::size_t alignment) { +#if defined(__TBB_USE_MEMALIGN) + return memalign(alignment, bytes); +#elif defined(__TBB_USE_POSIX_MEMALIGN) + void* p = nullptr; + int res = posix_memalign(&p, alignment, bytes); + if (res != 0) + p = nullptr; + return p; +#elif defined(__TBB_USE_MSVC_ALIGNED_MALLOC) + return _aligned_malloc(bytes, alignment); +#else + // TODO: make it common with cache_aligned_resource + std::size_t space = alignment + bytes; + std::uintptr_t base = reinterpret_cast<std::uintptr_t>(std::malloc(space)); + if (!base) { + return nullptr; + } + std::uintptr_t result = (base + nfs_size) & ~(nfs_size - 1); + // Round up to the next cache line (align the base address) + __TBB_ASSERT((result - base) >= sizeof(std::uintptr_t), "Cannot store a base pointer to the header"); + __TBB_ASSERT(space - (result - base) >= bytes, "Not enough space for the storage"); + + // Record where block actually starts. + (reinterpret_cast<std::uintptr_t*>(result))[-1] = base; + return reinterpret_cast<void*>(result); +#endif +} + +static void std_cache_aligned_deallocate(void* p) { +#if defined(__TBB_USE_MEMALIGN) || defined(__TBB_USE_POSIX_MEMALIGN) + free(p); +#elif defined(__TBB_USE_MSVC_ALIGNED_MALLOC) + _aligned_free(p); +#else + if (p) { + __TBB_ASSERT(reinterpret_cast<std::uintptr_t>(p) >= 0x4096, "attempt to free block not obtained from cache_aligned_allocator"); + // Recover where block actually starts + std::uintptr_t base = (reinterpret_cast<std::uintptr_t*>(p))[-1]; + __TBB_ASSERT(((base + nfs_size) & ~(nfs_size - 1)) == reinterpret_cast<std::uintptr_t>(p), "Incorrect alignment or not allocated by std_cache_aligned_deallocate?"); + std::free(reinterpret_cast<void*>(base)); + } +#endif +} + +void* __TBB_EXPORTED_FUNC allocate_memory(std::size_t size) { + void* result = allocate_handler.load(std::memory_order_acquire)(size); + if (!result) { + throw_exception(exception_id::bad_alloc); + } + return result; +} + +void __TBB_EXPORTED_FUNC deallocate_memory(void* p) { + if (p) { + __TBB_ASSERT(deallocate_handler, "Initialization has not been yet."); + (*deallocate_handler)(p); + } +} + +bool __TBB_EXPORTED_FUNC is_tbbmalloc_used() { + auto handler_snapshot = allocate_handler.load(std::memory_order_acquire); + if (handler_snapshot == &initialize_allocate_handler) { + initialize_cache_aligned_allocator(); + } + handler_snapshot = allocate_handler.load(std::memory_order_relaxed); + __TBB_ASSERT(handler_snapshot != &initialize_allocate_handler && deallocate_handler != nullptr, nullptr); + // Cast to void avoids type mismatch errors on some compilers (e.g. __IBMCPP__) + __TBB_ASSERT((reinterpret_cast<void*>(handler_snapshot) == reinterpret_cast<void*>(&std::malloc)) == (reinterpret_cast<void*>(deallocate_handler) == reinterpret_cast<void*>(&std::free)), + "Both shim pointers must refer to routines from the same package (either TBB or CRT)"); + return reinterpret_cast<void*>(handler_snapshot) == reinterpret_cast<void*>(&std::malloc); +} + +} // namespace r1 +} // namespace detail +} // namespace tbb diff --git a/src/tbb/src/tbb/arena.cpp b/src/tbb/src/tbb/arena.cpp index abe4d35be..6ca062d02 100644 --- a/src/tbb/src/tbb/arena.cpp +++ b/src/tbb/src/tbb/arena.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2024 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,1005 +12,888 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ -#include "tbb/global_control.h" // thread_stack_size - -#include "scheduler.h" +#include "task_dispatcher.h" #include "governor.h" +#include "threading_control.h" #include "arena.h" #include "itt_notify.h" #include "semaphore.h" -#include "tbb/internal/_flow_graph_impl.h" +#include "waiters.h" +#include "oneapi/tbb/detail/_task.h" +#include "oneapi/tbb/info.h" +#include "oneapi/tbb/tbb_allocator.h" +#include <atomic> +#include <cstring> #include <functional> -#if __TBB_STATISTICS_STDOUT -#include <cstdio> -#endif - namespace tbb { -namespace internal { - -// put it here in order to enable compiler to inline it into arena::process and nested_arena_entry -void generic_scheduler::attach_arena( arena* a, size_t index, bool is_master ) { - __TBB_ASSERT( a->my_market == my_market, NULL ); - my_arena = a; - my_arena_index = index; - my_arena_slot = a->my_slots + index; - attach_mailbox( affinity_id(index+1) ); - if ( is_master && my_inbox.is_idle_state( true ) ) { - // Master enters an arena with its own task to be executed. It means that master is not - // going to enter stealing loop and take affinity tasks. - my_inbox.set_is_idle( false ); - } -#if __TBB_TASK_GROUP_CONTEXT - // Context to be used by root tasks by default (if the user has not specified one). - if( !is_master ) - my_dummy_task->prefix().context = a->my_default_ctx; -#endif /* __TBB_TASK_GROUP_CONTEXT */ -#if __TBB_TASK_PRIORITY - // In the current implementation master threads continue processing even when - // there are other masters with higher priority. Only TBB worker threads are - // redistributed between arenas based on the latters' priority. Thus master - // threads use arena's top priority as a reference point (in contrast to workers - // that use my_market->my_global_top_priority). - if( is_master ) { - my_ref_top_priority = &a->my_top_priority; - my_ref_reload_epoch = &a->my_reload_epoch; - } - my_local_reload_epoch = *my_ref_reload_epoch; - __TBB_ASSERT( !my_offloaded_tasks, NULL ); -#endif /* __TBB_TASK_PRIORITY */ -} - -inline static bool occupy_slot( generic_scheduler*& slot, generic_scheduler& s ) { - return !slot && as_atomic( slot ).compare_and_swap( &s, NULL ) == NULL; -} - -size_t arena::occupy_free_slot_in_range( generic_scheduler& s, size_t lower, size_t upper ) { +namespace detail { +namespace r1 { + +#if __TBB_ARENA_BINDING +class numa_binding_observer : public tbb::task_scheduler_observer { + binding_handler* my_binding_handler; +public: + numa_binding_observer( d1::task_arena* ta, int num_slots, int numa_id, core_type_id core_type, int max_threads_per_core ) + : task_scheduler_observer(*ta) + , my_binding_handler(construct_binding_handler(num_slots, numa_id, core_type, max_threads_per_core)) + {} + + void on_scheduler_entry( bool ) override { + apply_affinity_mask(my_binding_handler, this_task_arena::current_thread_index()); + } + + void on_scheduler_exit( bool ) override { + restore_affinity_mask(my_binding_handler, this_task_arena::current_thread_index()); + } + + ~numa_binding_observer() override{ + destroy_binding_handler(my_binding_handler); + } +}; + +numa_binding_observer* construct_binding_observer( d1::task_arena* ta, int num_slots, int numa_id, core_type_id core_type, int max_threads_per_core ) { + numa_binding_observer* binding_observer = nullptr; + if ((core_type >= 0 && core_type_count() > 1) || (numa_id >= 0 && numa_node_count() > 1) || max_threads_per_core > 0) { + binding_observer = new(allocate_memory(sizeof(numa_binding_observer))) numa_binding_observer(ta, num_slots, numa_id, core_type, max_threads_per_core); + __TBB_ASSERT(binding_observer, "Failure during NUMA binding observer allocation and construction"); + } + return binding_observer; +} + +void destroy_binding_observer( numa_binding_observer* binding_observer ) { + __TBB_ASSERT(binding_observer, "Trying to deallocate nullptr pointer"); + binding_observer->observe(false); + binding_observer->~numa_binding_observer(); + deallocate_memory(binding_observer); +} +#endif /*!__TBB_ARENA_BINDING*/ + +void arena::on_thread_leaving(unsigned ref_param) { + // + // Implementation of arena destruction synchronization logic contained various + // bugs/flaws at the different stages of its evolution, so below is a detailed + // description of the issues taken into consideration in the framework of the + // current design. + // + // In case of using fire-and-forget tasks (scheduled via task::enqueue()) + // external thread is allowed to leave its arena before all its work is executed, + // and market may temporarily revoke all workers from this arena. Since revoked + // workers never attempt to reset arena state to EMPTY and cancel its request + // to RML for threads, the arena object is destroyed only when both the last + // thread is leaving it and arena's state is EMPTY (that is its external thread + // left and it does not contain any work). + // Thus resetting arena to EMPTY state (as earlier TBB versions did) should not + // be done here (or anywhere else in the external thread to that matter); doing so + // can result either in arena's premature destruction (at least without + // additional costly checks in workers) or in unnecessary arena state changes + // (and ensuing workers migration). + // + // A worker that checks for work presence and transitions arena to the EMPTY + // state (in snapshot taking procedure arena::out_of_work()) updates + // arena::my_pool_state first and only then arena::my_num_workers_requested. + // So the check for work absence must be done against the latter field. + // + // In a time window between decrementing the active threads count and checking + // if there is an outstanding request for workers. New worker thread may arrive, + // finish remaining work, set arena state to empty, and leave decrementing its + // refcount and destroying. Then the current thread will destroy the arena + // the second time. To preclude it a local copy of the outstanding request + // value can be stored before decrementing active threads count. + // + // But this technique may cause two other problem. When the stored request is + // zero, it is possible that arena still has threads and they can generate new + // tasks and thus re-establish non-zero requests. Then all the threads can be + // revoked (as described above) leaving this thread the last one, and causing + // it to destroy non-empty arena. + // + // The other problem takes place when the stored request is non-zero. Another + // thread may complete the work, set arena state to empty, and leave without + // arena destruction before this thread decrements the refcount. This thread + // cannot destroy the arena either. Thus the arena may be "orphaned". + // + // In both cases we cannot dereference arena pointer after the refcount is + // decremented, as our arena may already be destroyed. + // + // If this is the external thread, the market is protected by refcount to it. + // In case of workers market's liveness is ensured by the RML connection + // rundown protocol, according to which the client (i.e. the market) lives + // until RML server notifies it about connection termination, and this + // notification is fired only after all workers return into RML. + // + // Thus if we decremented refcount to zero we ask the market to check arena + // state (including the fact if it is alive) under the lock. + // + + __TBB_ASSERT(my_references.load(std::memory_order_relaxed) >= ref_param, "broken arena reference counter"); + + // When there is no workers someone must free arena, as + // without workers, no one calls out_of_work(). + if (ref_param == ref_external && !my_mandatory_concurrency.test()) { + out_of_work(); + } + + threading_control* tc = my_threading_control; + auto tc_client_snapshot = tc->prepare_client_destruction(my_tc_client); + // Release our reference to sync with destroy_client + unsigned remaining_ref = my_references.fetch_sub(ref_param, std::memory_order_release) - ref_param; + // do not access `this` it might be destroyed already + if (remaining_ref == 0) { + if (tc->try_destroy_client(tc_client_snapshot)) { + // We are requested to destroy ourself + free_arena(); + } + } +} + +std::size_t arena::occupy_free_slot_in_range( thread_data& tls, std::size_t lower, std::size_t upper ) { if ( lower >= upper ) return out_of_arena; // Start search for an empty slot from the one we occupied the last time - size_t index = s.my_arena_index; - if ( index < lower || index >= upper ) index = s.my_random.get() % (upper - lower) + lower; - __TBB_ASSERT( index >= lower && index < upper, NULL ); + std::size_t index = tls.my_arena_index; + if ( index < lower || index >= upper ) index = tls.my_random.get() % (upper - lower) + lower; + __TBB_ASSERT( index >= lower && index < upper, nullptr); // Find a free slot - for ( size_t i = index; i < upper; ++i ) - if ( occupy_slot(my_slots[i].my_scheduler, s) ) return i; - for ( size_t i = lower; i < index; ++i ) - if ( occupy_slot(my_slots[i].my_scheduler, s) ) return i; + for ( std::size_t i = index; i < upper; ++i ) + if (my_slots[i].try_occupy()) return i; + for ( std::size_t i = lower; i < index; ++i ) + if (my_slots[i].try_occupy()) return i; return out_of_arena; } template <bool as_worker> -size_t arena::occupy_free_slot( generic_scheduler& s ) { - // Firstly, masters try to occupy reserved slots - size_t index = as_worker ? out_of_arena : occupy_free_slot_in_range( s, 0, my_num_reserved_slots ); +std::size_t arena::occupy_free_slot(thread_data& tls) { + // Firstly, external threads try to occupy reserved slots + std::size_t index = as_worker ? out_of_arena : occupy_free_slot_in_range( tls, 0, my_num_reserved_slots ); if ( index == out_of_arena ) { // Secondly, all threads try to occupy all non-reserved slots - index = occupy_free_slot_in_range( s, my_num_reserved_slots, my_num_slots ); + index = occupy_free_slot_in_range(tls, my_num_reserved_slots, my_num_slots ); // Likely this arena is already saturated if ( index == out_of_arena ) return out_of_arena; } - ITT_NOTIFY(sync_acquired, my_slots + index); atomic_update( my_limit, (unsigned)(index + 1), std::less<unsigned>() ); return index; } -void arena::process( generic_scheduler& s ) { - __TBB_ASSERT( is_alive(my_guard), NULL ); - __TBB_ASSERT( governor::is_set(&s), NULL ); - __TBB_ASSERT( s.my_innermost_running_task == s.my_dummy_task, NULL ); - __TBB_ASSERT( s.worker_outermost_level(), NULL ); +std::uintptr_t arena::calculate_stealing_threshold() { + stack_anchor_type anchor; + return r1::calculate_stealing_threshold(reinterpret_cast<std::uintptr_t>(&anchor), my_threading_control->worker_stack_size()); +} - __TBB_ASSERT( my_num_slots > 1, NULL ); +void arena::process(thread_data& tls) { + governor::set_thread_data(tls); // TODO: consider moving to create_one_job. + __TBB_ASSERT( is_alive(my_guard), nullptr); + __TBB_ASSERT( my_num_slots >= 1, nullptr); - size_t index = occupy_free_slot</*as_worker*/true>( s ); - if ( index == out_of_arena ) - goto quit; + std::size_t index = occupy_free_slot</*as_worker*/true>(tls); + if (index == out_of_arena) { + on_thread_leaving(ref_worker); + return; + } __TBB_ASSERT( index >= my_num_reserved_slots, "Workers cannot occupy reserved slots" ); - s.attach_arena( this, index, /*is_master*/false ); + tls.attach_arena(*this, index); + // worker thread enters the dispatch loop to look for a work + tls.my_inbox.set_is_idle(true); + if (tls.my_arena_slot->is_task_pool_published()) { + tls.my_inbox.set_is_idle(false); + } -#if !__TBB_FP_CONTEXT - my_cpu_ctl_env.set_env(); -#endif + task_dispatcher& task_disp = tls.my_arena_slot->default_task_dispatcher(); + tls.enter_task_dispatcher(task_disp, calculate_stealing_threshold()); + __TBB_ASSERT(task_disp.can_steal(), nullptr); + + __TBB_ASSERT( !tls.my_last_observer, "There cannot be notified local observers when entering arena" ); + my_observers.notify_entry_observers(tls.my_last_observer, tls.my_is_worker); + + // Waiting on special object tied to this arena + outermost_worker_waiter waiter(*this); + d1::task* t = tls.my_task_dispatcher->local_wait_for_all(nullptr, waiter); + // For purposes of affinity support, the slot's mailbox is considered idle while no thread is + // attached to it. + tls.my_inbox.set_is_idle(true); + + __TBB_ASSERT_EX(t == nullptr, "Outermost worker must not leave dispatch loop with a task"); + __TBB_ASSERT(governor::is_thread_data_set(&tls), nullptr); + __TBB_ASSERT(tls.my_task_dispatcher == &task_disp, nullptr); + + my_observers.notify_exit_observers(tls.my_last_observer, tls.my_is_worker); + tls.my_last_observer = nullptr; + + tls.leave_task_dispatcher(); + + // Arena slot detach (arena may be used in market::process) + // TODO: Consider moving several calls below into a new method(e.g.detach_arena). + tls.my_arena_slot->release(); + tls.my_arena_slot = nullptr; + tls.my_inbox.detach(); + __TBB_ASSERT(tls.my_inbox.is_idle_state(true), nullptr); + __TBB_ASSERT(is_alive(my_guard), nullptr); -#if __TBB_ARENA_OBSERVER - __TBB_ASSERT( !s.my_last_local_observer, "There cannot be notified local observers when entering arena" ); - my_observers.notify_entry_observers( s.my_last_local_observer, /*worker=*/true ); -#endif /* __TBB_ARENA_OBSERVER */ - - // Task pool can be marked as non-empty if the worker occupies the slot left by a master. - if ( s.my_arena_slot->task_pool != EmptyTaskPool ) { - __TBB_ASSERT( s.my_inbox.is_idle_state(false), NULL ); - s.local_wait_for_all( *s.my_dummy_task, NULL ); - __TBB_ASSERT( s.my_inbox.is_idle_state(true), NULL ); - } - - for ( ;; ) { - __TBB_ASSERT( s.my_innermost_running_task == s.my_dummy_task, NULL ); - __TBB_ASSERT( s.worker_outermost_level(), NULL ); - __TBB_ASSERT( is_alive(my_guard), NULL ); - __TBB_ASSERT( s.is_quiescent_local_task_pool_reset(), - "Worker cannot leave arena while its task pool is not reset" ); - __TBB_ASSERT( s.my_arena_slot->task_pool == EmptyTaskPool, "Empty task pool is not marked appropriately" ); - // This check prevents relinquishing more than necessary workers because - // of the non-atomicity of the decision making procedure - if ( num_workers_active() > my_num_workers_allotted -#if __TBB_ENQUEUE_ENFORCED_CONCURRENCY - || recall_by_mandatory_request() -#endif - ) - break; - // Try to steal a task. - // Passing reference count is technically unnecessary in this context, - // but omitting it here would add checks inside the function. - task* t = s.receive_or_steal_task( __TBB_ISOLATION_ARG( s.my_dummy_task->prefix().ref_count, no_isolation ) ); - if (t) { - // A side effect of receive_or_steal_task is that my_innermost_running_task can be set. - // But for the outermost dispatch loop it has to be a dummy task. - s.my_innermost_running_task = s.my_dummy_task; - s.local_wait_for_all(*s.my_dummy_task,t); - } - } -#if __TBB_ARENA_OBSERVER - my_observers.notify_exit_observers( s.my_last_local_observer, /*worker=*/true ); - s.my_last_local_observer = NULL; -#endif /* __TBB_ARENA_OBSERVER */ -#if __TBB_TASK_PRIORITY - if ( s.my_offloaded_tasks ) - orphan_offloaded_tasks( s ); -#endif /* __TBB_TASK_PRIORITY */ -#if __TBB_STATISTICS - ++s.my_counters.arena_roundtrips; - *my_slots[index].my_counters += s.my_counters; - s.my_counters.reset(); -#endif /* __TBB_STATISTICS */ - __TBB_store_with_release( my_slots[index].my_scheduler, (generic_scheduler*)NULL ); - s.my_arena_slot = 0; // detached from slot - s.my_inbox.detach(); - __TBB_ASSERT( s.my_inbox.is_idle_state(true), NULL ); - __TBB_ASSERT( s.my_innermost_running_task == s.my_dummy_task, NULL ); - __TBB_ASSERT( s.worker_outermost_level(), NULL ); - __TBB_ASSERT( is_alive(my_guard), NULL ); -quit: // In contrast to earlier versions of TBB (before 3.0 U5) now it is possible // that arena may be temporarily left unpopulated by threads. See comments in // arena::on_thread_leaving() for more details. - on_thread_leaving<ref_worker>(); + on_thread_leaving(ref_worker); + __TBB_ASSERT(tls.my_arena == this, "my_arena is used as a hint when searching the arena to join"); } -arena::arena ( market& m, unsigned num_slots, unsigned num_reserved_slots ) { +arena::arena(threading_control* control, unsigned num_slots, unsigned num_reserved_slots, unsigned priority_level) { __TBB_ASSERT( !my_guard, "improperly allocated arena?" ); - __TBB_ASSERT( sizeof(my_slots[0]) % NFS_GetLineSize()==0, "arena::slot size not multiple of cache line size" ); - __TBB_ASSERT( (uintptr_t)this % NFS_GetLineSize()==0, "arena misaligned" ); -#if __TBB_TASK_PRIORITY - __TBB_ASSERT( !my_reload_epoch && !my_orphaned_tasks && !my_skipped_fifo_priority, "New arena object is not zeroed" ); -#endif /* __TBB_TASK_PRIORITY */ - my_market = &m; + __TBB_ASSERT( sizeof(my_slots[0]) % cache_line_size()==0, "arena::slot size not multiple of cache line size" ); + __TBB_ASSERT( is_aligned(this, cache_line_size()), "arena misaligned" ); + my_threading_control = control; my_limit = 1; - // Two slots are mandatory: for the master, and for 1 worker (required to support starvation resistant tasks). - my_num_slots = num_arena_slots(num_slots); + // Two slots are mandatory: for the external thread, and for 1 worker (required to support starvation resistant tasks). + my_num_slots = num_arena_slots(num_slots, num_reserved_slots); my_num_reserved_slots = num_reserved_slots; my_max_num_workers = num_slots-num_reserved_slots; - my_references = ref_external; // accounts for the master -#if __TBB_TASK_PRIORITY - my_bottom_priority = my_top_priority = normalized_normal_priority; -#endif /* __TBB_TASK_PRIORITY */ - my_aba_epoch = m.my_arenas_aba_epoch; -#if __TBB_ARENA_OBSERVER + my_priority_level = priority_level; + my_references = ref_external; // accounts for the external thread my_observers.my_arena = this; -#endif - __TBB_ASSERT ( my_max_num_workers <= my_num_slots, NULL ); + my_co_cache.init(4 * num_slots); + __TBB_ASSERT ( my_max_num_workers <= my_num_slots, nullptr); + // Initialize the default context. It should be allocated before task_dispatch construction. + my_default_ctx = new (cache_aligned_allocate(sizeof(d1::task_group_context))) + d1::task_group_context{ d1::task_group_context::isolated, d1::task_group_context::fp_settings }; // Construct slots. Mark internal synchronization elements for the tools. + task_dispatcher* base_td_pointer = reinterpret_cast<task_dispatcher*>(my_slots + my_num_slots); for( unsigned i = 0; i < my_num_slots; ++i ) { - __TBB_ASSERT( !my_slots[i].my_scheduler && !my_slots[i].task_pool, NULL ); - __TBB_ASSERT( !my_slots[i].task_pool_ptr, NULL ); - __TBB_ASSERT( !my_slots[i].my_task_pool_size, NULL ); - ITT_SYNC_CREATE(my_slots + i, SyncType_Scheduler, SyncObj_WorkerTaskPool); - mailbox(i+1).construct(); - ITT_SYNC_CREATE(&mailbox(i+1), SyncType_Scheduler, SyncObj_Mailbox); - my_slots[i].hint_for_pop = i; -#if __TBB_STATISTICS - my_slots[i].my_counters = new ( NFS_Allocate(1, sizeof(statistics_counters), NULL) ) statistics_counters; -#endif /* __TBB_STATISTICS */ - } - my_task_stream.initialize(my_num_slots); - ITT_SYNC_CREATE(&my_task_stream, SyncType_Scheduler, SyncObj_TaskStream); -#if __TBB_ENQUEUE_ENFORCED_CONCURRENCY - my_concurrency_mode = cm_normal; -#endif -#if !__TBB_FP_CONTEXT - my_cpu_ctl_env.get_env(); + // __TBB_ASSERT( !my_slots[i].my_scheduler && !my_slots[i].task_pool, nullptr); + __TBB_ASSERT( !my_slots[i].task_pool_ptr, nullptr); + __TBB_ASSERT( !my_slots[i].my_task_pool_size, nullptr); + mailbox(i).construct(); + my_slots[i].init_task_streams(i); + my_slots[i].my_default_task_dispatcher = new(base_td_pointer + i) task_dispatcher(this); + my_slots[i].my_is_occupied.store(false, std::memory_order_relaxed); + } + my_fifo_task_stream.initialize(my_num_slots); + my_resume_task_stream.initialize(my_num_slots); +#if __TBB_PREVIEW_CRITICAL_TASKS + my_critical_task_stream.initialize(my_num_slots); #endif + my_mandatory_requests = 0; } -arena& arena::allocate_arena( market& m, unsigned num_slots, unsigned num_reserved_slots ) { +arena& arena::allocate_arena(threading_control* control, unsigned num_slots, unsigned num_reserved_slots, + unsigned priority_level) +{ __TBB_ASSERT( sizeof(base_type) + sizeof(arena_slot) == sizeof(arena), "All arena data fields must go to arena_base" ); - __TBB_ASSERT( sizeof(base_type) % NFS_GetLineSize() == 0, "arena slots area misaligned: wrong padding" ); - __TBB_ASSERT( sizeof(mail_outbox) == NFS_MaxLineSize, "Mailbox padding is wrong" ); - size_t n = allocation_size(num_arena_slots(num_slots)); - unsigned char* storage = (unsigned char*)NFS_Allocate( 1, n, NULL ); + __TBB_ASSERT( sizeof(base_type) % cache_line_size() == 0, "arena slots area misaligned: wrong padding" ); + __TBB_ASSERT( sizeof(mail_outbox) == max_nfs_size, "Mailbox padding is wrong" ); + std::size_t n = allocation_size(num_arena_slots(num_slots, num_reserved_slots)); + unsigned char* storage = (unsigned char*)cache_aligned_allocate(n); // Zero all slots to indicate that they are empty - memset( storage, 0, n ); - return *new( storage + num_arena_slots(num_slots) * sizeof(mail_outbox) ) arena(m, num_slots, num_reserved_slots); + std::memset( storage, 0, n ); + + return *new( storage + num_arena_slots(num_slots, num_reserved_slots) * sizeof(mail_outbox) ) + arena(control, num_slots, num_reserved_slots, priority_level); } void arena::free_arena () { - __TBB_ASSERT( is_alive(my_guard), NULL ); - __TBB_ASSERT( !my_references, "There are threads in the dying arena" ); - __TBB_ASSERT( !my_num_workers_requested && !my_num_workers_allotted, "Dying arena requests workers" ); - __TBB_ASSERT( my_pool_state == SNAPSHOT_EMPTY || !my_max_num_workers, "Inconsistent state of a dying arena" ); -#if __TBB_ENQUEUE_ENFORCED_CONCURRENCY - __TBB_ASSERT( my_concurrency_mode != cm_enforced_global, NULL ); -#endif -#if !__TBB_STATISTICS_EARLY_DUMP - GATHER_STATISTIC( dump_arena_statistics() ); -#endif + __TBB_ASSERT( is_alive(my_guard), nullptr); + __TBB_ASSERT( !my_references.load(std::memory_order_relaxed), "There are threads in the dying arena" ); + __TBB_ASSERT( !my_total_num_workers_requested && !my_num_workers_allotted, "Dying arena requests workers" ); + __TBB_ASSERT( is_empty(), "Inconsistent state of a dying arena" ); +#if __TBB_ARENA_BINDING + if (my_numa_binding_observer != nullptr) { + destroy_binding_observer(my_numa_binding_observer); + my_numa_binding_observer = nullptr; + } +#endif /*__TBB_ARENA_BINDING*/ poison_value( my_guard ); - intptr_t drained = 0; for ( unsigned i = 0; i < my_num_slots; ++i ) { - __TBB_ASSERT( !my_slots[i].my_scheduler, "arena slot is not empty" ); + // __TBB_ASSERT( !my_slots[i].my_scheduler, "arena slot is not empty" ); // TODO: understand the assertion and modify - // __TBB_ASSERT( my_slots[i].task_pool == EmptyTaskPool, NULL ); - __TBB_ASSERT( my_slots[i].head == my_slots[i].tail, NULL ); // TODO: replace by is_quiescent_local_task_pool_empty + // __TBB_ASSERT( my_slots[i].task_pool == EmptyTaskPool, nullptr); + __TBB_ASSERT( my_slots[i].head == my_slots[i].tail, nullptr); // TODO: replace by is_quiescent_local_task_pool_empty my_slots[i].free_task_pool(); -#if __TBB_STATISTICS - NFS_Free( my_slots[i].my_counters ); -#endif /* __TBB_STATISTICS */ - drained += mailbox(i+1).drain(); - } - __TBB_ASSERT( my_task_stream.drain()==0, "Not all enqueued tasks were executed"); -#if __TBB_COUNT_TASK_NODES - my_market->update_task_node_count( -drained ); -#endif /* __TBB_COUNT_TASK_NODES */ - // remove an internal reference - my_market->release( /*is_public=*/false, /*blocking_terminate=*/false ); -#if __TBB_TASK_GROUP_CONTEXT - __TBB_ASSERT( my_default_ctx, "Master thread never entered the arena?" ); + mailbox(i).drain(); + my_slots[i].my_default_task_dispatcher->~task_dispatcher(); + } + __TBB_ASSERT(my_fifo_task_stream.empty(), "Not all enqueued tasks were executed"); + __TBB_ASSERT(my_resume_task_stream.empty(), "Not all enqueued tasks were executed"); + // Cleanup coroutines/schedulers cache + my_co_cache.cleanup(); my_default_ctx->~task_group_context(); - NFS_Free(my_default_ctx); -#endif /* __TBB_TASK_GROUP_CONTEXT */ -#if __TBB_ARENA_OBSERVER - if ( !my_observers.empty() ) - my_observers.clear(); -#endif /* __TBB_ARENA_OBSERVER */ - void* storage = &mailbox(my_num_slots); - __TBB_ASSERT( my_references == 0, NULL ); - __TBB_ASSERT( my_pool_state == SNAPSHOT_EMPTY || !my_max_num_workers, NULL ); + cache_aligned_deallocate(my_default_ctx); +#if __TBB_PREVIEW_CRITICAL_TASKS + __TBB_ASSERT( my_critical_task_stream.empty(), "Not all critical tasks were executed"); +#endif + // Clear enfources synchronization with observe(false) + my_observers.clear(); + + void* storage = &mailbox(my_num_slots-1); + __TBB_ASSERT( my_references.load(std::memory_order_relaxed) == 0, nullptr); this->~arena(); #if TBB_USE_ASSERT > 1 - memset( storage, 0, allocation_size(my_num_slots) ); + std::memset( storage, 0, allocation_size(my_num_slots) ); #endif /* TBB_USE_ASSERT */ - NFS_Free( storage ); + cache_aligned_deallocate( storage ); } -#if __TBB_STATISTICS -void arena::dump_arena_statistics () { - statistics_counters total; - for( unsigned i = 0; i < my_num_slots; ++i ) { -#if __TBB_STATISTICS_EARLY_DUMP - generic_scheduler* s = my_slots[i].my_scheduler; - if ( s ) - *my_slots[i].my_counters += s->my_counters; -#else - __TBB_ASSERT( !my_slots[i].my_scheduler, NULL ); -#endif - if ( i != 0 ) { - total += *my_slots[i].my_counters; - dump_statistics( *my_slots[i].my_counters, i ); - } +bool arena::has_enqueued_tasks() { + return !my_fifo_task_stream.empty(); +} + +void arena::request_workers(int mandatory_delta, int workers_delta, bool wakeup_threads) { + my_threading_control->adjust_demand(my_tc_client, mandatory_delta, workers_delta); + + if (wakeup_threads) { + // Notify all sleeping threads that work has appeared in the arena. + get_waiting_threads_monitor().notify([&] (market_context context) { + return this == context.my_arena_addr; + }); } - dump_statistics( *my_slots[0].my_counters, 0 ); -#if __TBB_STATISTICS_STDOUT -#if !__TBB_STATISTICS_TOTALS_ONLY - printf( "----------------------------------------------\n" ); -#endif - dump_statistics( total, workers_counters_total ); - total += *my_slots[0].my_counters; - dump_statistics( total, arena_counters_total ); -#if !__TBB_STATISTICS_TOTALS_ONLY - printf( "==============================================\n" ); -#endif -#endif /* __TBB_STATISTICS_STDOUT */ -} -#endif /* __TBB_STATISTICS */ - -#if __TBB_TASK_PRIORITY -// The method inspects a scheduler to determine: -// 1. if it has tasks that can be retrieved and executed (via the return value); -// 2. if it has any tasks at all, including those of lower priority (via tasks_present); -// 3. if it is able to work with enqueued tasks (via dequeuing_possible). -inline bool arena::may_have_tasks ( generic_scheduler* s, bool& tasks_present, bool& dequeuing_possible ) { - if ( !s || s->my_arena != this ) - return false; - dequeuing_possible |= s->worker_outermost_level(); - if ( s->my_pool_reshuffling_pending ) { - // This primary task pool is nonempty and may contain tasks at the current - // priority level. Its owner is winnowing lower priority tasks at the moment. - tasks_present = true; - return true; +} + +bool arena::has_tasks() { + // TODO: rework it to return at least a hint about where a task was found; better if the task itself. + std::size_t n = my_limit.load(std::memory_order_acquire); + bool tasks_are_available = false; + for (std::size_t k = 0; k < n && !tasks_are_available; ++k) { + tasks_are_available = !my_slots[k].is_empty(); } - if ( s->my_offloaded_tasks ) { - tasks_present = true; - if ( s->my_local_reload_epoch < *s->my_ref_reload_epoch ) { - // This scheduler's offload area is nonempty and may contain tasks at the - // current priority level. - return true; + tasks_are_available = tasks_are_available || has_enqueued_tasks() || !my_resume_task_stream.empty(); +#if __TBB_PREVIEW_CRITICAL_TASKS + tasks_are_available = tasks_are_available || !my_critical_task_stream.empty(); +#endif + return tasks_are_available; +} + +void arena::out_of_work() { + // We should try unset my_pool_state first due to keep arena invariants in consistent state + // Otherwise, we might have my_pool_state = false and my_mandatory_concurrency = true that is broken invariant + bool disable_mandatory = my_mandatory_concurrency.try_clear_if([this] { return !has_enqueued_tasks(); }); + bool release_workers = my_pool_state.try_clear_if([this] { return !has_tasks(); }); + + if (disable_mandatory || release_workers) { + int mandatory_delta = disable_mandatory ? -1 : 0; + int workers_delta = release_workers ? -(int)my_max_num_workers : 0; + + if (disable_mandatory && is_arena_workerless()) { + // We had set workers_delta to 1 when enabled mandatory concurrency, so revert it now + workers_delta = -1; } + request_workers(mandatory_delta, workers_delta); } - return false; } -void arena::orphan_offloaded_tasks(generic_scheduler& s) { - __TBB_ASSERT( s.my_offloaded_tasks, NULL ); - GATHER_STATISTIC( ++s.my_counters.prio_orphanings ); - ++my_abandonment_epoch; - __TBB_ASSERT( s.my_offloaded_task_list_tail_link && !*s.my_offloaded_task_list_tail_link, NULL ); - task* orphans; - do { - orphans = const_cast<task*>(my_orphaned_tasks); - *s.my_offloaded_task_list_tail_link = orphans; - } while ( as_atomic(my_orphaned_tasks).compare_and_swap(s.my_offloaded_tasks, orphans) != orphans ); - s.my_offloaded_tasks = NULL; -#if TBB_USE_ASSERT - s.my_offloaded_task_list_tail_link = NULL; -#endif /* TBB_USE_ASSERT */ +void arena::set_top_priority(bool is_top_priority) { + my_is_top_priority.store(is_top_priority, std::memory_order_relaxed); } -#endif /* __TBB_TASK_PRIORITY */ -bool arena::has_enqueued_tasks() { - // Look for enqueued tasks at all priority levels - for ( int p = 0; p < num_priority_levels; ++p ) - if ( !my_task_stream.empty(p) ) - return true; - return false; +bool arena::is_top_priority() const { + return my_is_top_priority.load(std::memory_order_relaxed); } -void arena::restore_priority_if_need() { - // Check for the presence of enqueued tasks "lost" on some of - // priority levels because updating arena priority and switching - // arena into "populated" (FULL) state happen non-atomically. - // Imposing atomicity would require task::enqueue() to use a lock, - // which is unacceptable. - if ( has_enqueued_tasks() ) { - advertise_new_work<work_enqueued>(); -#if __TBB_TASK_PRIORITY - // update_arena_priority() expects non-zero arena::my_num_workers_requested, - // so must be called after advertise_new_work<work_enqueued>() - for ( int p = 0; p < num_priority_levels; ++p ) - if ( !my_task_stream.empty(p) ) { - if ( p < my_bottom_priority || p > my_top_priority ) - my_market->update_arena_priority(*this, p); - } -#endif +bool arena::try_join() { + if (is_joinable()) { + my_references += arena::ref_worker; + return true; } + return false; } -bool arena::is_out_of_work() { - // TODO: rework it to return at least a hint about where a task was found; better if the task itself. - for(;;) { - pool_state_t snapshot = my_pool_state; - switch( snapshot ) { - case SNAPSHOT_EMPTY: - return true; - case SNAPSHOT_FULL: { - // Use unique id for "busy" in order to avoid ABA problems. - const pool_state_t busy = pool_state_t(&busy); - // Request permission to take snapshot - if( my_pool_state.compare_and_swap( busy, SNAPSHOT_FULL )==SNAPSHOT_FULL ) { - // Got permission. Take the snapshot. - // NOTE: This is not a lock, as the state can be set to FULL at - // any moment by a thread that spawns/enqueues new task. - size_t n = my_limit; - // Make local copies of volatile parameters. Their change during - // snapshot taking procedure invalidates the attempt, and returns - // this thread into the dispatch loop. -#if __TBB_TASK_PRIORITY - uintptr_t reload_epoch = __TBB_load_with_acquire( my_reload_epoch ); - intptr_t top_priority = my_top_priority; - // Inspect primary task pools first -#endif /* __TBB_TASK_PRIORITY */ - size_t k; - for( k=0; k<n; ++k ) { - if( my_slots[k].task_pool != EmptyTaskPool && - __TBB_load_relaxed(my_slots[k].head) < __TBB_load_relaxed(my_slots[k].tail) ) - { - // k-th primary task pool is nonempty and does contain tasks. - break; - } - if( my_pool_state!=busy ) - return false; // the work was published - } - __TBB_ASSERT( k <= n, NULL ); - bool work_absent = k == n; -#if __TBB_TASK_PRIORITY - // Variable tasks_present indicates presence of tasks at any priority - // level, while work_absent refers only to the current priority. - bool tasks_present = !work_absent || my_orphaned_tasks; - bool dequeuing_possible = false; - if ( work_absent ) { - // Check for the possibility that recent priority changes - // brought some tasks to the current priority level - - uintptr_t abandonment_epoch = my_abandonment_epoch; - // Master thread's scheduler needs special handling as it - // may be destroyed at any moment (workers' schedulers are - // guaranteed to be alive while at least one thread is in arena). - // The lock below excludes concurrency with task group state change - // propagation and guarantees lifetime of the master thread. - the_context_state_propagation_mutex.lock(); - work_absent = !may_have_tasks( my_slots[0].my_scheduler, tasks_present, dequeuing_possible ); - the_context_state_propagation_mutex.unlock(); - // The following loop is subject to data races. While k-th slot's - // scheduler is being examined, corresponding worker can either - // leave to RML or migrate to another arena. - // But the races are not prevented because all of them are benign. - // First, the code relies on the fact that worker thread's scheduler - // object persists until the whole library is deinitialized. - // Second, in the worst case the races can only cause another - // round of stealing attempts to be undertaken. Introducing complex - // synchronization into this coldest part of the scheduler's control - // flow does not seem to make sense because it both is unlikely to - // ever have any observable performance effect, and will require - // additional synchronization code on the hotter paths. - for( k = 1; work_absent && k < n; ++k ) { - if( my_pool_state!=busy ) - return false; // the work was published - work_absent = !may_have_tasks( my_slots[k].my_scheduler, tasks_present, dequeuing_possible ); - } - // Preclude premature switching arena off because of a race in the previous loop. - work_absent = work_absent - && !__TBB_load_with_acquire(my_orphaned_tasks) - && abandonment_epoch == my_abandonment_epoch; - } -#endif /* __TBB_TASK_PRIORITY */ - // Test and test-and-set. - if( my_pool_state==busy ) { -#if __TBB_TASK_PRIORITY - bool no_fifo_tasks = my_task_stream.empty(top_priority); - work_absent = work_absent && (!dequeuing_possible || no_fifo_tasks) - && top_priority == my_top_priority && reload_epoch == my_reload_epoch; -#else - bool no_fifo_tasks = my_task_stream.empty(0); - work_absent = work_absent && no_fifo_tasks; -#endif /* __TBB_TASK_PRIORITY */ - if( work_absent ) { -#if __TBB_TASK_PRIORITY - if ( top_priority > my_bottom_priority ) { - if ( my_market->lower_arena_priority(*this, top_priority - 1, reload_epoch) - && !my_task_stream.empty(top_priority) ) - { - atomic_update( my_skipped_fifo_priority, top_priority, std::less<intptr_t>()); - } - } - else if ( !tasks_present && !my_orphaned_tasks && no_fifo_tasks ) { -#endif /* __TBB_TASK_PRIORITY */ - // save current demand value before setting SNAPSHOT_EMPTY, - // to avoid race with advertise_new_work. - int current_demand = (int)my_max_num_workers; - if( my_pool_state.compare_and_swap( SNAPSHOT_EMPTY, busy )==busy ) { -#if __TBB_ENQUEUE_ENFORCED_CONCURRENCY - if( my_concurrency_mode==cm_enforced_global ) { - // adjust_demand() called inside, if needed - my_market->mandatory_concurrency_disable( this ); - } else -#endif /* __TBB_ENQUEUE_ENFORCED_CONCURRENCY */ - { - // This thread transitioned pool to empty state, and thus is - // responsible for telling the market that there is no work to do. - my_market->adjust_demand( *this, -current_demand ); - } - restore_priority_if_need(); - return true; - } - return false; -#if __TBB_TASK_PRIORITY - } -#endif /* __TBB_TASK_PRIORITY */ - } - // Undo previous transition SNAPSHOT_FULL-->busy, unless another thread undid it. - my_pool_state.compare_and_swap( SNAPSHOT_FULL, busy ); - } - } - return false; - } - default: - // Another thread is taking a snapshot. - return false; - } +void arena::set_allotment(unsigned allotment) { + if (my_num_workers_allotted.load(std::memory_order_relaxed) != allotment) { + my_num_workers_allotted.store(allotment, std::memory_order_relaxed); } } -#if __TBB_COUNT_TASK_NODES -intptr_t arena::workers_task_node_count() { - intptr_t result = 0; - for( unsigned i = 1; i < my_num_slots; ++i ) { - generic_scheduler* s = my_slots[i].my_scheduler; - if( s ) - result += s->my_task_node_count; +int arena::update_concurrency(unsigned allotment) { + int delta = allotment - my_num_workers_allotted.load(std::memory_order_relaxed); + if (delta != 0) { + my_num_workers_allotted.store(allotment, std::memory_order_relaxed); } - return result; + return delta; } -#endif /* __TBB_COUNT_TASK_NODES */ -void arena::enqueue_task( task& t, intptr_t prio, FastRandom &random ) -{ -#if __TBB_RECYCLE_TO_ENQUEUE - __TBB_ASSERT( t.state()==task::allocated || t.state()==task::to_enqueue, "attempt to enqueue task with inappropriate state" ); -#else - __TBB_ASSERT( t.state()==task::allocated, "attempt to enqueue task that is not in 'allocated' state" ); -#endif - t.prefix().state = task::ready; - t.prefix().extra_state |= es_task_enqueued; // enqueued task marker +std::pair<int, int> arena::update_request(int mandatory_delta, int workers_delta) { + __TBB_ASSERT(-1 <= mandatory_delta && mandatory_delta <= 1, nullptr); -#if TBB_USE_ASSERT - if( task* parent = t.parent() ) { - internal::reference_count ref_count = parent->prefix().ref_count; - __TBB_ASSERT( ref_count!=0, "attempt to enqueue task whose parent has a ref_count==0 (forgot to set_ref_count?)" ); - __TBB_ASSERT( ref_count>0, "attempt to enqueue task whose parent has a ref_count<0" ); - parent->prefix().extra_state |= es_ref_count_active; - } - __TBB_ASSERT(t.prefix().affinity==affinity_id(0), "affinity is ignored for enqueued tasks"); -#endif /* TBB_USE_ASSERT */ + int min_workers_request = 0; + int max_workers_request = 0; - ITT_NOTIFY(sync_releasing, &my_task_stream); -#if __TBB_TASK_PRIORITY - intptr_t p = prio ? normalize_priority(priority_t(prio)) : normalized_normal_priority; - assert_priority_valid(p); - my_task_stream.push( &t, p, random ); - if ( p != my_top_priority ) - my_market->update_arena_priority( *this, p ); -#else /* !__TBB_TASK_PRIORITY */ - __TBB_ASSERT_EX(prio == 0, "the library is not configured to respect the task priority"); - my_task_stream.push( &t, 0, random ); -#endif /* !__TBB_TASK_PRIORITY */ + // Calculate min request + my_mandatory_requests += mandatory_delta; + min_workers_request = my_mandatory_requests > 0 ? 1 : 0; + + // Calculate max request + my_total_num_workers_requested += workers_delta; + // Clamp worker request into interval [0, my_max_num_workers] + max_workers_request = clamp(my_total_num_workers_requested, 0, + min_workers_request > 0 && is_arena_workerless() ? 1 : (int)my_max_num_workers); + + return { min_workers_request, max_workers_request }; +} + +thread_control_monitor& arena::get_waiting_threads_monitor() { + return my_threading_control->get_waiting_threads_monitor(); +} + +void arena::enqueue_task(d1::task& t, d1::task_group_context& ctx, thread_data& td) { + task_group_context_impl::bind_to(ctx, &td); + task_accessor::context(t) = &ctx; + task_accessor::isolation(t) = no_isolation; + my_fifo_task_stream.push( &t, random_lane_selector(td.my_random) ); advertise_new_work<work_enqueued>(); -#if __TBB_TASK_PRIORITY - if ( p != my_top_priority ) - my_market->update_arena_priority( *this, p ); -#endif /* __TBB_TASK_PRIORITY */ } -class nested_arena_context : no_copy { -public: - nested_arena_context(generic_scheduler *s, arena* a, size_t slot_index, bool type, bool same) - : my_scheduler(*s), my_orig_ctx(NULL), same_arena(same) { - if (same_arena) { - my_orig_state.my_properties = my_scheduler.my_properties; - my_orig_state.my_innermost_running_task = my_scheduler.my_innermost_running_task; - mimic_outermost_level(a, type); - } else { - my_orig_state = *s; - mimic_outermost_level(a, type); - s->nested_arena_entry(a, slot_index); - } - } - ~nested_arena_context() { -#if __TBB_TASK_GROUP_CONTEXT - my_scheduler.my_dummy_task->prefix().context = my_orig_ctx; // restore context of dummy task -#endif - if (same_arena) { - my_scheduler.my_properties = my_orig_state.my_properties; - my_scheduler.my_innermost_running_task = my_orig_state.my_innermost_running_task; - } else { - my_scheduler.nested_arena_exit(); - static_cast<scheduler_state&>(my_scheduler) = my_orig_state; // restore arena settings -#if __TBB_TASK_PRIORITY - my_scheduler.my_local_reload_epoch = *my_orig_state.my_ref_reload_epoch; -#endif - governor::assume_scheduler(&my_scheduler); - } - } +arena& arena::create(threading_control* control, unsigned num_slots, unsigned num_reserved_slots, unsigned arena_priority_level, d1::constraints constraints) { + __TBB_ASSERT(num_slots > 0, NULL); + __TBB_ASSERT(num_reserved_slots <= num_slots, NULL); + // Add public market reference for an external thread/task_arena (that adds an internal reference in exchange). + arena& a = arena::allocate_arena(control, num_slots, num_reserved_slots, arena_priority_level); + a.my_tc_client = control->create_client(a); + // We should not publish arena until all fields are initialized + control->publish_client(a.my_tc_client, constraints); + return a; +} -private: - generic_scheduler &my_scheduler; - scheduler_state my_orig_state; - task_group_context *my_orig_ctx; - const bool same_arena; - - void mimic_outermost_level(arena* a, bool type) { - my_scheduler.my_properties.outermost = true; - my_scheduler.my_properties.type = type; - my_scheduler.my_innermost_running_task = my_scheduler.my_dummy_task; -#if __TBB_TASK_GROUP_CONTEXT - // Save dummy's context and replace it by arena's context - my_orig_ctx = my_scheduler.my_dummy_task->prefix().context; - my_scheduler.my_dummy_task->prefix().context = a->my_default_ctx; +} // namespace r1 +} // namespace detail +} // namespace tbb + +// Enable task_arena.h +#include "oneapi/tbb/task_arena.h" // task_arena_base + +namespace tbb { +namespace detail { +namespace r1 { + +#if TBB_USE_ASSERT +void assert_arena_priority_valid( tbb::task_arena::priority a_priority ) { + bool is_arena_priority_correct = + a_priority == tbb::task_arena::priority::high || + a_priority == tbb::task_arena::priority::normal || + a_priority == tbb::task_arena::priority::low; + __TBB_ASSERT( is_arena_priority_correct, + "Task arena priority should be equal to one of the predefined values." ); +} +#else +void assert_arena_priority_valid( tbb::task_arena::priority ) {} #endif - } + +unsigned arena_priority_level( tbb::task_arena::priority a_priority ) { + assert_arena_priority_valid( a_priority ); + return d1::num_priority_levels - unsigned(int(a_priority) / d1::priority_stride); +} + +tbb::task_arena::priority arena_priority( unsigned priority_level ) { + auto priority = tbb::task_arena::priority( + (d1::num_priority_levels - priority_level) * d1::priority_stride + ); + assert_arena_priority_valid( priority ); + return priority; +} + +struct task_arena_impl { + static void initialize(d1::task_arena_base&); + static void terminate(d1::task_arena_base&); + static bool attach(d1::task_arena_base&); + static void execute(d1::task_arena_base&, d1::delegate_base&); + static void wait(d1::task_arena_base&); + static int max_concurrency(const d1::task_arena_base*); + static void enqueue(d1::task&, d1::task_group_context*, d1::task_arena_base*); + static d1::slot_id execution_slot(const d1::task_arena_base&); }; -void generic_scheduler::nested_arena_entry(arena* a, size_t slot_index) { - __TBB_ASSERT( is_alive(a->my_guard), NULL ); - __TBB_ASSERT( a!=my_arena, NULL); - - // overwrite arena settings -#if __TBB_TASK_PRIORITY - if ( my_offloaded_tasks ) - my_arena->orphan_offloaded_tasks( *this ); - my_offloaded_tasks = NULL; -#endif /* __TBB_TASK_PRIORITY */ - attach_arena( a, slot_index, /*is_master*/true ); - __TBB_ASSERT( my_arena == a, NULL ); - governor::assume_scheduler( this ); -#if __TBB_ARENA_OBSERVER - my_last_local_observer = 0; // TODO: try optimize number of calls - my_arena->my_observers.notify_entry_observers( my_last_local_observer, /*worker=*/false ); -#endif - // TODO? ITT_NOTIFY(sync_acquired, a->my_slots + index); - // TODO: it requires market to have P workers (not P-1) - // TODO: a preempted worker should be excluded from assignment to other arenas e.g. my_slack-- - if( !is_worker() && slot_index >= my_arena->my_num_reserved_slots ) - my_arena->my_market->adjust_demand(*my_arena, -1); -} - -void generic_scheduler::nested_arena_exit() { - if( !is_worker() && my_arena_index >= my_arena->my_num_reserved_slots ) - my_arena->my_market->adjust_demand(*my_arena, 1); -#if __TBB_ARENA_OBSERVER - my_arena->my_observers.notify_exit_observers( my_last_local_observer, /*worker=*/false ); -#endif /* __TBB_ARENA_OBSERVER */ - -#if __TBB_TASK_PRIORITY - if ( my_offloaded_tasks ) - my_arena->orphan_offloaded_tasks( *this ); -#endif - // Free the master slot. - __TBB_ASSERT(my_arena->my_slots[my_arena_index].my_scheduler, "A slot is already empty"); - __TBB_store_with_release(my_arena->my_slots[my_arena_index].my_scheduler, (generic_scheduler*)NULL); - my_arena->my_exit_monitors.notify_one(); // do not relax! +void __TBB_EXPORTED_FUNC initialize(d1::task_arena_base& ta) { + task_arena_impl::initialize(ta); +} +void __TBB_EXPORTED_FUNC terminate(d1::task_arena_base& ta) { + task_arena_impl::terminate(ta); +} +bool __TBB_EXPORTED_FUNC attach(d1::task_arena_base& ta) { + return task_arena_impl::attach(ta); +} +void __TBB_EXPORTED_FUNC execute(d1::task_arena_base& ta, d1::delegate_base& d) { + task_arena_impl::execute(ta, d); +} +void __TBB_EXPORTED_FUNC wait(d1::task_arena_base& ta) { + task_arena_impl::wait(ta); } -void generic_scheduler::wait_until_empty() { - my_dummy_task->prefix().ref_count++; // prevents exit from local_wait_for_all when local work is done enforcing the stealing - while( my_arena->my_pool_state != arena::SNAPSHOT_EMPTY ) - local_wait_for_all(*my_dummy_task, NULL); - my_dummy_task->prefix().ref_count--; +int __TBB_EXPORTED_FUNC max_concurrency(const d1::task_arena_base* ta) { + return task_arena_impl::max_concurrency(ta); } -} // namespace internal -} // namespace tbb +void __TBB_EXPORTED_FUNC enqueue(d1::task& t, d1::task_arena_base* ta) { + task_arena_impl::enqueue(t, nullptr, ta); +} -#include "scheduler_utility.h" -#include "tbb/task_arena.h" // task_arena_base +void __TBB_EXPORTED_FUNC enqueue(d1::task& t, d1::task_group_context& ctx, d1::task_arena_base* ta) { + task_arena_impl::enqueue(t, &ctx, ta); +} -namespace tbb { -namespace interface7 { -namespace internal { - -void task_arena_base::internal_initialize( ) { - governor::one_time_init(); - if( my_max_concurrency < 1 ) - my_max_concurrency = (int)governor::default_num_threads(); - __TBB_ASSERT( my_master_slots <= (unsigned)my_max_concurrency, "Number of slots reserved for master should not exceed arena concurrency"); - arena* new_arena = market::create_arena( my_max_concurrency, my_master_slots, 0 ); - // add an internal market reference; a public reference was added in create_arena - market &m = market::global_market( /*is_public=*/false ); - // allocate default context for task_arena -#if __TBB_TASK_GROUP_CONTEXT - new_arena->my_default_ctx = new ( NFS_Allocate(1, sizeof(task_group_context), NULL) ) - task_group_context( task_group_context::isolated, task_group_context::default_traits ); -#if __TBB_FP_CONTEXT - new_arena->my_default_ctx->capture_fp_settings(); -#endif -#endif /* __TBB_TASK_GROUP_CONTEXT */ - // threads might race to initialize the arena - if(as_atomic(my_arena).compare_and_swap(new_arena, NULL) != NULL) { - __TBB_ASSERT(my_arena, NULL); // another thread won the race - // release public market reference - m.release( /*is_public=*/true, /*blocking_terminate=*/false ); - new_arena->on_thread_leaving<arena::ref_external>(); // destroy unneeded arena -#if __TBB_TASK_GROUP_CONTEXT - spin_wait_while_eq(my_context, (task_group_context*)NULL); - } else { - new_arena->my_default_ctx->my_version_and_traits |= my_version_and_traits & exact_exception_flag; - as_atomic(my_context) = new_arena->my_default_ctx; -#endif - } - // TODO: should it trigger automatic initialization of this thread? - governor::local_scheduler_weak(); +d1::slot_id __TBB_EXPORTED_FUNC execution_slot(const d1::task_arena_base& arena) { + return task_arena_impl::execution_slot(arena); } -void task_arena_base::internal_terminate( ) { - if( my_arena ) {// task_arena was initialized - my_arena->my_market->release( /*is_public=*/true, /*blocking_terminate=*/false ); - my_arena->on_thread_leaving<arena::ref_external>(); - my_arena = 0; -#if __TBB_TASK_GROUP_CONTEXT - my_context = 0; -#endif +void task_arena_impl::initialize(d1::task_arena_base& ta) { + // Enforce global market initialization to properly initialize soft limit + (void)governor::get_thread_data(); + d1::constraints arena_constraints; + +#if __TBB_ARENA_BINDING + arena_constraints = d1::constraints{} + .set_core_type(ta.core_type()) + .set_max_threads_per_core(ta.max_threads_per_core()) + .set_numa_id(ta.my_numa_id); +#endif /*__TBB_ARENA_BINDING*/ + + if (ta.my_max_concurrency < 1) { +#if __TBB_ARENA_BINDING + ta.my_max_concurrency = (int)default_concurrency(arena_constraints); +#else /*!__TBB_ARENA_BINDING*/ + ta.my_max_concurrency = (int)governor::default_num_threads(); +#endif /*!__TBB_ARENA_BINDING*/ } + +#if __TBB_CPUBIND_PRESENT + numa_binding_observer* observer = construct_binding_observer( + static_cast<d1::task_arena*>(&ta), arena::num_arena_slots(ta.my_max_concurrency, ta.my_num_reserved_slots), + ta.my_numa_id, ta.core_type(), ta.max_threads_per_core()); + if (observer) { + // TODO: Consider lazy initialization for internal arena so + // the direct calls to observer might be omitted until actual initialization. + observer->on_scheduler_entry(true); + } +#endif /*__TBB_CPUBIND_PRESENT*/ + + __TBB_ASSERT(ta.my_arena.load(std::memory_order_relaxed) == nullptr, "Arena already initialized"); + unsigned priority_level = arena_priority_level(ta.my_priority); + threading_control* thr_control = threading_control::register_public_reference(); + arena& a = arena::create(thr_control, unsigned(ta.my_max_concurrency), ta.my_num_reserved_slots, priority_level, arena_constraints); + + ta.my_arena.store(&a, std::memory_order_release); +#if __TBB_CPUBIND_PRESENT + a.my_numa_binding_observer = observer; + if (observer) { + observer->on_scheduler_exit(true); + observer->observe(true); + } +#endif /*__TBB_CPUBIND_PRESENT*/ } -void task_arena_base::internal_attach( ) { - __TBB_ASSERT(!my_arena, NULL); - generic_scheduler* s = governor::local_scheduler_if_initialized(); - if( s && s->my_arena ) { +void task_arena_impl::terminate(d1::task_arena_base& ta) { + arena* a = ta.my_arena.load(std::memory_order_relaxed); + assert_pointer_valid(a); + threading_control::unregister_public_reference(/*blocking_terminate=*/false); + a->on_thread_leaving(arena::ref_external); + ta.my_arena.store(nullptr, std::memory_order_relaxed); +} + +bool task_arena_impl::attach(d1::task_arena_base& ta) { + __TBB_ASSERT(!ta.my_arena.load(std::memory_order_relaxed), nullptr); + thread_data* td = governor::get_thread_data_if_initialized(); + if( td && td->my_arena ) { + arena* a = td->my_arena; // There is an active arena to attach to. // It's still used by s, so won't be destroyed right away. - my_arena = s->my_arena; - __TBB_ASSERT( my_arena->my_references > 0, NULL ); - my_arena->my_references += arena::ref_external; -#if __TBB_TASK_GROUP_CONTEXT - my_context = my_arena->my_default_ctx; - my_version_and_traits |= my_context->my_version_and_traits & exact_exception_flag; -#endif - my_master_slots = my_arena->my_num_reserved_slots; - my_max_concurrency = my_master_slots + my_arena->my_max_num_workers; - __TBB_ASSERT(arena::num_arena_slots(my_max_concurrency)==my_arena->my_num_slots, NULL); - // increases market's ref count for task_arena - market::global_market( /*is_public=*/true ); - } -} - -void task_arena_base::internal_enqueue( task& t, intptr_t prio ) const { - __TBB_ASSERT(my_arena, NULL); - generic_scheduler* s = governor::local_scheduler_if_initialized(); - __TBB_ASSERT(s, "Scheduler is not initialized"); // we allocated a task so can expect the scheduler -#if __TBB_TASK_GROUP_CONTEXT - __TBB_ASSERT(my_arena->my_default_ctx == t.prefix().context, NULL); - __TBB_ASSERT(!my_arena->my_default_ctx->is_group_execution_cancelled(), // TODO: any better idea? - "The task will not be executed because default task_group_context of task_arena is cancelled. Has previously enqueued task thrown an exception?"); -#endif - my_arena->enqueue_task( t, prio, s->my_random ); -} - -class delegated_task : public task { - internal::delegate_base & my_delegate; - concurrent_monitor & my_monitor; - task * my_root; - task* execute() __TBB_override { - generic_scheduler& s = *(generic_scheduler*)prefix().owner; - __TBB_ASSERT(s.outermost_level(), "expected to be enqueued and received on the outermost level"); - struct outermost_context : internal::no_copy { - delegated_task * t; - generic_scheduler & s; - task * orig_dummy; - task_group_context * orig_ctx; - scheduler_properties orig_props; - outermost_context(delegated_task *_t, generic_scheduler &_s) - : t(_t), s(_s), orig_dummy(s.my_dummy_task), orig_props(s.my_properties) { - __TBB_ASSERT(s.my_innermost_running_task == t, NULL); -#if __TBB_TASK_GROUP_CONTEXT - orig_ctx = t->prefix().context; - t->prefix().context = s.my_arena->my_default_ctx; -#endif - // Mimics outermost master - s.my_dummy_task = t; - s.my_properties.type = scheduler_properties::master; + __TBB_ASSERT(a->my_references > 0, nullptr); + a->my_references += arena::ref_external; + ta.my_num_reserved_slots = a->my_num_reserved_slots; + ta.my_priority = arena_priority(a->my_priority_level); + ta.my_max_concurrency = ta.my_num_reserved_slots + a->my_max_num_workers; + __TBB_ASSERT(arena::num_arena_slots(ta.my_max_concurrency, ta.my_num_reserved_slots) == a->my_num_slots, nullptr); + ta.my_arena.store(a, std::memory_order_release); + // increases threading_control's ref count for task_arena + threading_control::register_public_reference(); + return true; + } + return false; +} + +void task_arena_impl::enqueue(d1::task& t, d1::task_group_context* c, d1::task_arena_base* ta) { + thread_data* td = governor::get_thread_data(); // thread data is only needed for FastRandom instance + assert_pointer_valid(td, "thread_data pointer should not be null"); + arena* a = ta ? + ta->my_arena.load(std::memory_order_relaxed) + : td->my_arena + ; + assert_pointer_valid(a, "arena pointer should not be null"); + auto* ctx = c ? c : a->my_default_ctx; + assert_pointer_valid(ctx, "context pointer should not be null"); + // Is there a better place for checking the state of ctx? + __TBB_ASSERT(!a->my_default_ctx->is_group_execution_cancelled(), + "The task will not be executed because its task_group_context is cancelled."); + a->enqueue_task(t, *ctx, *td); +} + +d1::slot_id task_arena_impl::execution_slot(const d1::task_arena_base& ta) { + thread_data* td = governor::get_thread_data_if_initialized(); + if (td && (td->is_attached_to(ta.my_arena.load(std::memory_order_relaxed)))) { + return td->my_arena_index; + } + return d1::slot_id(-1); +} + +class nested_arena_context : no_copy { +public: + nested_arena_context(thread_data& td, arena& nested_arena, std::size_t slot_index) + : m_orig_execute_data_ext(td.my_task_dispatcher->m_execute_data_ext) + { + if (td.my_arena != &nested_arena) { + m_orig_arena = td.my_arena; + m_orig_slot_index = td.my_arena_index; + m_orig_last_observer = td.my_last_observer; + m_orig_is_thread_registered = td.my_is_registered; + + td.detach_task_dispatcher(); + td.attach_arena(nested_arena, slot_index); + td.my_is_registered = false; + if (td.my_inbox.is_idle_state(true)) + td.my_inbox.set_is_idle(false); + task_dispatcher& task_disp = td.my_arena_slot->default_task_dispatcher(); + td.enter_task_dispatcher(task_disp, m_orig_execute_data_ext.task_disp->m_stealing_threshold); + + // If the calling thread occupies the slots out of external thread reserve we need to notify the + // market that this arena requires one worker less. + if (td.my_arena_index >= td.my_arena->my_num_reserved_slots) { + td.my_arena->request_workers(/* mandatory_delta = */ 0, /* workers_delta = */ -1); } - ~outermost_context() { -#if __TBB_TASK_GROUP_CONTEXT - // Restore context for sake of registering potential exception - t->prefix().context = orig_ctx; -#endif - s.my_properties = orig_props; - s.my_dummy_task = orig_dummy; + + td.my_last_observer = nullptr; + // The task_arena::execute method considers each calling thread as an external thread. + td.my_arena->my_observers.notify_entry_observers(td.my_last_observer, /* worker*/false); + } + + m_task_dispatcher = td.my_task_dispatcher; + m_orig_fifo_tasks_allowed = m_task_dispatcher->allow_fifo_task(true); + m_orig_critical_task_allowed = m_task_dispatcher->m_properties.critical_task_allowed; + m_task_dispatcher->m_properties.critical_task_allowed = true; + + execution_data_ext& ed_ext = td.my_task_dispatcher->m_execute_data_ext; + ed_ext.context = td.my_arena->my_default_ctx; + ed_ext.original_slot = td.my_arena_index; + ed_ext.affinity_slot = d1::no_slot; + ed_ext.task_disp = td.my_task_dispatcher; + ed_ext.isolation = no_isolation; + + __TBB_ASSERT(td.my_arena_slot, nullptr); + __TBB_ASSERT(td.my_arena_slot->is_occupied(), nullptr); + __TBB_ASSERT(td.my_task_dispatcher, nullptr); + } + ~nested_arena_context() { + thread_data& td = *m_task_dispatcher->m_thread_data; + __TBB_ASSERT(governor::is_thread_data_set(&td), nullptr); + m_task_dispatcher->allow_fifo_task(m_orig_fifo_tasks_allowed); + m_task_dispatcher->m_properties.critical_task_allowed = m_orig_critical_task_allowed; + if (m_orig_arena) { + td.my_arena->my_observers.notify_exit_observers(td.my_last_observer, /*worker*/ false); + td.my_last_observer = m_orig_last_observer; + + // Notify the market that this thread releasing a one slot + // that can be used by a worker thread. + if (td.my_arena_index >= td.my_arena->my_num_reserved_slots) { + td.my_arena->request_workers(/* mandatory_delta = */ 0, /* workers_delta = */ 1); } - } scope(this, s); - my_delegate(); - return NULL; + + td.leave_task_dispatcher(); + td.my_arena_slot->release(); + td.my_arena->my_exit_monitors.notify_one(); // do not relax! + td.my_is_registered = m_orig_is_thread_registered; + td.attach_arena(*m_orig_arena, m_orig_slot_index); + td.attach_task_dispatcher(*m_orig_execute_data_ext.task_disp); + __TBB_ASSERT(td.my_inbox.is_idle_state(false), nullptr); + } + td.my_task_dispatcher->m_execute_data_ext = m_orig_execute_data_ext; } - ~delegated_task() { - // potential exception was already registered. It must happen before the notification - __TBB_ASSERT(my_root->ref_count()==2, NULL); - __TBB_store_with_release(my_root->prefix().ref_count, 1); // must precede the wakeup - my_monitor.notify(*this); // do not relax, it needs a fence! + +private: + execution_data_ext m_orig_execute_data_ext{}; + arena* m_orig_arena{ nullptr }; + observer_proxy* m_orig_last_observer{ nullptr }; + task_dispatcher* m_task_dispatcher{ nullptr }; + unsigned m_orig_slot_index{}; + bool m_orig_fifo_tasks_allowed{}; + bool m_orig_critical_task_allowed{}; + bool m_orig_is_thread_registered{}; +}; + +class delegated_task : public d1::task { + d1::delegate_base& m_delegate; + concurrent_monitor& m_monitor; + d1::wait_context& m_wait_ctx; + std::atomic<bool> m_completed; + d1::task* execute(d1::execution_data& ed) override { + const execution_data_ext& ed_ext = static_cast<const execution_data_ext&>(ed); + execution_data_ext orig_execute_data_ext = ed_ext.task_disp->m_execute_data_ext; + __TBB_ASSERT(&ed_ext.task_disp->m_execute_data_ext == &ed, + "The execute data shall point to the current task dispatcher execute data"); + __TBB_ASSERT(ed_ext.task_disp->m_execute_data_ext.isolation == no_isolation, nullptr); + + ed_ext.task_disp->m_execute_data_ext.context = ed_ext.task_disp->get_thread_data().my_arena->my_default_ctx; + bool fifo_task_allowed = ed_ext.task_disp->allow_fifo_task(true); + try_call([&] { + m_delegate(); + }).on_completion([&] { + ed_ext.task_disp->m_execute_data_ext = orig_execute_data_ext; + ed_ext.task_disp->allow_fifo_task(fifo_task_allowed); + }); + + finalize(); + return nullptr; + } + d1::task* cancel(d1::execution_data&) override { + finalize(); + return nullptr; + } + void finalize() { + m_wait_ctx.release(); // must precede the wakeup + m_monitor.notify([this] (std::uintptr_t ctx) { + return ctx == std::uintptr_t(&m_delegate); + }); // do not relax, it needs a fence! + m_completed.store(true, std::memory_order_release); } public: - delegated_task( internal::delegate_base & d, concurrent_monitor & s, task * t ) - : my_delegate(d), my_monitor(s), my_root(t) {} - // predicate for concurrent_monitor notification - bool operator()(uintptr_t ctx) const { return (void*)ctx == (void*)&my_delegate; } + delegated_task(d1::delegate_base& d, concurrent_monitor& s, d1::wait_context& wo) + : m_delegate(d), m_monitor(s), m_wait_ctx(wo), m_completed{ false }{} + ~delegated_task() override { + // The destructor can be called earlier than the m_monitor is notified + // because the waiting thread can be released after m_wait_ctx.release_wait. + // To close that race we wait for the m_completed signal. + spin_wait_until_eq(m_completed, true); + } }; -void task_arena_base::internal_execute(internal::delegate_base& d) const { - __TBB_ASSERT(my_arena, NULL); - generic_scheduler* s = governor::local_scheduler_weak(); - __TBB_ASSERT(s, "Scheduler is not initialized"); +void task_arena_impl::execute(d1::task_arena_base& ta, d1::delegate_base& d) { + arena* a = ta.my_arena.load(std::memory_order_relaxed); + __TBB_ASSERT(a != nullptr, nullptr); + thread_data* td = governor::get_thread_data(); - bool same_arena = s->my_arena == my_arena; - size_t index1 = s->my_arena_index; + bool same_arena = td->my_arena == a; + std::size_t index1 = td->my_arena_index; if (!same_arena) { - index1 = my_arena->occupy_free_slot</* as_worker*/false>(*s); + index1 = a->occupy_free_slot</*as_worker */false>(*td); if (index1 == arena::out_of_arena) { - -#if __TBB_USE_OPTIONAL_RTTI - // Workaround for the bug inside graph. If the thread can not occupy arena slot during task_arena::execute() - // and all aggregator operations depend on this task completion (all other threads are inside arena already) - // deadlock appears, because enqueued task will never enter arena. - // Workaround: check if the task came from graph via RTTI (casting to graph::spawn_functor) - // and enqueue this task with non-blocking internal_enqueue method. - // TODO: have to change behaviour later in next GOLD release (maybe to add new library entry point - try_execute) - typedef tbb::flow::interface10::graph::spawn_functor graph_funct; - internal::delegated_function< graph_funct, void >* deleg_funct = - dynamic_cast< internal::delegated_function< graph_funct, void>* >(&d); - - if (deleg_funct) { - internal_enqueue(*new(task::allocate_root(*my_context)) - internal::function_task< internal::strip< graph_funct >::type > - (internal::forward< graph_funct >(deleg_funct->my_func)), 0); - return; - } else { -#endif - concurrent_monitor::thread_context waiter; -#if __TBB_TASK_GROUP_CONTEXT - task_group_context exec_context(task_group_context::isolated, my_version_and_traits & exact_exception_flag); -#if __TBB_FP_CONTEXT - exec_context.copy_fp_settings(*my_context); -#endif -#endif - auto_empty_task root(__TBB_CONTEXT_ARG(s, &exec_context)); - root.prefix().ref_count = 2; - my_arena->enqueue_task(*new(task::allocate_root(__TBB_CONTEXT_ARG1(exec_context))) - delegated_task(d, my_arena->my_exit_monitors, &root), - 0, s->my_random); // TODO: priority? - size_t index2 = arena::out_of_arena; - do { - my_arena->my_exit_monitors.prepare_wait(waiter, (uintptr_t)&d); - if (__TBB_load_with_acquire(root.prefix().ref_count) < 2) { - my_arena->my_exit_monitors.cancel_wait(waiter); - break; - } - index2 = my_arena->occupy_free_slot</*as_worker*/false>(*s); - if (index2 != arena::out_of_arena) { - my_arena->my_exit_monitors.cancel_wait(waiter); - nested_arena_context scope(s, my_arena, index2, scheduler_properties::master, same_arena); - s->local_wait_for_all(root, NULL); -#if TBB_USE_EXCEPTIONS - __TBB_ASSERT(!exec_context.my_exception, NULL); // exception can be thrown above, not deferred -#endif - __TBB_ASSERT(root.prefix().ref_count == 0, NULL); - break; - } - my_arena->my_exit_monitors.commit_wait(waiter); - } while (__TBB_load_with_acquire(root.prefix().ref_count) == 2); - if (index2 == arena::out_of_arena) { - // notify a waiting thread even if this thread did not enter arena, - // in case it was woken by a leaving thread but did not need to enter - my_arena->my_exit_monitors.notify_one(); // do not relax! + concurrent_monitor::thread_context waiter((std::uintptr_t)&d); + d1::wait_context wo(1); + d1::task_group_context exec_context(d1::task_group_context::isolated); + task_group_context_impl::copy_fp_settings(exec_context, *a->my_default_ctx); + + delegated_task dt(d, a->my_exit_monitors, wo); + a->enqueue_task( dt, exec_context, *td); + size_t index2 = arena::out_of_arena; + do { + a->my_exit_monitors.prepare_wait(waiter); + if (!wo.continue_execution()) { + a->my_exit_monitors.cancel_wait(waiter); + break; } -#if TBB_USE_EXCEPTIONS - // process possible exception - if (task_group_context::exception_container_type *pe = exec_context.my_exception) - TbbRethrowException(pe); -#endif - return; -#if __TBB_USE_OPTIONAL_RTTI - } // if task came from graph -#endif + index2 = a->occupy_free_slot</*as_worker*/false>(*td); + if (index2 != arena::out_of_arena) { + a->my_exit_monitors.cancel_wait(waiter); + nested_arena_context scope(*td, *a, index2 ); + r1::wait(wo, exec_context); + __TBB_ASSERT(!exec_context.my_exception.load(std::memory_order_relaxed), nullptr); // exception can be thrown above, not deferred + break; + } + a->my_exit_monitors.commit_wait(waiter); + } while (wo.continue_execution()); + if (index2 == arena::out_of_arena) { + // notify a waiting thread even if this thread did not enter arena, + // in case it was woken by a leaving thread but did not need to enter + a->my_exit_monitors.notify_one(); // do not relax! + } + // process possible exception + auto exception = exec_context.my_exception.load(std::memory_order_acquire); + if (exception) { + __TBB_ASSERT(exec_context.is_group_execution_cancelled(), "The task group context with an exception should be canceled."); + exception->throw_self(); + } + __TBB_ASSERT(governor::is_thread_data_set(td), nullptr); + return; } // if (index1 == arena::out_of_arena) } // if (!same_arena) - cpu_ctl_env_helper cpu_ctl_helper; - cpu_ctl_helper.set_env(__TBB_CONTEXT_ARG1(my_context)); -#if TBB_USE_EXCEPTIONS + context_guard_helper</*report_tasks=*/false> context_guard; + context_guard.set_ctx(a->my_default_ctx); + nested_arena_context scope(*td, *a, index1); +#if _WIN64 try { #endif - //TODO: replace dummy tasks for workers as well to avoid using of the_dummy_context - nested_arena_context scope(s, my_arena, index1, scheduler_properties::master, same_arena); d(); -#if TBB_USE_EXCEPTIONS - } - catch (...) { - cpu_ctl_helper.restore_default(); // TODO: is it needed on Windows? - if (my_version_and_traits & exact_exception_flag) throw; - else { - task_group_context exception_container(task_group_context::isolated, - task_group_context::default_traits & ~task_group_context::exact_exception); - exception_container.register_pending_exception(); - __TBB_ASSERT(exception_container.my_exception, NULL); - TbbRethrowException(exception_container.my_exception); - } + __TBB_ASSERT(same_arena || governor::is_thread_data_set(td), nullptr); +#if _WIN64 + } catch (...) { + context_guard.restore_default(); + throw; } #endif } -// this wait task is a temporary approach to wait for arena emptiness for masters without slots -// TODO: it will be rather reworked for one source of notification from is_out_of_work -class wait_task : public task { - binary_semaphore & my_signal; - task* execute() __TBB_override { - generic_scheduler* s = governor::local_scheduler_if_initialized(); - __TBB_ASSERT( s, NULL ); - __TBB_ASSERT( s->outermost_level(), "The enqueued task can be processed only on outermost level" ); - if ( s->is_worker() ) { - __TBB_ASSERT( s->my_innermost_running_task == this, NULL ); - // Mimic worker on outermost level to run remaining tasks - s->my_innermost_running_task = s->my_dummy_task; - s->local_wait_for_all( *s->my_dummy_task, NULL ); - s->my_innermost_running_task = this; - } else s->my_arena->is_out_of_work(); // avoids starvation of internal_wait: issuing this task makes arena full - my_signal.V(); - return NULL; - } -public: - wait_task ( binary_semaphore & sema ) : my_signal(sema) {} -}; - -void task_arena_base::internal_wait() const { - __TBB_ASSERT(my_arena, NULL); - generic_scheduler* s = governor::local_scheduler_weak(); - __TBB_ASSERT(s, "Scheduler is not initialized"); - __TBB_ASSERT(s->my_arena != my_arena || s->my_arena_index == 0, "task_arena::wait_until_empty() is not supported within a worker context" ); - if( s->my_arena == my_arena ) { - //unsupported, but try do something for outermost master - __TBB_ASSERT(s->master_outermost_level(), "unsupported"); - if( !s->my_arena_index ) - while( my_arena->num_workers_active() ) - s->wait_until_empty(); - } else for(;;) { - while( my_arena->my_pool_state != arena::SNAPSHOT_EMPTY ) { - if( !__TBB_load_with_acquire(my_arena->my_slots[0].my_scheduler) // TODO TEMP: one master, make more masters - && as_atomic(my_arena->my_slots[0].my_scheduler).compare_and_swap(s, NULL) == NULL ) { - nested_arena_context a(s, my_arena, 0, scheduler_properties::worker, false); - s->wait_until_empty(); - } else { - binary_semaphore waiter; // TODO: replace by a single event notification from is_out_of_work - internal_enqueue( *new( task::allocate_root(__TBB_CONTEXT_ARG1(*my_context)) ) wait_task(waiter), 0 ); // TODO: priority? - waiter.P(); // TODO: concurrent_monitor - } +void task_arena_impl::wait(d1::task_arena_base& ta) { + arena* a = ta.my_arena.load(std::memory_order_relaxed); + __TBB_ASSERT(a != nullptr, nullptr); + thread_data* td = governor::get_thread_data(); + __TBB_ASSERT_EX(td, "Scheduler is not initialized"); + __TBB_ASSERT(td->my_arena != a || td->my_arena_index == 0, "internal_wait is not supported within a worker context" ); + if (a->my_max_num_workers != 0) { + while (a->num_workers_active() || !a->is_empty()) { + yield(); } - if( !my_arena->num_workers_active() && !my_arena->my_slots[0].my_scheduler) // no activity - break; // spin until workers active but avoid spinning in a worker - __TBB_Yield(); // wait until workers and master leave } } -/*static*/ int task_arena_base::internal_current_slot() { - generic_scheduler* s = governor::local_scheduler_if_initialized(); - return s? int(s->my_arena_index) : -1; -} +int task_arena_impl::max_concurrency(const d1::task_arena_base *ta) { + arena* a = nullptr; + if( ta ) // for special cases of ta->max_concurrency() + a = ta->my_arena.load(std::memory_order_relaxed); + else if( thread_data* td = governor::get_thread_data_if_initialized() ) + a = td->my_arena; // the current arena if any -#if __TBB_TASK_ISOLATION -class isolation_guard : tbb::internal::no_copy { - isolation_tag &guarded; - isolation_tag previous_value; -public: - isolation_guard( isolation_tag &isolation ) : guarded( isolation ), previous_value( isolation ) {} - ~isolation_guard() { - guarded = previous_value; + if( a ) { // Get parameters from the arena + __TBB_ASSERT( !ta || ta->my_max_concurrency==1, nullptr); + int mandatory_worker = 0; + if (a->is_arena_workerless() && a->my_num_reserved_slots == 1) { + mandatory_worker = a->my_mandatory_concurrency.test() ? 1 : 0; + } + return a->my_num_reserved_slots + a->my_max_num_workers + mandatory_worker; } -}; -void isolate_within_arena( delegate_base& d, intptr_t reserved ) { - __TBB_ASSERT( reserved == 0, NULL ); - // TODO: Decide what to do if the scheduler is not initialized. Is there a use case for it? - generic_scheduler* s = governor::local_scheduler_weak(); - __TBB_ASSERT( s, "this_task_arena::isolate() needs an initialized scheduler" ); - // Theoretically, we can keep the current isolation in the scheduler; however, it makes sense to store it in innermost - // running task because it can in principle be queried via task::self(). - isolation_tag& current_isolation = s->my_innermost_running_task->prefix().isolation; - // We temporarily change the isolation tag of the currently running task. It will be restored in the destructor of the guard. - isolation_guard guard( current_isolation ); - current_isolation = reinterpret_cast<isolation_tag>(&d); - d(); -} -#endif /* __TBB_TASK_ISOLATION */ - -int task_arena_base::internal_max_concurrency(const task_arena *ta) { - arena* a = NULL; - if( ta ) // for special cases of ta->max_concurrency() - a = ta->my_arena; - else if( generic_scheduler* s = governor::local_scheduler_if_initialized() ) - a = s->my_arena; // the current arena if any + if (ta && ta->my_max_concurrency == 1) { + return 1; + } - if( a ) { // Get parameters from the arena - __TBB_ASSERT( !ta || ta->my_max_concurrency==1, NULL ); - return a->my_num_reserved_slots + a->my_max_num_workers; - } else { - __TBB_ASSERT( !ta || ta->my_max_concurrency==automatic, NULL ); - return int(governor::default_num_threads()); +#if __TBB_ARENA_BINDING + if (ta) { + d1::constraints arena_constraints = d1::constraints{} + .set_numa_id(ta->my_numa_id) + .set_core_type(ta->core_type()) + .set_max_threads_per_core(ta->max_threads_per_core()); + return (int)default_concurrency(arena_constraints); } +#endif /*!__TBB_ARENA_BINDING*/ + + __TBB_ASSERT(!ta || ta->my_max_concurrency==d1::task_arena_base::automatic, nullptr); + return int(governor::default_num_threads()); } -} // tbb::interfaceX::internal -} // tbb::interfaceX -} // tbb + +void isolate_within_arena(d1::delegate_base& d, std::intptr_t isolation) { + // TODO: Decide what to do if the scheduler is not initialized. Is there a use case for it? + thread_data* tls = governor::get_thread_data(); + assert_pointers_valid(tls, tls->my_task_dispatcher); + task_dispatcher* dispatcher = tls->my_task_dispatcher; + isolation_type previous_isolation = dispatcher->m_execute_data_ext.isolation; + try_call([&] { + // We temporarily change the isolation tag of the currently running task. It will be restored in the destructor of the guard. + isolation_type current_isolation = isolation ? isolation : reinterpret_cast<isolation_type>(&d); + // Save the current isolation value and set new one + previous_isolation = dispatcher->set_isolation(current_isolation); + // Isolation within this callable + d(); + }).on_completion([&] { + __TBB_ASSERT(governor::get_thread_data()->my_task_dispatcher == dispatcher, nullptr); + dispatcher->set_isolation(previous_isolation); + }); +} + +} // namespace r1 +} // namespace detail +} // namespace tbb diff --git a/src/tbb/src/tbb/arena.h b/src/tbb/src/tbb/arena.h index 2bc23c4bb..1e95f117b 100644 --- a/src/tbb/src/tbb/arena.h +++ b/src/tbb/src/tbb/arena.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2024 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,159 +12,266 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef _TBB_arena_H #define _TBB_arena_H -#include "tbb/tbb_stddef.h" -#include "tbb/atomic.h" +#include <atomic> +#include <cstring> -#include "tbb/tbb_machine.h" +#include "oneapi/tbb/detail/_task.h" +#include "oneapi/tbb/detail/_utils.h" +#include "oneapi/tbb/spin_mutex.h" #include "scheduler_common.h" #include "intrusive_list.h" #include "task_stream.h" -#include "../rml/include/rml_tbb.h" +#include "arena_slot.h" +#include "rml_tbb.h" #include "mailbox.h" -#include "observer_proxy.h" -#include "market.h" #include "governor.h" #include "concurrent_monitor.h" +#include "observer_proxy.h" +#include "thread_control_monitor.h" +#include "threading_control_client.h" namespace tbb { +namespace detail { +namespace r1 { +class task_dispatcher; class task_group_context; +class threading_control; class allocate_root_with_context_proxy; -namespace internal { +#if __TBB_ARENA_BINDING +class numa_binding_observer; +#endif /*__TBB_ARENA_BINDING*/ + +//! Bounded coroutines cache LIFO ring buffer +class arena_co_cache { + //! Ring buffer storage + task_dispatcher** my_co_scheduler_cache; + //! Current cache index + unsigned my_head; + //! Cache capacity for arena + unsigned my_max_index; + //! Accessor lock for modification operations + tbb::spin_mutex my_co_cache_mutex; + + unsigned next_index() { + return ( my_head == my_max_index ) ? 0 : my_head + 1; + } + + unsigned prev_index() { + return ( my_head == 0 ) ? my_max_index : my_head - 1; + } + + bool internal_empty() { + return my_co_scheduler_cache[prev_index()] == nullptr; + } + + void internal_task_dispatcher_cleanup(task_dispatcher* to_cleanup) { + to_cleanup->~task_dispatcher(); + cache_aligned_deallocate(to_cleanup); + } + +public: + void init(unsigned cache_capacity) { + std::size_t alloc_size = cache_capacity * sizeof(task_dispatcher*); + my_co_scheduler_cache = (task_dispatcher**)cache_aligned_allocate(alloc_size); + std::memset( my_co_scheduler_cache, 0, alloc_size ); + my_head = 0; + my_max_index = cache_capacity - 1; + } + + void cleanup() { + while (task_dispatcher* to_cleanup = pop()) { + internal_task_dispatcher_cleanup(to_cleanup); + } + cache_aligned_deallocate(my_co_scheduler_cache); + } + + //! Insert scheduler to the current available place. + //! Replace an old value, if necessary. + void push(task_dispatcher* s) { + task_dispatcher* to_cleanup = nullptr; + { + tbb::spin_mutex::scoped_lock lock(my_co_cache_mutex); + // Check if we are replacing some existing buffer entrance + if (my_co_scheduler_cache[my_head] != nullptr) { + to_cleanup = my_co_scheduler_cache[my_head]; + } + // Store the cached value + my_co_scheduler_cache[my_head] = s; + // Move head index to the next slot + my_head = next_index(); + } + // Cleanup replaced buffer if any + if (to_cleanup) { + internal_task_dispatcher_cleanup(to_cleanup); + } + } + + //! Get a cached scheduler if any + task_dispatcher* pop() { + tbb::spin_mutex::scoped_lock lock(my_co_cache_mutex); + // No cached coroutine + if (internal_empty()) { + return nullptr; + } + // Move head index to the currently available value + my_head = prev_index(); + // Retrieve the value from the buffer + task_dispatcher* to_return = my_co_scheduler_cache[my_head]; + // Clear the previous entrance value + my_co_scheduler_cache[my_head] = nullptr; + return to_return; + } +}; + +struct stack_anchor_type { + stack_anchor_type() = default; + stack_anchor_type(const stack_anchor_type&) = delete; +}; + +class atomic_flag { + static const std::uintptr_t SET = 1; + static const std::uintptr_t UNSET = 0; + std::atomic<std::uintptr_t> my_state{UNSET}; +public: + bool test_and_set() { + std::uintptr_t state = my_state.load(std::memory_order_acquire); + switch (state) { + case SET: + return false; + default: /* busy */ + if (my_state.compare_exchange_strong(state, SET)) { + // We interrupted clear transaction + return false; + } + if (state != UNSET) { + // We lost our epoch + return false; + } + // We are too late but still in the same epoch + __TBB_fallthrough; + case UNSET: + return my_state.compare_exchange_strong(state, SET); + } + } + template <typename Pred> + bool try_clear_if(Pred&& pred) { + std::uintptr_t busy = std::uintptr_t(&busy); + std::uintptr_t state = my_state.load(std::memory_order_acquire); + if (state == SET && my_state.compare_exchange_strong(state, busy)) { + if (pred()) { + return my_state.compare_exchange_strong(busy, UNSET); + } + // The result of the next operation is discarded, always false should be returned. + my_state.compare_exchange_strong(busy, SET); + } + return false; + } + bool test(std::memory_order order = std::memory_order_acquire) { + return my_state.load(order) != UNSET; + } +}; //! The structure of an arena, except the array of slots. /** Separated in order to simplify padding. Intrusive list node base class is used by market to form a list of arenas. **/ +// TODO: Analyze arena_base cache lines placement struct arena_base : padded<intrusive_list_node> { //! The number of workers that have been marked out by the resource manager to service the arena. - unsigned my_num_workers_allotted; // heavy use in stealing loop + std::atomic<unsigned> my_num_workers_allotted; // heavy use in stealing loop //! Reference counter for the arena. - /** Worker and master references are counted separately: first several bits are for references - from master threads or explicit task_arenas (see arena::ref_external_bits below); + /** Worker and external thread references are counted separately: first several bits are for references + from external thread threads or explicit task_arenas (see arena::ref_external_bits below); the rest counts the number of workers servicing the arena. */ - atomic<unsigned> my_references; // heavy use in stealing loop - -#if __TBB_TASK_PRIORITY - //! The highest priority of recently spawned or enqueued tasks. - volatile intptr_t my_top_priority; // heavy use in stealing loop -#endif /* !__TBB_TASK_PRIORITY */ + std::atomic<unsigned> my_references; // heavy use in stealing loop //! The maximal number of currently busy slots. - atomic<unsigned> my_limit; // heavy use in stealing loop + std::atomic<unsigned> my_limit; // heavy use in stealing loop //! Task pool for the tasks scheduled via task::enqueue() method. /** Such scheduling guarantees eventual execution even if - new tasks are constantly coming (by extracting scheduled tasks in relaxed FIFO order); - - the enqueuing thread does not call any of wait_for_all methods. - Depending on __TBB_TASK_PRIORITY, num_priority_levels can be 1 or more. **/ - task_stream<num_priority_levels> my_task_stream; // heavy use in stealing loop + - the enqueuing thread does not call any of wait_for_all methods. **/ + task_stream<front_accessor> my_fifo_task_stream; // heavy use in stealing loop + + //! Task pool for the tasks scheduled via tbb::resume() function. + task_stream<front_accessor> my_resume_task_stream; // heavy use in stealing loop + +#if __TBB_PREVIEW_CRITICAL_TASKS + //! Task pool for the tasks with critical property set. + /** Critical tasks are scheduled for execution ahead of other sources (including local task pool + and even bypassed tasks) unless the thread already executes a critical task in an outer + dispatch loop **/ + // used on the hot path of the task dispatch loop + task_stream<back_nonnull_accessor> my_critical_task_stream; +#endif - //! The number of workers requested by the master thread owning the arena. - unsigned my_max_num_workers; + //! The total number of workers that are requested from the resource manager. + int my_total_num_workers_requested; - //! The number of workers that are currently requested from the resource manager. - int my_num_workers_requested; + //! The index in the array of per priority lists of arenas this object is in. + /*const*/ unsigned my_priority_level; + + //! The max priority level of arena in permit manager. + std::atomic<bool> my_is_top_priority{false}; //! Current task pool state and estimate of available tasks amount. - /** The estimate is either 0 (SNAPSHOT_EMPTY) or infinity (SNAPSHOT_FULL). - Special state is "busy" (any other unsigned value). - Note that the implementation of arena::is_busy_or_empty() requires - my_pool_state to be unsigned. */ - tbb::atomic<uintptr_t> my_pool_state; + atomic_flag my_pool_state; -#if __TBB_ARENA_OBSERVER //! The list of local observers attached to this arena. observer_list my_observers; -#endif - -#if __TBB_TASK_PRIORITY - //! The lowest normalized priority of available spawned or enqueued tasks. - intptr_t my_bottom_priority; - //! Tracks events that may bring tasks in offload areas to the top priority level. - /** Incremented when arena top priority changes or a task group priority - is elevated to the current arena's top level. **/ - uintptr_t my_reload_epoch; - - //! The list of offloaded tasks abandoned by workers revoked by the market. - task* my_orphaned_tasks; - - //! Counter used to track the occurrence of recent orphaning and re-sharing operations. - tbb::atomic<uintptr_t> my_abandonment_epoch; - - //! The highest priority level containing enqueued tasks. - /** It being greater than 0 means that high priority enqueued tasks had to be - bypassed because all workers were blocked in nested dispatch loops and - were unable to progress at then current priority level. **/ - tbb::atomic<intptr_t> my_skipped_fifo_priority; -#endif /* !__TBB_TASK_PRIORITY */ +#if __TBB_ARENA_BINDING + //! Pointer to internal observer that allows to bind threads in arena to certain NUMA node. + numa_binding_observer* my_numa_binding_observer{nullptr}; +#endif /*__TBB_ARENA_BINDING*/ // Below are rarely modified members - //! The market that owns this arena. - market* my_market; + threading_control* my_threading_control; - //! ABA prevention marker. - uintptr_t my_aba_epoch; + //! Default task group context. + d1::task_group_context* my_default_ctx; -#if !__TBB_FP_CONTEXT - //! FPU control settings of arena's master thread captured at the moment of arena instantiation. - cpu_ctl_env my_cpu_ctl_env; -#endif + //! Waiting object for external threads that cannot join the arena. + concurrent_monitor my_exit_monitors; -#if __TBB_TASK_GROUP_CONTEXT - //! Default task group context. - /** Used by root tasks allocated directly by the master thread (not from inside - a TBB task) without explicit context specification. **/ - task_group_context* my_default_ctx; -#endif /* __TBB_TASK_GROUP_CONTEXT */ + //! Coroutines (task_dispathers) cache buffer + arena_co_cache my_co_cache; + + // arena needs an extra worker despite the arena limit + atomic_flag my_mandatory_concurrency; + // the number of local mandatory concurrency requests + int my_mandatory_requests; //! The number of slots in the arena. unsigned my_num_slots; - - //! The number of reserved slots (can be occupied only by masters). + //! The number of reserved slots (can be occupied only by external threads). unsigned my_num_reserved_slots; + //! The number of workers requested by the external thread owning the arena. + unsigned my_max_num_workers; -#if __TBB_ENQUEUE_ENFORCED_CONCURRENCY - //! Possible states for the concurrency mode of an arena. - enum concurrency_mode { - cm_normal = 0, // arena is served by workers as usual - cm_enforced_local, // arena needs an extra worker despite the arena limit - cm_enforced_global // arena needs an extra worker despite a global limit - }; - - //! The concurrency mode of an arena. - concurrency_mode my_concurrency_mode; -#endif /* __TBB_ENQUEUE_ENFORCED_CONCURRENCY */ - - //! Waiting object for master threads that cannot join the arena. - concurrent_monitor my_exit_monitors; + threading_control_client my_tc_client; #if TBB_USE_ASSERT //! Used to trap accesses to the object after its destruction. - uintptr_t my_guard; + std::uintptr_t my_guard; #endif /* TBB_USE_ASSERT */ }; // struct arena_base class arena: public padded<arena_base> { - //! If enqueued tasks found, restore arena priority and task presence status - void restore_priority_if_need(); public: - typedef padded<arena_base> base_type; + using base_type = padded<arena_base>; //! Types of work advertised by advertise_new_work() enum new_work_type { @@ -174,286 +281,232 @@ class arena: public padded<arena_base> }; //! Constructor - arena ( market&, unsigned max_num_workers, unsigned num_reserved_slots ); + arena(threading_control* control, unsigned max_num_workers, unsigned num_reserved_slots, unsigned priority_level); //! Allocate an instance of arena. - static arena& allocate_arena( market&, unsigned num_slots, unsigned num_reserved_slots ); + static arena& allocate_arena(threading_control* control, unsigned num_slots, unsigned num_reserved_slots, + unsigned priority_level); - static int unsigned num_arena_slots ( unsigned num_slots ) { - return max(2u, num_slots); + static arena& create(threading_control* control, unsigned num_slots, unsigned num_reserved_slots, unsigned arena_priority_level, d1::constraints constraints = d1::constraints{}); + + static int unsigned num_arena_slots ( unsigned num_slots, unsigned num_reserved_slots ) { + return num_reserved_slots == 0 ? num_slots : max(2u, num_slots); } - static int allocation_size ( unsigned num_slots ) { - return sizeof(base_type) + num_slots * (sizeof(mail_outbox) + sizeof(arena_slot)); + static int allocation_size( unsigned num_slots ) { + return sizeof(base_type) + num_slots * (sizeof(mail_outbox) + sizeof(arena_slot) + sizeof(task_dispatcher)); } - //! Get reference to mailbox corresponding to given affinity_id. - mail_outbox& mailbox( affinity_id id ) { - __TBB_ASSERT( 0<id, "affinity id must be positive integer" ); - __TBB_ASSERT( id <= my_num_slots, "affinity id out of bounds" ); + //! Get reference to mailbox corresponding to given slot_id + mail_outbox& mailbox( d1::slot_id slot ) { + __TBB_ASSERT( slot != d1::no_slot, "affinity should be specified" ); - return ((mail_outbox*)this)[-(int)id]; + return reinterpret_cast<mail_outbox*>(this)[-(int)(slot+1)]; // cast to 'int' is redundant but left for readability } //! Completes arena shutdown, destructs and deallocates it. - void free_arena (); - - typedef uintptr_t pool_state_t; - - //! No tasks to steal since last snapshot was taken - static const pool_state_t SNAPSHOT_EMPTY = 0; - - //! At least one task has been offered for stealing since the last snapshot started - static const pool_state_t SNAPSHOT_FULL = pool_state_t(-1); + void free_arena(); //! The number of least significant bits for external references static const unsigned ref_external_bits = 12; // up to 4095 external and 1M workers //! Reference increment values for externals and workers static const unsigned ref_external = 1; - static const unsigned ref_worker = 1<<ref_external_bits; - - //! No tasks to steal or snapshot is being taken. - static bool is_busy_or_empty( pool_state_t s ) { return s < SNAPSHOT_FULL; } + static const unsigned ref_worker = 1 << ref_external_bits; //! The number of workers active in the arena. - unsigned num_workers_active( ) { - return my_references >> ref_external_bits; + unsigned num_workers_active() const { + return my_references.load(std::memory_order_acquire) >> ref_external_bits; + } + + //! Check if the recall is requested by the market. + bool is_recall_requested() const { + return num_workers_active() > my_num_workers_allotted.load(std::memory_order_relaxed); } + void request_workers(int mandatory_delta, int workers_delta, bool wakeup_threads = false); + //! If necessary, raise a flag that there is new job in arena. template<arena::new_work_type work_type> void advertise_new_work(); + //! Attempts to steal a task from a randomly chosen arena slot + d1::task* steal_task(unsigned arena_index, FastRandom& frnd, execution_data_ext& ed, isolation_type isolation); + + //! Get a task from a global starvation resistant queue + template<task_stream_accessor_type accessor> + d1::task* get_stream_task(task_stream<accessor>& stream, unsigned& hint); + +#if __TBB_PREVIEW_CRITICAL_TASKS + //! Tries to find a critical task in global critical task stream + d1::task* get_critical_task(unsigned& hint, isolation_type isolation); +#endif + //! Check if there is job anywhere in arena. - /** Return true if no job or if arena is being cleaned up. */ - bool is_out_of_work(); + void out_of_work(); //! enqueue a task into starvation-resistance queue - void enqueue_task( task&, intptr_t, FastRandom & ); + void enqueue_task(d1::task&, d1::task_group_context&, thread_data&); //! Registers the worker with the arena and enters TBB scheduler dispatch loop - void process( generic_scheduler& ); + void process(thread_data&); - //! Notification that worker or master leaves its arena - template<unsigned ref_param> - inline void on_thread_leaving ( ); + //! Notification that the thread leaves its arena -#if __TBB_STATISTICS - //! Outputs internal statistics accumulated by the arena - void dump_arena_statistics (); -#endif /* __TBB_STATISTICS */ + void on_thread_leaving(unsigned ref_param); -#if __TBB_TASK_PRIORITY - //! Check if recent priority changes may bring some tasks to the current priority level soon - /** /param tasks_present indicates presence of tasks at any priority level. **/ - inline bool may_have_tasks ( generic_scheduler*, bool& tasks_present, bool& dequeuing_possible ); + //! Check for the presence of enqueued tasks + bool has_enqueued_tasks(); - //! Puts offloaded tasks into global list of orphaned tasks - void orphan_offloaded_tasks ( generic_scheduler& s ); -#endif /* __TBB_TASK_PRIORITY */ + //! Check for the presence of any tasks + bool has_tasks(); -#if __TBB_COUNT_TASK_NODES - //! Returns the number of task objects "living" in worker threads - intptr_t workers_task_node_count(); -#endif + bool is_empty() { return my_pool_state.test() == /* EMPTY */ false; } - //! Check for the presence of enqueued tasks at all priority levels - bool has_enqueued_tasks(); + thread_control_monitor& get_waiting_threads_monitor(); -#if __TBB_ENQUEUE_ENFORCED_CONCURRENCY - //! Recall worker if global mandatory is enabled, but not for this arena - bool recall_by_mandatory_request() const { - return my_market->my_mandatory_num_requested && my_concurrency_mode==cm_normal; - } - - //! The arena is currently in an enforced concurrency mode - bool must_have_concurrency() const { - return my_num_workers_requested && - ( my_concurrency_mode==cm_enforced_local || my_concurrency_mode==cm_enforced_global ); - } -#endif - static const size_t out_of_arena = ~size_t(0); + static const std::size_t out_of_arena = ~size_t(0); //! Tries to occupy a slot in the arena. On success, returns the slot index; if no slot is available, returns out_of_arena. template <bool as_worker> - size_t occupy_free_slot( generic_scheduler& s ); + std::size_t occupy_free_slot(thread_data&); //! Tries to occupy a slot in the specified range. - size_t occupy_free_slot_in_range( generic_scheduler& s, size_t lower, size_t upper ); + std::size_t occupy_free_slot_in_range(thread_data& tls, std::size_t lower, std::size_t upper); + + std::uintptr_t calculate_stealing_threshold(); + + unsigned priority_level() { return my_priority_level; } + + bool has_request() { return my_total_num_workers_requested; } + + unsigned references() const { return my_references.load(std::memory_order_acquire); } + + bool is_arena_workerless() const { return my_max_num_workers == 0; } + + void set_top_priority(bool); + + bool is_top_priority() const; + + bool is_joinable() const { + return num_workers_active() < my_num_workers_allotted.load(std::memory_order_relaxed); + } + + bool try_join(); + + void set_allotment(unsigned allotment); + + int update_concurrency(unsigned concurrency); + + std::pair</*min workers = */ int, /*max workers = */ int> update_request(int mandatory_delta, int workers_delta); /** Must be the last data field */ arena_slot my_slots[1]; }; // class arena -template<unsigned ref_param> -inline void arena::on_thread_leaving ( ) { - // - // Implementation of arena destruction synchronization logic contained various - // bugs/flaws at the different stages of its evolution, so below is a detailed - // description of the issues taken into consideration in the framework of the - // current design. - // - // In case of using fire-and-forget tasks (scheduled via task::enqueue()) - // master thread is allowed to leave its arena before all its work is executed, - // and market may temporarily revoke all workers from this arena. Since revoked - // workers never attempt to reset arena state to EMPTY and cancel its request - // to RML for threads, the arena object is destroyed only when both the last - // thread is leaving it and arena's state is EMPTY (that is its master thread - // left and it does not contain any work). - // Thus resetting arena to EMPTY state (as earlier TBB versions did) should not - // be done here (or anywhere else in the master thread to that matter); doing so - // can result either in arena's premature destruction (at least without - // additional costly checks in workers) or in unnecessary arena state changes - // (and ensuing workers migration). - // - // A worker that checks for work presence and transitions arena to the EMPTY - // state (in snapshot taking procedure arena::is_out_of_work()) updates - // arena::my_pool_state first and only then arena::my_num_workers_requested. - // So the check for work absence must be done against the latter field. - // - // In a time window between decrementing the active threads count and checking - // if there is an outstanding request for workers. New worker thread may arrive, - // finish remaining work, set arena state to empty, and leave decrementing its - // refcount and destroying. Then the current thread will destroy the arena - // the second time. To preclude it a local copy of the outstanding request - // value can be stored before decrementing active threads count. - // - // But this technique may cause two other problem. When the stored request is - // zero, it is possible that arena still has threads and they can generate new - // tasks and thus re-establish non-zero requests. Then all the threads can be - // revoked (as described above) leaving this thread the last one, and causing - // it to destroy non-empty arena. - // - // The other problem takes place when the stored request is non-zero. Another - // thread may complete the work, set arena state to empty, and leave without - // arena destruction before this thread decrements the refcount. This thread - // cannot destroy the arena either. Thus the arena may be "orphaned". - // - // In both cases we cannot dereference arena pointer after the refcount is - // decremented, as our arena may already be destroyed. - // - // If this is the master thread, the market is protected by refcount to it. - // In case of workers market's liveness is ensured by the RML connection - // rundown protocol, according to which the client (i.e. the market) lives - // until RML server notifies it about connection termination, and this - // notification is fired only after all workers return into RML. - // - // Thus if we decremented refcount to zero we ask the market to check arena - // state (including the fact if it is alive) under the lock. - // - uintptr_t aba_epoch = my_aba_epoch; - market* m = my_market; - __TBB_ASSERT(my_references >= ref_param, "broken arena reference counter"); -#if __TBB_STATISTICS_EARLY_DUMP - // While still holding a reference to the arena, compute how many external references are left. - // If just one, dump statistics. - if ( modulo_power_of_two(my_references,ref_worker)==ref_param ) // may only be true with ref_external - GATHER_STATISTIC( dump_arena_statistics() ); -#endif -#if __TBB_ENQUEUE_ENFORCED_CONCURRENCY - // When there is no workers someone must free arena, as - // without workers, no one calls is_out_of_work(). - // Skip workerless arenas because they have no demand for workers. - // TODO: consider more strict conditions for the cleanup, - // because it can create the demand of workers, - // but the arena can be already empty (and so ready for destroying) - if( ref_param==ref_external && my_num_slots != my_num_reserved_slots - && 0 == m->my_num_workers_soft_limit && my_concurrency_mode==cm_normal ) { - bool is_out = false; - for (int i=0; i<num_priority_levels; i++) { - is_out = is_out_of_work(); - if (is_out) - break; - } - // We expect, that in worst case it's enough to have num_priority_levels-1 - // calls to restore priorities and and yet another is_out_of_work() to conform - // that no work was found. But as market::set_active_num_workers() can be called - // concurrently, can't guarantee last is_out_of_work() return true. - } -#endif - if ( (my_references -= ref_param ) == 0 ) - m->try_destroy_arena( this, aba_epoch ); -} +template <arena::new_work_type work_type> +void arena::advertise_new_work() { + bool is_mandatory_needed = false; + bool are_workers_needed = false; -template<arena::new_work_type work_type> void arena::advertise_new_work() { - if( work_type == work_enqueued ) { -#if __TBB_ENQUEUE_ENFORCED_CONCURRENCY - if( my_market->my_num_workers_soft_limit == 0 ) { - if( my_concurrency_mode!=cm_enforced_global ) { - if( my_market->mandatory_concurrency_enable( this ) ) { - my_pool_state = SNAPSHOT_FULL; - return; - } - } - } else if( my_max_num_workers==0 && my_num_reserved_slots==1 ) { - my_max_num_workers = 1; - __TBB_ASSERT(my_concurrency_mode==cm_normal, NULL); - my_concurrency_mode = cm_enforced_local; - my_pool_state = SNAPSHOT_FULL; - my_market->adjust_demand( *this, 1 ); - return; - } -#endif /* __TBB_ENQUEUE_ENFORCED_CONCURRENCY */ + if (work_type != work_spawned) { // Local memory fence here and below is required to avoid missed wakeups; see the comment below. // Starvation resistant tasks require concurrency, so missed wakeups are unacceptable. - atomic_fence(); + atomic_fence_seq_cst(); } - else if( work_type == wakeup ) { - __TBB_ASSERT(my_max_num_workers!=0, "Unexpected worker wakeup request"); - atomic_fence(); + + if (work_type == work_enqueued && my_num_slots > my_num_reserved_slots) { + is_mandatory_needed = my_mandatory_concurrency.test_and_set(); } + // Double-check idiom that, in case of spawning, is deliberately sloppy about memory fences. // Technically, to avoid missed wakeups, there should be a full memory fence between the point we // released the task pool (i.e. spawned task) and read the arena's state. However, adding such a // fence might hurt overall performance more than it helps, because the fence would be executed // on every task pool release, even when stealing does not occur. Since TBB allows parallelism, // but never promises parallelism, the missed wakeup is not a correctness problem. - pool_state_t snapshot = my_pool_state; - if( is_busy_or_empty(snapshot) ) { - // Attempt to mark as full. The compare_and_swap below is a little unusual because the - // result is compared to a value that can be different than the comparand argument. - if( my_pool_state.compare_and_swap( SNAPSHOT_FULL, snapshot )==SNAPSHOT_EMPTY ) { - if( snapshot!=SNAPSHOT_EMPTY ) { - // This thread read "busy" into snapshot, and then another thread transitioned - // my_pool_state to "empty" in the meantime, which caused the compare_and_swap above - // to fail. Attempt to transition my_pool_state from "empty" to "full". - if( my_pool_state.compare_and_swap( SNAPSHOT_FULL, SNAPSHOT_EMPTY )!=SNAPSHOT_EMPTY ) { - // Some other thread transitioned my_pool_state from "empty", and hence became - // responsible for waking up workers. - return; - } - } - // This thread transitioned pool from empty to full state, and thus is responsible for - // telling the market that there is work to do. -#if __TBB_ENQUEUE_ENFORCED_CONCURRENCY - if( work_type == work_spawned ) { - if( my_concurrency_mode!=cm_normal ) { - switch( my_concurrency_mode ) { - case cm_enforced_local: - __TBB_ASSERT(my_max_num_workers==1, ""); - __TBB_ASSERT(!governor::local_scheduler()->is_worker(), ""); - // There was deliberate oversubscription on 1 core for sake of starvation-resistant tasks. - // Now a single active thread (must be the master) supposedly starts a new parallel region - // with relaxed sequential semantics, and oversubscription should be avoided. - // Demand for workers has been decreased to 0 during SNAPSHOT_EMPTY, so just keep it. - my_max_num_workers = 0; - my_concurrency_mode = cm_normal; - break; - case cm_enforced_global: - my_market->mandatory_concurrency_disable( this ); - restore_priority_if_need(); - break; - default: - break; - } - return; - } - } -#endif /* __TBB_ENQUEUE_ENFORCED_CONCURRENCY */ - my_market->adjust_demand( *this, my_max_num_workers ); + are_workers_needed = my_pool_state.test_and_set(); + + if (is_mandatory_needed || are_workers_needed) { + int mandatory_delta = is_mandatory_needed ? 1 : 0; + int workers_delta = are_workers_needed ? my_max_num_workers : 0; + + if (is_mandatory_needed && is_arena_workerless()) { + // Set workers_delta to 1 to keep arena invariants consistent + workers_delta = 1; + } + + request_workers(mandatory_delta, workers_delta, /* wakeup_threads = */ true); + } +} + +inline d1::task* arena::steal_task(unsigned arena_index, FastRandom& frnd, execution_data_ext& ed, isolation_type isolation) { + auto slot_num_limit = my_limit.load(std::memory_order_relaxed); + if (slot_num_limit == 1) { + // No slots to steal from + return nullptr; + } + // Try to steal a task from a random victim. + std::size_t k = frnd.get() % (slot_num_limit - 1); + // The following condition excludes the external thread that might have + // already taken our previous place in the arena from the list . + // of potential victims. But since such a situation can take + // place only in case of significant oversubscription, keeping + // the checks simple seems to be preferable to complicating the code. + if (k >= arena_index) { + ++k; // Adjusts random distribution to exclude self + } + arena_slot* victim = &my_slots[k]; + d1::task **pool = victim->task_pool.load(std::memory_order_relaxed); + d1::task *t = nullptr; + if (pool == EmptyTaskPool || !(t = victim->steal_task(*this, isolation, k))) { + return nullptr; + } + if (task_accessor::is_proxy_task(*t)) { + task_proxy &tp = *(task_proxy*)t; + d1::slot_id slot = tp.slot; + t = tp.extract_task<task_proxy::pool_bit>(); + if (!t) { + // Proxy was empty, so it's our responsibility to free it + tp.allocator.delete_object(&tp, ed); + return nullptr; } + // Note affinity is called for any stolen task (proxy or general) + ed.affinity_slot = slot; + } else { + // Note affinity is called for any stolen task (proxy or general) + ed.affinity_slot = d1::any_slot; + } + // Update task owner thread id to identify stealing + ed.original_slot = k; + return t; +} + +template<task_stream_accessor_type accessor> +inline d1::task* arena::get_stream_task(task_stream<accessor>& stream, unsigned& hint) { + if (stream.empty()) + return nullptr; + return stream.pop(subsequent_lane_selector(hint)); +} + +#if __TBB_PREVIEW_CRITICAL_TASKS +// Retrieves critical task respecting isolation level, if provided. The rule is: +// 1) If no outer critical task and no isolation => take any critical task +// 2) If working on an outer critical task and no isolation => cannot take any critical task +// 3) If no outer critical task but isolated => respect isolation +// 4) If working on an outer critical task and isolated => respect isolation +// Hint is used to keep some LIFO-ness, start search with the lane that was used during push operation. +inline d1::task* arena::get_critical_task(unsigned& hint, isolation_type isolation) { + if (my_critical_task_stream.empty()) + return nullptr; + + if ( isolation != no_isolation ) { + return my_critical_task_stream.pop_specific( hint, isolation ); + } else { + return my_critical_task_stream.pop(preceding_lane_selector(hint)); } } +#endif // __TBB_PREVIEW_CRITICAL_TASKS -} // namespace internal +} // namespace r1 +} // namespace detail } // namespace tbb #endif /* _TBB_arena_H */ diff --git a/src/tbb/src/tbb/arena_slot.cpp b/src/tbb/src/tbb/arena_slot.cpp new file mode 100644 index 000000000..bce5701db --- /dev/null +++ b/src/tbb/src/tbb/arena_slot.cpp @@ -0,0 +1,218 @@ +/* + Copyright (c) 2005-2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "arena_slot.h" +#include "arena.h" +#include "thread_data.h" + +namespace tbb { +namespace detail { +namespace r1 { + +//------------------------------------------------------------------------ +// Arena Slot +//------------------------------------------------------------------------ +d1::task* arena_slot::get_task_impl(size_t T, execution_data_ext& ed, bool& tasks_omitted, isolation_type isolation) { + __TBB_ASSERT(tail.load(std::memory_order_relaxed) <= T || is_local_task_pool_quiescent(), + "Is it safe to get a task at position T?"); + + d1::task* result = task_pool_ptr[T]; + __TBB_ASSERT(!is_poisoned( result ), "The poisoned task is going to be processed"); + + if (!result) { + return nullptr; + } + bool omit = isolation != no_isolation && isolation != task_accessor::isolation(*result); + if (!omit && !task_accessor::is_proxy_task(*result)) { + return result; + } else if (omit) { + tasks_omitted = true; + return nullptr; + } + + task_proxy& tp = static_cast<task_proxy&>(*result); + d1::slot_id aff_id = tp.slot; + if ( d1::task *t = tp.extract_task<task_proxy::pool_bit>() ) { + ed.affinity_slot = aff_id; + return t; + } + // Proxy was empty, so it's our responsibility to free it + tp.allocator.delete_object(&tp, ed); + + if ( tasks_omitted ) { + task_pool_ptr[T] = nullptr; + } + return nullptr; +} + +d1::task* arena_slot::get_task(execution_data_ext& ed, isolation_type isolation) { + __TBB_ASSERT(is_task_pool_published(), nullptr); + // The current task position in the task pool. + std::size_t T0 = tail.load(std::memory_order_relaxed); + // The bounds of available tasks in the task pool. H0 is only used when the head bound is reached. + std::size_t H0 = (std::size_t)-1, T = T0; + d1::task* result = nullptr; + bool task_pool_empty = false; + bool tasks_omitted = false; + do { + __TBB_ASSERT( !result, nullptr ); + // The full fence is required to sync the store of `tail` with the load of `head` (write-read barrier) + T = --tail; + // The acquire load of head is required to guarantee consistency of our task pool + // when a thief rolls back the head. + if ( (std::intptr_t)( head.load(std::memory_order_acquire) ) > (std::intptr_t)T ) { + acquire_task_pool(); + H0 = head.load(std::memory_order_relaxed); + if ( (std::intptr_t)H0 > (std::intptr_t)T ) { + // The thief has not backed off - nothing to grab. + __TBB_ASSERT( H0 == head.load(std::memory_order_relaxed) + && T == tail.load(std::memory_order_relaxed) + && H0 == T + 1, "victim/thief arbitration algorithm failure" ); + reset_task_pool_and_leave(); + // No tasks in the task pool. + task_pool_empty = true; + break; + } else if ( H0 == T ) { + // There is only one task in the task pool. + reset_task_pool_and_leave(); + task_pool_empty = true; + } else { + // Release task pool if there are still some tasks. + // After the release, the tail will be less than T, thus a thief + // will not attempt to get a task at position T. + release_task_pool(); + } + } + result = get_task_impl( T, ed, tasks_omitted, isolation ); + if ( result ) { + poison_pointer( task_pool_ptr[T] ); + break; + } else if ( !tasks_omitted ) { + poison_pointer( task_pool_ptr[T] ); + __TBB_ASSERT( T0 == T+1, nullptr ); + T0 = T; + } + } while ( !result && !task_pool_empty ); + + if ( tasks_omitted ) { + if ( task_pool_empty ) { + // All tasks have been checked. The task pool should be in reset state. + // We just restore the bounds for the available tasks. + // TODO: Does it have sense to move them to the beginning of the task pool? + __TBB_ASSERT( is_quiescent_local_task_pool_reset(), nullptr ); + if ( result ) { + // If we have a task, it should be at H0 position. + __TBB_ASSERT( H0 == T, nullptr ); + ++H0; + } + __TBB_ASSERT( H0 <= T0, nullptr ); + if ( H0 < T0 ) { + // Restore the task pool if there are some tasks. + head.store(H0, std::memory_order_relaxed); + tail.store(T0, std::memory_order_relaxed); + // The release fence is used in publish_task_pool. + publish_task_pool(); + // Synchronize with snapshot as we published some tasks. + ed.task_disp->m_thread_data->my_arena->advertise_new_work<arena::wakeup>(); + } + } else { + // A task has been obtained. We need to make a hole in position T. + __TBB_ASSERT( is_task_pool_published(), nullptr ); + __TBB_ASSERT( result, nullptr ); + task_pool_ptr[T] = nullptr; + tail.store(T0, std::memory_order_release); + // Synchronize with snapshot as we published some tasks. + // TODO: consider some approach not to call wakeup for each time. E.g. check if the tail reached the head. + ed.task_disp->m_thread_data->my_arena->advertise_new_work<arena::wakeup>(); + } + } + + __TBB_ASSERT( (std::intptr_t)tail.load(std::memory_order_relaxed) >= 0, nullptr ); + __TBB_ASSERT( result || tasks_omitted || is_quiescent_local_task_pool_reset(), nullptr ); + return result; +} + +d1::task* arena_slot::steal_task(arena& a, isolation_type isolation, std::size_t slot_index) { + d1::task** victim_pool = lock_task_pool(); + if (!victim_pool) { + return nullptr; + } + d1::task* result = nullptr; + std::size_t H = head.load(std::memory_order_relaxed); // mirror + std::size_t H0 = H; + bool tasks_omitted = false; + do { + // The full fence is required to sync the store of `head` with the load of `tail` (write-read barrier) + H = ++head; + // The acquire load of tail is required to guarantee consistency of victim_pool + // because the owner synchronizes task spawning via tail. + if ((std::intptr_t)H > (std::intptr_t)(tail.load(std::memory_order_acquire))) { + // Stealing attempt failed, deque contents has not been changed by us + head.store( /*dead: H = */ H0, std::memory_order_relaxed ); + __TBB_ASSERT( !result, nullptr ); + goto unlock; + } + result = victim_pool[H-1]; + __TBB_ASSERT( !is_poisoned( result ), nullptr ); + + if (result) { + if (isolation == no_isolation || isolation == task_accessor::isolation(*result)) { + if (!task_accessor::is_proxy_task(*result)) { + break; + } + task_proxy& tp = *static_cast<task_proxy*>(result); + // If mailed task is likely to be grabbed by its destination thread, skip it. + if (!task_proxy::is_shared(tp.task_and_tag) || !tp.outbox->recipient_is_idle() || a.mailbox(slot_index).recipient_is_idle()) { + break; + } + } + // The task cannot be executed either due to isolation or proxy constraints. + result = nullptr; + tasks_omitted = true; + } else if (!tasks_omitted) { + // Cleanup the task pool from holes until a task is skipped. + __TBB_ASSERT( H0 == H-1, nullptr ); + poison_pointer( victim_pool[H0] ); + H0 = H; + } + } while (!result); + __TBB_ASSERT( result, nullptr ); + + // emit "task was consumed" signal + poison_pointer( victim_pool[H-1] ); + if (tasks_omitted) { + // Some proxies in the task pool have been omitted. Set the stolen task to nullptr. + victim_pool[H-1] = nullptr; + // The release store synchronizes the victim_pool update(the store of nullptr). + head.store( /*dead: H = */ H0, std::memory_order_release ); + } +unlock: + unlock_task_pool(victim_pool); + +#if __TBB_PREFETCHING + __TBB_cl_evict(&victim_slot.head); + __TBB_cl_evict(&victim_slot.tail); +#endif + if (tasks_omitted) { + // Synchronize with snapshot as the head and tail can be bumped which can falsely trigger EMPTY state + a.advertise_new_work<arena::wakeup>(); + } + return result; +} + +} // namespace r1 +} // namespace detail +} // namespace tbb diff --git a/src/tbb/src/tbb/arena_slot.h b/src/tbb/src/tbb/arena_slot.h new file mode 100644 index 000000000..c526e4743 --- /dev/null +++ b/src/tbb/src/tbb/arena_slot.h @@ -0,0 +1,414 @@ +/* + Copyright (c) 2005-2023 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef _TBB_arena_slot_H +#define _TBB_arena_slot_H + +#include "oneapi/tbb/detail/_config.h" +#include "oneapi/tbb/detail/_utils.h" +#include "oneapi/tbb/detail/_template_helpers.h" +#include "oneapi/tbb/detail/_task.h" + +#include "oneapi/tbb/cache_aligned_allocator.h" + +#include "misc.h" +#include "mailbox.h" +#include "scheduler_common.h" + +#include <atomic> + +namespace tbb { +namespace detail { +namespace r1 { + +class arena; +class task_group_context; + +//-------------------------------------------------------------------------------------------------------- +// Arena Slot +//-------------------------------------------------------------------------------------------------------- + +static d1::task** const EmptyTaskPool = nullptr; +static d1::task** const LockedTaskPool = reinterpret_cast<d1::task**>(~std::intptr_t(0)); + +struct alignas(max_nfs_size) arena_slot_shared_state { + //! Scheduler of the thread attached to the slot + /** Marks the slot as busy, and is used to iterate through the schedulers belonging to this arena **/ + std::atomic<bool> my_is_occupied; + + // Synchronization of access to Task pool + /** Also is used to specify if the slot is empty or locked: + 0 - empty + -1 - locked **/ + std::atomic<d1::task**> task_pool; + + //! Index of the first ready task in the deque. + /** Modified by thieves, and by the owner during compaction/reallocation **/ + std::atomic<std::size_t> head; +}; + +struct alignas(max_nfs_size) arena_slot_private_state { + //! Hint provided for operations with the container of starvation-resistant tasks. + /** Modified by the owner thread (during these operations). **/ + unsigned hint_for_fifo_stream; + +#if __TBB_PREVIEW_CRITICAL_TASKS + //! Similar to 'hint_for_fifo_stream' but for critical tasks. + unsigned hint_for_critical_stream; +#endif + + //! Similar to 'hint_for_fifo_stream' but for the resume tasks. + unsigned hint_for_resume_stream; + + //! Index of the element following the last ready task in the deque. + /** Modified by the owner thread. **/ + std::atomic<std::size_t> tail; + + //! Capacity of the primary task pool (number of elements - pointers to task). + std::size_t my_task_pool_size; + + //! Task pool of the scheduler that owns this slot + // TODO: previously was task**__TBB_atomic, but seems like not accessed on other thread + d1::task** task_pool_ptr; +}; + +class arena_slot : private arena_slot_shared_state, private arena_slot_private_state { + friend class arena; + friend class outermost_worker_waiter; + friend class task_dispatcher; + friend class thread_data; + friend class nested_arena_context; + + //! The original task dispather associated with this slot + task_dispatcher* my_default_task_dispatcher; + +#if TBB_USE_ASSERT + void fill_with_canary_pattern ( std::size_t first, std::size_t last ) { + for ( std::size_t i = first; i < last; ++i ) + poison_pointer(task_pool_ptr[i]); + } +#else + void fill_with_canary_pattern ( size_t, std::size_t ) {} +#endif /* TBB_USE_ASSERT */ + + static constexpr std::size_t min_task_pool_size = 64; + + void allocate_task_pool( std::size_t n ) { + std::size_t byte_size = ((n * sizeof(d1::task*) + max_nfs_size - 1) / max_nfs_size) * max_nfs_size; + my_task_pool_size = byte_size / sizeof(d1::task*); + task_pool_ptr = (d1::task**)cache_aligned_allocate(byte_size); + // No need to clear the fresh deque since valid items are designated by the head and tail members. + // But fill it with a canary pattern in the high vigilance debug mode. + fill_with_canary_pattern( 0, my_task_pool_size ); + } + +public: + //! Deallocate task pool that was allocated by means of allocate_task_pool. + void free_task_pool( ) { + // TODO: understand the assertion and modify + // __TBB_ASSERT( !task_pool /* TODO: == EmptyTaskPool */, nullptr); + if( task_pool_ptr ) { + __TBB_ASSERT( my_task_pool_size, nullptr); + cache_aligned_deallocate( task_pool_ptr ); + task_pool_ptr = nullptr; + my_task_pool_size = 0; + } + } + + //! Get a task from the local pool. + /** Called only by the pool owner. + Returns the pointer to the task or nullptr if a suitable task is not found. + Resets the pool if it is empty. **/ + d1::task* get_task(execution_data_ext&, isolation_type); + + //! Steal task from slot's ready pool + d1::task* steal_task(arena&, isolation_type, std::size_t); + + //! Some thread is now the owner of this slot + void occupy() { + __TBB_ASSERT(!my_is_occupied.load(std::memory_order_relaxed), nullptr); + my_is_occupied.store(true, std::memory_order_release); + } + + //! Try to occupy the slot + bool try_occupy() { + return !is_occupied() && my_is_occupied.exchange(true) == false; + } + + //! Some thread is now the owner of this slot + void release() { + __TBB_ASSERT(my_is_occupied.load(std::memory_order_relaxed), nullptr); + my_is_occupied.store(false, std::memory_order_release); + } + + //! Spawn newly created tasks + void spawn(d1::task& t) { + std::size_t T = prepare_task_pool(1); + __TBB_ASSERT(is_poisoned(task_pool_ptr[T]), nullptr); + task_pool_ptr[T] = &t; + commit_spawned_tasks(T + 1); + if (!is_task_pool_published()) { + publish_task_pool(); + } + } + + bool is_task_pool_published() const { + return task_pool.load(std::memory_order_relaxed) != EmptyTaskPool; + } + + bool is_empty() const { + return task_pool.load(std::memory_order_relaxed) == EmptyTaskPool || + head.load(std::memory_order_relaxed) >= tail.load(std::memory_order_relaxed); + } + + bool is_occupied() const { + return my_is_occupied.load(std::memory_order_relaxed); + } + + task_dispatcher& default_task_dispatcher() { + __TBB_ASSERT(my_default_task_dispatcher != nullptr, nullptr); + return *my_default_task_dispatcher; + } + + void init_task_streams(unsigned h) { + hint_for_fifo_stream = h; +#if __TBB_RESUMABLE_TASKS + hint_for_resume_stream = h; +#endif +#if __TBB_PREVIEW_CRITICAL_TASKS + hint_for_critical_stream = h; +#endif + } + +#if __TBB_PREVIEW_CRITICAL_TASKS + unsigned& critical_hint() { + return hint_for_critical_stream; + } +#endif +private: + //! Get a task from the local pool at specified location T. + /** Returns the pointer to the task or nullptr if the task cannot be executed, + e.g. proxy has been deallocated or isolation constraint is not met. + tasks_omitted tells if some tasks have been omitted. + Called only by the pool owner. The caller should guarantee that the + position T is not available for a thief. **/ + d1::task* get_task_impl(size_t T, execution_data_ext& ed, bool& tasks_omitted, isolation_type isolation); + + //! Makes sure that the task pool can accommodate at least n more elements + /** If necessary relocates existing task pointers or grows the ready task deque. + * Returns (possible updated) tail index (not accounting for n). **/ + std::size_t prepare_task_pool(std::size_t num_tasks) { + std::size_t T = tail.load(std::memory_order_relaxed); // mirror + if ( T + num_tasks <= my_task_pool_size ) { + return T; + } + + std::size_t new_size = num_tasks; + if ( !my_task_pool_size ) { + __TBB_ASSERT( !is_task_pool_published() && is_quiescent_local_task_pool_reset(), nullptr); + __TBB_ASSERT( !task_pool_ptr, nullptr); + if ( num_tasks < min_task_pool_size ) new_size = min_task_pool_size; + allocate_task_pool( new_size ); + return 0; + } + acquire_task_pool(); + std::size_t H = head.load(std::memory_order_relaxed); // mirror + d1::task** new_task_pool = task_pool_ptr; + __TBB_ASSERT( my_task_pool_size >= min_task_pool_size, nullptr); + // Count not skipped tasks. Consider using std::count_if. + for ( std::size_t i = H; i < T; ++i ) + if ( new_task_pool[i] ) ++new_size; + // If the free space at the beginning of the task pool is too short, we + // are likely facing a pathological single-producer-multiple-consumers + // scenario, and thus it's better to expand the task pool + bool allocate = new_size > my_task_pool_size - min_task_pool_size/4; + if ( allocate ) { + // Grow task pool. As this operation is rare, and its cost is asymptotically + // amortizable, we can tolerate new task pool allocation done under the lock. + if ( new_size < 2 * my_task_pool_size ) + new_size = 2 * my_task_pool_size; + allocate_task_pool( new_size ); // updates my_task_pool_size + } + // Filter out skipped tasks. Consider using std::copy_if. + std::size_t T1 = 0; + for ( std::size_t i = H; i < T; ++i ) { + if ( new_task_pool[i] ) { + task_pool_ptr[T1++] = new_task_pool[i]; + } + } + // Deallocate the previous task pool if a new one has been allocated. + if ( allocate ) + cache_aligned_deallocate( new_task_pool ); + else + fill_with_canary_pattern( T1, tail ); + // Publish the new state. + commit_relocated_tasks( T1 ); + // assert_task_pool_valid(); + return T1; + } + + //! Makes newly spawned tasks visible to thieves + void commit_spawned_tasks(std::size_t new_tail) { + __TBB_ASSERT (new_tail <= my_task_pool_size, "task deque end was overwritten"); + // emit "task was released" signal + // Release fence is necessary to make sure that previously stored task pointers + // are visible to thieves. + tail.store(new_tail, std::memory_order_release); + } + + //! Used by workers to enter the task pool + /** Does not lock the task pool in case if arena slot has been successfully grabbed. **/ + void publish_task_pool() { + __TBB_ASSERT ( task_pool == EmptyTaskPool, "someone else grabbed my arena slot?" ); + __TBB_ASSERT ( head.load(std::memory_order_relaxed) < tail.load(std::memory_order_relaxed), + "entering arena without tasks to share" ); + // Release signal on behalf of previously spawned tasks (when this thread was not in arena yet) + task_pool.store(task_pool_ptr, std::memory_order_release ); + } + + //! Locks the local task pool + /** Garbles task_pool for the duration of the lock. Requires correctly set task_pool_ptr. + ATTENTION: This method is mostly the same as generic_scheduler::lock_task_pool(), with + a little different logic of slot state checks (slot is either locked or points + to our task pool). Thus if either of them is changed, consider changing the counterpart as well. **/ + void acquire_task_pool() { + if (!is_task_pool_published()) { + return; // we are not in arena - nothing to lock + } + bool sync_prepare_done = false; + for( atomic_backoff b;;b.pause() ) { +#if TBB_USE_ASSERT + // Local copy of the arena slot task pool pointer is necessary for the next + // assertion to work correctly to exclude asynchronous state transition effect. + d1::task** tp = task_pool.load(std::memory_order_relaxed); + __TBB_ASSERT( tp == LockedTaskPool || tp == task_pool_ptr, "slot ownership corrupt?" ); +#endif + d1::task** expected = task_pool_ptr; + if( task_pool.load(std::memory_order_relaxed) != LockedTaskPool && + task_pool.compare_exchange_strong(expected, LockedTaskPool ) ) { + // We acquired our own slot + break; + } else if( !sync_prepare_done ) { + // Start waiting + sync_prepare_done = true; + } + // Someone else acquired a lock, so pause and do exponential backoff. + } + __TBB_ASSERT( task_pool.load(std::memory_order_relaxed) == LockedTaskPool, "not really acquired task pool" ); + } + + //! Unlocks the local task pool + /** Restores task_pool munged by acquire_task_pool. Requires + correctly set task_pool_ptr. **/ + void release_task_pool() { + if ( !(task_pool.load(std::memory_order_relaxed) != EmptyTaskPool) ) + return; // we are not in arena - nothing to unlock + __TBB_ASSERT( task_pool.load(std::memory_order_relaxed) == LockedTaskPool, "arena slot is not locked" ); + task_pool.store( task_pool_ptr, std::memory_order_release ); + } + + //! Locks victim's task pool, and returns pointer to it. The pointer can be nullptr. + /** Garbles victim_arena_slot->task_pool for the duration of the lock. **/ + d1::task** lock_task_pool() { + d1::task** victim_task_pool; + for ( atomic_backoff backoff;; /*backoff pause embedded in the loop*/) { + victim_task_pool = task_pool.load(std::memory_order_relaxed); + // Microbenchmarks demonstrated that aborting stealing attempt when the + // victim's task pool is locked degrade performance. + // NOTE: Do not use comparison of head and tail indices to check for + // the presence of work in the victim's task pool, as they may give + // incorrect indication because of task pool relocations and resizes. + if (victim_task_pool == EmptyTaskPool) { + break; + } + d1::task** expected = victim_task_pool; + if (victim_task_pool != LockedTaskPool && task_pool.compare_exchange_strong(expected, LockedTaskPool) ) { + // We've locked victim's task pool + break; + } + // Someone else acquired a lock, so pause and do exponential backoff. + backoff.pause(); + } + __TBB_ASSERT(victim_task_pool == EmptyTaskPool || + (task_pool.load(std::memory_order_relaxed) == LockedTaskPool && + victim_task_pool != LockedTaskPool), "not really locked victim's task pool?"); + return victim_task_pool; + } + + //! Unlocks victim's task pool + /** Restores victim_arena_slot->task_pool munged by lock_task_pool. **/ + void unlock_task_pool(d1::task** victim_task_pool) { + __TBB_ASSERT(task_pool.load(std::memory_order_relaxed) == LockedTaskPool, "victim arena slot is not locked"); + __TBB_ASSERT(victim_task_pool != LockedTaskPool, nullptr); + task_pool.store(victim_task_pool, std::memory_order_release); + } + +#if TBB_USE_ASSERT + bool is_local_task_pool_quiescent() const { + d1::task** tp = task_pool.load(std::memory_order_relaxed); + return tp == EmptyTaskPool || tp == LockedTaskPool; + } + + bool is_quiescent_local_task_pool_empty() const { + __TBB_ASSERT(is_local_task_pool_quiescent(), "Task pool is not quiescent"); + return head.load(std::memory_order_relaxed) == tail.load(std::memory_order_relaxed); + } + + bool is_quiescent_local_task_pool_reset() const { + __TBB_ASSERT(is_local_task_pool_quiescent(), "Task pool is not quiescent"); + return head.load(std::memory_order_relaxed) == 0 && tail.load(std::memory_order_relaxed) == 0; + } +#endif // TBB_USE_ASSERT + + //! Leave the task pool + /** Leaving task pool automatically releases the task pool if it is locked. **/ + void leave_task_pool() { + __TBB_ASSERT(is_task_pool_published(), "Not in arena"); + // Do not reset my_arena_index. It will be used to (attempt to) re-acquire the slot next time + __TBB_ASSERT(task_pool.load(std::memory_order_relaxed) == LockedTaskPool, "Task pool must be locked when leaving arena"); + __TBB_ASSERT(is_quiescent_local_task_pool_empty(), "Cannot leave arena when the task pool is not empty"); + // No release fence is necessary here as this assignment precludes external + // accesses to the local task pool when becomes visible. Thus it is harmless + // if it gets hoisted above preceding local bookkeeping manipulations. + task_pool.store(EmptyTaskPool, std::memory_order_relaxed); + } + + //! Resets head and tail indices to 0, and leaves task pool + /** The task pool must be locked by the owner (via acquire_task_pool).**/ + void reset_task_pool_and_leave() { + __TBB_ASSERT(task_pool.load(std::memory_order_relaxed) == LockedTaskPool, "Task pool must be locked when resetting task pool"); + tail.store(0, std::memory_order_relaxed); + head.store(0, std::memory_order_relaxed); + leave_task_pool(); + } + + //! Makes relocated tasks visible to thieves and releases the local task pool. + /** Obviously, the task pool must be locked when calling this method. **/ + void commit_relocated_tasks(std::size_t new_tail) { + __TBB_ASSERT(is_local_task_pool_quiescent(), "Task pool must be locked when calling commit_relocated_tasks()"); + head.store(0, std::memory_order_relaxed); + // Tail is updated last to minimize probability of a thread making arena + // snapshot being misguided into thinking that this task pool is empty. + tail.store(new_tail, std::memory_order_release); + release_task_pool(); + } +}; + +} // namespace r1 +} // namespace detail +} // namespace tbb + +#endif // __TBB_arena_slot_H diff --git a/src/tbb/src/tbb/assert_impl.h b/src/tbb/src/tbb/assert_impl.h new file mode 100644 index 000000000..5628658b7 --- /dev/null +++ b/src/tbb/src/tbb/assert_impl.h @@ -0,0 +1,97 @@ +/* + Copyright (c) 2005-2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_assert_impl_H +#define __TBB_assert_impl_H + +#include "oneapi/tbb/detail/_config.h" +#include "oneapi/tbb/detail/_utils.h" + +#include <cstdio> +#include <cstdlib> +#include <cstring> +#include <cstdarg> +#if _MSC_VER && _DEBUG +#include <crtdbg.h> +#endif + +#include <mutex> + +#if __TBBMALLOC_BUILD +namespace rml { namespace internal { +#else +namespace tbb { +namespace detail { +namespace r1 { +#endif +// TODO: consider extension for formatted error description string +static void assertion_failure_impl(const char* location, int line, const char* expression, const char* comment) { + + std::fprintf(stderr, "Assertion %s failed (located in the %s function, line in file: %d)\n", + expression, location, line); + + if (comment) { + std::fprintf(stderr, "Detailed description: %s\n", comment); + } +#if _MSC_VER && _DEBUG + if (1 == _CrtDbgReport(_CRT_ASSERT, location, line, "tbb_debug.dll", "%s\r\n%s", expression, comment?comment:"")) { + _CrtDbgBreak(); + } else +#endif + { + std::fflush(stderr); + std::abort(); + } +} + +// Do not move the definition into the assertion_failure function because it will require "magic statics". +// It will bring a dependency on C++ runtime on some platforms while assert_impl.h is reused in tbbmalloc +// that should not depend on C++ runtime +static std::atomic<tbb::detail::do_once_state> assertion_state; + +void __TBB_EXPORTED_FUNC assertion_failure(const char* location, int line, const char* expression, const char* comment) { +#if __TBB_MSVC_UNREACHABLE_CODE_IGNORED + // Workaround for erroneous "unreachable code" during assertion throwing using call_once + // #pragma warning (push) + // #pragma warning (disable: 4702) +#endif + // We cannot use std::call_once because it brings a dependency on C++ runtime on some platforms + // while assert_impl.h is reused in tbbmalloc that should not depend on C++ runtime + atomic_do_once([&](){ assertion_failure_impl(location, line, expression, comment); }, assertion_state); +#if __TBB_MSVC_UNREACHABLE_CODE_IGNORED + // #pragma warning (pop) +#endif +} + +//! Report a runtime warning. +void runtime_warning( const char* format, ... ) { + char str[1024]; std::memset(str, 0, 1024); + va_list args; va_start(args, format); + vsnprintf( str, 1024-1, format, args); + va_end(args); + fprintf(stderr, "TBB Warning: %s\n", str); +} + +#if __TBBMALLOC_BUILD +}} // namespaces rml::internal +#else +} // namespace r1 +} // namespace detail +} // namespace tbb +#endif + +#endif // __TBB_assert_impl_H + diff --git a/src/tbb/src/tbb/cache_aligned_allocator.cpp b/src/tbb/src/tbb/cache_aligned_allocator.cpp deleted file mode 100644 index e1e871320..000000000 --- a/src/tbb/src/tbb/cache_aligned_allocator.cpp +++ /dev/null @@ -1,258 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#include "tbb/tbb_config.h" -#include "tbb/cache_aligned_allocator.h" -#include "tbb/tbb_allocator.h" -#include "tbb/tbb_exception.h" -#include "tbb_misc.h" -#include "dynamic_link.h" -#include <cstdlib> - -#if _WIN32||_WIN64 -#include "tbb/machine/windows_api.h" -#else -#include <dlfcn.h> -#endif /* _WIN32||_WIN64 */ - -using namespace std; - -#if __TBB_WEAK_SYMBOLS_PRESENT - -#pragma weak scalable_malloc -#pragma weak scalable_free -#pragma weak scalable_aligned_malloc -#pragma weak scalable_aligned_free - -extern "C" { - void* scalable_malloc( size_t ); - void scalable_free( void* ); - void* scalable_aligned_malloc( size_t, size_t ); - void scalable_aligned_free( void* ); -} - -#endif /* __TBB_WEAK_SYMBOLS_PRESENT */ - -namespace tbb { - -namespace internal { - -//! Dummy routine used for first indirect call via MallocHandler. -static void* DummyMalloc( size_t size ); - -//! Dummy routine used for first indirect call via FreeHandler. -static void DummyFree( void * ptr ); - -//! Handler for memory allocation -static void* (*MallocHandler)( size_t size ) = &DummyMalloc; - -//! Handler for memory deallocation -static void (*FreeHandler)( void* pointer ) = &DummyFree; - -//! Dummy routine used for first indirect call via padded_allocate_handler. -static void* dummy_padded_allocate( size_t bytes, size_t alignment ); - -//! Dummy routine used for first indirect call via padded_free_handler. -static void dummy_padded_free( void * ptr ); - -// ! Allocates memory using standard malloc. It is used when scalable_allocator is not available -static void* padded_allocate( size_t bytes, size_t alignment ); - -// ! Allocates memory using standard free. It is used when scalable_allocator is not available -static void padded_free( void* p ); - -//! Handler for padded memory allocation -static void* (*padded_allocate_handler)( size_t bytes, size_t alignment ) = &dummy_padded_allocate; - -//! Handler for padded memory deallocation -static void (*padded_free_handler)( void* p ) = &dummy_padded_free; - -//! Table describing how to link the handlers. -static const dynamic_link_descriptor MallocLinkTable[] = { - DLD(scalable_malloc, MallocHandler), - DLD(scalable_free, FreeHandler), - DLD(scalable_aligned_malloc, padded_allocate_handler), - DLD(scalable_aligned_free, padded_free_handler), -}; - - -#if TBB_USE_DEBUG -#define DEBUG_SUFFIX "_debug" -#else -#define DEBUG_SUFFIX -#endif /* TBB_USE_DEBUG */ - -// MALLOCLIB_NAME is the name of the TBB memory allocator library. -#if _WIN32||_WIN64 -#define MALLOCLIB_NAME "tbbmalloc" DEBUG_SUFFIX ".dll" -#elif __APPLE__ -#define MALLOCLIB_NAME "libtbbmalloc" DEBUG_SUFFIX ".dylib" -#elif __FreeBSD__ || __NetBSD__ || __sun || _AIX || __ANDROID__ -#define MALLOCLIB_NAME "libtbbmalloc" DEBUG_SUFFIX ".so" -#elif __linux__ // Note that order of these #elif's is important! -#define MALLOCLIB_NAME "libtbbmalloc" DEBUG_SUFFIX __TBB_STRING(.so.TBB_COMPATIBLE_INTERFACE_VERSION) -#else -#error Unknown OS -#endif - -//! Initialize the allocation/free handler pointers. -/** Caller is responsible for ensuring this routine is called exactly once. - The routine attempts to dynamically link with the TBB memory allocator. - If that allocator is not found, it links to malloc and free. */ -void initialize_handler_pointers() { - __TBB_ASSERT( MallocHandler==&DummyMalloc, NULL ); - bool success = dynamic_link( MALLOCLIB_NAME, MallocLinkTable, 4 ); - if( !success ) { - // If unsuccessful, set the handlers to the default routines. - // This must be done now, and not before FillDynamicLinks runs, because if other - // threads call the handlers, we want them to go through the DoOneTimeInitializations logic, - // which forces them to wait. - FreeHandler = &free; - MallocHandler = &malloc; - padded_allocate_handler = &padded_allocate; - padded_free_handler = &padded_free; - } -#if !__TBB_RML_STATIC - PrintExtraVersionInfo( "ALLOCATOR", success?"scalable_malloc":"malloc" ); -#endif -} - -static tbb::atomic<do_once_state> initialization_state; -void initialize_cache_aligned_allocator() { - atomic_do_once( &initialize_handler_pointers, initialization_state ); -} - -//! Executed on very first call through MallocHandler -static void* DummyMalloc( size_t size ) { - initialize_cache_aligned_allocator(); - __TBB_ASSERT( MallocHandler!=&DummyMalloc, NULL ); - return (*MallocHandler)( size ); -} - -//! Executed on very first call through FreeHandler -static void DummyFree( void * ptr ) { - initialize_cache_aligned_allocator(); - __TBB_ASSERT( FreeHandler!=&DummyFree, NULL ); - (*FreeHandler)( ptr ); -} - -//! Executed on very first call through padded_allocate_handler -static void* dummy_padded_allocate( size_t bytes, size_t alignment ) { - initialize_cache_aligned_allocator(); - __TBB_ASSERT( padded_allocate_handler!=&dummy_padded_allocate, NULL ); - return (*padded_allocate_handler)(bytes, alignment); -} - -//! Executed on very first call through padded_free_handler -static void dummy_padded_free( void * ptr ) { - initialize_cache_aligned_allocator(); - __TBB_ASSERT( padded_free_handler!=&dummy_padded_free, NULL ); - (*padded_free_handler)( ptr ); -} - -// TODO: use CPUID to find actual line size, though consider backward compatibility -static size_t NFS_LineSize = 128; - -size_t NFS_GetLineSize() { - return NFS_LineSize; -} - -#if _MSC_VER && !defined(__INTEL_COMPILER) - // unary minus operator applied to unsigned type, result still unsigned - #pragma warning( disable: 4146 4706 ) -#endif - -void* NFS_Allocate( size_t n, size_t element_size, void* /*hint*/ ) { - //TODO: make this functionality available via an adaptor over generic STL like allocator - const size_t nfs_cache_line_size = NFS_LineSize; - __TBB_ASSERT( nfs_cache_line_size <= NFS_MaxLineSize, "illegal value for NFS_LineSize" ); - __TBB_ASSERT( is_power_of_two(nfs_cache_line_size), "must be power of two" ); - size_t bytes = n*element_size; - - if (bytes<n || bytes+nfs_cache_line_size<bytes) { - // Overflow - throw_exception(eid_bad_alloc); - } - // scalable_aligned_malloc considers zero size request an error, and returns NULL - if (bytes==0) bytes = 1; - - void* result = (*padded_allocate_handler)( bytes, nfs_cache_line_size ); - if (!result) - throw_exception(eid_bad_alloc); - - __TBB_ASSERT( is_aligned(result, nfs_cache_line_size), "The address returned isn't aligned to cache line size" ); - return result; -} - -void NFS_Free( void* p ) { - (*padded_free_handler)( p ); -} - -static void* padded_allocate( size_t bytes, size_t alignment ) { - unsigned char* result = NULL; - unsigned char* base = (unsigned char*)malloc(alignment+bytes); - if( base ) { - // Round up to the next line - result = (unsigned char*)((uintptr_t)(base+alignment)&-alignment); - // Record where block actually starts. - ((uintptr_t*)result)[-1] = uintptr_t(base); - } - return result; -} - -static void padded_free( void* p ) { - if( p ) { - __TBB_ASSERT( (uintptr_t)p>=0x4096, "attempt to free block not obtained from cache_aligned_allocator" ); - // Recover where block actually starts - unsigned char* base = ((unsigned char**)p)[-1]; - __TBB_ASSERT( (void*)((uintptr_t)(base+NFS_LineSize)&-NFS_LineSize)==p, "not allocated by NFS_Allocate?" ); - free(base); - } -} - -void* __TBB_EXPORTED_FUNC allocate_via_handler_v3( size_t n ) { - void* result = (*MallocHandler) (n); - if (!result) { - throw_exception(eid_bad_alloc); - } - return result; -} - -void __TBB_EXPORTED_FUNC deallocate_via_handler_v3( void *p ) { - if( p ) { - (*FreeHandler)( p ); - } -} - -bool __TBB_EXPORTED_FUNC is_malloc_used_v3() { - if (MallocHandler == &DummyMalloc) { - void* void_ptr = (*MallocHandler)(1); - (*FreeHandler)(void_ptr); - } - __TBB_ASSERT( MallocHandler!=&DummyMalloc && FreeHandler!=&DummyFree, NULL ); - // Cast to void avoids type mismatch errors on some compilers (e.g. __IBMCPP__) - __TBB_ASSERT( !(((void*)MallocHandler==(void*)&malloc) ^ ((void*)FreeHandler==(void*)&free)), - "Both shim pointers must refer to routines from the same package (either TBB or CRT)" ); - return (void*)MallocHandler == (void*)&malloc; -} - -} // namespace internal - -} // namespace tbb diff --git a/src/tbb/src/tbb/cancellation_disseminator.h b/src/tbb/src/tbb/cancellation_disseminator.h new file mode 100644 index 000000000..724458296 --- /dev/null +++ b/src/tbb/src/tbb/cancellation_disseminator.h @@ -0,0 +1,85 @@ +/* + Copyright (c) 2022-2023 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef _TBB_cancellation_disseminator_H +#define _TBB_cancellation_disseminator_H + +#include "oneapi/tbb/mutex.h" +#include "oneapi/tbb/task_group.h" + +#include "intrusive_list.h" +#include "thread_data.h" + +namespace tbb { +namespace detail { +namespace r1 { + +class cancellation_disseminator { +public: + //! Finds all contexts affected by the state change and propagates the new state to them. + /* The propagation is relayed to the cancellation_disseminator because tasks created by one + external thread can be passed to and executed by other external threads. This means + that context trees can span several arenas at once and thus state change + propagation cannot be generally localized to one arena only. + */ + bool propagate_task_group_state(std::atomic<uint32_t> d1::task_group_context::*mptr_state, d1::task_group_context& src, uint32_t new_state) { + if (src.my_may_have_children.load(std::memory_order_relaxed) != d1::task_group_context::may_have_children) { + return true; + } + + // The whole propagation algorithm is under the lock in order to ensure correctness + // in case of concurrent state changes at the different levels of the context tree. + threads_list_mutex_type::scoped_lock lock(my_threads_list_mutex); + // TODO: consider to use double-check idiom + if ((src.*mptr_state).load(std::memory_order_relaxed) != new_state) { + // Another thread has concurrently changed the state. Back down. + return false; + } + + // Advance global state propagation epoch + ++the_context_state_propagation_epoch; + // Propagate to all workers and external threads and sync up their local epochs with the global one + // The whole propagation sequence is locked, thus no contention is expected + for (auto& thr_data : my_threads_list) { + thr_data.propagate_task_group_state(mptr_state, src, new_state); + } + + return true; + } + + void register_thread(thread_data& td) { + threads_list_mutex_type::scoped_lock lock(my_threads_list_mutex); + my_threads_list.push_front(td); + } + + void unregister_thread(thread_data& td) { + threads_list_mutex_type::scoped_lock lock(my_threads_list_mutex); + my_threads_list.remove(td); + } + +private: + using thread_data_list_type = intrusive_list<thread_data>; + using threads_list_mutex_type = d1::mutex; + + threads_list_mutex_type my_threads_list_mutex; + thread_data_list_type my_threads_list; +}; + +} // namespace r1 +} // namespace detail +} // namespace tbb + +#endif // _TBB_cancellation_disseminator_H diff --git a/src/tbb/src/tbb/cilk-tbb-interop.h b/src/tbb/src/tbb/cilk-tbb-interop.h deleted file mode 100644 index 19db6ea9b..000000000 --- a/src/tbb/src/tbb/cilk-tbb-interop.h +++ /dev/null @@ -1,115 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -/* The API to enable interoperability between Intel(R) Cilk(TM) Plus and - Intel(R) Threading Building Blocks. */ - -#ifndef CILK_TBB_INTEROP_H -#define CILK_TBB_INTEROP_H - -#ifndef _WIN32 -#ifdef IN_CILK_RUNTIME -#define CILK_EXPORT __attribute__((visibility("protected"))) -#else -#define CILK_EXPORT /* nothing */ -#endif -#else -#ifdef IN_CILK_RUNTIME -#define CILK_EXPORT __declspec(dllexport) -#else -#define CILK_EXPORT __declspec(dllimport) -#endif // IN_CILK_RUNTIME -#endif // _WIN32 - -#ifdef __cplusplus -extern "C" { -#endif /* __cplusplus */ - -/* A return code. 0 indicates success */ -typedef int __cilk_tbb_retcode; - -enum __cilk_tbb_stack_op { - CILK_TBB_STACK_ORPHAN, // disconnecting stack from a thread - CILK_TBB_STACK_ADOPT, // reconnecting orphaned stack to a trhead - CILK_TBB_STACK_RELEASE // releasing stack -}; - -typedef __cilk_tbb_retcode (*__cilk_tbb_pfn_stack_op)(enum __cilk_tbb_stack_op, void* data); - -typedef __cilk_tbb_retcode (*__cilk_tbb_pfn_unwatch_stacks)(void *data); - -/* Each thunk structure has two pointers: "routine" and "data". - The caller of the thunk invokes *routine, passing "data" as the void* parameter. */ - -/* Thunk invoked by Intel Cilk Plus runtime (cilkrts) when it changes the relationship - between a stack and a thread. It does not matter what stack the thunk runs on. - The thread (not fiber) on which the thunk runs is important. - - CILK_TBB_STACK_ORPHAN - The thunk must be invoked on the thread disconnecting itself from the stack. - Must "happen before" the stack is adopted elsewhere. - CILK_TBB_STACK_ADOPT - The thunk must be invoked on the thread adopting the stack. - CILK_TBB_STACK_RELEASE - The thunk must be invoked on the thread doing the releasing, - Must "happen before" the stack is used elsewhere. - - When a non-empty stack is transferred between threads, the first thread must orphan it - and the second thread must adopt it. - - An empty stack can be transferred similarly, or simply released by the first thread. - - Here is a summary of the actions as transitions on a state machine. - - watch ORPHAN - -->--> -->-- - / \ / \ - (freed empty stack) (TBB sees stack running on thread) (stack in limbo) - | \ / \ / | - | --<-- --<-- | - ^ RELEASE or ADOPT V - \ unwatch / - \ / - --------------------------<--------------------------- - RELEASE -*/ -struct __cilk_tbb_stack_op_thunk { - __cilk_tbb_pfn_stack_op routine; - void* data; /* Set by TBB */ -}; - -/* Thunk invoked by TBB when it is no longer interested in watching the stack bound to the current thread. */ -struct __cilk_tbb_unwatch_thunk { - __cilk_tbb_pfn_unwatch_stacks routine; - void* data; -}; - -/* Defined by cilkrts, called by TBB. - Requests that cilkrts invoke __cilk_tbb_stack_op_thunk when it orphans a stack. - cilkrts sets *u to a thunk that TBB should call when it is no longer interested in watching the stack. */ -CILK_EXPORT -__cilk_tbb_retcode __cilkrts_watch_stack(struct __cilk_tbb_unwatch_thunk* u, - struct __cilk_tbb_stack_op_thunk o); - -#ifdef __cplusplus -} -#endif /* __cplusplus */ - -#endif // CILK_TBB_INTEROP_H diff --git a/src/tbb/src/tbb/co_context.h b/src/tbb/src/tbb/co_context.h new file mode 100644 index 000000000..ed5143275 --- /dev/null +++ b/src/tbb/src/tbb/co_context.h @@ -0,0 +1,378 @@ +/* + Copyright (c) 2005-2022 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef _TBB_co_context_H +#define _TBB_co_context_H + +#include "oneapi/tbb/detail/_config.h" + +#if __TBB_RESUMABLE_TASKS + +#include <cstddef> +#include <cstdint> + +#if __TBB_RESUMABLE_TASKS_USE_THREADS + +#if _WIN32 || _WIN64 +#include <windows.h> +#else +#include <pthread.h> +#endif + +#include <condition_variable> +#include "governor.h" + +#elif _WIN32 || _WIN64 +#include <windows.h> +#else +// ucontext.h API is deprecated since macOS 10.6 +#if __APPLE__ + #if __INTEL_COMPILER + // #pragma warning(push) + // #pragma warning(disable:1478) + #elif __clang__ + // #pragma clang diagnostic push + // #pragma clang diagnostic ignored "-Wdeprecated-declarations" + #endif +#endif // __APPLE__ + +#include <ucontext.h> +#include <sys/mman.h> // mprotect + +#include "governor.h" // default_page_size() + +#ifndef MAP_STACK +// macOS* does not define MAP_STACK +#define MAP_STACK 0 +#endif +#ifndef MAP_ANONYMOUS +// macOS* defines MAP_ANON, which is deprecated in Linux*. +#define MAP_ANONYMOUS MAP_ANON +#endif +#endif // _WIN32 || _WIN64 + +namespace tbb { +namespace detail { +namespace r1 { + +#if __TBB_RESUMABLE_TASKS_USE_THREADS + struct coroutine_type { +#if _WIN32 || _WIN64 + using handle_type = HANDLE; +#else + using handle_type = pthread_t; +#endif + + handle_type my_thread; + std::condition_variable my_condvar; + std::mutex my_mutex; + thread_data* my_thread_data{ nullptr }; + bool my_is_active{ true }; + }; +#elif _WIN32 || _WIN64 + typedef LPVOID coroutine_type; +#else + struct coroutine_type { + coroutine_type() : my_context(), my_stack(), my_stack_size() {} + ucontext_t my_context; + void* my_stack; + std::size_t my_stack_size; + }; +#endif + + // Forward declaration of the coroutine API. + void create_coroutine(coroutine_type& c, std::size_t stack_size, void* arg); + void current_coroutine(coroutine_type& c); + void swap_coroutine(coroutine_type& prev_coroutine, coroutine_type& new_coroutine); + void destroy_coroutine(coroutine_type& c); + +class co_context { + enum co_state { + co_invalid, + co_suspended, + co_executing, + co_destroyed + }; + coroutine_type my_coroutine; + co_state my_state; + +public: + co_context(std::size_t stack_size, void* arg) + : my_state(stack_size ? co_suspended : co_executing) + { + if (stack_size) { + __TBB_ASSERT(arg != nullptr, nullptr); + create_coroutine(my_coroutine, stack_size, arg); + } else { + current_coroutine(my_coroutine); + } + } + + ~co_context() { + __TBB_ASSERT(1 << my_state & (1 << co_suspended | 1 << co_executing), nullptr); + if (my_state == co_suspended) { +#if __TBB_RESUMABLE_TASKS_USE_THREADS + my_state = co_executing; +#endif + destroy_coroutine(my_coroutine); + } + my_state = co_destroyed; + } + + void resume(co_context& target) { + // Do not create non-trivial objects on the stack of this function. They might never be destroyed. + __TBB_ASSERT(my_state == co_executing, nullptr); + __TBB_ASSERT(target.my_state == co_suspended, nullptr); + + my_state = co_suspended; + target.my_state = co_executing; + + // 'target' can reference an invalid object after swap_coroutine. Do not access it. + swap_coroutine(my_coroutine, target.my_coroutine); + + __TBB_ASSERT(my_state == co_executing, nullptr); + } +}; + +#if _WIN32 || _WIN64 +/* [[noreturn]] */ void __stdcall co_local_wait_for_all(void* arg) noexcept; +#else +/* [[noreturn]] */ void co_local_wait_for_all(unsigned hi, unsigned lo) noexcept; +#endif + +#if __TBB_RESUMABLE_TASKS_USE_THREADS +void handle_perror(int error_code, const char* what); + +inline void check(int error_code, const char* routine) { + if (error_code) { + handle_perror(error_code, routine); + } +} + +using thread_data_t = std::pair<coroutine_type&, void*&>; + +#if _WIN32 || _WIN64 +inline unsigned WINAPI coroutine_thread_func(void* d) +#else +inline void* coroutine_thread_func(void* d) +#endif +{ + thread_data_t& data = *static_cast<thread_data_t*>(d); + coroutine_type& c = data.first; + void* arg = data.second; + { + std::unique_lock<std::mutex> lock(c.my_mutex); + __TBB_ASSERT(c.my_thread_data == nullptr, nullptr); + c.my_is_active = false; + + // We read the data notify the waiting thread + data.second = nullptr; + c.my_condvar.notify_one(); + + c.my_condvar.wait(lock, [&c] { return c.my_is_active == true; }); + } + __TBB_ASSERT(c.my_thread_data != nullptr, nullptr); + governor::set_thread_data(*c.my_thread_data); + +#if _WIN32 || _WIN64 + co_local_wait_for_all(arg); + + return 0; +#else + std::uintptr_t addr = std::uintptr_t(arg); + unsigned lo = unsigned(addr); + unsigned hi = unsigned(std::uint64_t(addr) >> 32); + __TBB_ASSERT(sizeof(addr) == 8 || hi == 0, nullptr); + + co_local_wait_for_all(hi, lo); + + return nullptr; +#endif +}; + +inline void create_coroutine(coroutine_type& c, std::size_t stack_size, void* arg) { + thread_data_t data{ c, arg }; + +#if _WIN32 || _WIN64 + c.my_thread = (HANDLE)_beginthreadex(nullptr, unsigned(stack_size), coroutine_thread_func, &data, STACK_SIZE_PARAM_IS_A_RESERVATION, nullptr); + if (!c.my_thread) { + handle_perror(0, "create_coroutine: _beginthreadex failed\n"); + } +#else + pthread_attr_t s; + check(pthread_attr_init(&s), "pthread_attr_init has failed"); + if (stack_size > 0) { + check(pthread_attr_setstacksize(&s, stack_size), "pthread_attr_setstack_size has failed"); + } + check(pthread_create(&c.my_thread, &s, coroutine_thread_func, &data), "pthread_create has failed"); + check(pthread_attr_destroy(&s), "pthread_attr_destroy has failed"); +#endif + + // Wait for the just created thread to read the data + std::unique_lock<std::mutex> lock(c.my_mutex); + c.my_condvar.wait(lock, [&arg] { return arg == nullptr; }); +} + +inline void current_coroutine(coroutine_type& c) { +#if _WIN32 || _WIN64 + c.my_thread = GetCurrentThread(); +#else + c.my_thread = pthread_self(); +#endif +} + +inline void swap_coroutine(coroutine_type& prev_coroutine, coroutine_type& new_coroutine) { + thread_data* td = governor::get_thread_data(); + __TBB_ASSERT(prev_coroutine.my_is_active == true, "The current thread should be active"); + + // Detach our state before notification other thread + // (because we might be notified just after other thread notification) + prev_coroutine.my_thread_data = nullptr; + prev_coroutine.my_is_active = false; + governor::clear_thread_data(); + + { + std::unique_lock<std::mutex> lock(new_coroutine.my_mutex); + __TBB_ASSERT(new_coroutine.my_is_active == false, "The sleeping thread should not be active"); + __TBB_ASSERT(new_coroutine.my_thread_data == nullptr, "The sleeping thread should not be active"); + + new_coroutine.my_thread_data = td; + new_coroutine.my_is_active = true; + new_coroutine.my_condvar.notify_one(); + } + + std::unique_lock<std::mutex> lock(prev_coroutine.my_mutex); + prev_coroutine.my_condvar.wait(lock, [&prev_coroutine] { return prev_coroutine.my_is_active == true; }); + __TBB_ASSERT(governor::get_thread_data() != nullptr, nullptr); + governor::set_thread_data(*prev_coroutine.my_thread_data); +} + +inline void destroy_coroutine(coroutine_type& c) { + { + std::unique_lock<std::mutex> lock(c.my_mutex); + __TBB_ASSERT(c.my_thread_data == nullptr, "The sleeping thread should not be active"); + __TBB_ASSERT(c.my_is_active == false, "The sleeping thread should not be active"); + c.my_is_active = true; + c.my_condvar.notify_one(); + } +#if _WIN32 || _WIN64 + WaitForSingleObject(c.my_thread, INFINITE); + CloseHandle(c.my_thread); +#else + check(pthread_join(c.my_thread, nullptr), "pthread_join has failed"); +#endif +} +#elif _WIN32 || _WIN64 +inline void create_coroutine(coroutine_type& c, std::size_t stack_size, void* arg) { + __TBB_ASSERT(arg, nullptr); + c = CreateFiber(stack_size, co_local_wait_for_all, arg); + __TBB_ASSERT(c, nullptr); +} + +inline void current_coroutine(coroutine_type& c) { + c = IsThreadAFiber() ? GetCurrentFiber() : + ConvertThreadToFiberEx(nullptr, FIBER_FLAG_FLOAT_SWITCH); + __TBB_ASSERT(c, nullptr); +} + +inline void swap_coroutine(coroutine_type& prev_coroutine, coroutine_type& new_coroutine) { + if (!IsThreadAFiber()) { + ConvertThreadToFiberEx(nullptr, FIBER_FLAG_FLOAT_SWITCH); + } + __TBB_ASSERT(new_coroutine, nullptr); + prev_coroutine = GetCurrentFiber(); + __TBB_ASSERT(prev_coroutine, nullptr); + SwitchToFiber(new_coroutine); +} + +inline void destroy_coroutine(coroutine_type& c) { + __TBB_ASSERT(c, nullptr); + DeleteFiber(c); +} +#else // !(_WIN32 || _WIN64) + +inline void create_coroutine(coroutine_type& c, std::size_t stack_size, void* arg) { + const std::size_t REG_PAGE_SIZE = governor::default_page_size(); + const std::size_t page_aligned_stack_size = (stack_size + (REG_PAGE_SIZE - 1)) & ~(REG_PAGE_SIZE - 1); + const std::size_t protected_stack_size = page_aligned_stack_size + 2 * REG_PAGE_SIZE; + + // Allocate the stack with protection property + std::uintptr_t stack_ptr = (std::uintptr_t)mmap(nullptr, protected_stack_size, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0); + __TBB_ASSERT((void*)stack_ptr != MAP_FAILED, nullptr); + + // Allow read write on our stack (guarded pages are still protected) + int err = mprotect((void*)(stack_ptr + REG_PAGE_SIZE), page_aligned_stack_size, PROT_READ | PROT_WRITE); + __TBB_ASSERT_EX(!err, nullptr); + + // Remember the stack state + c.my_stack = (void*)(stack_ptr + REG_PAGE_SIZE); + c.my_stack_size = page_aligned_stack_size; + + err = getcontext(&c.my_context); + __TBB_ASSERT_EX(!err, nullptr); + + c.my_context.uc_link = nullptr; + // cast to char* to disable FreeBSD clang-3.4.1 'incompatible type' error + c.my_context.uc_stack.ss_sp = (char*)c.my_stack; + c.my_context.uc_stack.ss_size = c.my_stack_size; + c.my_context.uc_stack.ss_flags = 0; + + typedef void(*coroutine_func_t)(); + + std::uintptr_t addr = std::uintptr_t(arg); + unsigned lo = unsigned(addr); + unsigned hi = unsigned(std::uint64_t(addr) >> 32); + __TBB_ASSERT(sizeof(addr) == 8 || hi == 0, nullptr); + + makecontext(&c.my_context, (coroutine_func_t)co_local_wait_for_all, 2, hi, lo); +} + +inline void current_coroutine(coroutine_type& c) { + int err = getcontext(&c.my_context); + __TBB_ASSERT_EX(!err, nullptr); +} + +inline void swap_coroutine(coroutine_type& prev_coroutine, coroutine_type& new_coroutine) { + int err = swapcontext(&prev_coroutine.my_context, &new_coroutine.my_context); + __TBB_ASSERT_EX(!err, nullptr); +} + +inline void destroy_coroutine(coroutine_type& c) { + const std::size_t REG_PAGE_SIZE = governor::default_page_size(); + // Free stack memory with guarded pages + munmap((void*)((std::uintptr_t)c.my_stack - REG_PAGE_SIZE), c.my_stack_size + 2 * REG_PAGE_SIZE); + // Clear the stack state afterwards + c.my_stack = nullptr; + c.my_stack_size = 0; +} + +#if __APPLE__ + #if __INTEL_COMPILER + // #pragma warning(pop) // 1478 warning + #elif __clang__ + // #pragma clang diagnostic pop // "-Wdeprecated-declarations" + #endif +#endif + +#endif // _WIN32 || _WIN64 + +} // namespace r1 +} // namespace detail +} // namespace tbb + +#endif /* __TBB_RESUMABLE_TASKS */ + +#endif /* _TBB_co_context_H */ diff --git a/src/tbb/src/tbb/concurrent_bounded_queue.cpp b/src/tbb/src/tbb/concurrent_bounded_queue.cpp new file mode 100644 index 000000000..14617175d --- /dev/null +++ b/src/tbb/src/tbb/concurrent_bounded_queue.cpp @@ -0,0 +1,84 @@ +/* + Copyright (c) 2020-2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "oneapi/tbb/detail/_utils.h" +#include "oneapi/tbb/concurrent_queue.h" +#include "oneapi/tbb/cache_aligned_allocator.h" +#include "concurrent_monitor.h" + +namespace tbb { +namespace detail { +namespace r1 { + +static constexpr std::size_t monitors_number = 2; + +std::uint8_t* __TBB_EXPORTED_FUNC allocate_bounded_queue_rep( std::size_t queue_rep_size ) +{ + std::size_t monitors_mem_size = sizeof(concurrent_monitor) * monitors_number; + std::uint8_t* mem = static_cast<std::uint8_t*>(cache_aligned_allocate(queue_rep_size + monitors_mem_size)); + + concurrent_monitor* monitors = reinterpret_cast<concurrent_monitor*>(mem + queue_rep_size); + for (std::size_t i = 0; i < monitors_number; ++i) { + new (monitors + i) concurrent_monitor(); + } + + return mem; +} + +void __TBB_EXPORTED_FUNC deallocate_bounded_queue_rep( std::uint8_t* mem, std::size_t queue_rep_size ) +{ + concurrent_monitor* monitors = reinterpret_cast<concurrent_monitor*>(mem + queue_rep_size); + for (std::size_t i = 0; i < monitors_number; ++i) { + monitors[i].~concurrent_monitor(); + } + + cache_aligned_deallocate(mem); +} + +void __TBB_EXPORTED_FUNC wait_bounded_queue_monitor( concurrent_monitor* monitors, std::size_t monitor_tag, + std::ptrdiff_t target, d1::delegate_base& predicate ) +{ + __TBB_ASSERT(monitor_tag < monitors_number, nullptr); + concurrent_monitor& monitor = monitors[monitor_tag]; + + monitor.wait<concurrent_monitor::thread_context>([&] { return !predicate(); }, std::uintptr_t(target)); +} + +void __TBB_EXPORTED_FUNC abort_bounded_queue_monitors( concurrent_monitor* monitors ) { + concurrent_monitor& items_avail = monitors[d2::cbq_items_avail_tag]; + concurrent_monitor& slots_avail = monitors[d2::cbq_slots_avail_tag]; + + items_avail.abort_all(); + slots_avail.abort_all(); +} + +struct predicate_leq { + std::size_t my_ticket; + predicate_leq( std::size_t ticket ) : my_ticket(ticket) {} + bool operator() ( std::uintptr_t ticket ) const { return static_cast<std::size_t>(ticket) <= my_ticket; } +}; + +void __TBB_EXPORTED_FUNC notify_bounded_queue_monitor( concurrent_monitor* monitors, + std::size_t monitor_tag, std::size_t ticket) +{ + __TBB_ASSERT(monitor_tag < monitors_number, nullptr); + concurrent_monitor& monitor = monitors[monitor_tag]; + monitor.notify(predicate_leq(ticket)); +} + +} // namespace r1 +} // namespace detail +} // namespace tbb diff --git a/src/tbb/src/tbb/concurrent_hash_map.cpp b/src/tbb/src/tbb/concurrent_hash_map.cpp deleted file mode 100644 index d2dcb23d7..000000000 --- a/src/tbb/src/tbb/concurrent_hash_map.cpp +++ /dev/null @@ -1,58 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#include "tbb/concurrent_hash_map.h" - -namespace tbb { - -namespace internal { -#if !TBB_NO_LEGACY -struct hash_map_segment_base { - typedef spin_rw_mutex segment_mutex_t; - //! Type of a hash code. - typedef size_t hashcode_t; - //! Log2 of n_segment - static const size_t n_segment_bits = 6; - //! Maximum size of array of chains - static const size_t max_physical_size = size_t(1)<<(8*sizeof(hashcode_t)-n_segment_bits); - //! Mutex that protects this segment - segment_mutex_t my_mutex; - // Number of nodes - atomic<size_t> my_logical_size; - // Size of chains - /** Always zero or a power of two */ - size_t my_physical_size; - //! True if my_logical_size>=my_physical_size. - /** Used to support Intel(R) Thread Checker. */ - bool __TBB_EXPORTED_METHOD internal_grow_predicate() const; -}; - -bool hash_map_segment_base::internal_grow_predicate() const { - // Intel(R) Thread Checker considers the following reads to be races, so we hide them in the - // library so that Intel(R) Thread Checker will ignore them. The reads are used in a double-check - // context, so the program is nonetheless correct despite the race. - return my_logical_size >= my_physical_size && my_physical_size < max_physical_size; -} -#endif//!TBB_NO_LEGACY - -} // namespace internal - -} // namespace tbb - diff --git a/src/tbb/src/tbb/concurrent_monitor.cpp b/src/tbb/src/tbb/concurrent_monitor.cpp deleted file mode 100644 index 5ae3784c6..000000000 --- a/src/tbb/src/tbb/concurrent_monitor.cpp +++ /dev/null @@ -1,137 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#include "concurrent_monitor.h" - -namespace tbb { -namespace internal { - -void concurrent_monitor::thread_context::init() { - new (sema.begin()) binary_semaphore; - ready = true; -} - -concurrent_monitor::~concurrent_monitor() { - abort_all(); - __TBB_ASSERT( waitset_ec.empty(), "waitset not empty?" ); -} - -void concurrent_monitor::prepare_wait( thread_context& thr, uintptr_t ctx ) { - if( !thr.ready ) - thr.init(); - // this is good place to pump previous spurious wakeup - else if( thr.spurious ) { - thr.spurious = false; - thr.semaphore().P(); - } - thr.context = ctx; - thr.in_waitset = true; - { - tbb::spin_mutex::scoped_lock l( mutex_ec ); - __TBB_store_relaxed( thr.epoch, __TBB_load_relaxed(epoch) ); - waitset_ec.add( (waitset_t::node_t*)&thr ); - } - atomic_fence(); -} - -void concurrent_monitor::cancel_wait( thread_context& thr ) { - // spurious wakeup will be pumped in the following prepare_wait() - thr.spurious = true; - // try to remove node from waitset - bool th_in_waitset = thr.in_waitset; - if( th_in_waitset ) { - tbb::spin_mutex::scoped_lock l( mutex_ec ); - if (thr.in_waitset) { - // successfully removed from waitset, - // so there will be no spurious wakeup - thr.in_waitset = false; - thr.spurious = false; - waitset_ec.remove( (waitset_t::node_t&)thr ); - } - } -} - -void concurrent_monitor::notify_one_relaxed() { - if( waitset_ec.empty() ) - return; - waitset_node_t* n; - const waitset_node_t* end = waitset_ec.end(); - { - tbb::spin_mutex::scoped_lock l( mutex_ec ); - __TBB_store_relaxed( epoch, __TBB_load_relaxed(epoch) + 1 ); - n = waitset_ec.front(); - if( n!=end ) { - waitset_ec.remove( *n ); - to_thread_context(n)->in_waitset = false; - } - } - if( n!=end ) - to_thread_context(n)->semaphore().V(); -} - -void concurrent_monitor::notify_all_relaxed() { - if( waitset_ec.empty() ) - return; - waitset_t temp; - const waitset_node_t* end; - { - tbb::spin_mutex::scoped_lock l( mutex_ec ); - __TBB_store_relaxed( epoch, __TBB_load_relaxed(epoch) + 1 ); - waitset_ec.flush_to( temp ); - end = temp.end(); - for( waitset_node_t* n=temp.front(); n!=end; n=n->next ) - to_thread_context(n)->in_waitset = false; - } - waitset_node_t* nxt; - for( waitset_node_t* n=temp.front(); n!=end; n=nxt ) { - nxt = n->next; - to_thread_context(n)->semaphore().V(); - } -#if TBB_USE_ASSERT - temp.clear(); -#endif -} - -void concurrent_monitor::abort_all_relaxed() { - if( waitset_ec.empty() ) - return; - waitset_t temp; - const waitset_node_t* end; - { - tbb::spin_mutex::scoped_lock l( mutex_ec ); - __TBB_store_relaxed( epoch, __TBB_load_relaxed(epoch) + 1 ); - waitset_ec.flush_to( temp ); - end = temp.end(); - for( waitset_node_t* n=temp.front(); n!=end; n=n->next ) - to_thread_context(n)->in_waitset = false; - } - waitset_node_t* nxt; - for( waitset_node_t* n=temp.front(); n!=end; n=nxt ) { - nxt = n->next; - to_thread_context(n)->aborted = true; - to_thread_context(n)->semaphore().V(); - } -#if TBB_USE_ASSERT - temp.clear(); -#endif -} - -} // namespace internal -} // namespace tbb diff --git a/src/tbb/src/tbb/concurrent_monitor.h b/src/tbb/src/tbb/concurrent_monitor.h index 55c4dc920..b9f8a5897 100644 --- a/src/tbb/src/tbb/concurrent_monitor.h +++ b/src/tbb/src/tbb/concurrent_monitor.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2023 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,51 +12,50 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_concurrent_monitor_H #define __TBB_concurrent_monitor_H -#include "tbb/tbb_stddef.h" -#include "tbb/atomic.h" -#include "tbb/spin_mutex.h" -#include "tbb/tbb_exception.h" -#include "tbb/aligned_space.h" - +#include "oneapi/tbb/spin_mutex.h" +#include "oneapi/tbb/detail/_exception.h" +#include "oneapi/tbb/detail/_aligned_space.h" +#include "concurrent_monitor_mutex.h" #include "semaphore.h" +#include <atomic> + namespace tbb { -namespace internal { +namespace detail { +namespace r1 { //! Circular doubly-linked list with sentinel /** head.next points to the front and head.prev points to the back */ -class circular_doubly_linked_list_with_sentinel : no_copy { +class circular_doubly_linked_list_with_sentinel { public: - struct node_t { - node_t* next; - node_t* prev; - explicit node_t() : next((node_t*)(uintptr_t)0xcdcdcdcd), prev((node_t*)(uintptr_t)0xcdcdcdcd) {} + struct base_node { + base_node* next; + base_node* prev; + + constexpr base_node(base_node* n, base_node* p) : next(n), prev(p) {} + explicit base_node() : next((base_node*)(uintptr_t)0xcdcdcdcd), prev((base_node*)(uintptr_t)0xcdcdcdcd) {} }; // ctor - circular_doubly_linked_list_with_sentinel() {clear();} - // dtor - ~circular_doubly_linked_list_with_sentinel() {__TBB_ASSERT( head.next==&head && head.prev==&head, "the list is not empty" );} + constexpr circular_doubly_linked_list_with_sentinel() : count(0), head(&head, &head) {} + + circular_doubly_linked_list_with_sentinel(const circular_doubly_linked_list_with_sentinel&) = delete; + circular_doubly_linked_list_with_sentinel& operator=(const circular_doubly_linked_list_with_sentinel&) = delete; - inline size_t size() const {return count;} - inline bool empty() const {return size()==0;} - inline node_t* front() const {return head.next;} - inline node_t* last() const {return head.prev;} - inline node_t* begin() const {return front();} - inline const node_t* end() const {return &head;} + inline std::size_t size() const { return count.load(std::memory_order_relaxed); } + inline bool empty() const { return size() == 0; } + inline base_node* front() const { return head.next; } + inline base_node* last() const { return head.prev; } + inline const base_node* end() const { return &head; } //! add to the back of the list - inline void add( node_t* n ) { - __TBB_store_relaxed(count, __TBB_load_relaxed(count) + 1); + inline void add( base_node* n ) { + count.store(count.load(std::memory_order_relaxed) + 1, std::memory_order_relaxed); n->prev = head.prev; n->next = &head; head.prev->next = n; @@ -64,17 +63,18 @@ class circular_doubly_linked_list_with_sentinel : no_copy { } //! remove node 'n' - inline void remove( node_t& n ) { - __TBB_ASSERT( count > 0, "attempt to remove an item from an empty list" ); - __TBB_store_relaxed(count, __TBB_load_relaxed(count) - 1); + inline void remove( base_node& n ) { + __TBB_ASSERT(count.load(std::memory_order_relaxed) > 0, "attempt to remove an item from an empty list"); + count.store(count.load( std::memory_order_relaxed ) - 1, std::memory_order_relaxed); n.prev->next = n.next; n.next->prev = n.prev; } //! move all elements to 'lst' and initialize the 'this' list inline void flush_to( circular_doubly_linked_list_with_sentinel& lst ) { - if( const size_t l_count = __TBB_load_relaxed(count) ) { - __TBB_store_relaxed(lst.count, l_count); + const std::size_t l_count = size(); + if (l_count > 0) { + lst.count.store(l_count, std::memory_order_relaxed); lst.head.next = head.next; lst.head.prev = head.prev; head.next->prev = &lst.head; @@ -83,159 +83,416 @@ class circular_doubly_linked_list_with_sentinel : no_copy { } } - void clear() {head.next = head.prev = &head; __TBB_store_relaxed(count, 0);} + void clear() { + head.next = &head; + head.prev = &head; + count.store(0, std::memory_order_relaxed); + } private: - __TBB_atomic size_t count; - node_t head; + std::atomic<std::size_t> count; + base_node head; }; -typedef circular_doubly_linked_list_with_sentinel waitset_t; -typedef circular_doubly_linked_list_with_sentinel::node_t waitset_node_t; +using base_list = circular_doubly_linked_list_with_sentinel; +using base_node = circular_doubly_linked_list_with_sentinel::base_node; -//! concurrent_monitor -/** fine-grained concurrent_monitor implementation */ -class concurrent_monitor : no_copy { +template <typename Context> +class concurrent_monitor_base; + +template <typename Context> +class wait_node : public base_node { public: - /** per-thread descriptor for concurrent_monitor */ - class thread_context : waitset_node_t, no_copy { - friend class concurrent_monitor; - public: - thread_context() : spurious(false), aborted(false), ready(false), context(0) { - epoch = 0; - in_waitset = false; - } - ~thread_context() { - if (ready) { - if( spurious ) semaphore().P(); - semaphore().~binary_semaphore(); - } + +#if __TBB_GLIBCXX_VERSION >= 40800 && __TBB_GLIBCXX_VERSION < 40900 + wait_node(Context ctx) : my_context(ctx), my_is_in_list(false) {} +#else + wait_node(Context ctx) : my_context(ctx) {} +#endif + + virtual ~wait_node() = default; + + virtual void init() { + __TBB_ASSERT(!my_initialized, nullptr); + my_initialized = true; + } + + virtual void wait() = 0; + + virtual void reset() { + __TBB_ASSERT(my_skipped_wakeup, nullptr); + my_skipped_wakeup = false; + } + + virtual void notify() = 0; + +protected: + friend class concurrent_monitor_base<Context>; + friend class thread_data; + + Context my_context{}; +#if __TBB_GLIBCXX_VERSION >= 40800 && __TBB_GLIBCXX_VERSION < 40900 + std::atomic<bool> my_is_in_list; +#else + std::atomic<bool> my_is_in_list{false}; +#endif + + bool my_initialized{false}; + bool my_skipped_wakeup{false}; + bool my_aborted{false}; + unsigned my_epoch{0}; +}; + +template <typename Context> +class sleep_node : public wait_node<Context> { + using base_type = wait_node<Context>; +public: + using base_type::base_type; + + ~sleep_node() override { + if (this->my_initialized) { + if (this->my_skipped_wakeup) semaphore().P(); + semaphore().~binary_semaphore(); } - binary_semaphore& semaphore() { return *sema.begin(); } - private: - //! The method for lazy initialization of the thread_context's semaphore. - // Inlining of the method is undesirable, due to extra instructions for - // exception support added at caller side. - __TBB_NOINLINE( void init() ); - tbb::aligned_space<binary_semaphore> sema; - __TBB_atomic unsigned epoch; - tbb::atomic<bool> in_waitset; - bool spurious; - bool aborted; - bool ready; - uintptr_t context; - }; + } - //! ctor - concurrent_monitor() {__TBB_store_relaxed(epoch, 0);} + binary_semaphore& semaphore() { return *sema.begin(); } + + void init() override { + if (!this->my_initialized) { + new (sema.begin()) binary_semaphore; + base_type::init(); + } + } + void wait() override { + __TBB_ASSERT(this->my_initialized, + "Use of commit_wait() without prior prepare_wait()"); + semaphore().P(); + __TBB_ASSERT(!this->my_is_in_list.load(std::memory_order_relaxed), "Still in the queue?"); + if (this->my_aborted) + throw_exception(exception_id::user_abort); + } + + void reset() override { + base_type::reset(); + semaphore().P(); + } + + void notify() override { + semaphore().V(); + } + +private: + tbb::detail::aligned_space<binary_semaphore> sema; +}; + +//! concurrent_monitor +/** fine-grained concurrent_monitor implementation */ +template <typename Context> +class concurrent_monitor_base { +public: + //! ctor + constexpr concurrent_monitor_base() {} //! dtor - ~concurrent_monitor() ; + ~concurrent_monitor_base() = default; + + concurrent_monitor_base(const concurrent_monitor_base&) = delete; + concurrent_monitor_base& operator=(const concurrent_monitor_base&) = delete; //! prepare wait by inserting 'thr' into the wait queue - void prepare_wait( thread_context& thr, uintptr_t ctx = 0 ); + void prepare_wait( wait_node<Context>& node) { + // TODO: consider making even more lazy instantiation of the semaphore, that is only when it is actually needed, e.g. move it in node::wait() + if (!node.my_initialized) { + node.init(); + } + // this is good place to pump previous skipped wakeup + else if (node.my_skipped_wakeup) { + node.reset(); + } + + node.my_is_in_list.store(true, std::memory_order_relaxed); + + { + concurrent_monitor_mutex::scoped_lock l(my_mutex); + node.my_epoch = my_epoch.load(std::memory_order_relaxed); + my_waitset.add(&node); + } + + // Prepare wait guarantees Write Read memory barrier. + // In C++ only full fence covers this type of barrier. + atomic_fence_seq_cst(); + } //! Commit wait if event count has not changed; otherwise, cancel wait. /** Returns true if committed, false if canceled. */ - inline bool commit_wait( thread_context& thr ) { - const bool do_it = thr.epoch == __TBB_load_relaxed(epoch); + inline bool commit_wait( wait_node<Context>& node ) { + const bool do_it = node.my_epoch == my_epoch.load(std::memory_order_relaxed); // this check is just an optimization - if( do_it ) { - __TBB_ASSERT( thr.ready, "use of commit_wait() without prior prepare_wait()"); - thr.semaphore().P(); - __TBB_ASSERT( !thr.in_waitset, "still in the queue?" ); - if( thr.aborted ) - throw_exception( eid_user_abort ); + if (do_it) { + node.wait(); } else { - cancel_wait( thr ); + cancel_wait( node ); } return do_it; } + //! Cancel the wait. Removes the thread from the wait queue if not removed yet. - void cancel_wait( thread_context& thr ); + void cancel_wait( wait_node<Context>& node ) { + // possible skipped wakeup will be pumped in the following prepare_wait() + node.my_skipped_wakeup = true; + // try to remove node from waitset + // Cancel wait guarantees acquire memory barrier. + bool in_list = node.my_is_in_list.load(std::memory_order_acquire); + if (in_list) { + concurrent_monitor_mutex::scoped_lock l(my_mutex); + if (node.my_is_in_list.load(std::memory_order_relaxed)) { + my_waitset.remove(node); + // node is removed from waitset, so there will be no wakeup + node.my_is_in_list.store(false, std::memory_order_relaxed); + node.my_skipped_wakeup = false; + } + } + } + + //! Wait for a condition to be satisfied with waiting-on my_context + template <typename NodeType, typename Pred> + bool wait(Pred&& pred, NodeType&& node) { + prepare_wait(node); + while (!guarded_call(std::forward<Pred>(pred), node)) { + if (commit_wait(node)) { + return true; + } - //! Wait for a condition to be satisfied with waiting-on context - template<typename WaitUntil, typename Context> - void wait( WaitUntil until, Context on ); + prepare_wait(node); + } + + cancel_wait(node); + return false; + } //! Notify one thread about the event - void notify_one() {atomic_fence(); notify_one_relaxed();} + void notify_one() { + atomic_fence_seq_cst(); + notify_one_relaxed(); + } //! Notify one thread about the event. Relaxed version. - void notify_one_relaxed(); + void notify_one_relaxed() { + if (my_waitset.empty()) { + return; + } - //! Notify all waiting threads of the event - void notify_all() {atomic_fence(); notify_all_relaxed();} + base_node* n; + const base_node* end = my_waitset.end(); + { + concurrent_monitor_mutex::scoped_lock l(my_mutex); + my_epoch.store(my_epoch.load(std::memory_order_relaxed) + 1, std::memory_order_relaxed); + n = my_waitset.front(); + if (n != end) { + my_waitset.remove(*n); + +// GCC 12.x-13.x issues a warning here that to_wait_node(n)->my_is_in_list might have size 0, since n is +// a base_node pointer. (This cannot happen, because only wait_node pointers are added to my_waitset.) +#if (__TBB_GCC_VERSION >= 120100 && __TBB_GCC_VERSION < 140000 ) && !__clang__ && !__INTEL_COMPILER +// #pragma GCC diagnostic push +// #pragma GCC diagnostic ignored "-Wstringop-overflow" +#endif + to_wait_node(n)->my_is_in_list.store(false, std::memory_order_relaxed); +#if (__TBB_GCC_VERSION >= 120100 && __TBB_GCC_VERSION < 140000 ) && !__clang__ && !__INTEL_COMPILER +// #pragma GCC diagnostic pop +#endif + } + } - //! Notify all waiting threads of the event; Relaxed version - void notify_all_relaxed(); + if (n != end) { + to_wait_node(n)->notify(); + } + } - //! Notify waiting threads of the event that satisfies the given predicate - template<typename P> void notify( const P& predicate ) {atomic_fence(); notify_relaxed( predicate );} + //! Notify all waiting threads of the event + void notify_all() { + atomic_fence_seq_cst(); + notify_all_relaxed(); + } - //! Notify waiting threads of the event that satisfies the given predicate; Relaxed version - template<typename P> void notify_relaxed( const P& predicate ); + // ! Notify all waiting threads of the event; Relaxed version + void notify_all_relaxed() { + if (my_waitset.empty()) { + return; + } - //! Abort any sleeping threads at the time of the call - void abort_all() {atomic_fence(); abort_all_relaxed(); } + base_list temp; + const base_node* end; + { + concurrent_monitor_mutex::scoped_lock l(my_mutex); + my_epoch.store(my_epoch.load(std::memory_order_relaxed) + 1, std::memory_order_relaxed); + // TODO: Possible optimization, don't change node state under lock, just do flush + my_waitset.flush_to(temp); + end = temp.end(); + for (base_node* n = temp.front(); n != end; n = n->next) { + to_wait_node(n)->my_is_in_list.store(false, std::memory_order_relaxed); + } + } - //! Abort any sleeping threads at the time of the call; Relaxed version - void abort_all_relaxed(); + base_node* nxt; + for (base_node* n = temp.front(); n != end; n=nxt) { + nxt = n->next; + to_wait_node(n)->notify(); + } +#if TBB_USE_ASSERT + temp.clear(); +#endif + } -private: - tbb::spin_mutex mutex_ec; - waitset_t waitset_ec; - __TBB_atomic unsigned epoch; - thread_context* to_thread_context( waitset_node_t* n ) { return static_cast<thread_context*>(n); } -}; + //! Notify waiting threads of the event that satisfies the given predicate + template <typename P> + void notify( const P& predicate ) { + atomic_fence_seq_cst(); + notify_relaxed( predicate ); + } -template<typename WaitUntil, typename Context> -void concurrent_monitor::wait( WaitUntil until, Context on ) -{ - bool slept = false; - thread_context thr_ctx; - prepare_wait( thr_ctx, on() ); - while( !until() ) { - if( (slept = commit_wait( thr_ctx ) )==true ) - if( until() ) break; - slept = false; - prepare_wait( thr_ctx, on() ); - } - if( !slept ) - cancel_wait( thr_ctx ); -} - -template<typename P> -void concurrent_monitor::notify_relaxed( const P& predicate ) { - if( waitset_ec.empty() ) + //! Notify waiting threads of the event that satisfies the given predicate; + //! the predicate is called under the lock. Relaxed version. + template<typename P> + void notify_relaxed( const P& predicate ) { + if (my_waitset.empty()) { return; - waitset_t temp; - waitset_node_t* nxt; - const waitset_node_t* end = waitset_ec.end(); + } + + base_list temp; + base_node* nxt; + const base_node* end = my_waitset.end(); { - tbb::spin_mutex::scoped_lock l( mutex_ec ); - __TBB_store_relaxed(epoch, __TBB_load_relaxed(epoch) + 1); - for( waitset_node_t* n=waitset_ec.last(); n!=end; n=nxt ) { + concurrent_monitor_mutex::scoped_lock l(my_mutex); + my_epoch.store(my_epoch.load( std::memory_order_relaxed ) + 1, std::memory_order_relaxed); + for (base_node* n = my_waitset.last(); n != end; n = nxt) { nxt = n->prev; - thread_context* thr = to_thread_context( n ); - if( predicate( thr->context ) ) { - waitset_ec.remove( *n ); - thr->in_waitset = false; - temp.add( n ); + auto* node = static_cast<wait_node<Context>*>(n); + if (predicate(node->my_context)) { + my_waitset.remove(*n); + node->my_is_in_list.store(false, std::memory_order_relaxed); + temp.add(n); } } } end = temp.end(); - for( waitset_node_t* n=temp.front(); n!=end; n=nxt ) { + for (base_node* n=temp.front(); n != end; n = nxt) { nxt = n->next; - to_thread_context(n)->semaphore().V(); + to_wait_node(n)->notify(); } #if TBB_USE_ASSERT temp.clear(); #endif -} + } + + //! Notify waiting threads of the event that satisfies the given predicate; + //! the predicate is called under the lock. Relaxed version. + template<typename P> + void notify_one_relaxed( const P& predicate ) { + if (my_waitset.empty()) { + return; + } + + base_node* tmp = nullptr; + base_node* next{}; + const base_node* end = my_waitset.end(); + { + concurrent_monitor_mutex::scoped_lock l(my_mutex); + my_epoch.store(my_epoch.load( std::memory_order_relaxed ) + 1, std::memory_order_relaxed); + for (base_node* n = my_waitset.last(); n != end; n = next) { + next = n->prev; + auto* node = static_cast<wait_node<Context>*>(n); + if (predicate(node->my_context)) { + my_waitset.remove(*n); + node->my_is_in_list.store(false, std::memory_order_relaxed); + tmp = n; + break; + } + } + } + + if (tmp) { + to_wait_node(tmp)->notify(); + } + } + + //! Abort any sleeping threads at the time of the call + void abort_all() { + atomic_fence_seq_cst(); + abort_all_relaxed(); + } + + //! Abort any sleeping threads at the time of the call; Relaxed version + void abort_all_relaxed() { + if (my_waitset.empty()) { + return; + } + + base_list temp; + const base_node* end; + { + concurrent_monitor_mutex::scoped_lock l(my_mutex); + my_epoch.store(my_epoch.load(std::memory_order_relaxed) + 1, std::memory_order_relaxed); + my_waitset.flush_to(temp); + end = temp.end(); + for (base_node* n = temp.front(); n != end; n = n->next) { + to_wait_node(n)->my_is_in_list.store(false, std::memory_order_relaxed); + } + } + + base_node* nxt; + for (base_node* n = temp.front(); n != end; n = nxt) { + nxt = n->next; + to_wait_node(n)->my_aborted = true; + to_wait_node(n)->notify(); + } +#if TBB_USE_ASSERT + temp.clear(); +#endif + } + + void destroy() { + this->abort_all(); + my_mutex.destroy(); + __TBB_ASSERT(this->my_waitset.empty(), "waitset not empty?"); + } + +private: + template <typename NodeType, typename Pred> + bool guarded_call(Pred&& predicate, NodeType& node) { + bool res = false; + tbb::detail::d0::try_call( [&] { + res = std::forward<Pred>(predicate)(); + }).on_exception( [&] { + cancel_wait(node); + }); + + return res; + } + + concurrent_monitor_mutex my_mutex{}; + base_list my_waitset{}; + std::atomic<unsigned> my_epoch{}; + + wait_node<Context>* to_wait_node( base_node* node ) { return static_cast<wait_node<Context>*>(node); } +}; + +class concurrent_monitor : public concurrent_monitor_base<std::uintptr_t> { + using base_type = concurrent_monitor_base<std::uintptr_t>; +public: + using base_type::base_type; + + ~concurrent_monitor() { + destroy(); + } + + /** per-thread descriptor for concurrent_monitor */ + using thread_context = sleep_node<std::uintptr_t>; +}; -} // namespace internal +} // namespace r1 +} // namespace detail } // namespace tbb #endif /* __TBB_concurrent_monitor_H */ diff --git a/src/tbb/src/tbb/concurrent_monitor_mutex.h b/src/tbb/src/tbb/concurrent_monitor_mutex.h new file mode 100644 index 000000000..cae6526dd --- /dev/null +++ b/src/tbb/src/tbb/concurrent_monitor_mutex.h @@ -0,0 +1,113 @@ +/* + Copyright (c) 2005-2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_monitor_mutex_H +#define __TBB_monitor_mutex_H + +#include "oneapi/tbb/detail/_utils.h" +#include "oneapi/tbb/detail/_aligned_space.h" +#include "semaphore.h" + +#include <mutex> + +namespace tbb { +namespace detail { +namespace r1 { + +class concurrent_monitor_mutex { +public: + using scoped_lock = std::lock_guard<concurrent_monitor_mutex>; + + constexpr concurrent_monitor_mutex() {} + + ~concurrent_monitor_mutex() = default; + + void destroy() { +#if !__TBB_USE_FUTEX + if (my_init_flag.load(std::memory_order_relaxed)) { + get_semaphore().~semaphore(); + } +#endif + } + + void lock() { + auto wakeup_condition = [&] { + return my_flag.load(std::memory_order_relaxed) == 0; + }; + + while (my_flag.exchange(1)) { + if (!timed_spin_wait_until(wakeup_condition)) { + ++my_waiters; + while (!wakeup_condition()) { + wait(); + } + --my_waiters; + } + } + } + + void unlock() { + my_flag.exchange(0); // full fence, so the next load is relaxed + if (my_waiters.load(std::memory_order_relaxed)) { + wakeup(); + } + } + +private: + void wait() { +#if __TBB_USE_FUTEX + futex_wait(&my_flag, 1); +#else + get_semaphore().P(); +#endif + } + + void wakeup() { +#if __TBB_USE_FUTEX + futex_wakeup_one(&my_flag); +#else + get_semaphore().V(); +#endif + } + + // The flag should be int for the futex operations + std::atomic<int> my_flag{0}; + std::atomic<int> my_waiters{0}; + +#if !__TBB_USE_FUTEX + semaphore& get_semaphore() { + if (!my_init_flag.load(std::memory_order_acquire)) { + std::lock_guard<std::mutex> lock(my_init_mutex); + if (!my_init_flag.load(std::memory_order_relaxed)) { + new (my_semaphore.begin()) semaphore(); + my_init_flag.store(true, std::memory_order_release); + } + } + + return *my_semaphore.begin(); + } + + static std::mutex my_init_mutex; + std::atomic<bool> my_init_flag{false}; + aligned_space<semaphore> my_semaphore{}; +#endif +}; + +} // namespace r1 +} // namespace detail +} // namespace tbb + +#endif // __TBB_monitor_mutex_H diff --git a/src/tbb/src/tbb/concurrent_queue.cpp b/src/tbb/src/tbb/concurrent_queue.cpp deleted file mode 100644 index c41be3fb3..000000000 --- a/src/tbb/src/tbb/concurrent_queue.cpp +++ /dev/null @@ -1,676 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#include "tbb/tbb_stddef.h" -#include "tbb/tbb_machine.h" -#include "tbb/tbb_exception.h" -// Define required to satisfy test in internal file. -#define __TBB_concurrent_queue_H -#include "tbb/internal/_concurrent_queue_impl.h" -#include "concurrent_monitor.h" -#include "itt_notify.h" -#include <new> -#include <cstring> // for memset() - -using namespace std; - -#if defined(_MSC_VER) && defined(_Wp64) - // Workaround for overzealous compiler warnings in /Wp64 mode - #pragma warning (disable: 4267) -#endif - -#define RECORD_EVENTS 0 - - -namespace tbb { - -namespace internal { - -typedef concurrent_queue_base_v3 concurrent_queue_base; - -typedef size_t ticket; - -//! A queue using simple locking. -/** For efficiency, this class has no constructor. - The caller is expected to zero-initialize it. */ -struct micro_queue { - typedef concurrent_queue_base::page page; - - friend class micro_queue_pop_finalizer; - - atomic<page*> head_page; - atomic<ticket> head_counter; - - atomic<page*> tail_page; - atomic<ticket> tail_counter; - - spin_mutex page_mutex; - - void push( const void* item, ticket k, concurrent_queue_base& base, - concurrent_queue_base::copy_specifics op_type ); - - void abort_push( ticket k, concurrent_queue_base& base ); - - bool pop( void* dst, ticket k, concurrent_queue_base& base ); - - micro_queue& assign( const micro_queue& src, concurrent_queue_base& base, - concurrent_queue_base::copy_specifics op_type ); - - page* make_copy ( concurrent_queue_base& base, const page* src_page, size_t begin_in_page, - size_t end_in_page, ticket& g_index, concurrent_queue_base::copy_specifics op_type ) ; - - void make_invalid( ticket k ); -}; - -// we need to yank it out of micro_queue because of concurrent_queue_base::deallocate_page being virtual. -class micro_queue_pop_finalizer: no_copy { - typedef concurrent_queue_base::page page; - ticket my_ticket; - micro_queue& my_queue; - page* my_page; - concurrent_queue_base &base; -public: - micro_queue_pop_finalizer( micro_queue& queue, concurrent_queue_base& b, ticket k, page* p ) : - my_ticket(k), my_queue(queue), my_page(p), base(b) - {} - ~micro_queue_pop_finalizer() { - page* p = my_page; - if( p ) { - spin_mutex::scoped_lock lock( my_queue.page_mutex ); - page* q = p->next; - my_queue.head_page = q; - if( !q ) { - my_queue.tail_page = NULL; - } - } - my_queue.head_counter = my_ticket; - if( p ) - base.deallocate_page( p ); - } -}; - -struct predicate_leq { - ticket t; - predicate_leq( ticket t_ ) : t(t_) {} - bool operator() ( uintptr_t p ) const {return (ticket)p<=t;} -}; - -//! Internal representation of a ConcurrentQueue. -/** For efficiency, this class has no constructor. - The caller is expected to zero-initialize it. */ -class concurrent_queue_rep { -public: -private: - friend struct micro_queue; - - //! Approximately n_queue/golden ratio - static const size_t phi = 3; - -public: - //! Must be power of 2 - static const size_t n_queue = 8; - - //! Map ticket to an array index - static size_t index( ticket k ) { - return k*phi%n_queue; - } - - atomic<ticket> head_counter; - concurrent_monitor items_avail; - atomic<size_t> n_invalid_entries; - char pad1[NFS_MaxLineSize-((sizeof(atomic<ticket>)+sizeof(concurrent_monitor)+sizeof(atomic<size_t>))&(NFS_MaxLineSize-1))]; - - atomic<ticket> tail_counter; - concurrent_monitor slots_avail; - char pad2[NFS_MaxLineSize-((sizeof(atomic<ticket>)+sizeof(concurrent_monitor))&(NFS_MaxLineSize-1))]; - micro_queue array[n_queue]; - - micro_queue& choose( ticket k ) { - // The formula here approximates LRU in a cache-oblivious way. - return array[index(k)]; - } - - atomic<unsigned> abort_counter; - - //! Value for effective_capacity that denotes unbounded queue. - static const ptrdiff_t infinite_capacity = ptrdiff_t(~size_t(0)/2); -}; - -#if _MSC_VER && !defined(__INTEL_COMPILER) - // unary minus operator applied to unsigned type, result still unsigned - #pragma warning( push ) - #pragma warning( disable: 4146 ) -#endif - -static void* static_invalid_page; - -//------------------------------------------------------------------------ -// micro_queue -//------------------------------------------------------------------------ -void micro_queue::push( const void* item, ticket k, concurrent_queue_base& base, - concurrent_queue_base::copy_specifics op_type ) { - k &= -concurrent_queue_rep::n_queue; - page* p = NULL; - // find index on page where we would put the data - size_t index = modulo_power_of_two( k/concurrent_queue_rep::n_queue, base.items_per_page ); - if( !index ) { // make a new page - __TBB_TRY { - p = base.allocate_page(); - } __TBB_CATCH(...) { - ++base.my_rep->n_invalid_entries; - make_invalid( k ); - __TBB_RETHROW(); - } - p->mask = 0; - p->next = NULL; - } - - // wait for my turn - if( tail_counter!=k ) // The developer insisted on keeping first check out of the backoff loop - for( atomic_backoff b(true);;b.pause() ) { - ticket tail = tail_counter; - if( tail==k ) break; - else if( tail&0x1 ) { - // no memory. throws an exception; assumes concurrent_queue_rep::n_queue>1 - ++base.my_rep->n_invalid_entries; - throw_exception( eid_bad_last_alloc ); - } - } - - if( p ) { // page is newly allocated; insert in micro_queue - spin_mutex::scoped_lock lock( page_mutex ); - if( page* q = tail_page ) - q->next = p; - else - head_page = p; - tail_page = p; - } - - if (item) { - p = tail_page; - ITT_NOTIFY( sync_acquired, p ); - __TBB_TRY { - if( concurrent_queue_base::copy == op_type ) { - base.copy_item( *p, index, item ); - } else { - __TBB_ASSERT( concurrent_queue_base::move == op_type, NULL ); - static_cast<concurrent_queue_base_v8&>(base).move_item( *p, index, item ); - } - } __TBB_CATCH(...) { - ++base.my_rep->n_invalid_entries; - tail_counter += concurrent_queue_rep::n_queue; - __TBB_RETHROW(); - } - ITT_NOTIFY( sync_releasing, p ); - // If no exception was thrown, mark item as present. - p->mask |= uintptr_t(1)<<index; - } - else // no item; this was called from abort_push - ++base.my_rep->n_invalid_entries; - - tail_counter += concurrent_queue_rep::n_queue; -} - - -void micro_queue::abort_push( ticket k, concurrent_queue_base& base ) { - push(NULL, k, base, concurrent_queue_base::copy); -} - -bool micro_queue::pop( void* dst, ticket k, concurrent_queue_base& base ) { - k &= -concurrent_queue_rep::n_queue; - spin_wait_until_eq( head_counter, k ); - spin_wait_while_eq( tail_counter, k ); - page *p = head_page; - __TBB_ASSERT( p, NULL ); - size_t index = modulo_power_of_two( k/concurrent_queue_rep::n_queue, base.items_per_page ); - bool success = false; - { - micro_queue_pop_finalizer finalizer( *this, base, k+concurrent_queue_rep::n_queue, index==base.items_per_page-1 ? p : NULL ); - if( p->mask & uintptr_t(1)<<index ) { - success = true; - ITT_NOTIFY( sync_acquired, dst ); - ITT_NOTIFY( sync_acquired, head_page ); - base.assign_and_destroy_item( dst, *p, index ); - ITT_NOTIFY( sync_releasing, head_page ); - } else { - --base.my_rep->n_invalid_entries; - } - } - return success; -} - -micro_queue& micro_queue::assign( const micro_queue& src, concurrent_queue_base& base, - concurrent_queue_base::copy_specifics op_type ) -{ - head_counter = src.head_counter; - tail_counter = src.tail_counter; - - const page* srcp = src.head_page; - if( srcp ) { - ticket g_index = head_counter; - __TBB_TRY { - size_t n_items = (tail_counter-head_counter)/concurrent_queue_rep::n_queue; - size_t index = modulo_power_of_two( head_counter/concurrent_queue_rep::n_queue, base.items_per_page ); - size_t end_in_first_page = (index+n_items<base.items_per_page)?(index+n_items):base.items_per_page; - - head_page = make_copy( base, srcp, index, end_in_first_page, g_index, op_type ); - page* cur_page = head_page; - - if( srcp != src.tail_page ) { - for( srcp = srcp->next; srcp!=src.tail_page; srcp=srcp->next ) { - cur_page->next = make_copy( base, srcp, 0, base.items_per_page, g_index, op_type ); - cur_page = cur_page->next; - } - - __TBB_ASSERT( srcp==src.tail_page, NULL ); - - size_t last_index = modulo_power_of_two( tail_counter/concurrent_queue_rep::n_queue, base.items_per_page ); - if( last_index==0 ) last_index = base.items_per_page; - - cur_page->next = make_copy( base, srcp, 0, last_index, g_index, op_type ); - cur_page = cur_page->next; - } - tail_page = cur_page; - } __TBB_CATCH(...) { - make_invalid( g_index ); - __TBB_RETHROW(); - } - } else { - head_page = tail_page = NULL; - } - return *this; -} - -concurrent_queue_base::page* micro_queue::make_copy( concurrent_queue_base& base, - const concurrent_queue_base::page* src_page, size_t begin_in_page, size_t end_in_page, - ticket& g_index, concurrent_queue_base::copy_specifics op_type ) -{ - page* new_page = base.allocate_page(); - new_page->next = NULL; - new_page->mask = src_page->mask; - for( ; begin_in_page!=end_in_page; ++begin_in_page, ++g_index ) - if( new_page->mask & uintptr_t(1)<<begin_in_page ) { - if( concurrent_queue_base::copy == op_type ) { - base.copy_page_item( *new_page, begin_in_page, *src_page, begin_in_page ); - } else { - __TBB_ASSERT( concurrent_queue_base::move == op_type, NULL ); - static_cast<concurrent_queue_base_v8&>(base).move_page_item( *new_page, begin_in_page, *src_page, begin_in_page ); - } - } - return new_page; -} - -void micro_queue::make_invalid( ticket k ) -{ - static concurrent_queue_base::page dummy = {static_cast<page*>((void*)1), 0}; - // mark it so that no more pushes are allowed. - static_invalid_page = &dummy; - { - spin_mutex::scoped_lock lock( page_mutex ); - tail_counter = k+concurrent_queue_rep::n_queue+1; - if( page* q = tail_page ) - q->next = static_cast<page*>(static_invalid_page); - else - head_page = static_cast<page*>(static_invalid_page); - tail_page = static_cast<page*>(static_invalid_page); - } -} - -#if _MSC_VER && !defined(__INTEL_COMPILER) - #pragma warning( pop ) -#endif // warning 4146 is back - -//------------------------------------------------------------------------ -// concurrent_queue_base -//------------------------------------------------------------------------ -concurrent_queue_base_v3::concurrent_queue_base_v3( size_t item_sz ) { - items_per_page = item_sz<= 8 ? 32 : - item_sz<= 16 ? 16 : - item_sz<= 32 ? 8 : - item_sz<= 64 ? 4 : - item_sz<=128 ? 2 : - 1; - my_capacity = size_t(-1)/(item_sz>1 ? item_sz : 2); - my_rep = cache_aligned_allocator<concurrent_queue_rep>().allocate(1); - __TBB_ASSERT( is_aligned(my_rep, NFS_GetLineSize()), "alignment error" ); - __TBB_ASSERT( is_aligned(&my_rep->head_counter, NFS_GetLineSize()), "alignment error" ); - __TBB_ASSERT( is_aligned(&my_rep->tail_counter, NFS_GetLineSize()), "alignment error" ); - __TBB_ASSERT( is_aligned(&my_rep->array, NFS_GetLineSize()), "alignment error" ); - memset((void*)my_rep,0,sizeof(concurrent_queue_rep)); - new ( &my_rep->items_avail ) concurrent_monitor(); - new ( &my_rep->slots_avail ) concurrent_monitor(); - this->item_size = item_sz; -} - -concurrent_queue_base_v3::~concurrent_queue_base_v3() { - size_t nq = my_rep->n_queue; - for( size_t i=0; i<nq; i++ ) - __TBB_ASSERT( my_rep->array[i].tail_page==NULL, "pages were not freed properly" ); - cache_aligned_allocator<concurrent_queue_rep>().deallocate(my_rep,1); -} - -void concurrent_queue_base_v3::internal_push( const void* src ) { - internal_insert_item( src, copy ); -} - -void concurrent_queue_base_v8::internal_push_move( const void* src ) { - internal_insert_item( src, move ); -} - -void concurrent_queue_base_v3::internal_insert_item( const void* src, copy_specifics op_type ) { - concurrent_queue_rep& r = *my_rep; - unsigned old_abort_counter = r.abort_counter; - ticket k = r.tail_counter++; - ptrdiff_t e = my_capacity; -#if DO_ITT_NOTIFY - bool sync_prepare_done = false; -#endif - if( (ptrdiff_t)(k-r.head_counter)>=e ) { // queue is full -#if DO_ITT_NOTIFY - if( !sync_prepare_done ) { - ITT_NOTIFY( sync_prepare, &sync_prepare_done ); - sync_prepare_done = true; - } -#endif - bool slept = false; - concurrent_monitor::thread_context thr_ctx; - r.slots_avail.prepare_wait( thr_ctx, ((ptrdiff_t)(k-e)) ); - while( (ptrdiff_t)(k-r.head_counter)>=const_cast<volatile ptrdiff_t&>(e = my_capacity) ) { - __TBB_TRY { - if( r.abort_counter!=old_abort_counter ) { - r.slots_avail.cancel_wait( thr_ctx ); - throw_exception( eid_user_abort ); - } - slept = r.slots_avail.commit_wait( thr_ctx ); - } __TBB_CATCH( tbb::user_abort& ) { - r.choose(k).abort_push(k, *this); - __TBB_RETHROW(); - } __TBB_CATCH(...) { - __TBB_RETHROW(); - } - if (slept == true) break; - r.slots_avail.prepare_wait( thr_ctx, ((ptrdiff_t)(k-e)) ); - } - if( !slept ) - r.slots_avail.cancel_wait( thr_ctx ); - } - ITT_NOTIFY( sync_acquired, &sync_prepare_done ); - __TBB_ASSERT( (ptrdiff_t)(k-r.head_counter)<my_capacity, NULL); - r.choose( k ).push( src, k, *this, op_type ); - r.items_avail.notify( predicate_leq(k) ); -} - -void concurrent_queue_base_v3::internal_pop( void* dst ) { - concurrent_queue_rep& r = *my_rep; - ticket k; -#if DO_ITT_NOTIFY - bool sync_prepare_done = false; -#endif - unsigned old_abort_counter = r.abort_counter; - // This loop is a single pop operation; abort_counter should not be re-read inside - do { - k=r.head_counter++; - if ( (ptrdiff_t)(r.tail_counter-k)<=0 ) { // queue is empty -#if DO_ITT_NOTIFY - if( !sync_prepare_done ) { - ITT_NOTIFY( sync_prepare, dst ); - sync_prepare_done = true; - } -#endif - bool slept = false; - concurrent_monitor::thread_context thr_ctx; - r.items_avail.prepare_wait( thr_ctx, k ); - while( (ptrdiff_t)(r.tail_counter-k)<=0 ) { - __TBB_TRY { - if( r.abort_counter!=old_abort_counter ) { - r.items_avail.cancel_wait( thr_ctx ); - throw_exception( eid_user_abort ); - } - slept = r.items_avail.commit_wait( thr_ctx ); - } __TBB_CATCH( tbb::user_abort& ) { - r.head_counter--; - __TBB_RETHROW(); - } __TBB_CATCH(...) { - __TBB_RETHROW(); - } - if (slept == true) break; - r.items_avail.prepare_wait( thr_ctx, k ); - } - if( !slept ) - r.items_avail.cancel_wait( thr_ctx ); - } - __TBB_ASSERT((ptrdiff_t)(r.tail_counter-k)>0, NULL); - } while( !r.choose(k).pop(dst,k,*this) ); - - // wake up a producer.. - r.slots_avail.notify( predicate_leq(k) ); -} - -void concurrent_queue_base_v3::internal_abort() { - concurrent_queue_rep& r = *my_rep; - ++r.abort_counter; - r.items_avail.abort_all(); - r.slots_avail.abort_all(); -} - -bool concurrent_queue_base_v3::internal_pop_if_present( void* dst ) { - concurrent_queue_rep& r = *my_rep; - ticket k; - do { - k = r.head_counter; - for(;;) { - if( (ptrdiff_t)(r.tail_counter-k)<=0 ) { - // Queue is empty - return false; - } - // Queue had item with ticket k when we looked. Attempt to get that item. - ticket tk=k; - k = r.head_counter.compare_and_swap( tk+1, tk ); - if( k==tk ) - break; - // Another thread snatched the item, retry. - } - } while( !r.choose( k ).pop( dst, k, *this ) ); - - r.slots_avail.notify( predicate_leq(k) ); - - return true; -} - -bool concurrent_queue_base_v3::internal_push_if_not_full( const void* src ) { - return internal_insert_if_not_full( src, copy ); -} - -bool concurrent_queue_base_v8::internal_push_move_if_not_full( const void* src ) { - return internal_insert_if_not_full( src, move ); -} - -bool concurrent_queue_base_v3::internal_insert_if_not_full( const void* src, copy_specifics op_type ) { - concurrent_queue_rep& r = *my_rep; - ticket k = r.tail_counter; - for(;;) { - if( (ptrdiff_t)(k-r.head_counter)>=my_capacity ) { - // Queue is full - return false; - } - // Queue had empty slot with ticket k when we looked. Attempt to claim that slot. - ticket tk=k; - k = r.tail_counter.compare_and_swap( tk+1, tk ); - if( k==tk ) - break; - // Another thread claimed the slot, so retry. - } - r.choose(k).push(src, k, *this, op_type); - r.items_avail.notify( predicate_leq(k) ); - return true; -} - -ptrdiff_t concurrent_queue_base_v3::internal_size() const { - __TBB_ASSERT( sizeof(ptrdiff_t)<=sizeof(size_t), NULL ); - return ptrdiff_t(my_rep->tail_counter-my_rep->head_counter-my_rep->n_invalid_entries); -} - -bool concurrent_queue_base_v3::internal_empty() const { - ticket tc = my_rep->tail_counter; - ticket hc = my_rep->head_counter; - // if tc!=r.tail_counter, the queue was not empty at some point between the two reads. - return ( tc==my_rep->tail_counter && ptrdiff_t(tc-hc-my_rep->n_invalid_entries)<=0 ); -} - -void concurrent_queue_base_v3::internal_set_capacity( ptrdiff_t capacity, size_t /*item_sz*/ ) { - my_capacity = capacity<0 ? concurrent_queue_rep::infinite_capacity : capacity; -} - -void concurrent_queue_base_v3::internal_finish_clear() { - size_t nq = my_rep->n_queue; - for( size_t i=0; i<nq; ++i ) { - page* tp = my_rep->array[i].tail_page; - __TBB_ASSERT( my_rep->array[i].head_page==tp, "at most one page should remain" ); - if( tp!=NULL) { - if( tp!=static_invalid_page ) deallocate_page( tp ); - my_rep->array[i].tail_page = NULL; - } - } -} - -void concurrent_queue_base_v3::internal_throw_exception() const { - throw_exception( eid_bad_alloc ); -} - -void concurrent_queue_base_v3::internal_assign( const concurrent_queue_base& src, copy_specifics op_type ) { - items_per_page = src.items_per_page; - my_capacity = src.my_capacity; - - // copy concurrent_queue_rep. - my_rep->head_counter = src.my_rep->head_counter; - my_rep->tail_counter = src.my_rep->tail_counter; - my_rep->n_invalid_entries = src.my_rep->n_invalid_entries; - my_rep->abort_counter = src.my_rep->abort_counter; - - // copy micro_queues - for( size_t i = 0; i<my_rep->n_queue; ++i ) - my_rep->array[i].assign( src.my_rep->array[i], *this, op_type ); - - __TBB_ASSERT( my_rep->head_counter==src.my_rep->head_counter && my_rep->tail_counter==src.my_rep->tail_counter, - "the source concurrent queue should not be concurrently modified." ); -} - -void concurrent_queue_base_v3::assign( const concurrent_queue_base& src ) { - internal_assign( src, copy ); -} - -void concurrent_queue_base_v8::move_content( concurrent_queue_base_v8& src ) { - internal_assign( src, move ); -} - -//------------------------------------------------------------------------ -// concurrent_queue_iterator_rep -//------------------------------------------------------------------------ -class concurrent_queue_iterator_rep: no_assign { -public: - ticket head_counter; - const concurrent_queue_base& my_queue; - const size_t offset_of_last; - concurrent_queue_base::page* array[concurrent_queue_rep::n_queue]; - concurrent_queue_iterator_rep( const concurrent_queue_base& queue, size_t offset_of_last_ ) : - head_counter(queue.my_rep->head_counter), - my_queue(queue), - offset_of_last(offset_of_last_) - { - const concurrent_queue_rep& rep = *queue.my_rep; - for( size_t k=0; k<concurrent_queue_rep::n_queue; ++k ) - array[k] = rep.array[k].head_page; - } - //! Set item to point to kth element. Return true if at end of queue or item is marked valid; false otherwise. - bool get_item( void*& item, size_t k ) { - if( k==my_queue.my_rep->tail_counter ) { - item = NULL; - return true; - } else { - concurrent_queue_base::page* p = array[concurrent_queue_rep::index(k)]; - __TBB_ASSERT(p,NULL); - size_t i = modulo_power_of_two( k/concurrent_queue_rep::n_queue, my_queue.items_per_page ); - item = static_cast<unsigned char*>(static_cast<void*>(p)) + offset_of_last + my_queue.item_size*i; - return (p->mask & uintptr_t(1)<<i)!=0; - } - } -}; - -//------------------------------------------------------------------------ -// concurrent_queue_iterator_base -//------------------------------------------------------------------------ - -void concurrent_queue_iterator_base_v3::initialize( const concurrent_queue_base& queue, size_t offset_of_last ) { - my_rep = cache_aligned_allocator<concurrent_queue_iterator_rep>().allocate(1); - new( my_rep ) concurrent_queue_iterator_rep(queue,offset_of_last); - size_t k = my_rep->head_counter; - if( !my_rep->get_item(my_item, k) ) advance(); -} - -concurrent_queue_iterator_base_v3::concurrent_queue_iterator_base_v3( const concurrent_queue_base& queue ) { - initialize(queue,0); -} - -concurrent_queue_iterator_base_v3::concurrent_queue_iterator_base_v3( const concurrent_queue_base& queue, size_t offset_of_last ) { - initialize(queue,offset_of_last); -} - -void concurrent_queue_iterator_base_v3::assign( const concurrent_queue_iterator_base& other ) { - if( my_rep!=other.my_rep ) { - if( my_rep ) { - cache_aligned_allocator<concurrent_queue_iterator_rep>().deallocate(my_rep, 1); - my_rep = NULL; - } - if( other.my_rep ) { - my_rep = cache_aligned_allocator<concurrent_queue_iterator_rep>().allocate(1); - new( my_rep ) concurrent_queue_iterator_rep( *other.my_rep ); - } - } - my_item = other.my_item; -} - -void concurrent_queue_iterator_base_v3::advance() { - __TBB_ASSERT( my_item, "attempt to increment iterator past end of queue" ); - size_t k = my_rep->head_counter; - const concurrent_queue_base& queue = my_rep->my_queue; -#if TBB_USE_ASSERT - void* tmp; - my_rep->get_item(tmp,k); - __TBB_ASSERT( my_item==tmp, NULL ); -#endif /* TBB_USE_ASSERT */ - size_t i = modulo_power_of_two( k/concurrent_queue_rep::n_queue, queue.items_per_page ); - if( i==queue.items_per_page-1 ) { - concurrent_queue_base::page*& root = my_rep->array[concurrent_queue_rep::index(k)]; - root = root->next; - } - // advance k - my_rep->head_counter = ++k; - if( !my_rep->get_item(my_item, k) ) advance(); -} - -concurrent_queue_iterator_base_v3::~concurrent_queue_iterator_base_v3() { - //delete my_rep; - cache_aligned_allocator<concurrent_queue_iterator_rep>().deallocate(my_rep, 1); - my_rep = NULL; -} - -} // namespace internal - -} // namespace tbb diff --git a/src/tbb/src/tbb/concurrent_vector.cpp b/src/tbb/src/tbb/concurrent_vector.cpp deleted file mode 100644 index 4ecf09903..000000000 --- a/src/tbb/src/tbb/concurrent_vector.cpp +++ /dev/null @@ -1,618 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#if (_MSC_VER) - //MSVC 10 "deprecated" application of some std:: algorithms to raw pointers as not safe. - //The reason is that destination is not checked against bounds/having enough place. - #define _SCL_SECURE_NO_WARNINGS -#endif - -#include "tbb/concurrent_vector.h" -#include "tbb/cache_aligned_allocator.h" -#include "tbb/tbb_exception.h" -#include "tbb_misc.h" -#include "itt_notify.h" - -#include <cstring> -#include <memory> //for uninitialized_fill_n - -#if defined(_MSC_VER) && defined(_Wp64) - // Workaround for overzealous compiler warnings in /Wp64 mode - #pragma warning (disable: 4267) -#endif - -using namespace std; - -namespace tbb { - -namespace internal { - class concurrent_vector_base_v3::helper :no_assign { -public: - //! memory page size - static const size_type page_size = 4096; - - inline static bool incompact_predicate(size_type size) { // assert size != 0, see source/test/test_vector_layout.cpp - return size < page_size || ((size-1)%page_size < page_size/2 && size < page_size * 128); // for more details - } - - inline static size_type find_segment_end(const concurrent_vector_base_v3 &v) { - segment_t *s = v.my_segment; - segment_index_t u = s==v.my_storage? pointers_per_short_table : pointers_per_long_table; - segment_index_t k = 0; - while( k < u && (s[k].load<relaxed>()==segment_allocated() )) - ++k; - return k; - } - - // TODO: optimize accesses to my_first_block - //! assign first segment size. k - is index of last segment to be allocated, not a count of segments - inline static void assign_first_segment_if_necessary(concurrent_vector_base_v3 &v, segment_index_t k) { - if( !v.my_first_block ) { - /* There was a suggestion to set first segment according to incompact_predicate: - while( k && !helper::incompact_predicate(segment_size( k ) * element_size) ) - --k; // while previous vector size is compact, decrement - // reasons to not do it: - // * constructor(n) is not ready to accept fragmented segments - // * backward compatibility due to that constructor - // * current version gives additional guarantee and faster init. - // * two calls to reserve() will give the same effect. - */ - v.my_first_block.compare_and_swap(k+1, 0); // store number of segments - } - } - - inline static void *allocate_segment(concurrent_vector_base_v3 &v, size_type n) { - void *ptr = v.vector_allocator_ptr(v, n); - if(!ptr) throw_exception(eid_bad_alloc); // check for bad allocation, throw exception - return ptr; - } - - //! Publish segment so other threads can see it. - template<typename argument_type> - inline static void publish_segment( segment_t& s, argument_type rhs ) { - // see also itt_store_pointer_with_release_v3() - ITT_NOTIFY( sync_releasing, &s ); - s.store<release>(rhs); - } - - static size_type enable_segment(concurrent_vector_base_v3 &v, size_type k, size_type element_size, bool mark_as_not_used_on_failure = false); - - // TODO: rename as get_segments_table() and return segment pointer - inline static void extend_table_if_necessary(concurrent_vector_base_v3 &v, size_type k, size_type start ) { - if(k >= pointers_per_short_table && v.my_segment == v.my_storage) - extend_segment_table(v, start ); - } - - static void extend_segment_table(concurrent_vector_base_v3 &v, size_type start); - - struct segment_not_used_predicate: no_assign { - segment_t &s; - segment_not_used_predicate(segment_t &segment) : s(segment) {} - bool operator()() const { return s.load<relaxed>() == segment_not_used ();} - }; - inline static segment_t& acquire_segment(concurrent_vector_base_v3 &v, size_type index, size_type element_size, bool owner) { - segment_t &s = v.my_segment[index]; // TODO: pass v.my_segment as argument - if( s.load<acquire>() == segment_not_used() ) { // do not check for segment_allocation_failed state - if( owner ) { - enable_segment( v, index, element_size ); - } else { - ITT_NOTIFY(sync_prepare, &s); - spin_wait_while(segment_not_used_predicate(s)); - ITT_NOTIFY(sync_acquired, &s); - } - } else { - ITT_NOTIFY(sync_acquired, &s); - } - enforce_segment_allocated(s.load<relaxed>()); //it's hard to recover correctly after segment_allocation_failed state - return s; - } - - ///// non-static fields of helper for exception-safe iteration across segments - segment_t *table;// TODO: review all segment_index_t as just short type - size_type first_block, k, sz, start, finish, element_size; - helper(segment_t *segments, size_type fb, size_type esize, size_type index, size_type s, size_type f) throw() - : table(segments), first_block(fb), k(index), sz(0), start(s), finish(f), element_size(esize) {} - inline void first_segment() throw() { - __TBB_ASSERT( start <= finish, NULL ); - __TBB_ASSERT( first_block || !finish, NULL ); - if( k < first_block ) k = 0; // process solid segment at a time - size_type base = segment_base( k ); - __TBB_ASSERT( base <= start, NULL ); - finish -= base; start -= base; // rebase as offsets from segment k - sz = k ? base : segment_size( first_block ); // sz==base for k>0 - } - inline void next_segment() throw() { - finish -= sz; start = 0; // offsets from next segment - if( !k ) k = first_block; - else { ++k; sz = segment_size( k ); } - } - template<typename F> - inline size_type apply(const F &func) { - first_segment(); - while( sz < finish ) { // work for more than one segment - //TODO: remove extra load() of table[k] inside func - func( table[k], table[k].load<relaxed>().pointer<char>() + element_size*start, sz - start ); - next_segment(); - } - func( table[k], table[k].load<relaxed>().pointer<char>() + element_size*start, finish - start ); - return k; - } - inline segment_value_t get_segment_value(size_type index, bool wait) { - segment_t &s = table[index]; - if( wait && (s.load<acquire>() == segment_not_used()) ) { - ITT_NOTIFY(sync_prepare, &s); - spin_wait_while(segment_not_used_predicate(s)); - ITT_NOTIFY(sync_acquired, &s); - } - return s.load<relaxed>(); - } - ~helper() { - if( sz >= finish ) return; // the work is done correctly - cleanup(); - } - - //! Out of line code to assists destructor in infrequent cases. - void cleanup(); - - /// TODO: turn into lambda functions when available - struct init_body { - internal_array_op2 func; - const void *arg; - init_body(internal_array_op2 init, const void *src) : func(init), arg(src) {} - void operator()(segment_t &, void *begin, size_type n) const { - func( begin, arg, n ); - } - }; - struct safe_init_body { - internal_array_op2 func; - const void *arg; - safe_init_body(internal_array_op2 init, const void *src) : func(init), arg(src) {} - void operator()(segment_t &s, void *begin, size_type n) const { - enforce_segment_allocated(s.load<relaxed>()); - func( begin, arg, n ); - } - }; - struct destroy_body { - internal_array_op1 func; - destroy_body(internal_array_op1 destroy) : func(destroy) {} - void operator()(segment_t &s, void *begin, size_type n) const { - if(s.load<relaxed>() == segment_allocated()) - func( begin, n ); - } - }; -}; - -void concurrent_vector_base_v3::helper::extend_segment_table(concurrent_vector_base_v3 &v, concurrent_vector_base_v3::size_type start) { - if( start > segment_size(pointers_per_short_table) ) start = segment_size(pointers_per_short_table); - // If other threads are trying to set pointers in the short segment, wait for them to finish their - // assignments before we copy the short segment to the long segment. Note: grow_to_at_least depends on it - for( segment_index_t i = 0; segment_base(i) < start && v.my_segment == v.my_storage; i++ ){ - if(v.my_storage[i].load<relaxed>() == segment_not_used()) { - ITT_NOTIFY(sync_prepare, &v.my_storage[i]); - atomic_backoff backoff(true); - while( v.my_segment == v.my_storage && (v.my_storage[i].load<relaxed>() == segment_not_used()) ) - backoff.pause(); - ITT_NOTIFY(sync_acquired, &v.my_storage[i]); - } - } - if( v.my_segment != v.my_storage ) return; - - segment_t* new_segment_table = (segment_t*)NFS_Allocate( pointers_per_long_table, sizeof(segment_t), NULL ); - __TBB_ASSERT(new_segment_table, "NFS_Allocate should throws exception if it cannot allocate the requested storage, and not returns zero pointer" ); - std::uninitialized_fill_n(new_segment_table,size_t(pointers_per_long_table),segment_t()); //init newly allocated table - //TODO: replace with static assert - __TBB_STATIC_ASSERT(pointers_per_long_table >= pointers_per_short_table, "size of the big table should be not lesser than of the small one, as we copy values to it" ); - std::copy(v.my_storage, v.my_storage+pointers_per_short_table, new_segment_table);//copy values from old table, here operator= of segment_t is used - if( v.my_segment.compare_and_swap( new_segment_table, v.my_storage ) != v.my_storage ) - NFS_Free( new_segment_table ); - // else TODO: add ITT_NOTIFY signals for v.my_segment? -} - -concurrent_vector_base_v3::size_type concurrent_vector_base_v3::helper::enable_segment(concurrent_vector_base_v3 &v, concurrent_vector_base_v3::size_type k, concurrent_vector_base_v3::size_type element_size, - bool mark_as_not_used_on_failure ) { - - struct segment_scope_guard : no_copy{ - segment_t* my_segment_ptr; - bool my_mark_as_not_used; - segment_scope_guard(segment_t& segment, bool mark_as_not_used) : my_segment_ptr(&segment), my_mark_as_not_used(mark_as_not_used){} - void dismiss(){ my_segment_ptr = 0;} - ~segment_scope_guard(){ - if (my_segment_ptr){ - if (!my_mark_as_not_used){ - publish_segment(*my_segment_ptr, segment_allocation_failed()); - }else{ - publish_segment(*my_segment_ptr, segment_not_used()); - } - } - } - }; - - segment_t* s = v.my_segment; // TODO: optimize out as argument? Optimize accesses to my_first_block - __TBB_ASSERT(s[k].load<relaxed>() != segment_allocated(), "concurrent operation during growth?"); - - size_type size_of_enabled_segment = segment_size(k); - size_type size_to_allocate = size_of_enabled_segment; - if( !k ) { - assign_first_segment_if_necessary(v, default_initial_segments-1); - size_of_enabled_segment = 2 ; - size_to_allocate = segment_size(v.my_first_block); - - } else { - spin_wait_while_eq( v.my_first_block, segment_index_t(0) ); - } - - if( k && (k < v.my_first_block)){ //no need to allocate anything - // s[0].array is changed only once ( 0 -> !0 ) and points to uninitialized memory - segment_value_t array0 = s[0].load<acquire>(); - if(array0 == segment_not_used()){ - // sync_prepare called only if there is a wait - ITT_NOTIFY(sync_prepare, &s[0]); - spin_wait_while( segment_not_used_predicate(s[0])); - array0 = s[0].load<acquire>(); - } - ITT_NOTIFY(sync_acquired, &s[0]); - - segment_scope_guard k_segment_guard(s[k], false); - enforce_segment_allocated(array0); // initial segment should be allocated - k_segment_guard.dismiss(); - - publish_segment( s[k], - static_cast<void*>(array0.pointer<char>() + segment_base(k)*element_size ) - ); - } else { - segment_scope_guard k_segment_guard(s[k], mark_as_not_used_on_failure); - publish_segment(s[k], allocate_segment(v, size_to_allocate)); - k_segment_guard.dismiss(); - } - return size_of_enabled_segment; -} - -void concurrent_vector_base_v3::helper::cleanup() { - if( !sz ) { // allocation failed, restore the table - segment_index_t k_start = k, k_end = segment_index_of(finish-1); - if( segment_base( k_start ) < start ) - get_segment_value(k_start++, true); // wait - if( k_start < first_block ) { - segment_value_t segment0 = get_segment_value(0, start>0); // wait if necessary - if((segment0 != segment_not_used()) && !k_start ) ++k_start; - if(segment0 != segment_allocated()) - for(; k_start < first_block && k_start <= k_end; ++k_start ) - publish_segment(table[k_start], segment_allocation_failed()); - else for(; k_start < first_block && k_start <= k_end; ++k_start ) - publish_segment(table[k_start], static_cast<void*>( - (segment0.pointer<char>()) + segment_base(k_start)*element_size) ); - } - for(; k_start <= k_end; ++k_start ) // not in first block - if(table[k_start].load<acquire>() == segment_not_used()) - publish_segment(table[k_start], segment_allocation_failed()); - // fill allocated items - first_segment(); - goto recover; - } - while( sz <= finish ) { // there is still work for at least one segment - next_segment(); -recover: - segment_value_t array = table[k].load<relaxed>(); - if(array == segment_allocated()) - std::memset( (array.pointer<char>()) + element_size*start, 0, ((sz<finish?sz:finish) - start)*element_size ); - else __TBB_ASSERT( array == segment_allocation_failed(), NULL ); - } -} - -concurrent_vector_base_v3::~concurrent_vector_base_v3() { - segment_t* s = my_segment; - if( s != my_storage ) { -#if TBB_USE_ASSERT - //to please assert in segment_t destructor - std::fill_n(my_storage,size_t(pointers_per_short_table),segment_t()); -#endif /* TBB_USE_ASSERT */ -#if TBB_USE_DEBUG - for( segment_index_t i = 0; i < pointers_per_long_table; i++) - __TBB_ASSERT( my_segment[i].load<relaxed>() != segment_allocated(), "Segment should have been freed. Please recompile with new TBB before using exceptions."); -#endif - my_segment = my_storage; - NFS_Free( s ); - } -} - -concurrent_vector_base_v3::size_type concurrent_vector_base_v3::internal_capacity() const { - return segment_base( helper::find_segment_end(*this) ); -} - -void concurrent_vector_base_v3::internal_throw_exception(size_type t) const { - switch(t) { - case 0: throw_exception(eid_out_of_range); - case 1: throw_exception(eid_segment_range_error); - case 2: throw_exception(eid_index_range_error); - } -} - -void concurrent_vector_base_v3::internal_reserve( size_type n, size_type element_size, size_type max_size ) { - if( n>max_size ) - throw_exception(eid_reservation_length_error); - __TBB_ASSERT( n, NULL ); - helper::assign_first_segment_if_necessary(*this, segment_index_of(n-1)); - segment_index_t k = helper::find_segment_end(*this); - - for( ; segment_base(k)<n; ++k ) { - helper::extend_table_if_necessary(*this, k, 0); - if(my_segment[k].load<relaxed>() != segment_allocated()) - helper::enable_segment(*this, k, element_size, true ); //in case of failure mark segments as not used - } -} - -//TODO: Looks like atomic loads can be done relaxed here, as the only place this method is called from -//is the constructor, which does not require synchronization (for more details see comment in the -// concurrent_vector_base constructor). -void concurrent_vector_base_v3::internal_copy( const concurrent_vector_base_v3& src, size_type element_size, internal_array_op2 copy ) { - size_type n = src.my_early_size; - __TBB_ASSERT( my_segment == my_storage, NULL); - if( n ) { - helper::assign_first_segment_if_necessary(*this, segment_index_of(n-1)); - size_type b; - for( segment_index_t k=0; (b=segment_base(k))<n; ++k ) { - if( (src.my_segment.load<acquire>() == src.my_storage && k >= pointers_per_short_table) - || (src.my_segment[k].load<relaxed>() != segment_allocated())) { - my_early_size = b; break; - } - helper::extend_table_if_necessary(*this, k, 0); - size_type m = helper::enable_segment(*this, k, element_size); - if( m > n-b ) m = n-b; - my_early_size = b+m; - copy( my_segment[k].load<relaxed>().pointer<void>(), src.my_segment[k].load<relaxed>().pointer<void>(), m ); - } - } -} - -void concurrent_vector_base_v3::internal_assign( const concurrent_vector_base_v3& src, size_type element_size, internal_array_op1 destroy, internal_array_op2 assign, internal_array_op2 copy ) { - size_type n = src.my_early_size; - while( my_early_size>n ) { // TODO: improve - segment_index_t k = segment_index_of( my_early_size-1 ); - size_type b=segment_base(k); - size_type new_end = b>=n ? b : n; - __TBB_ASSERT( my_early_size>new_end, NULL ); - enforce_segment_allocated(my_segment[k].load<relaxed>()); //if vector was broken before - // destructors are supposed to not throw any exceptions - destroy( my_segment[k].load<relaxed>().pointer<char>() + element_size*(new_end-b), my_early_size-new_end ); - my_early_size = new_end; - } - size_type dst_initialized_size = my_early_size; - my_early_size = n; - helper::assign_first_segment_if_necessary(*this, segment_index_of(n)); - size_type b; - for( segment_index_t k=0; (b=segment_base(k))<n; ++k ) { - if( (src.my_segment.load<acquire>() == src.my_storage && k >= pointers_per_short_table) - || src.my_segment[k].load<relaxed>() != segment_allocated() ) { // if source is damaged - my_early_size = b; break; // TODO: it may cause undestructed items - } - helper::extend_table_if_necessary(*this, k, 0); - if( my_segment[k].load<relaxed>() == segment_not_used()) - helper::enable_segment(*this, k, element_size); - else - enforce_segment_allocated(my_segment[k].load<relaxed>()); - size_type m = k? segment_size(k) : 2; - if( m > n-b ) m = n-b; - size_type a = 0; - if( dst_initialized_size>b ) { - a = dst_initialized_size-b; - if( a>m ) a = m; - assign( my_segment[k].load<relaxed>().pointer<void>(), src.my_segment[k].load<relaxed>().pointer<void>(), a ); - m -= a; - a *= element_size; - } - if( m>0 ) - copy( my_segment[k].load<relaxed>().pointer<char>() + a, src.my_segment[k].load<relaxed>().pointer<char>() + a, m ); - } - __TBB_ASSERT( src.my_early_size==n, "detected use of concurrent_vector::operator= with right side that was concurrently modified" ); -} - -void* concurrent_vector_base_v3::internal_push_back( size_type element_size, size_type& index ) { - __TBB_ASSERT( sizeof(my_early_size)==sizeof(uintptr_t), NULL ); - size_type tmp = my_early_size.fetch_and_increment<acquire>(); - index = tmp; - segment_index_t k_old = segment_index_of( tmp ); - size_type base = segment_base(k_old); - helper::extend_table_if_necessary(*this, k_old, tmp); - segment_t& s = helper::acquire_segment(*this, k_old, element_size, base==tmp); - size_type j_begin = tmp-base; - return (void*)(s.load<relaxed>().pointer<char>() + element_size*j_begin); -} - -void concurrent_vector_base_v3::internal_grow_to_at_least( size_type new_size, size_type element_size, internal_array_op2 init, const void *src ) { - internal_grow_to_at_least_with_result( new_size, element_size, init, src ); -} - -concurrent_vector_base_v3::size_type concurrent_vector_base_v3::internal_grow_to_at_least_with_result( size_type new_size, size_type element_size, internal_array_op2 init, const void *src ) { - size_type e = my_early_size; - while( e<new_size ) { - size_type f = my_early_size.compare_and_swap(new_size,e); - if( f==e ) { - internal_grow( e, new_size, element_size, init, src ); - break; - } - e = f; - } - // Check/wait for segments allocation completes - segment_index_t i, k_old = segment_index_of( new_size-1 ); - if( k_old >= pointers_per_short_table && my_segment == my_storage ) { - spin_wait_while_eq( my_segment, my_storage ); - } - for( i = 0; i <= k_old; ++i ) { - segment_t &s = my_segment[i]; - if(s.load<relaxed>() == segment_not_used()) { - ITT_NOTIFY(sync_prepare, &s); - atomic_backoff backoff(true); - while( my_segment[i].load<acquire>() == segment_not_used() ) // my_segment may change concurrently - backoff.pause(); - ITT_NOTIFY(sync_acquired, &s); - } - enforce_segment_allocated(my_segment[i].load<relaxed>()); - } -#if TBB_USE_DEBUG - size_type capacity = internal_capacity(); - __TBB_ASSERT( capacity >= new_size, NULL); -#endif - return e; -} - -concurrent_vector_base_v3::size_type concurrent_vector_base_v3::internal_grow_by( size_type delta, size_type element_size, internal_array_op2 init, const void *src ) { - size_type result = my_early_size.fetch_and_add(delta); - internal_grow( result, result+delta, element_size, init, src ); - return result; -} - -void concurrent_vector_base_v3::internal_grow( const size_type start, size_type finish, size_type element_size, internal_array_op2 init, const void *src ) { - __TBB_ASSERT( start<finish, "start must be less than finish" ); - segment_index_t k_start = segment_index_of(start), k_end = segment_index_of(finish-1); - helper::assign_first_segment_if_necessary(*this, k_end); - helper::extend_table_if_necessary(*this, k_end, start); - helper range(my_segment, my_first_block, element_size, k_start, start, finish); - for(; k_end > k_start && k_end >= range.first_block; --k_end ) // allocate segments in reverse order - helper::acquire_segment(*this, k_end, element_size, true/*for k_end>k_start*/); - for(; k_start <= k_end; ++k_start ) // but allocate first block in straight order - helper::acquire_segment(*this, k_start, element_size, segment_base( k_start ) >= start ); - range.apply( helper::init_body(init, src) ); -} - -void concurrent_vector_base_v3::internal_resize( size_type n, size_type element_size, size_type max_size, const void *src, - internal_array_op1 destroy, internal_array_op2 init ) { - size_type j = my_early_size; - if( n > j ) { // construct items - internal_reserve(n, element_size, max_size); - my_early_size = n; - helper for_each(my_segment, my_first_block, element_size, segment_index_of(j), j, n); - for_each.apply( helper::safe_init_body(init, src) ); - } else { - my_early_size = n; - helper for_each(my_segment, my_first_block, element_size, segment_index_of(n), n, j); - for_each.apply( helper::destroy_body(destroy) ); - } -} - -concurrent_vector_base_v3::segment_index_t concurrent_vector_base_v3::internal_clear( internal_array_op1 destroy ) { - __TBB_ASSERT( my_segment, NULL ); - size_type j = my_early_size; - my_early_size = 0; - helper for_each(my_segment, my_first_block, 0, 0, 0, j); // element_size is safe to be zero if 'start' is zero - j = for_each.apply( helper::destroy_body(destroy) ); - size_type i = helper::find_segment_end(*this); - return j < i? i : j+1; -} - -void *concurrent_vector_base_v3::internal_compact( size_type element_size, void *table, internal_array_op1 destroy, internal_array_op2 copy ) -{ - const size_type my_size = my_early_size; - const segment_index_t k_end = helper::find_segment_end(*this); // allocated segments - const segment_index_t k_stop = my_size? segment_index_of(my_size-1) + 1 : 0; // number of segments to store existing items: 0=>0; 1,2=>1; 3,4=>2; [5-8]=>3;.. - const segment_index_t first_block = my_first_block; // number of merged segments, getting values from atomics - - segment_index_t k = first_block; - if(k_stop < first_block) - k = k_stop; - else - while (k < k_stop && helper::incompact_predicate(segment_size( k ) * element_size) ) k++; - if(k_stop == k_end && k == first_block) - return NULL; - - segment_t *const segment_table = my_segment; - internal_segments_table &old = *static_cast<internal_segments_table*>( table ); - //this call is left here for sake of backward compatibility, and as a placeholder for table initialization - std::fill_n(old.table,sizeof(old.table)/sizeof(old.table[0]),segment_t()); - old.first_block=0; - - if ( k != first_block && k ) // first segment optimization - { - // exception can occur here - void *seg = helper::allocate_segment(*this, segment_size(k)); - old.table[0].store<relaxed>(seg); - old.first_block = k; // fill info for freeing new segment if exception occurs - // copy items to the new segment - size_type my_segment_size = segment_size( first_block ); - for (segment_index_t i = 0, j = 0; i < k && j < my_size; j = my_segment_size) { - __TBB_ASSERT( segment_table[i].load<relaxed>() == segment_allocated(), NULL); - void *s = static_cast<void*>( - static_cast<char*>(seg) + segment_base(i)*element_size ); - //TODO: refactor to use std::min - if(j + my_segment_size >= my_size) my_segment_size = my_size - j; - __TBB_TRY { // exception can occur here - copy( s, segment_table[i].load<relaxed>().pointer<void>(), my_segment_size ); - } __TBB_CATCH(...) { // destroy all the already copied items - helper for_each(&old.table[0], old.first_block, element_size, - 0, 0, segment_base(i)+ my_segment_size); - for_each.apply( helper::destroy_body(destroy) ); - __TBB_RETHROW(); - } - my_segment_size = i? segment_size( ++i ) : segment_size( i = first_block ); - } - // commit the changes - std::copy(segment_table,segment_table + k,old.table); - for (segment_index_t i = 0; i < k; i++) { - segment_table[i].store<relaxed>(static_cast<void*>( - static_cast<char*>(seg) + segment_base(i)*element_size )); - } - old.first_block = first_block; my_first_block = k; // now, first_block != my_first_block - // destroy original copies - my_segment_size = segment_size( first_block ); // old.first_block actually - for (segment_index_t i = 0, j = 0; i < k && j < my_size; j = my_segment_size) { - if(j + my_segment_size >= my_size) my_segment_size = my_size - j; - // destructors are supposed to not throw any exceptions - destroy( old.table[i].load<relaxed>().pointer<void>(), my_segment_size ); - my_segment_size = i? segment_size( ++i ) : segment_size( i = first_block ); - } - } - // free unnecessary segments allocated by reserve() call - if ( k_stop < k_end ) { - old.first_block = first_block; - std::copy(segment_table+k_stop, segment_table+k_end, old.table+k_stop ); - std::fill_n(segment_table+k_stop, (k_end-k_stop), segment_t()); - if( !k ) my_first_block = 0; - } - return table; -} - -void concurrent_vector_base_v3::internal_swap(concurrent_vector_base_v3& v) -{ - size_type my_sz = my_early_size.load<acquire>(); - size_type v_sz = v.my_early_size.load<relaxed>(); - if(!my_sz && !v_sz) return; - - bool my_was_short = (my_segment.load<relaxed>() == my_storage); - bool v_was_short = (v.my_segment.load<relaxed>() == v.my_storage); - - //In C++11, this would be: swap(my_storage, v.my_storage); - for (int i=0; i < pointers_per_short_table; ++i){ - swap(my_storage[i], v.my_storage[i]); - } - tbb::internal::swap<relaxed>(my_first_block, v.my_first_block); - tbb::internal::swap<relaxed>(my_segment, v.my_segment); - if (my_was_short){ - v.my_segment.store<relaxed>(v.my_storage); - } - if(v_was_short){ - my_segment.store<relaxed>(my_storage); - } - - my_early_size.store<relaxed>(v_sz); - v.my_early_size.store<release>(my_sz); -} - -} // namespace internal - -} // tbb diff --git a/src/tbb/src/tbb/condition_variable.cpp b/src/tbb/src/tbb/condition_variable.cpp deleted file mode 100644 index 49ed609f4..000000000 --- a/src/tbb/src/tbb/condition_variable.cpp +++ /dev/null @@ -1,199 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#include "tbb/tbb_config.h" -#include "tbb/compat/condition_variable" -#include "tbb/atomic.h" -#include "tbb_misc.h" -#include "dynamic_link.h" -#include "itt_notify.h" - -namespace tbb { - -namespace internal { - -//condition_variable -#if _WIN32||_WIN64 -using tbb::interface5::internal::condition_variable_using_event; - -static atomic<do_once_state> condvar_api_state; - -void WINAPI init_condvar_using_event( condition_variable_using_event* cv_event ) -{ - // TODO: For Metro port, we can always use the API for condition variables, without dynamic_link etc. - cv_event->event = CreateEventEx(NULL, NULL, 0x1 /*CREATE_EVENT_MANUAL_RESET*/, EVENT_ALL_ACCESS ); - InitializeCriticalSectionEx( &cv_event->mutex, 4000, 0 ); - cv_event->n_waiters = 0; - cv_event->release_count = 0; - cv_event->epoch = 0; -} - -BOOL WINAPI sleep_condition_variable_cs_using_event( condition_variable_using_event* cv_event, LPCRITICAL_SECTION cs, DWORD dwMilliseconds ) -{ - EnterCriticalSection( &cv_event->mutex ); - ++cv_event->n_waiters; - unsigned my_generation = cv_event->epoch; - LeaveCriticalSection( &cv_event->mutex ); - LeaveCriticalSection( cs ); - for (;;) { - // should come here at least once - DWORD rc = WaitForSingleObjectEx( cv_event->event, dwMilliseconds, FALSE ); - EnterCriticalSection( &cv_event->mutex ); - if( rc!=WAIT_OBJECT_0 ) { - --cv_event->n_waiters; - LeaveCriticalSection( &cv_event->mutex ); - if( rc==WAIT_TIMEOUT ) { - SetLastError( WAIT_TIMEOUT ); - EnterCriticalSection( cs ); - } - return false; - } - __TBB_ASSERT( rc==WAIT_OBJECT_0, NULL ); - if( cv_event->release_count>0 && cv_event->epoch!=my_generation ) - break; - LeaveCriticalSection( &cv_event->mutex ); - } - - // still in the critical section - --cv_event->n_waiters; - int count = --cv_event->release_count; - LeaveCriticalSection( &cv_event->mutex ); - - if( count==0 ) { - __TBB_ASSERT( cv_event->event, "Premature destruction of condition variable?" ); - ResetEvent( cv_event->event ); - } - EnterCriticalSection( cs ); - return true; -} - -void WINAPI wake_condition_variable_using_event( condition_variable_using_event* cv_event ) -{ - EnterCriticalSection( &cv_event->mutex ); - if( cv_event->n_waiters>cv_event->release_count ) { - SetEvent( cv_event->event ); // Signal the manual-reset event. - ++cv_event->release_count; - ++cv_event->epoch; - } - LeaveCriticalSection( &cv_event->mutex ); -} - -void WINAPI wake_all_condition_variable_using_event( condition_variable_using_event* cv_event ) -{ - EnterCriticalSection( &cv_event->mutex ); - if( cv_event->n_waiters>0 ) { - SetEvent( cv_event->event ); - cv_event->release_count = cv_event->n_waiters; - ++cv_event->epoch; - } - LeaveCriticalSection( &cv_event->mutex ); -} - -void WINAPI destroy_condvar_using_event( condition_variable_using_event* cv_event ) -{ - HANDLE my_event = cv_event->event; - EnterCriticalSection( &cv_event->mutex ); - // NULL is an invalid HANDLE value - cv_event->event = NULL; - if( cv_event->n_waiters>0 ) { - LeaveCriticalSection( &cv_event->mutex ); - spin_wait_until_eq( cv_event->n_waiters, 0 ); - // make sure the last thread completes its access to cv - EnterCriticalSection( &cv_event->mutex ); - } - LeaveCriticalSection( &cv_event->mutex ); - CloseHandle( my_event ); -} - -void WINAPI destroy_condvar_noop( CONDITION_VARIABLE* /*cv*/ ) { /*no op*/ } - -static void (WINAPI *__TBB_init_condvar)( PCONDITION_VARIABLE ) = (void (WINAPI *)(PCONDITION_VARIABLE))&init_condvar_using_event; -static BOOL (WINAPI *__TBB_condvar_wait)( PCONDITION_VARIABLE, LPCRITICAL_SECTION, DWORD ) = (BOOL (WINAPI *)(PCONDITION_VARIABLE,LPCRITICAL_SECTION, DWORD))&sleep_condition_variable_cs_using_event; -static void (WINAPI *__TBB_condvar_notify_one)( PCONDITION_VARIABLE ) = (void (WINAPI *)(PCONDITION_VARIABLE))&wake_condition_variable_using_event; -static void (WINAPI *__TBB_condvar_notify_all)( PCONDITION_VARIABLE ) = (void (WINAPI *)(PCONDITION_VARIABLE))&wake_all_condition_variable_using_event; -static void (WINAPI *__TBB_destroy_condvar)( PCONDITION_VARIABLE ) = (void (WINAPI *)(PCONDITION_VARIABLE))&destroy_condvar_using_event; - -//! Table describing how to link the handlers. -static const dynamic_link_descriptor CondVarLinkTable[] = { - DLD(InitializeConditionVariable, __TBB_init_condvar), - DLD(SleepConditionVariableCS, __TBB_condvar_wait), - DLD(WakeConditionVariable, __TBB_condvar_notify_one), - DLD(WakeAllConditionVariable, __TBB_condvar_notify_all) -}; - -void init_condvar_module() -{ - __TBB_ASSERT( (uintptr_t)__TBB_init_condvar==(uintptr_t)&init_condvar_using_event, NULL ); - if( dynamic_link( "Kernel32.dll", CondVarLinkTable, 4 ) ) - __TBB_destroy_condvar = (void (WINAPI *)(PCONDITION_VARIABLE))&destroy_condvar_noop; -} -#endif /* _WIN32||_WIN64 */ - -} // namespace internal - -#if _WIN32||_WIN64 - -namespace interface5 { -namespace internal { - -using tbb::internal::condvar_api_state; -using tbb::internal::__TBB_init_condvar; -using tbb::internal::__TBB_condvar_wait; -using tbb::internal::__TBB_condvar_notify_one; -using tbb::internal::__TBB_condvar_notify_all; -using tbb::internal::__TBB_destroy_condvar; -using tbb::internal::init_condvar_module; - -void internal_initialize_condition_variable( condvar_impl_t& cv ) -{ - atomic_do_once( &init_condvar_module, condvar_api_state ); - __TBB_init_condvar( &cv.cv_native ); -} - -void internal_destroy_condition_variable( condvar_impl_t& cv ) -{ - __TBB_destroy_condvar( &cv.cv_native ); -} - -void internal_condition_variable_notify_one( condvar_impl_t& cv ) -{ - __TBB_condvar_notify_one ( &cv.cv_native ); -} - -void internal_condition_variable_notify_all( condvar_impl_t& cv ) -{ - __TBB_condvar_notify_all( &cv.cv_native ); -} - -bool internal_condition_variable_wait( condvar_impl_t& cv, mutex* mtx, const tick_count::interval_t* i ) -{ - DWORD duration = i ? DWORD((i->seconds()*1000)) : INFINITE; - mtx->set_state( mutex::INITIALIZED ); - BOOL res = __TBB_condvar_wait( &cv.cv_native, mtx->native_handle(), duration ); - mtx->set_state( mutex::HELD ); - return res?true:false; -} - -} // namespace internal -} // nameespace interface5 - -#endif /* _WIN32||_WIN64 */ - -} // namespace tbb diff --git a/src/tbb/src/tbb/custom_scheduler.h b/src/tbb/src/tbb/custom_scheduler.h deleted file mode 100644 index c6511b271..000000000 --- a/src/tbb/src/tbb/custom_scheduler.h +++ /dev/null @@ -1,706 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef _TBB_custom_scheduler_H -#define _TBB_custom_scheduler_H - -#include "scheduler.h" -#include "observer_proxy.h" -#include "itt_notify.h" - -namespace tbb { -namespace internal { - -//------------------------------------------------------------------------ -//! Traits classes for scheduler -//------------------------------------------------------------------------ - -struct DefaultSchedulerTraits { - static const bool itt_possible = true; - static const bool has_slow_atomic = false; -}; - -struct IntelSchedulerTraits { - static const bool itt_possible = false; -#if __TBB_x86_32||__TBB_x86_64 - static const bool has_slow_atomic = true; -#else - static const bool has_slow_atomic = false; -#endif /* __TBB_x86_32||__TBB_x86_64 */ -}; - -//------------------------------------------------------------------------ -// custom_scheduler -//------------------------------------------------------------------------ - -//! A scheduler with a customized evaluation loop. -/** The customization can use SchedulerTraits to make decisions without needing a run-time check. */ -template<typename SchedulerTraits> -class custom_scheduler: private generic_scheduler { - typedef custom_scheduler<SchedulerTraits> scheduler_type; - - custom_scheduler( market& m ) : generic_scheduler(m) {} - - //! Scheduler loop that dispatches tasks. - /** If child is non-NULL, it is dispatched first. - Then, until "parent" has a reference count of 1, other task are dispatched or stolen. */ - void local_wait_for_all( task& parent, task* child ) __TBB_override; - - //! Entry point from client code to the scheduler loop that dispatches tasks. - /** The method is virtual, but the *this object is used only for sake of dispatching on the correct vtable, - not necessarily the correct *this object. The correct *this object is looked up in TLS. */ - void wait_for_all( task& parent, task* child ) __TBB_override { - static_cast<custom_scheduler*>(governor::local_scheduler())->scheduler_type::local_wait_for_all( parent, child ); - } - - //! Decrements ref_count of a predecessor. - /** If it achieves 0, the predecessor is scheduled for execution. - When changing, remember that this is a hot path function. */ - void tally_completion_of_predecessor( task& s, __TBB_ISOLATION_ARG( task*& bypass_slot, isolation_tag isolation ) ) { - task_prefix& p = s.prefix(); - if( SchedulerTraits::itt_possible ) - ITT_NOTIFY(sync_releasing, &p.ref_count); - if( SchedulerTraits::has_slow_atomic && p.ref_count==1 ) - p.ref_count=0; - else if( __TBB_FetchAndDecrementWrelease(&p.ref_count) > 1 ) {// more references exist - // '__TBB_cl_evict(&p)' degraded performance of parallel_preorder example - return; - } - - // Ordering on p.ref_count (superfluous if SchedulerTraits::has_slow_atomic) - __TBB_control_consistency_helper(); - __TBB_ASSERT(p.ref_count==0, "completion of task caused predecessor's reference count to underflow"); - if( SchedulerTraits::itt_possible ) - ITT_NOTIFY(sync_acquired, &p.ref_count); -#if TBB_USE_ASSERT - p.extra_state &= ~es_ref_count_active; -#endif /* TBB_USE_ASSERT */ -#if __TBB_TASK_ISOLATION - if ( isolation != no_isolation ) { - // The parent is allowed not to have isolation (even if a child has isolation) because it has never spawned. - __TBB_ASSERT(p.isolation == no_isolation || p.isolation == isolation, NULL); - p.isolation = isolation; - } -#endif /* __TBB_TASK_ISOLATION */ - -#if __TBB_RECYCLE_TO_ENQUEUE - if (p.state==task::to_enqueue) { - // related to __TBB_TASK_ARENA TODO: try keep priority of the task - // e.g. rework task_prefix to remember priority of received task and use here - my_arena->enqueue_task(s, 0, my_random ); - } else -#endif /*__TBB_RECYCLE_TO_ENQUEUE*/ - if( bypass_slot==NULL ) - bypass_slot = &s; - else - local_spawn( &s, s.prefix().next ); - } - -public: - static generic_scheduler* allocate_scheduler( market& m ) { - void* p = NFS_Allocate(1, sizeof(scheduler_type), NULL); - std::memset(p, 0, sizeof(scheduler_type)); - scheduler_type* s = new( p ) scheduler_type( m ); - s->assert_task_pool_valid(); - ITT_SYNC_CREATE(s, SyncType_Scheduler, SyncObj_TaskPoolSpinning); - return s; - } - - //! Try getting a task from the mailbox or stealing from another scheduler. - /** Returns the stolen task or NULL if all attempts fail. */ - task* receive_or_steal_task( __TBB_ISOLATION_ARG( __TBB_atomic reference_count& completion_ref_count, isolation_tag isolation ) ) __TBB_override; - -}; // class custom_scheduler<> - -//------------------------------------------------------------------------ -// custom_scheduler methods -//------------------------------------------------------------------------ -template<typename SchedulerTraits> -task* custom_scheduler<SchedulerTraits>::receive_or_steal_task( __TBB_ISOLATION_ARG(__TBB_atomic reference_count& completion_ref_count, isolation_tag isolation) ) { - task* t = NULL; - bool outermost_worker_level = worker_outermost_level(); - bool outermost_dispatch_level = outermost_worker_level || master_outermost_level(); - bool can_steal_here = can_steal(); - my_inbox.set_is_idle( true ); -#if __TBB_HOARD_NONLOCAL_TASKS - __TBB_ASSERT(!my_nonlocal_free_list, NULL); -#endif -#if __TBB_TASK_PRIORITY - if ( outermost_dispatch_level ) { - if ( intptr_t skipped_priority = my_arena->my_skipped_fifo_priority ) { - // This thread can dequeue FIFO tasks, and some priority levels of - // FIFO tasks have been bypassed (to prevent deadlock caused by - // dynamic priority changes in nested task group hierarchy). - if ( my_arena->my_skipped_fifo_priority.compare_and_swap(0, skipped_priority) == skipped_priority - && skipped_priority > my_arena->my_top_priority ) - { - my_market->update_arena_priority( *my_arena, skipped_priority ); - } - } - } -#endif /* !__TBB_TASK_PRIORITY */ - // TODO: Try to find a place to reset my_limit (under market's lock) - // The number of slots potentially used in the arena. Updated once in a while, as my_limit changes rarely. - size_t n = my_arena->my_limit-1; - int yield_count = 0; - // The state "failure_count==-1" is used only when itt_possible is true, - // and denotes that a sync_prepare has not yet been issued. - for( int failure_count = -static_cast<int>(SchedulerTraits::itt_possible);; ++failure_count) { - __TBB_ASSERT( my_arena->my_limit > 0, NULL ); - __TBB_ASSERT( my_arena_index <= n, NULL ); - if( completion_ref_count==1 ) { - if( SchedulerTraits::itt_possible ) { - if( failure_count!=-1 ) { - ITT_NOTIFY(sync_prepare, &completion_ref_count); - // Notify Intel(R) Thread Profiler that thread has stopped spinning. - ITT_NOTIFY(sync_acquired, this); - } - ITT_NOTIFY(sync_acquired, &completion_ref_count); - } - __TBB_ASSERT( !t, NULL ); - // A worker thread in its outermost dispatch loop (i.e. its execution stack is empty) should - // exit it either when there is no more work in the current arena, or when revoked by the market. - __TBB_ASSERT( !outermost_worker_level, NULL ); - __TBB_control_consistency_helper(); // on ref_count - break; // exit stealing loop and return; - } - // Check if the resource manager requires our arena to relinquish some threads - if ( outermost_worker_level && (my_arena->my_num_workers_allotted < my_arena->num_workers_active() -#if __TBB_ENQUEUE_ENFORCED_CONCURRENCY - || my_arena->recall_by_mandatory_request() -#endif - ) ) { - if( SchedulerTraits::itt_possible && failure_count != -1 ) - ITT_NOTIFY(sync_cancel, this); - return NULL; - } -#if __TBB_TASK_PRIORITY - const int p = int(my_arena->my_top_priority); -#else /* !__TBB_TASK_PRIORITY */ - static const int p = 0; -#endif - // Check if there are tasks mailed to this thread via task-to-thread affinity mechanism. - __TBB_ASSERT(my_affinity_id, NULL); - if ( n && !my_inbox.empty() ) { - t = get_mailbox_task( __TBB_ISOLATION_EXPR( isolation ) ); -#if __TBB_TASK_ISOLATION - // There is a race with a thread adding a new task (possibly with suitable isolation) - // to our mailbox, so the below conditions might result in a false positive. - // Then set_is_idle(false) allows that task to be stolen; it's OK. - if ( isolation != no_isolation && !t && !my_inbox.empty() - && my_inbox.is_idle_state( true ) ) { - // We have proxy tasks in our mailbox but the isolation blocks their execution. - // So publish the proxy tasks in mailbox to be available for stealing from owner's task pool. - my_inbox.set_is_idle( false ); - } -#endif /* __TBB_TASK_ISOLATION */ - } - if ( t ) { - GATHER_STATISTIC( ++my_counters.mails_received ); - } - // Check if there are tasks in starvation-resistant stream. - // Only allowed at the outermost dispatch level. - else if ( outermost_dispatch_level && !my_arena->my_task_stream.empty(p) - && (t = my_arena->my_task_stream.pop( p, my_arena_slot->hint_for_pop)) ) { - ITT_NOTIFY(sync_acquired, &my_arena->my_task_stream); - // just proceed with the obtained task - } -#if __TBB_TASK_PRIORITY - // Check if any earlier offloaded non-top priority tasks become returned to the top level - else if ( my_offloaded_tasks && (t = reload_tasks( __TBB_ISOLATION_EXPR( isolation ) )) ) { - __TBB_ASSERT( !is_proxy(*t), "The proxy task cannot be offloaded" ); - // just proceed with the obtained task - } -#endif /* __TBB_TASK_PRIORITY */ - else if ( can_steal_here && n ) { - // Try to steal a task from a random victim. - size_t k = my_random.get() % n; - arena_slot* victim = &my_arena->my_slots[k]; - // The following condition excludes the master that might have - // already taken our previous place in the arena from the list . - // of potential victims. But since such a situation can take - // place only in case of significant oversubscription, keeping - // the checks simple seems to be preferable to complicating the code. - if( k >= my_arena_index ) - ++victim; // Adjusts random distribution to exclude self - task **pool = victim->task_pool; - if( pool == EmptyTaskPool || !(t = steal_task( __TBB_ISOLATION_ARG(*victim, isolation) )) ) - goto fail; - if( is_proxy(*t) ) { - task_proxy &tp = *(task_proxy*)t; - t = tp.extract_task<task_proxy::pool_bit>(); - if ( !t ) { - // Proxy was empty, so it's our responsibility to free it - free_task<no_cache_small_task>(tp); - goto fail; - } - GATHER_STATISTIC( ++my_counters.proxies_stolen ); - } - t->prefix().extra_state |= es_task_is_stolen; - if( is_version_3_task(*t) ) { - my_innermost_running_task = t; - t->prefix().owner = this; - t->note_affinity( my_affinity_id ); - } - GATHER_STATISTIC( ++my_counters.steals_committed ); - } // end of stealing branch - else - goto fail; - // A task was successfully obtained somewhere - __TBB_ASSERT(t,NULL); -#if __TBB_ARENA_OBSERVER - my_arena->my_observers.notify_entry_observers( my_last_local_observer, is_worker() ); -#endif -#if __TBB_SCHEDULER_OBSERVER - the_global_observer_list.notify_entry_observers( my_last_global_observer, is_worker() ); -#endif /* __TBB_SCHEDULER_OBSERVER */ - if ( SchedulerTraits::itt_possible && failure_count != -1 ) { - // FIXME - might be victim, or might be selected from a mailbox - // Notify Intel(R) Thread Profiler that thread has stopped spinning. - ITT_NOTIFY(sync_acquired, this); - } - break; // exit stealing loop and return -fail: - GATHER_STATISTIC( ++my_counters.steals_failed ); - if( SchedulerTraits::itt_possible && failure_count==-1 ) { - // The first attempt to steal work failed, so notify Intel(R) Thread Profiler that - // the thread has started spinning. Ideally, we would do this notification - // *before* the first failed attempt to steal, but at that point we do not - // know that the steal will fail. - ITT_NOTIFY(sync_prepare, this); - failure_count = 0; - } - // Pause, even if we are going to yield, because the yield might return immediately. - prolonged_pause(); - const int failure_threshold = 2*int(n+1); - if( failure_count>=failure_threshold ) { -#if __TBB_YIELD2P - failure_count = 0; -#else - failure_count = failure_threshold; -#endif - __TBB_Yield(); -#if __TBB_TASK_PRIORITY - // Check if there are tasks abandoned by other workers - if ( my_arena->my_orphaned_tasks ) { - // Epoch must be advanced before seizing the list pointer - ++my_arena->my_abandonment_epoch; - task* orphans = (task*)__TBB_FetchAndStoreW( &my_arena->my_orphaned_tasks, 0 ); - if ( orphans ) { - task** link = NULL; - // Get local counter out of the way (we've just brought in external tasks) - my_local_reload_epoch--; - t = reload_tasks( orphans, link, __TBB_ISOLATION_ARG( effective_reference_priority(), isolation ) ); - if ( orphans ) { - *link = my_offloaded_tasks; - if ( !my_offloaded_tasks ) - my_offloaded_task_list_tail_link = link; - my_offloaded_tasks = orphans; - } - __TBB_ASSERT( !my_offloaded_tasks == !my_offloaded_task_list_tail_link, NULL ); - if ( t ) { - if( SchedulerTraits::itt_possible ) - ITT_NOTIFY(sync_cancel, this); - __TBB_ASSERT( !is_proxy(*t), "The proxy task cannot be offloaded" ); - break; // exit stealing loop and return - } - } - } -#endif /* __TBB_TASK_PRIORITY */ - const int yield_threshold = 100; - if( yield_count++ >= yield_threshold ) { - // When a worker thread has nothing to do, return it to RML. - // For purposes of affinity support, the thread is considered idle while in RML. -#if __TBB_TASK_PRIORITY - if( outermost_worker_level || my_arena->my_top_priority > my_arena->my_bottom_priority ) { - if ( my_arena->is_out_of_work() && outermost_worker_level ) { -#else /* !__TBB_TASK_PRIORITY */ - if ( outermost_worker_level && my_arena->is_out_of_work() ) { -#endif /* !__TBB_TASK_PRIORITY */ - if( SchedulerTraits::itt_possible ) - ITT_NOTIFY(sync_cancel, this); - return NULL; - } -#if __TBB_TASK_PRIORITY - } - if ( my_offloaded_tasks ) { - // Safeguard against any sloppiness in managing reload epoch - // counter (e.g. on the hot path because of performance reasons). - my_local_reload_epoch--; - // Break the deadlock caused by a higher priority dispatch loop - // stealing and offloading a lower priority task. Priority check - // at the stealing moment cannot completely preclude such cases - // because priorities can changes dynamically. - if ( !outermost_worker_level && *my_ref_top_priority > my_arena->my_top_priority ) { - GATHER_STATISTIC( ++my_counters.prio_ref_fixups ); - my_ref_top_priority = &my_arena->my_top_priority; - // it's expected that only outermost workers can use global reload epoch - __TBB_ASSERT(my_ref_reload_epoch == &my_arena->my_reload_epoch, NULL); - } - } -#endif /* __TBB_TASK_PRIORITY */ - } // end of arena snapshot branch - // If several attempts did not find work, re-read the arena limit. - n = my_arena->my_limit-1; - } // end of yielding branch - } // end of nonlocal task retrieval loop - if ( my_inbox.is_idle_state( true ) ) - my_inbox.set_is_idle( false ); - return t; -} - -template<typename SchedulerTraits> -void custom_scheduler<SchedulerTraits>::local_wait_for_all( task& parent, task* child ) { - __TBB_ASSERT( governor::is_set(this), NULL ); - __TBB_ASSERT( parent.ref_count() >= (child && child->parent() == &parent ? 2 : 1), "ref_count is too small" ); - __TBB_ASSERT( my_innermost_running_task, NULL ); - assert_task_pool_valid(); - // Using parent's refcount in sync_prepare (in the stealing loop below) is - // a workaround for TP. We need to name it here to display correctly in Ampl. - if( SchedulerTraits::itt_possible ) - ITT_SYNC_CREATE(&parent.prefix().ref_count, SyncType_Scheduler, SyncObj_TaskStealingLoop); -#if __TBB_TASK_GROUP_CONTEXT - __TBB_ASSERT( parent.prefix().context, "parent task does not have context" ); -#endif /* __TBB_TASK_GROUP_CONTEXT */ - task* t = child; - // Constant all_local_work_done is an unreachable refcount value that prevents - // early quitting the dispatch loop. It is defined to be in the middle of the range - // of negative values representable by the reference_count type. - static const reference_count - // For normal dispatch loops - parents_work_done = 1, - // For termination dispatch loops in masters - all_local_work_done = (reference_count)3 << (sizeof(reference_count) * 8 - 2); - reference_count quit_point; -#if __TBB_TASK_PRIORITY - __TBB_ASSERT( (uintptr_t)*my_ref_top_priority < (uintptr_t)num_priority_levels, NULL ); - volatile intptr_t *old_ref_top_priority = my_ref_top_priority; - // When entering nested parallelism level market level counter - // must be replaced with the one local to this arena. - volatile uintptr_t *old_ref_reload_epoch = my_ref_reload_epoch; -#endif /* __TBB_TASK_PRIORITY */ - task* old_innermost_running_task = my_innermost_running_task; - scheduler_properties old_properties = my_properties; - // Remove outermost property to indicate nested level. - __TBB_ASSERT( my_properties.outermost || my_innermost_running_task!=my_dummy_task, "The outermost property should be set out of a dispatch loop" ); - my_properties.outermost &= my_innermost_running_task==my_dummy_task; -#if __TBB_TASK_ISOLATION - isolation_tag isolation = my_innermost_running_task->prefix().isolation; -#endif /* __TBB_TASK_ISOLATION */ - if( master_outermost_level() ) { - // We are in the outermost task dispatch loop of a master thread or a worker which mimics master - quit_point = &parent == my_dummy_task ? all_local_work_done : parents_work_done; - } else { - quit_point = parents_work_done; -#if __TBB_TASK_PRIORITY - if ( &parent != my_dummy_task ) { - // We are in a nested dispatch loop. - // Market or arena priority must not prevent child tasks from being - // executed so that dynamic priority changes did not cause deadlock. - my_ref_top_priority = &parent.prefix().context->my_priority; - my_ref_reload_epoch = &my_arena->my_reload_epoch; - if(my_ref_reload_epoch != old_ref_reload_epoch) - my_local_reload_epoch = *my_ref_reload_epoch-1; - } -#endif /* __TBB_TASK_PRIORITY */ - } - - cpu_ctl_env_helper cpu_ctl_helper; - if ( t ) { - cpu_ctl_helper.set_env( __TBB_CONTEXT_ARG1(t->prefix().context) ); -#if __TBB_TASK_ISOLATION - if ( isolation != no_isolation ) { - __TBB_ASSERT( t->prefix().isolation == no_isolation, NULL ); - // Propagate the isolation to the task executed without spawn. - t->prefix().isolation = isolation; - } -#endif /* __TBB_TASK_ISOLATION */ - } - -#if TBB_USE_EXCEPTIONS - // Infinite safeguard EH loop - for (;;) { - try { -#endif /* TBB_USE_EXCEPTIONS */ - // Outer loop receives tasks from global environment (via mailbox, FIFO queue(s), - // and by stealing from other threads' task pools). - // All exit points from the dispatch loop are located in its immediate scope. - for(;;) { - // Middle loop retrieves tasks from the local task pool. - for(;;) { - // Inner loop evaluates tasks coming from nesting loops and those returned - // by just executed tasks (bypassing spawn or enqueue calls). - while(t) { - __TBB_ASSERT( my_inbox.is_idle_state(false), NULL ); - __TBB_ASSERT(!is_proxy(*t),"unexpected proxy"); - __TBB_ASSERT( t->prefix().owner, NULL ); -#if __TBB_TASK_ISOLATION - __TBB_ASSERT( isolation == no_isolation || isolation == t->prefix().isolation, - "A task from another isolated region is going to be executed" ); -#endif /* __TBB_TASK_ISOLATION */ - assert_task_valid(t); -#if __TBB_TASK_GROUP_CONTEXT && TBB_USE_ASSERT - assert_context_valid(t->prefix().context); - if ( !t->prefix().context->my_cancellation_requested ) -#endif - // TODO: make the assert stronger by prohibiting allocated state. - __TBB_ASSERT( 1L<<t->state() & (1L<<task::allocated|1L<<task::ready|1L<<task::reexecute), NULL ); - assert_task_pool_valid(); -#if __TBB_TASK_PRIORITY - intptr_t p = priority(*t); - if ( p != *my_ref_top_priority && (t->prefix().extra_state & es_task_enqueued) == 0) { - assert_priority_valid(p); - if ( p != my_arena->my_top_priority ) { - my_market->update_arena_priority( *my_arena, p ); - } - if ( p < effective_reference_priority() ) { - if ( !my_offloaded_tasks ) { - my_offloaded_task_list_tail_link = &t->prefix().next_offloaded; - // Erase possible reference to the owner scheduler (next_offloaded is a union member) - *my_offloaded_task_list_tail_link = NULL; - } - offload_task( *t, p ); - if ( is_task_pool_published() ) { - t = winnow_task_pool( __TBB_ISOLATION_EXPR( isolation ) ); - if ( t ) - continue; - } else { - // Mark arena as full to unlock arena priority level adjustment - // by arena::is_out_of_work(), and ensure worker's presence. - my_arena->advertise_new_work<arena::wakeup>(); - } - goto stealing_ground; - } - } -#endif /* __TBB_TASK_PRIORITY */ - task* t_next = NULL; - my_innermost_running_task = t; - t->prefix().owner = this; - t->prefix().state = task::executing; -#if __TBB_TASK_GROUP_CONTEXT - if ( !t->prefix().context->my_cancellation_requested ) -#endif - { - GATHER_STATISTIC( ++my_counters.tasks_executed ); - GATHER_STATISTIC( my_counters.avg_arena_concurrency += my_arena->num_workers_active() ); - GATHER_STATISTIC( my_counters.avg_assigned_workers += my_arena->my_num_workers_allotted ); -#if __TBB_TASK_PRIORITY - GATHER_STATISTIC( my_counters.avg_arena_prio += p ); - GATHER_STATISTIC( my_counters.avg_market_prio += my_market->my_global_top_priority ); -#endif /* __TBB_TASK_PRIORITY */ - ITT_STACK(SchedulerTraits::itt_possible, callee_enter, t->prefix().context->itt_caller); - t_next = t->execute(); - ITT_STACK(SchedulerTraits::itt_possible, callee_leave, t->prefix().context->itt_caller); - if (t_next) { - __TBB_ASSERT( t_next->state()==task::allocated, - "if task::execute() returns task, it must be marked as allocated" ); - reset_extra_state(t_next); - __TBB_ISOLATION_EXPR( t_next->prefix().isolation = t->prefix().isolation ); -#if TBB_USE_ASSERT - affinity_id next_affinity=t_next->prefix().affinity; - if (next_affinity != 0 && next_affinity != my_affinity_id) - GATHER_STATISTIC( ++my_counters.affinity_ignored ); -#endif - } - } - assert_task_pool_valid(); - switch( t->state() ) { - case task::executing: { - task* s = t->parent(); - __TBB_ASSERT( my_innermost_running_task==t, NULL ); - __TBB_ASSERT( t->prefix().ref_count==0, "Task still has children after it has been executed" ); - t->~task(); - if( s ) - tally_completion_of_predecessor( *s, __TBB_ISOLATION_ARG( t_next, t->prefix().isolation ) ); - free_task<no_hint>( *t ); - poison_pointer( my_innermost_running_task ); - assert_task_pool_valid(); - break; - } - - case task::recycle: // set by recycle_as_safe_continuation() - t->prefix().state = task::allocated; -#if __TBB_RECYCLE_TO_ENQUEUE - case task::to_enqueue: // set by recycle_to_enqueue() -#endif - __TBB_ASSERT( t_next != t, "a task returned from method execute() can not be recycled in another way" ); - reset_extra_state(t); - // for safe continuation, need atomically decrement ref_count; - tally_completion_of_predecessor(*t, __TBB_ISOLATION_ARG( t_next, t->prefix().isolation ) ); - assert_task_pool_valid(); - break; - - case task::reexecute: // set by recycle_to_reexecute() - __TBB_ASSERT( t_next, "reexecution requires that method execute() return another task" ); - __TBB_ASSERT( t_next != t, "a task returned from method execute() can not be recycled in another way" ); - t->prefix().state = task::allocated; - reset_extra_state(t); - local_spawn( t, t->prefix().next ); - assert_task_pool_valid(); - break; - case task::allocated: - reset_extra_state(t); - break; -#if TBB_USE_ASSERT - case task::ready: - __TBB_ASSERT( false, "task is in READY state upon return from method execute()" ); - break; - default: - __TBB_ASSERT( false, "illegal state" ); -#else - default: // just to shut up some compilation warnings - break; -#endif /* TBB_USE_ASSERT */ - } - GATHER_STATISTIC( t_next ? ++my_counters.spawns_bypassed : 0 ); - t = t_next; - } // end of scheduler bypass loop - - assert_task_pool_valid(); - if ( parent.prefix().ref_count == quit_point ) { - __TBB_ASSERT( quit_point != all_local_work_done, NULL ); - __TBB_control_consistency_helper(); // on ref_count - ITT_NOTIFY(sync_acquired, &parent.prefix().ref_count); - goto done; - } - if ( is_task_pool_published() ) { - t = get_task( __TBB_ISOLATION_EXPR( isolation ) ); - } else { - __TBB_ASSERT( is_quiescent_local_task_pool_reset(), NULL ); - break; - } - assert_task_pool_valid(); - - if ( !t ) break; - - cpu_ctl_helper.set_env( __TBB_CONTEXT_ARG1(t->prefix().context) ); - }; // end of local task pool retrieval loop - -#if __TBB_TASK_PRIORITY -stealing_ground: -#endif /* __TBB_TASK_PRIORITY */ -#if __TBB_HOARD_NONLOCAL_TASKS - // before stealing, previously stolen task objects are returned - for (; my_nonlocal_free_list; my_nonlocal_free_list = t ) { - t = my_nonlocal_free_list->prefix().next; - free_nonlocal_small_task( *my_nonlocal_free_list ); - } -#endif - if ( quit_point == all_local_work_done ) { - __TBB_ASSERT( !is_task_pool_published() && is_quiescent_local_task_pool_reset(), NULL ); - __TBB_ASSERT( !worker_outermost_level(), NULL ); - my_innermost_running_task = old_innermost_running_task; - my_properties = old_properties; -#if __TBB_TASK_PRIORITY - my_ref_top_priority = old_ref_top_priority; - if(my_ref_reload_epoch != old_ref_reload_epoch) - my_local_reload_epoch = *old_ref_reload_epoch-1; - my_ref_reload_epoch = old_ref_reload_epoch; -#endif /* __TBB_TASK_PRIORITY */ - return; - } - - t = receive_or_steal_task( __TBB_ISOLATION_ARG( parent.prefix().ref_count, isolation ) ); - if ( !t ) - goto done; - - // The user can capture another the FPU settings to the context so the - // cached data in the helper can be out-of-date and we cannot do fast - // check. - cpu_ctl_helper.set_env( __TBB_CONTEXT_ARG1(t->prefix().context) ); - } // end of infinite stealing loop -#if TBB_USE_EXCEPTIONS - __TBB_ASSERT( false, "Must never get here" ); - } // end of try-block - TbbCatchAll( t->prefix().context ); - // Complete post-processing ... - if( t->state() == task::recycle -#if __TBB_RECYCLE_TO_ENQUEUE - // TODO: the enqueue semantics gets lost below, consider reimplementing - || t->state() == task::to_enqueue -#endif - ) { - // ... for recycled tasks to atomically decrement ref_count - t->prefix().state = task::allocated; - if( SchedulerTraits::itt_possible ) - ITT_NOTIFY(sync_releasing, &t->prefix().ref_count); - if( __TBB_FetchAndDecrementWrelease(&t->prefix().ref_count)==1 ) { - if( SchedulerTraits::itt_possible ) - ITT_NOTIFY(sync_acquired, &t->prefix().ref_count); - }else{ - t = NULL; - } - } - } // end of infinite EH loop - __TBB_ASSERT( false, "Must never get here too" ); -#endif /* TBB_USE_EXCEPTIONS */ -done: - my_innermost_running_task = old_innermost_running_task; - my_properties = old_properties; -#if __TBB_TASK_PRIORITY - my_ref_top_priority = old_ref_top_priority; - if(my_ref_reload_epoch != old_ref_reload_epoch) - my_local_reload_epoch = *old_ref_reload_epoch-1; - my_ref_reload_epoch = old_ref_reload_epoch; -#endif /* __TBB_TASK_PRIORITY */ - if ( !ConcurrentWaitsEnabled(parent) ) { - if ( parent.prefix().ref_count != parents_work_done ) { - // This is a worker that was revoked by the market. - __TBB_ASSERT( worker_outermost_level(), - "Worker thread exits nested dispatch loop prematurely" ); - return; - } - parent.prefix().ref_count = 0; - } -#if TBB_USE_ASSERT - parent.prefix().extra_state &= ~es_ref_count_active; -#endif /* TBB_USE_ASSERT */ -#if __TBB_TASK_GROUP_CONTEXT - __TBB_ASSERT(parent.prefix().context && default_context(), NULL); - task_group_context* parent_ctx = parent.prefix().context; - if ( parent_ctx->my_cancellation_requested ) { - task_group_context::exception_container_type *pe = parent_ctx->my_exception; - if ( master_outermost_level() && parent_ctx == default_context() ) { - // We are in the outermost task dispatch loop of a master thread, and - // the whole task tree has been collapsed. So we may clear cancellation data. - parent_ctx->my_cancellation_requested = 0; - // TODO: Add assertion that master's dummy task context does not have children - parent_ctx->my_state &= ~(uintptr_t)task_group_context::may_have_children; - } - if ( pe ) { - // On Windows, FPU control settings changed in the helper destructor are not visible - // outside a catch block. So restore the default settings manually before rethrowing - // the exception. - cpu_ctl_helper.restore_default(); - TbbRethrowException( pe ); - } - } - __TBB_ASSERT(!is_worker() || !CancellationInfoPresent(*my_dummy_task), - "Worker's dummy task context modified"); - __TBB_ASSERT(!master_outermost_level() || !CancellationInfoPresent(*my_dummy_task), - "Unexpected exception or cancellation data in the master's dummy task"); -#endif /* __TBB_TASK_GROUP_CONTEXT */ - assert_task_pool_valid(); -} - -} // namespace internal -} // namespace tbb - -#endif /* _TBB_custom_scheduler_H */ diff --git a/src/tbb/src/tbb/def/lin32-tbb.def b/src/tbb/src/tbb/def/lin32-tbb.def new file mode 100644 index 000000000..2aef269a6 --- /dev/null +++ b/src/tbb/src/tbb/def/lin32-tbb.def @@ -0,0 +1,183 @@ +/* + Copyright (c) 2005-2024 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +{ +global: + +/* Needed for backwards compatibility */ +_ZNSt13runtime_errorD1Ev; +_ZTISt13runtime_error; +_ZTSSt13runtime_error; +_ZNSt16invalid_argumentD1Ev; +_ZTISt16invalid_argument; +_ZTSSt16invalid_argument; +_ZNSt11range_errorD1Ev; +_ZTISt11range_error; +_ZTSSt11range_error; +_ZNSt12length_errorD1Ev; +_ZTISt12length_error; +_ZTSSt12length_error; +_ZNSt12out_of_rangeD1Ev; +_ZTISt12out_of_range; +_ZTSSt12out_of_range; + +/* Needed by rstan */ +_ZN3tbb8internal26task_scheduler_observer_v37observeEb; + +/* Assertions (assert.cpp) */ +_ZN3tbb6detail2r117assertion_failureEPKciS3_S3_; + +/* ITT (profiling.cpp) */ +_ZN3tbb6detail2r112itt_task_endENS0_2d115itt_domain_enumE; +_ZN3tbb6detail2r114itt_region_endENS0_2d115itt_domain_enumEPvy; +_ZN3tbb6detail2r114itt_task_beginENS0_2d115itt_domain_enumEPvyS4_yNS0_2d021string_resource_indexE; +_ZN3tbb6detail2r115call_itt_notifyEiPv; +_ZN3tbb6detail2r115create_itt_syncEPvPKcS4_; +_ZN3tbb6detail2r116itt_region_beginENS0_2d115itt_domain_enumEPvyS4_yNS0_2d021string_resource_indexE; +_ZN3tbb6detail2r116itt_relation_addENS0_2d115itt_domain_enumEPvyNS0_2d012itt_relationES4_y; +_ZN3tbb6detail2r117itt_set_sync_nameEPvPKc; +_ZN3tbb6detail2r119itt_make_task_groupENS0_2d115itt_domain_enumEPvyS4_yNS0_2d021string_resource_indexE; +_ZN3tbb6detail2r120itt_metadata_str_addENS0_2d115itt_domain_enumEPvyNS0_2d021string_resource_indexEPKc; +_ZN3tbb6detail2r120itt_metadata_ptr_addENS0_2d115itt_domain_enumEPvyNS0_2d021string_resource_indexES4_; + +/* Allocators (allocator.cpp) */ +_ZN3tbb6detail2r115allocate_memoryEj; +_ZN3tbb6detail2r117deallocate_memoryEPv; +_ZN3tbb6detail2r122cache_aligned_allocateEj; +_ZN3tbb6detail2r124cache_aligned_deallocateEPv; +_ZN3tbb6detail2r115cache_line_sizeEv; +_ZN3tbb6detail2r117is_tbbmalloc_usedEv; + +/* Small object pool (small_object_pool.cpp) */ +_ZN3tbb6detail2r18allocateERPNS0_2d117small_object_poolEj; +_ZN3tbb6detail2r18allocateERPNS0_2d117small_object_poolEjRKNS2_14execution_dataE; +_ZN3tbb6detail2r110deallocateERNS0_2d117small_object_poolEPvj; +_ZN3tbb6detail2r110deallocateERNS0_2d117small_object_poolEPvjRKNS2_14execution_dataE; + +/* Error handling (exception.cpp) */ +_ZN3tbb6detail2r115throw_exceptionENS0_2d012exception_idE; +_ZTIN3tbb6detail2r114bad_last_allocE; +_ZTVN3tbb6detail2r114bad_last_allocE; +_ZTIN3tbb6detail2r112missing_waitE; +_ZTVN3tbb6detail2r112missing_waitE; +_ZTIN3tbb6detail2r110user_abortE; +_ZTVN3tbb6detail2r110user_abortE; +_ZTIN3tbb6detail2r111unsafe_waitE; +_ZTVN3tbb6detail2r111unsafe_waitE; + +/* RTM Mutex (rtm_mutex.cpp) */ +_ZN3tbb6detail2r17acquireERNS0_2d19rtm_mutexERNS3_11scoped_lockEb; +_ZN3tbb6detail2r17releaseERNS0_2d19rtm_mutex11scoped_lockE; +_ZN3tbb6detail2r111try_acquireERNS0_2d19rtm_mutexERNS3_11scoped_lockE; + +/* RTM RW Mutex (rtm_rw_mutex.cpp) */ +_ZN3tbb6detail2r114acquire_readerERNS0_2d112rtm_rw_mutexERNS3_11scoped_lockEb; +_ZN3tbb6detail2r114acquire_writerERNS0_2d112rtm_rw_mutexERNS3_11scoped_lockEb; +_ZN3tbb6detail2r118try_acquire_readerERNS0_2d112rtm_rw_mutexERNS3_11scoped_lockE; +_ZN3tbb6detail2r118try_acquire_writerERNS0_2d112rtm_rw_mutexERNS3_11scoped_lockE; +_ZN3tbb6detail2r17releaseERNS0_2d112rtm_rw_mutex11scoped_lockE; +_ZN3tbb6detail2r17upgradeERNS0_2d112rtm_rw_mutex11scoped_lockE; +_ZN3tbb6detail2r19downgradeERNS0_2d112rtm_rw_mutex11scoped_lockE; + +/* Tasks and partitioners (task.cpp) */ +_ZN3tbb6detail2r17suspendEPFvPvPNS1_18suspend_point_typeEES2_; +_ZN3tbb6detail2r16resumeEPNS1_18suspend_point_typeE; +_ZN3tbb6detail2r121current_suspend_pointEv; +_ZN3tbb6detail2r114notify_waitersEj; +_ZN3tbb6detail2r127get_thread_reference_vertexEPNS0_2d126wait_tree_vertex_interfaceE; + +/* Task dispatcher (task_dispatcher.cpp) */ +_ZN3tbb6detail2r114execution_slotEPKNS0_2d114execution_dataE; +_ZN3tbb6detail2r14waitERNS0_2d112wait_contextERNS2_18task_group_contextE; +_ZN3tbb6detail2r15spawnERNS0_2d14taskERNS2_18task_group_contextE; +_ZN3tbb6detail2r15spawnERNS0_2d14taskERNS2_18task_group_contextEt; +_ZN3tbb6detail2r116execute_and_waitERNS0_2d14taskERNS2_18task_group_contextERNS2_12wait_contextES6_; +_ZN3tbb6detail2r16submitERNS0_2d14taskERNS2_18task_group_contextEPNS1_5arenaEj; +_ZN3tbb6detail2r115current_contextEv; + +/* Task group context (task_group_context.cpp) */ +_ZN3tbb6detail2r110initializeERNS0_2d118task_group_contextE; +_ZN3tbb6detail2r122cancel_group_executionERNS0_2d118task_group_contextE; +_ZN3tbb6detail2r128is_group_execution_cancelledERNS0_2d118task_group_contextE; +_ZN3tbb6detail2r15resetERNS0_2d118task_group_contextE; +_ZN3tbb6detail2r17destroyERNS0_2d118task_group_contextE; +_ZN3tbb6detail2r119capture_fp_settingsERNS0_2d118task_group_contextE; + +/* Task arena (arena.cpp) */ +_ZN3tbb6detail2r115max_concurrencyEPKNS0_2d115task_arena_baseE; +_ZN3tbb6detail2r110initializeERNS0_2d115task_arena_baseE; +_ZN3tbb6detail2r16attachERNS0_2d115task_arena_baseE; +_ZN3tbb6detail2r17executeERNS0_2d115task_arena_baseERNS2_13delegate_baseE; +_ZN3tbb6detail2r19terminateERNS0_2d115task_arena_baseE; +_ZN3tbb6detail2r120isolate_within_arenaERNS0_2d113delegate_baseEi; +_ZN3tbb6detail2r17enqueueERNS0_2d14taskEPNS2_15task_arena_baseE; +_ZN3tbb6detail2r17enqueueERNS0_2d14taskERNS2_18task_group_contextEPNS2_15task_arena_baseE; +_ZN3tbb6detail2r14waitERNS0_2d115task_arena_baseE; +_ZN3tbb6detail2r114execution_slotERKNS0_2d115task_arena_baseE; + +/* System topology parsing and threads pinning (governor.cpp) */ +_ZN3tbb6detail2r115numa_node_countEv; +_ZN3tbb6detail2r117fill_numa_indicesEPi; +_ZN3tbb6detail2r115core_type_countEi; +_ZN3tbb6detail2r122fill_core_type_indicesEPii; +_ZN3tbb6detail2r131constraints_default_concurrencyERKNS0_2d111constraintsEi; +_ZN3tbb6detail2r128constraints_threads_per_coreERKNS0_2d111constraintsEi; +_ZN3tbb6detail2r124numa_default_concurrencyEi; + +/* Observer (observer_proxy.cpp) */ +_ZN3tbb6detail2r17observeERNS0_2d123task_scheduler_observerEb; + +/* Queuing RW Mutex (queuing_rw_mutex.cpp) */ +_ZN3tbb6detail2r111try_acquireERNS0_2d116queuing_rw_mutexERNS3_11scoped_lockEb; +_ZN3tbb6detail2r117upgrade_to_writerERNS0_2d116queuing_rw_mutex11scoped_lockE; +_ZN3tbb6detail2r119downgrade_to_readerERNS0_2d116queuing_rw_mutex11scoped_lockE; +_ZN3tbb6detail2r17acquireERNS0_2d116queuing_rw_mutexERNS3_11scoped_lockEb; +_ZN3tbb6detail2r17releaseERNS0_2d116queuing_rw_mutex11scoped_lockE; +_ZN3tbb6detail2r19constructERNS0_2d116queuing_rw_mutexE; +_ZN3tbb6detail2r19is_writerERKNS0_2d116queuing_rw_mutex11scoped_lockE; + +/* Global control (global_control.cpp) */ +_ZN3tbb6detail2r16createERNS0_2d114global_controlE; +_ZN3tbb6detail2r17destroyERNS0_2d114global_controlE; +_ZN3tbb6detail2r127global_control_active_valueEi; +_ZN3tbb6detail2r18finalizeERNS0_2d121task_scheduler_handleEi; +_ZN3tbb6detail2r13getERNS0_2d121task_scheduler_handleE; + +/* Parallel pipeline (parallel_pipeline.cpp) */ +_ZN3tbb6detail2r117parallel_pipelineERNS0_2d118task_group_contextEjRKNS2_11filter_nodeE; +_ZN3tbb6detail2r116set_end_of_inputERNS0_2d111base_filterE; + +/* Concurrent bounded queue (concurrent_bounded_queue.cpp) */ +_ZN3tbb6detail2r126allocate_bounded_queue_repEj; +_ZN3tbb6detail2r126wait_bounded_queue_monitorEPNS1_18concurrent_monitorEjiRNS0_2d113delegate_baseE; +_ZN3tbb6detail2r128abort_bounded_queue_monitorsEPNS1_18concurrent_monitorE; +_ZN3tbb6detail2r128deallocate_bounded_queue_repEPhj; +_ZN3tbb6detail2r128notify_bounded_queue_monitorEPNS1_18concurrent_monitorEjj; + +/* Concurrent monitor (address_waiter.cpp) */ +_ZN3tbb6detail2r115wait_on_addressEPvRNS0_2d113delegate_baseEj; +_ZN3tbb6detail2r117notify_by_addressEPvj; +_ZN3tbb6detail2r121notify_by_address_oneEPv; +_ZN3tbb6detail2r121notify_by_address_allEPv; + +/* Versioning (version.cpp) */ +TBB_runtime_interface_version; +TBB_runtime_version; + +local: +/* TODO: fill more precisely */ +*; +}; diff --git a/src/tbb/src/tbb/def/lin64-tbb.def b/src/tbb/src/tbb/def/lin64-tbb.def new file mode 100644 index 000000000..365ef6c35 --- /dev/null +++ b/src/tbb/src/tbb/def/lin64-tbb.def @@ -0,0 +1,183 @@ +/* + Copyright (c) 2005-2024 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +{ +global: + +/* Needed for backwards compatibility */ +_ZNSt13runtime_errorD1Ev; +_ZTISt13runtime_error; +_ZTSSt13runtime_error; +_ZNSt16invalid_argumentD1Ev; +_ZTISt16invalid_argument; +_ZTSSt16invalid_argument; +_ZNSt11range_errorD1Ev; +_ZTISt11range_error; +_ZTSSt11range_error; +_ZNSt12length_errorD1Ev; +_ZTISt12length_error; +_ZTSSt12length_error; +_ZNSt12out_of_rangeD1Ev; +_ZTISt12out_of_range; +_ZTSSt12out_of_range; + +/* Needed by rstan */ +_ZN3tbb8internal26task_scheduler_observer_v37observeEb; + +/* Assertions (assert.cpp) */ +_ZN3tbb6detail2r117assertion_failureEPKciS3_S3_; + +/* ITT (profiling.cpp) */ +_ZN3tbb6detail2r112itt_task_endENS0_2d115itt_domain_enumE; +_ZN3tbb6detail2r114itt_region_endENS0_2d115itt_domain_enumEPvy; +_ZN3tbb6detail2r114itt_task_beginENS0_2d115itt_domain_enumEPvyS4_yNS0_2d021string_resource_indexE; +_ZN3tbb6detail2r115call_itt_notifyEiPv; +_ZN3tbb6detail2r115create_itt_syncEPvPKcS4_; +_ZN3tbb6detail2r116itt_region_beginENS0_2d115itt_domain_enumEPvyS4_yNS0_2d021string_resource_indexE; +_ZN3tbb6detail2r116itt_relation_addENS0_2d115itt_domain_enumEPvyNS0_2d012itt_relationES4_y; +_ZN3tbb6detail2r117itt_set_sync_nameEPvPKc; +_ZN3tbb6detail2r119itt_make_task_groupENS0_2d115itt_domain_enumEPvyS4_yNS0_2d021string_resource_indexE; +_ZN3tbb6detail2r120itt_metadata_str_addENS0_2d115itt_domain_enumEPvyNS0_2d021string_resource_indexEPKc; +_ZN3tbb6detail2r120itt_metadata_ptr_addENS0_2d115itt_domain_enumEPvyNS0_2d021string_resource_indexES4_; + +/* Allocators (allocator.cpp) */ +_ZN3tbb6detail2r115allocate_memoryEm; +_ZN3tbb6detail2r117deallocate_memoryEPv; +_ZN3tbb6detail2r122cache_aligned_allocateEm; +_ZN3tbb6detail2r124cache_aligned_deallocateEPv; +_ZN3tbb6detail2r115cache_line_sizeEv; +_ZN3tbb6detail2r117is_tbbmalloc_usedEv; + +/* Small object pool (small_object_pool.cpp) */ +_ZN3tbb6detail2r18allocateERPNS0_2d117small_object_poolEm; +_ZN3tbb6detail2r18allocateERPNS0_2d117small_object_poolEmRKNS2_14execution_dataE; +_ZN3tbb6detail2r110deallocateERNS0_2d117small_object_poolEPvm; +_ZN3tbb6detail2r110deallocateERNS0_2d117small_object_poolEPvmRKNS2_14execution_dataE; + +/* Error handling (exception.cpp) */ +_ZN3tbb6detail2r115throw_exceptionENS0_2d012exception_idE; +_ZTIN3tbb6detail2r114bad_last_allocE; +_ZTVN3tbb6detail2r114bad_last_allocE; +_ZTIN3tbb6detail2r112missing_waitE; +_ZTVN3tbb6detail2r112missing_waitE; +_ZTIN3tbb6detail2r110user_abortE; +_ZTVN3tbb6detail2r110user_abortE; +_ZTIN3tbb6detail2r111unsafe_waitE; +_ZTVN3tbb6detail2r111unsafe_waitE; + +/* RTM Mutex (rtm_mutex.cpp) */ +_ZN3tbb6detail2r17acquireERNS0_2d19rtm_mutexERNS3_11scoped_lockEb; +_ZN3tbb6detail2r17releaseERNS0_2d19rtm_mutex11scoped_lockE; +_ZN3tbb6detail2r111try_acquireERNS0_2d19rtm_mutexERNS3_11scoped_lockE; + +/* RTM RW Mutex (rtm_rw_mutex.cpp) */ +_ZN3tbb6detail2r114acquire_readerERNS0_2d112rtm_rw_mutexERNS3_11scoped_lockEb; +_ZN3tbb6detail2r114acquire_writerERNS0_2d112rtm_rw_mutexERNS3_11scoped_lockEb; +_ZN3tbb6detail2r118try_acquire_readerERNS0_2d112rtm_rw_mutexERNS3_11scoped_lockE; +_ZN3tbb6detail2r118try_acquire_writerERNS0_2d112rtm_rw_mutexERNS3_11scoped_lockE; +_ZN3tbb6detail2r17releaseERNS0_2d112rtm_rw_mutex11scoped_lockE; +_ZN3tbb6detail2r17upgradeERNS0_2d112rtm_rw_mutex11scoped_lockE; +_ZN3tbb6detail2r19downgradeERNS0_2d112rtm_rw_mutex11scoped_lockE; + +/* Tasks and partitioners (task.cpp) */ +_ZN3tbb6detail2r17suspendEPFvPvPNS1_18suspend_point_typeEES2_; +_ZN3tbb6detail2r16resumeEPNS1_18suspend_point_typeE; +_ZN3tbb6detail2r121current_suspend_pointEv; +_ZN3tbb6detail2r114notify_waitersEm; +_ZN3tbb6detail2r127get_thread_reference_vertexEPNS0_2d126wait_tree_vertex_interfaceE; + +/* Task dispatcher (task_dispatcher.cpp) */ +_ZN3tbb6detail2r114execution_slotEPKNS0_2d114execution_dataE; +_ZN3tbb6detail2r14waitERNS0_2d112wait_contextERNS2_18task_group_contextE; +_ZN3tbb6detail2r15spawnERNS0_2d14taskERNS2_18task_group_contextE; +_ZN3tbb6detail2r15spawnERNS0_2d14taskERNS2_18task_group_contextEt; +_ZN3tbb6detail2r116execute_and_waitERNS0_2d14taskERNS2_18task_group_contextERNS2_12wait_contextES6_; +_ZN3tbb6detail2r16submitERNS0_2d14taskERNS2_18task_group_contextEPNS1_5arenaEm; +_ZN3tbb6detail2r115current_contextEv; + +/* Task group context (task_group_context.cpp) */ +_ZN3tbb6detail2r110initializeERNS0_2d118task_group_contextE; +_ZN3tbb6detail2r122cancel_group_executionERNS0_2d118task_group_contextE; +_ZN3tbb6detail2r128is_group_execution_cancelledERNS0_2d118task_group_contextE; +_ZN3tbb6detail2r15resetERNS0_2d118task_group_contextE; +_ZN3tbb6detail2r17destroyERNS0_2d118task_group_contextE; +_ZN3tbb6detail2r119capture_fp_settingsERNS0_2d118task_group_contextE; + +/* Task arena (arena.cpp) */ +_ZN3tbb6detail2r115max_concurrencyEPKNS0_2d115task_arena_baseE; +_ZN3tbb6detail2r110initializeERNS0_2d115task_arena_baseE; +_ZN3tbb6detail2r16attachERNS0_2d115task_arena_baseE; +_ZN3tbb6detail2r17executeERNS0_2d115task_arena_baseERNS2_13delegate_baseE; +_ZN3tbb6detail2r19terminateERNS0_2d115task_arena_baseE; +_ZN3tbb6detail2r120isolate_within_arenaERNS0_2d113delegate_baseEl; +_ZN3tbb6detail2r17enqueueERNS0_2d14taskEPNS2_15task_arena_baseE; +_ZN3tbb6detail2r17enqueueERNS0_2d14taskERNS2_18task_group_contextEPNS2_15task_arena_baseE; +_ZN3tbb6detail2r14waitERNS0_2d115task_arena_baseE; +_ZN3tbb6detail2r114execution_slotERKNS0_2d115task_arena_baseE; + +/* System topology parsing and threads pinning (governor.cpp) */ +_ZN3tbb6detail2r115numa_node_countEv; +_ZN3tbb6detail2r117fill_numa_indicesEPi; +_ZN3tbb6detail2r115core_type_countEl; +_ZN3tbb6detail2r122fill_core_type_indicesEPil; +_ZN3tbb6detail2r131constraints_default_concurrencyERKNS0_2d111constraintsEl; +_ZN3tbb6detail2r128constraints_threads_per_coreERKNS0_2d111constraintsEl; +_ZN3tbb6detail2r124numa_default_concurrencyEi; + +/* Observer (observer_proxy.cpp) */ +_ZN3tbb6detail2r17observeERNS0_2d123task_scheduler_observerEb; + +/* Queuing RW Mutex (queuing_rw_mutex.cpp) */ +_ZN3tbb6detail2r111try_acquireERNS0_2d116queuing_rw_mutexERNS3_11scoped_lockEb; +_ZN3tbb6detail2r117upgrade_to_writerERNS0_2d116queuing_rw_mutex11scoped_lockE; +_ZN3tbb6detail2r119downgrade_to_readerERNS0_2d116queuing_rw_mutex11scoped_lockE; +_ZN3tbb6detail2r17acquireERNS0_2d116queuing_rw_mutexERNS3_11scoped_lockEb; +_ZN3tbb6detail2r17releaseERNS0_2d116queuing_rw_mutex11scoped_lockE; +_ZN3tbb6detail2r19constructERNS0_2d116queuing_rw_mutexE; +_ZN3tbb6detail2r19is_writerERKNS0_2d116queuing_rw_mutex11scoped_lockE; + +/* Global control (global_control.cpp) */ +_ZN3tbb6detail2r16createERNS0_2d114global_controlE; +_ZN3tbb6detail2r17destroyERNS0_2d114global_controlE; +_ZN3tbb6detail2r127global_control_active_valueEi; +_ZN3tbb6detail2r18finalizeERNS0_2d121task_scheduler_handleEl; +_ZN3tbb6detail2r13getERNS0_2d121task_scheduler_handleE; + +/* Parallel pipeline (parallel_pipeline.cpp) */ +_ZN3tbb6detail2r117parallel_pipelineERNS0_2d118task_group_contextEmRKNS2_11filter_nodeE; +_ZN3tbb6detail2r116set_end_of_inputERNS0_2d111base_filterE; + +/* Concurrent bounded queue (concurrent_bounded_queue.cpp) */ +_ZN3tbb6detail2r126allocate_bounded_queue_repEm; +_ZN3tbb6detail2r126wait_bounded_queue_monitorEPNS1_18concurrent_monitorEmlRNS0_2d113delegate_baseE; +_ZN3tbb6detail2r128abort_bounded_queue_monitorsEPNS1_18concurrent_monitorE; +_ZN3tbb6detail2r128deallocate_bounded_queue_repEPhm; +_ZN3tbb6detail2r128notify_bounded_queue_monitorEPNS1_18concurrent_monitorEmm; + +/* Concurrent monitor (address_waiter.cpp) */ +_ZN3tbb6detail2r115wait_on_addressEPvRNS0_2d113delegate_baseEm; +_ZN3tbb6detail2r117notify_by_addressEPvm; +_ZN3tbb6detail2r121notify_by_address_oneEPv; +_ZN3tbb6detail2r121notify_by_address_allEPv; + +/* Versioning (version.cpp) */ +TBB_runtime_interface_version; +TBB_runtime_version; + +local: +/* TODO: fill more precisely */ +*; +}; diff --git a/src/tbb/src/tbb/def/mac64-tbb.def b/src/tbb/src/tbb/def/mac64-tbb.def new file mode 100644 index 000000000..c98e886ff --- /dev/null +++ b/src/tbb/src/tbb/def/mac64-tbb.def @@ -0,0 +1,180 @@ +# Copyright (c) 2005-2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# TODO: check the legacy comment below, currently use extra leading underscore everywhere. +# Sometimes macOS* requires leading underscore (e. g. in export list file), but sometimes not +# (e. g. when searching symbol in a dynamic library via dlsym()). Symbols in this file SHOULD +# be listed WITHOUT one leading underscore. __TBB_SYMBOL macro should add underscore when +# necessary, depending on the intended usage. + +# Needed for backwards compatibility +__ZNSt13runtime_errorD1Ev +__ZTISt13runtime_error +__ZTSSt13runtime_error +__ZNSt16invalid_argumentD1Ev +__ZTISt16invalid_argument +__ZTSSt16invalid_argument +__ZNSt11range_errorD1Ev +__ZTISt11range_error +__ZTSSt11range_error +__ZNSt12length_errorD1Ev +__ZTISt12length_error +__ZTSSt12length_error +__ZNSt12out_of_rangeD1Ev +__ZTISt12out_of_range +__ZTSSt12out_of_range + +# Needed by rstan +__ZN3tbb8internal26task_scheduler_observer_v37observeEb + +# Assertions (assert.cpp) +__ZN3tbb6detail2r117assertion_failureEPKciS3_S3_ + +# ITT (profiling.cpp) +__ZN3tbb6detail2r112itt_task_endENS0_2d115itt_domain_enumE +__ZN3tbb6detail2r114itt_region_endENS0_2d115itt_domain_enumEPvy +__ZN3tbb6detail2r114itt_task_beginENS0_2d115itt_domain_enumEPvyS4_yNS0_2d021string_resource_indexE +__ZN3tbb6detail2r115call_itt_notifyEiPv +__ZN3tbb6detail2r115create_itt_syncEPvPKcS4_ +__ZN3tbb6detail2r116itt_region_beginENS0_2d115itt_domain_enumEPvyS4_yNS0_2d021string_resource_indexE +__ZN3tbb6detail2r116itt_relation_addENS0_2d115itt_domain_enumEPvyNS0_2d012itt_relationES4_y +__ZN3tbb6detail2r117itt_set_sync_nameEPvPKc +__ZN3tbb6detail2r119itt_make_task_groupENS0_2d115itt_domain_enumEPvyS4_yNS0_2d021string_resource_indexE +__ZN3tbb6detail2r120itt_metadata_str_addENS0_2d115itt_domain_enumEPvyNS0_2d021string_resource_indexEPKc +__ZN3tbb6detail2r120itt_metadata_ptr_addENS0_2d115itt_domain_enumEPvyNS0_2d021string_resource_indexES4_ + +# Allocators (allocator.cpp) +__ZN3tbb6detail2r115allocate_memoryEm +__ZN3tbb6detail2r117deallocate_memoryEPv +__ZN3tbb6detail2r122cache_aligned_allocateEm +__ZN3tbb6detail2r124cache_aligned_deallocateEPv +__ZN3tbb6detail2r115cache_line_sizeEv +__ZN3tbb6detail2r117is_tbbmalloc_usedEv + +# Small object pool (small_object_pool.cpp) +__ZN3tbb6detail2r18allocateERPNS0_2d117small_object_poolEm +__ZN3tbb6detail2r18allocateERPNS0_2d117small_object_poolEmRKNS2_14execution_dataE +__ZN3tbb6detail2r110deallocateERNS0_2d117small_object_poolEPvm +__ZN3tbb6detail2r110deallocateERNS0_2d117small_object_poolEPvmRKNS2_14execution_dataE + +# Error handling (exception.cpp) +__ZN3tbb6detail2r115throw_exceptionENS0_2d012exception_idE +__ZTIN3tbb6detail2r114bad_last_allocE +__ZTVN3tbb6detail2r114bad_last_allocE +__ZTIN3tbb6detail2r112missing_waitE +__ZTVN3tbb6detail2r112missing_waitE +__ZTIN3tbb6detail2r110user_abortE +__ZTVN3tbb6detail2r110user_abortE +__ZTIN3tbb6detail2r111unsafe_waitE +__ZTVN3tbb6detail2r111unsafe_waitE + +# RTM Mutex (rtm_mutex.cpp) +__ZN3tbb6detail2r17acquireERNS0_2d19rtm_mutexERNS3_11scoped_lockEb +__ZN3tbb6detail2r17releaseERNS0_2d19rtm_mutex11scoped_lockE +__ZN3tbb6detail2r111try_acquireERNS0_2d19rtm_mutexERNS3_11scoped_lockE + +# RTM RW Mutex (rtm_rw_mutex.cpp) +__ZN3tbb6detail2r114acquire_readerERNS0_2d112rtm_rw_mutexERNS3_11scoped_lockEb +__ZN3tbb6detail2r114acquire_writerERNS0_2d112rtm_rw_mutexERNS3_11scoped_lockEb +__ZN3tbb6detail2r118try_acquire_readerERNS0_2d112rtm_rw_mutexERNS3_11scoped_lockE +__ZN3tbb6detail2r118try_acquire_writerERNS0_2d112rtm_rw_mutexERNS3_11scoped_lockE +__ZN3tbb6detail2r17releaseERNS0_2d112rtm_rw_mutex11scoped_lockE +__ZN3tbb6detail2r17upgradeERNS0_2d112rtm_rw_mutex11scoped_lockE +__ZN3tbb6detail2r19downgradeERNS0_2d112rtm_rw_mutex11scoped_lockE + +# Tasks and partitioners (task.cpp) +__ZN3tbb6detail2r17suspendEPFvPvPNS1_18suspend_point_typeEES2_ +__ZN3tbb6detail2r16resumeEPNS1_18suspend_point_typeE +__ZN3tbb6detail2r121current_suspend_pointEv +__ZN3tbb6detail2r114notify_waitersEm +__ZN3tbb6detail2r127get_thread_reference_vertexEPNS0_2d126wait_tree_vertex_interfaceE + +# Task dispatcher (task_dispatcher.cpp) +__ZN3tbb6detail2r114execution_slotEPKNS0_2d114execution_dataE +__ZN3tbb6detail2r14waitERNS0_2d112wait_contextERNS2_18task_group_contextE +__ZN3tbb6detail2r15spawnERNS0_2d14taskERNS2_18task_group_contextE +__ZN3tbb6detail2r15spawnERNS0_2d14taskERNS2_18task_group_contextEt +__ZN3tbb6detail2r116execute_and_waitERNS0_2d14taskERNS2_18task_group_contextERNS2_12wait_contextES6_ +__ZN3tbb6detail2r16submitERNS0_2d14taskERNS2_18task_group_contextEPNS1_5arenaEm +__ZN3tbb6detail2r115current_contextEv + +# Task group context (task_group_context.cpp) +__ZN3tbb6detail2r110initializeERNS0_2d118task_group_contextE +__ZN3tbb6detail2r122cancel_group_executionERNS0_2d118task_group_contextE +__ZN3tbb6detail2r128is_group_execution_cancelledERNS0_2d118task_group_contextE +__ZN3tbb6detail2r15resetERNS0_2d118task_group_contextE +__ZN3tbb6detail2r17destroyERNS0_2d118task_group_contextE +__ZN3tbb6detail2r119capture_fp_settingsERNS0_2d118task_group_contextE + +# Task arena (arena.cpp) +__ZN3tbb6detail2r115max_concurrencyEPKNS0_2d115task_arena_baseE +__ZN3tbb6detail2r110initializeERNS0_2d115task_arena_baseE +__ZN3tbb6detail2r16attachERNS0_2d115task_arena_baseE +__ZN3tbb6detail2r17executeERNS0_2d115task_arena_baseERNS2_13delegate_baseE +__ZN3tbb6detail2r19terminateERNS0_2d115task_arena_baseE +__ZN3tbb6detail2r120isolate_within_arenaERNS0_2d113delegate_baseEl +__ZN3tbb6detail2r17enqueueERNS0_2d14taskEPNS2_15task_arena_baseE +__ZN3tbb6detail2r17enqueueERNS0_2d14taskERNS2_18task_group_contextEPNS2_15task_arena_baseE +__ZN3tbb6detail2r14waitERNS0_2d115task_arena_baseE +__ZN3tbb6detail2r114execution_slotERKNS0_2d115task_arena_baseE + +# System topology parsing and threads pinning (governor.cpp) +__ZN3tbb6detail2r115numa_node_countEv +__ZN3tbb6detail2r117fill_numa_indicesEPi +__ZN3tbb6detail2r115core_type_countEl +__ZN3tbb6detail2r122fill_core_type_indicesEPil +__ZN3tbb6detail2r131constraints_default_concurrencyERKNS0_2d111constraintsEl +__ZN3tbb6detail2r128constraints_threads_per_coreERKNS0_2d111constraintsEl +__ZN3tbb6detail2r124numa_default_concurrencyEi + +# Observer (observer_proxy.cpp) +__ZN3tbb6detail2r17observeERNS0_2d123task_scheduler_observerEb + +# Queuing RW Mutex (queuing_rw_mutex.cpp) +__ZN3tbb6detail2r111try_acquireERNS0_2d116queuing_rw_mutexERNS3_11scoped_lockEb +__ZN3tbb6detail2r117upgrade_to_writerERNS0_2d116queuing_rw_mutex11scoped_lockE +__ZN3tbb6detail2r119downgrade_to_readerERNS0_2d116queuing_rw_mutex11scoped_lockE +__ZN3tbb6detail2r17acquireERNS0_2d116queuing_rw_mutexERNS3_11scoped_lockEb +__ZN3tbb6detail2r17releaseERNS0_2d116queuing_rw_mutex11scoped_lockE +__ZN3tbb6detail2r19constructERNS0_2d116queuing_rw_mutexE +__ZN3tbb6detail2r19is_writerERKNS0_2d116queuing_rw_mutex11scoped_lockE + +# Global control (global_control.cpp) +__ZN3tbb6detail2r16createERNS0_2d114global_controlE +__ZN3tbb6detail2r17destroyERNS0_2d114global_controlE +__ZN3tbb6detail2r127global_control_active_valueEi +__ZN3tbb6detail2r18finalizeERNS0_2d121task_scheduler_handleEl +__ZN3tbb6detail2r13getERNS0_2d121task_scheduler_handleE + +# Parallel pipeline (parallel_pipeline.cpp) +__ZN3tbb6detail2r117parallel_pipelineERNS0_2d118task_group_contextEmRKNS2_11filter_nodeE +__ZN3tbb6detail2r116set_end_of_inputERNS0_2d111base_filterE + +# Concurrent bounded queue (concurrent_bounded_queue.cpp) +__ZN3tbb6detail2r126allocate_bounded_queue_repEm +__ZN3tbb6detail2r126wait_bounded_queue_monitorEPNS1_18concurrent_monitorEmlRNS0_2d113delegate_baseE +__ZN3tbb6detail2r128abort_bounded_queue_monitorsEPNS1_18concurrent_monitorE +__ZN3tbb6detail2r128deallocate_bounded_queue_repEPhm +__ZN3tbb6detail2r128notify_bounded_queue_monitorEPNS1_18concurrent_monitorEmm + +# Concurrent monitor (address_waiter.cpp) +__ZN3tbb6detail2r115wait_on_addressEPvRNS0_2d113delegate_baseEm +__ZN3tbb6detail2r117notify_by_addressEPvm +__ZN3tbb6detail2r121notify_by_address_oneEPv +__ZN3tbb6detail2r121notify_by_address_allEPv + +# Versioning (version.cpp) +_TBB_runtime_interface_version +_TBB_runtime_version diff --git a/src/tbb/src/tbb/def/win32-tbb.def b/src/tbb/src/tbb/def/win32-tbb.def new file mode 100644 index 000000000..94b544170 --- /dev/null +++ b/src/tbb/src/tbb/def/win32-tbb.def @@ -0,0 +1,152 @@ +; Copyright (c) 2005-2024 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the "License"); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, software +; distributed under the License is distributed on an "AS IS" BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions and +; limitations under the License. + +; This file is organized with a section for each .cpp file. + +EXPORTS + +; Assertions (assert.cpp) +?assertion_failure@r1@detail@tbb@@YAXPBDH00@Z + +; ITT (tbb_profiling.cpp) +?call_itt_notify@r1@detail@tbb@@YAXHPAX@Z +?create_itt_sync@r1@detail@tbb@@YAXPAXPB_W1@Z +?itt_make_task_group@r1@detail@tbb@@YAXW4itt_domain_enum@d1@23@PAX_K12W4string_resource_index@d0@23@@Z +?itt_task_begin@r1@detail@tbb@@YAXW4itt_domain_enum@d1@23@PAX_K12W4string_resource_index@d0@23@@Z +?itt_task_end@r1@detail@tbb@@YAXW4itt_domain_enum@d1@23@@Z +?itt_metadata_str_add@r1@detail@tbb@@YAXW4itt_domain_enum@d1@23@PAX_KW4string_resource_index@d0@23@PBD@Z +?itt_relation_add@r1@detail@tbb@@YAXW4itt_domain_enum@d1@23@PAX_KW4itt_relation@d0@23@12@Z +?itt_region_begin@r1@detail@tbb@@YAXW4itt_domain_enum@d1@23@PAX_K12W4string_resource_index@d0@23@@Z +?itt_region_end@r1@detail@tbb@@YAXW4itt_domain_enum@d1@23@PAX_K@Z +?itt_set_sync_name@r1@detail@tbb@@YAXPAXPB_W@Z +?itt_metadata_ptr_add@r1@detail@tbb@@YAXW4itt_domain_enum@d1@23@PAX_KW4string_resource_index@d0@23@1@Z + +; Allocators (tbb_allocator.cpp) +?cache_aligned_allocate@r1@detail@tbb@@YAPAXI@Z +?cache_aligned_deallocate@r1@detail@tbb@@YAXPAX@Z +?cache_line_size@r1@detail@tbb@@YAIXZ +?allocate_memory@r1@detail@tbb@@YAPAXI@Z +?deallocate_memory@r1@detail@tbb@@YAXPAX@Z +?is_tbbmalloc_used@r1@detail@tbb@@YA_NXZ + +; Small object pool (small_object_pool.cpp) +?allocate@r1@detail@tbb@@YAPAXAAPAVsmall_object_pool@d1@23@IABUexecution_data@523@@Z +?allocate@r1@detail@tbb@@YAPAXAAPAVsmall_object_pool@d1@23@I@Z +?deallocate@r1@detail@tbb@@YAXAAVsmall_object_pool@d1@23@PAXIABUexecution_data@523@@Z +?deallocate@r1@detail@tbb@@YAXAAVsmall_object_pool@d1@23@PAXI@Z + +; Error handling (exception.cpp) +?throw_exception@r1@detail@tbb@@YAXW4exception_id@d0@23@@Z +?what@bad_last_alloc@r1@detail@tbb@@UBEPBDXZ +?what@user_abort@r1@detail@tbb@@UBEPBDXZ +?what@missing_wait@r1@detail@tbb@@UBEPBDXZ + +; RTM Mutex (rtm_mutex.cpp) +?acquire@r1@detail@tbb@@YAXAAVrtm_mutex@d1@23@AAVscoped_lock@4523@_N@Z +?release@r1@detail@tbb@@YAXAAVscoped_lock@rtm_mutex@d1@23@@Z +?try_acquire@r1@detail@tbb@@YA_NAAVrtm_mutex@d1@23@AAVscoped_lock@4523@@Z + +; RTM RW Mutex (rtm_rw_mutex.cpp) +?acquire_reader@r1@detail@tbb@@YAXAAVrtm_rw_mutex@d1@23@AAVscoped_lock@4523@_N@Z +?acquire_writer@r1@detail@tbb@@YAXAAVrtm_rw_mutex@d1@23@AAVscoped_lock@4523@_N@Z +?downgrade@r1@detail@tbb@@YA_NAAVscoped_lock@rtm_rw_mutex@d1@23@@Z +?release@r1@detail@tbb@@YAXAAVscoped_lock@rtm_rw_mutex@d1@23@@Z +?try_acquire_reader@r1@detail@tbb@@YA_NAAVrtm_rw_mutex@d1@23@AAVscoped_lock@4523@@Z +?try_acquire_writer@r1@detail@tbb@@YA_NAAVrtm_rw_mutex@d1@23@AAVscoped_lock@4523@@Z +?upgrade@r1@detail@tbb@@YA_NAAVscoped_lock@rtm_rw_mutex@d1@23@@Z + +; Tasks and partitioners (task.cpp) +?current_suspend_point@r1@detail@tbb@@YAPAUsuspend_point_type@123@XZ +?resume@r1@detail@tbb@@YAXPAUsuspend_point_type@123@@Z +?suspend@r1@detail@tbb@@YAXP6AXPAXPAUsuspend_point_type@123@@Z0@Z +?notify_waiters@r1@detail@tbb@@YAXI@Z +?get_thread_reference_vertex@r1@detail@tbb@@YAPAVwait_tree_vertex_interface@d1@23@PAV4523@@Z + +; Task dispatcher (task_dispatcher.cpp) +?spawn@r1@detail@tbb@@YAXAAVtask@d1@23@AAVtask_group_context@523@G@Z +?spawn@r1@detail@tbb@@YAXAAVtask@d1@23@AAVtask_group_context@523@@Z +?execute_and_wait@r1@detail@tbb@@YAXAAVtask@d1@23@AAVtask_group_context@523@AAVwait_context@523@1@Z +?execution_slot@r1@detail@tbb@@YAGPBUexecution_data@d1@23@@Z +?wait@r1@detail@tbb@@YAXAAVwait_context@d1@23@AAVtask_group_context@523@@Z +?submit@r1@detail@tbb@@YAXAAVtask@d1@23@AAVtask_group_context@523@PAVarena@123@I@Z +?current_context@r1@detail@tbb@@YAPAVtask_group_context@d1@23@XZ + +; Task group context (task_group_context.cpp) +?cancel_group_execution@r1@detail@tbb@@YA_NAAVtask_group_context@d1@23@@Z +?capture_fp_settings@r1@detail@tbb@@YAXAAVtask_group_context@d1@23@@Z +?destroy@r1@detail@tbb@@YAXAAVtask_group_context@d1@23@@Z +?initialize@r1@detail@tbb@@YAXAAVtask_group_context@d1@23@@Z +?is_group_execution_cancelled@r1@detail@tbb@@YA_NAAVtask_group_context@d1@23@@Z +?reset@r1@detail@tbb@@YAXAAVtask_group_context@d1@23@@Z + +; Task arena (arena.cpp) +?attach@r1@detail@tbb@@YA_NAAVtask_arena_base@d1@23@@Z +?enqueue@r1@detail@tbb@@YAXAAVtask@d1@23@PAVtask_arena_base@523@@Z +?execute@r1@detail@tbb@@YAXAAVtask_arena_base@d1@23@AAVdelegate_base@523@@Z +?initialize@r1@detail@tbb@@YAXAAVtask_arena_base@d1@23@@Z +?isolate_within_arena@r1@detail@tbb@@YAXAAVdelegate_base@d1@23@H@Z +?max_concurrency@r1@detail@tbb@@YAHPBVtask_arena_base@d1@23@@Z +?terminate@r1@detail@tbb@@YAXAAVtask_arena_base@d1@23@@Z +?wait@r1@detail@tbb@@YAXAAVtask_arena_base@d1@23@@Z +?enqueue@r1@detail@tbb@@YAXAAVtask@d1@23@AAVtask_group_context@523@PAVtask_arena_base@523@@Z +?execution_slot@r1@detail@tbb@@YAGABVtask_arena_base@d1@23@@Z + +; System topology parsing and threads pinning (governor.cpp) +?numa_node_count@r1@detail@tbb@@YAIXZ +?fill_numa_indices@r1@detail@tbb@@YAXPAH@Z +?core_type_count@r1@detail@tbb@@YAIH@Z +?fill_core_type_indices@r1@detail@tbb@@YAXPAHH@Z +?numa_default_concurrency@r1@detail@tbb@@YAHH@Z +?constraints_default_concurrency@r1@detail@tbb@@YAHABUconstraints@d1@23@H@Z +?constraints_threads_per_core@r1@detail@tbb@@YAHABUconstraints@d1@23@H@Z + +; Observer (observer_proxy.cpp) +?observe@r1@detail@tbb@@YAXAAVtask_scheduler_observer@d1@23@_N@Z + +; Queuing RW Mutex (queuing_rw_mutex.cpp) +?acquire@r1@detail@tbb@@YAXAAVqueuing_rw_mutex@d1@23@AAVscoped_lock@4523@_N@Z +?construct@r1@detail@tbb@@YAXAAVqueuing_rw_mutex@d1@23@@Z +?downgrade_to_reader@r1@detail@tbb@@YA_NAAVscoped_lock@queuing_rw_mutex@d1@23@@Z +?release@r1@detail@tbb@@YAXAAVscoped_lock@queuing_rw_mutex@d1@23@@Z +?try_acquire@r1@detail@tbb@@YA_NAAVqueuing_rw_mutex@d1@23@AAVscoped_lock@4523@_N@Z +?upgrade_to_writer@r1@detail@tbb@@YA_NAAVscoped_lock@queuing_rw_mutex@d1@23@@Z +?is_writer@r1@detail@tbb@@YA_NABVscoped_lock@queuing_rw_mutex@d1@23@@Z + +; Global control (global_control.cpp) +?create@r1@detail@tbb@@YAXAAVglobal_control@d1@23@@Z +?destroy@r1@detail@tbb@@YAXAAVglobal_control@d1@23@@Z +?global_control_active_value@r1@detail@tbb@@YAIH@Z +?get@r1@detail@tbb@@YAXAAVtask_scheduler_handle@d1@23@@Z +?finalize@r1@detail@tbb@@YA_NAAVtask_scheduler_handle@d1@23@H@Z + +; Parallel pipeline (parallel_pipeline.cpp) +?parallel_pipeline@r1@detail@tbb@@YAXAAVtask_group_context@d1@23@IABVfilter_node@523@@Z +?set_end_of_input@r1@detail@tbb@@YAXAAVbase_filter@d1@23@@Z + +; Concurrent bounded queue (concurrent_bounded_queue.cpp) +?abort_bounded_queue_monitors@r1@detail@tbb@@YAXPAVconcurrent_monitor@123@@Z +?allocate_bounded_queue_rep@r1@detail@tbb@@YAPAEI@Z +?deallocate_bounded_queue_rep@r1@detail@tbb@@YAXPAEI@Z +?notify_bounded_queue_monitor@r1@detail@tbb@@YAXPAVconcurrent_monitor@123@II@Z +?wait_bounded_queue_monitor@r1@detail@tbb@@YAXPAVconcurrent_monitor@123@IHAAVdelegate_base@d1@23@@Z + +; Concurrent monitor (address_waiter.cpp) +?wait_on_address@r1@detail@tbb@@YAXPAXAAVdelegate_base@d1@23@I@Z +?notify_by_address@r1@detail@tbb@@YAXPAXI@Z +?notify_by_address_one@r1@detail@tbb@@YAXPAX@Z +?notify_by_address_all@r1@detail@tbb@@YAXPAX@Z + +;; Versioning (version.cpp) +TBB_runtime_interface_version +TBB_runtime_version diff --git a/src/tbb/src/tbb/def/win64-tbb.def b/src/tbb/src/tbb/def/win64-tbb.def new file mode 100644 index 000000000..96bafc016 --- /dev/null +++ b/src/tbb/src/tbb/def/win64-tbb.def @@ -0,0 +1,152 @@ +; Copyright (c) 2005-2024 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the "License"); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, software +; distributed under the License is distributed on an "AS IS" BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions and +; limitations under the License. + +; This file is organized with a section for each .cpp file. + +EXPORTS + +; Assertions (assert.cpp) +?assertion_failure@r1@detail@tbb@@YAXPEBDH00@Z + +; ITT (tbb_profiling.cpp) +?call_itt_notify@r1@detail@tbb@@YAXHPEAX@Z +?create_itt_sync@r1@detail@tbb@@YAXPEAXPEB_W1@Z +?itt_make_task_group@r1@detail@tbb@@YAXW4itt_domain_enum@d1@23@PEAX_K12W4string_resource_index@d0@23@@Z +?itt_task_begin@r1@detail@tbb@@YAXW4itt_domain_enum@d1@23@PEAX_K12W4string_resource_index@d0@23@@Z +?itt_task_end@r1@detail@tbb@@YAXW4itt_domain_enum@d1@23@@Z +?itt_set_sync_name@r1@detail@tbb@@YAXPEAXPEB_W@Z +?itt_metadata_str_add@r1@detail@tbb@@YAXW4itt_domain_enum@d1@23@PEAX_KW4string_resource_index@d0@23@PEBD@Z +?itt_relation_add@r1@detail@tbb@@YAXW4itt_domain_enum@d1@23@PEAX_KW4itt_relation@d0@23@12@Z +?itt_region_begin@r1@detail@tbb@@YAXW4itt_domain_enum@d1@23@PEAX_K12W4string_resource_index@d0@23@@Z +?itt_region_end@r1@detail@tbb@@YAXW4itt_domain_enum@d1@23@PEAX_K@Z +?itt_metadata_ptr_add@r1@detail@tbb@@YAXW4itt_domain_enum@d1@23@PEAX_KW4string_resource_index@d0@23@1@Z + +; Allocators (tbb_allocator.cpp) +?cache_aligned_allocate@r1@detail@tbb@@YAPEAX_K@Z +?cache_aligned_deallocate@r1@detail@tbb@@YAXPEAX@Z +?cache_line_size@r1@detail@tbb@@YA_KXZ +?allocate_memory@r1@detail@tbb@@YAPEAX_K@Z +?deallocate_memory@r1@detail@tbb@@YAXPEAX@Z +?is_tbbmalloc_used@r1@detail@tbb@@YA_NXZ + +; Small object pool (small_object_pool.cpp) +?allocate@r1@detail@tbb@@YAPEAXAEAPEAVsmall_object_pool@d1@23@_KAEBUexecution_data@523@@Z +?allocate@r1@detail@tbb@@YAPEAXAEAPEAVsmall_object_pool@d1@23@_K@Z +?deallocate@r1@detail@tbb@@YAXAEAVsmall_object_pool@d1@23@PEAX_KAEBUexecution_data@523@@Z +?deallocate@r1@detail@tbb@@YAXAEAVsmall_object_pool@d1@23@PEAX_K@Z + +; Error handling (exception.cpp) +?throw_exception@r1@detail@tbb@@YAXW4exception_id@d0@23@@Z +?what@bad_last_alloc@r1@detail@tbb@@UEBAPEBDXZ +?what@user_abort@r1@detail@tbb@@UEBAPEBDXZ +?what@missing_wait@r1@detail@tbb@@UEBAPEBDXZ + +; RTM Mutex (rtm_mutex.cpp) +?try_acquire@r1@detail@tbb@@YA_NAEAVrtm_mutex@d1@23@AEAVscoped_lock@4523@@Z +?acquire@r1@detail@tbb@@YAXAEAVrtm_mutex@d1@23@AEAVscoped_lock@4523@_N@Z +?release@r1@detail@tbb@@YAXAEAVscoped_lock@rtm_mutex@d1@23@@Z + +; RTM RW Mutex (rtm_rw_mutex.cpp) +?acquire_writer@r1@detail@tbb@@YAXAEAVrtm_rw_mutex@d1@23@AEAVscoped_lock@4523@_N@Z +?acquire_reader@r1@detail@tbb@@YAXAEAVrtm_rw_mutex@d1@23@AEAVscoped_lock@4523@_N@Z +?upgrade@r1@detail@tbb@@YA_NAEAVscoped_lock@rtm_rw_mutex@d1@23@@Z +?downgrade@r1@detail@tbb@@YA_NAEAVscoped_lock@rtm_rw_mutex@d1@23@@Z +?try_acquire_writer@r1@detail@tbb@@YA_NAEAVrtm_rw_mutex@d1@23@AEAVscoped_lock@4523@@Z +?try_acquire_reader@r1@detail@tbb@@YA_NAEAVrtm_rw_mutex@d1@23@AEAVscoped_lock@4523@@Z +?release@r1@detail@tbb@@YAXAEAVscoped_lock@rtm_rw_mutex@d1@23@@Z + +; Tasks and partitioners (task.cpp) +?suspend@r1@detail@tbb@@YAXP6AXPEAXPEAUsuspend_point_type@123@@Z0@Z +?resume@r1@detail@tbb@@YAXPEAUsuspend_point_type@123@@Z +?current_suspend_point@r1@detail@tbb@@YAPEAUsuspend_point_type@123@XZ +?notify_waiters@r1@detail@tbb@@YAX_K@Z +?get_thread_reference_vertex@r1@detail@tbb@@YAPEAVwait_tree_vertex_interface@d1@23@PEAV4523@@Z + +; Task dispatcher (task_dispatcher.cpp) +?spawn@r1@detail@tbb@@YAXAEAVtask@d1@23@AEAVtask_group_context@523@@Z +?spawn@r1@detail@tbb@@YAXAEAVtask@d1@23@AEAVtask_group_context@523@G@Z +?execute_and_wait@r1@detail@tbb@@YAXAEAVtask@d1@23@AEAVtask_group_context@523@AEAVwait_context@523@1@Z +?execution_slot@r1@detail@tbb@@YAGPEBUexecution_data@d1@23@@Z +?wait@r1@detail@tbb@@YAXAEAVwait_context@d1@23@AEAVtask_group_context@523@@Z +?submit@r1@detail@tbb@@YAXAEAVtask@d1@23@AEAVtask_group_context@523@PEAVarena@123@_K@Z +?current_context@r1@detail@tbb@@YAPEAVtask_group_context@d1@23@XZ + +; Task group context (task_group_context.cpp) +?initialize@r1@detail@tbb@@YAXAEAVtask_group_context@d1@23@@Z +?destroy@r1@detail@tbb@@YAXAEAVtask_group_context@d1@23@@Z +?is_group_execution_cancelled@r1@detail@tbb@@YA_NAEAVtask_group_context@d1@23@@Z +?reset@r1@detail@tbb@@YAXAEAVtask_group_context@d1@23@@Z +?cancel_group_execution@r1@detail@tbb@@YA_NAEAVtask_group_context@d1@23@@Z +?capture_fp_settings@r1@detail@tbb@@YAXAEAVtask_group_context@d1@23@@Z + +; Task arena (arena.cpp) +?max_concurrency@r1@detail@tbb@@YAHPEBVtask_arena_base@d1@23@@Z +?initialize@r1@detail@tbb@@YAXAEAVtask_arena_base@d1@23@@Z +?terminate@r1@detail@tbb@@YAXAEAVtask_arena_base@d1@23@@Z +?execute@r1@detail@tbb@@YAXAEAVtask_arena_base@d1@23@AEAVdelegate_base@523@@Z +?wait@r1@detail@tbb@@YAXAEAVtask_arena_base@d1@23@@Z +?attach@r1@detail@tbb@@YA_NAEAVtask_arena_base@d1@23@@Z +?isolate_within_arena@r1@detail@tbb@@YAXAEAVdelegate_base@d1@23@_J@Z +?enqueue@r1@detail@tbb@@YAXAEAVtask@d1@23@PEAVtask_arena_base@523@@Z +?enqueue@r1@detail@tbb@@YAXAEAVtask@d1@23@AEAVtask_group_context@523@PEAVtask_arena_base@523@@Z +?execution_slot@r1@detail@tbb@@YAGAEBVtask_arena_base@d1@23@@Z + +; System topology parsing and threads pinning (governor.cpp) +?numa_node_count@r1@detail@tbb@@YAIXZ +?fill_numa_indices@r1@detail@tbb@@YAXPEAH@Z +?core_type_count@r1@detail@tbb@@YAI_J@Z +?fill_core_type_indices@r1@detail@tbb@@YAXPEAH_J@Z +?numa_default_concurrency@r1@detail@tbb@@YAHH@Z +?constraints_default_concurrency@r1@detail@tbb@@YAHAEBUconstraints@d1@23@_J@Z +?constraints_threads_per_core@r1@detail@tbb@@YAHAEBUconstraints@d1@23@_J@Z + +; Observer (observer_proxy.cpp) +?observe@r1@detail@tbb@@YAXAEAVtask_scheduler_observer@d1@23@_N@Z + +; Queuing RW Mutex (queuing_rw_mutex.cpp) +?construct@r1@detail@tbb@@YAXAEAVqueuing_rw_mutex@d1@23@@Z +?try_acquire@r1@detail@tbb@@YA_NAEAVqueuing_rw_mutex@d1@23@AEAVscoped_lock@4523@_N@Z +?acquire@r1@detail@tbb@@YAXAEAVqueuing_rw_mutex@d1@23@AEAVscoped_lock@4523@_N@Z +?release@r1@detail@tbb@@YAXAEAVscoped_lock@queuing_rw_mutex@d1@23@@Z +?upgrade_to_writer@r1@detail@tbb@@YA_NAEAVscoped_lock@queuing_rw_mutex@d1@23@@Z +?downgrade_to_reader@r1@detail@tbb@@YA_NAEAVscoped_lock@queuing_rw_mutex@d1@23@@Z +?is_writer@r1@detail@tbb@@YA_NAEBVscoped_lock@queuing_rw_mutex@d1@23@@Z + +; Global control (global_control.cpp) +?global_control_active_value@r1@detail@tbb@@YA_KH@Z +?create@r1@detail@tbb@@YAXAEAVglobal_control@d1@23@@Z +?destroy@r1@detail@tbb@@YAXAEAVglobal_control@d1@23@@Z +?get@r1@detail@tbb@@YAXAEAVtask_scheduler_handle@d1@23@@Z +?finalize@r1@detail@tbb@@YA_NAEAVtask_scheduler_handle@d1@23@_J@Z + +; Parallel pipeline (parallel_pipeline.cpp) +?set_end_of_input@r1@detail@tbb@@YAXAEAVbase_filter@d1@23@@Z +?parallel_pipeline@r1@detail@tbb@@YAXAEAVtask_group_context@d1@23@_KAEBVfilter_node@523@@Z + +; Concurrent bounded queue (concurrent_bounded_queue.cpp) +?allocate_bounded_queue_rep@r1@detail@tbb@@YAPEAE_K@Z +?deallocate_bounded_queue_rep@r1@detail@tbb@@YAXPEAE_K@Z +?wait_bounded_queue_monitor@r1@detail@tbb@@YAXPEAVconcurrent_monitor@123@_K_JAEAVdelegate_base@d1@23@@Z +?abort_bounded_queue_monitors@r1@detail@tbb@@YAXPEAVconcurrent_monitor@123@@Z +?notify_bounded_queue_monitor@r1@detail@tbb@@YAXPEAVconcurrent_monitor@123@_K1@Z + +; Concurrent monitor (address_waiter.cpp) +?wait_on_address@r1@detail@tbb@@YAXPEAXAEAVdelegate_base@d1@23@_K@Z +?notify_by_address@r1@detail@tbb@@YAXPEAX_K@Z +?notify_by_address_one@r1@detail@tbb@@YAXPEAX@Z +?notify_by_address_all@r1@detail@tbb@@YAXPEAX@Z + +;; Versioning (version.cpp) +TBB_runtime_interface_version +TBB_runtime_version diff --git a/src/tbb/src/tbb/dynamic_link.cpp b/src/tbb/src/tbb/dynamic_link.cpp index b51280d0c..4911b97df 100644 --- a/src/tbb/src/tbb/dynamic_link.cpp +++ b/src/tbb/src/tbb/dynamic_link.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2024 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,39 +12,40 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include "dynamic_link.h" -#include "tbb/tbb_config.h" +#include "environment.h" + +#include "oneapi/tbb/detail/_template_helpers.h" +#include "oneapi/tbb/detail/_utils.h" /* This file is used by both TBB and OpenMP RTL. Do not use __TBB_ASSERT() macro and runtime_warning() function because they are not available in OpenMP. Use - LIBRARY_ASSERT and DYNAMIC_LINK_WARNING instead. + __TBB_ASSERT_EX and DYNAMIC_LINK_WARNING instead. */ #include <cstdarg> // va_list etc. +#include <cstring> // strrchr #if _WIN32 #include <malloc.h> // Unify system calls #define dlopen( name, flags ) LoadLibrary( name ) #define dlsym( handle, name ) GetProcAddress( handle, name ) - #define dlclose( handle ) ( ! FreeLibrary( handle ) ) + // FreeLibrary return bool value that is not used. + #define dlclose( handle ) (void)( ! FreeLibrary( handle ) ) #define dlerror() GetLastError() #ifndef PATH_MAX #define PATH_MAX MAX_PATH #endif #else /* _WIN32 */ #include <dlfcn.h> - #include <string.h> #include <unistd.h> - #include <limits.h> - #include <stdlib.h> + + #include <climits> + #include <cstdlib> #endif /* _WIN32 */ #if __TBB_WEAK_SYMBOLS_PRESENT && !__TBB_DYNAMIC_LOAD_ENABLED @@ -54,14 +55,9 @@ #pragma weak dlclose #endif /* __TBB_WEAK_SYMBOLS_PRESENT && !__TBB_DYNAMIC_LOAD_ENABLED */ -#include "tbb/tbb_misc.h" -#define __USE_TBB_ATOMICS ( !(__linux__&&__ia64__) || __TBB_BUILD ) #define __USE_STATIC_DL_INIT ( !__ANDROID__ ) -#if !__USE_TBB_ATOMICS -#include <pthread.h> -#endif /* dynamic_link is a common interface for searching for required symbols in an @@ -105,7 +101,9 @@ soon as all of the symbols have been resolved. 3. Weak symbols: if weak symbols are available they are returned. */ -OPEN_INTERNAL_NAMESPACE +namespace tbb { +namespace detail { +namespace r1 { #if __TBB_WEAK_SYMBOLS_PRESENT || __TBB_DYNAMIC_LOAD_ENABLED @@ -113,11 +111,11 @@ OPEN_INTERNAL_NAMESPACE // Report runtime errors and continue. #define DYNAMIC_LINK_WARNING dynamic_link_warning static void dynamic_link_warning( dynamic_link_error_t code, ... ) { - (void) code; + suppress_unused_warning(code); } // library_warning #endif /* !defined(DYNAMIC_LINK_WARNING) && !__TBB_WIN8UI_SUPPORT && __TBB_DYNAMIC_LOAD_ENABLED */ - static bool resolve_symbols( dynamic_link_handle module, const dynamic_link_descriptor descriptors[], size_t required ) + static bool resolve_symbols( dynamic_link_handle module, const dynamic_link_descriptor descriptors[], std::size_t required ) { if ( !module ) return false; @@ -126,12 +124,12 @@ OPEN_INTERNAL_NAMESPACE if ( !dlsym ) return false; #endif /* !__TBB_DYNAMIC_LOAD_ENABLED */ - const size_t n_desc=20; // Usually we don't have more than 20 descriptors per library - LIBRARY_ASSERT( required <= n_desc, "Too many descriptors is required" ); + const std::size_t n_desc=20; // Usually we don't have more than 20 descriptors per library + __TBB_ASSERT_EX( required <= n_desc, "Too many descriptors is required" ); if ( required > n_desc ) return false; pointer_to_handler h[n_desc]; - for ( size_t k = 0; k < required; ++k ) { + for ( std::size_t k = 0; k < required; ++k ) { dynamic_link_descriptor const & desc = descriptors[k]; pointer_to_handler addr = (pointer_to_handler)dlsym( module, desc.name ); if ( !addr ) { @@ -142,19 +140,19 @@ OPEN_INTERNAL_NAMESPACE // Commit the entry points. // Cannot use memset here, because the writes must be atomic. - for( size_t k = 0; k < required; ++k ) + for( std::size_t k = 0; k < required; ++k ) *descriptors[k].handler = h[k]; return true; } #if __TBB_WIN8UI_SUPPORT - bool dynamic_link( const char* library, const dynamic_link_descriptor descriptors[], size_t required, dynamic_link_handle*, int flags ) { - dynamic_link_handle tmp_handle = NULL; + bool dynamic_link( const char* library, const dynamic_link_descriptor descriptors[], std::size_t required, dynamic_link_handle*, int flags ) { + dynamic_link_handle tmp_handle = nullptr; TCHAR wlibrary[256]; if ( MultiByteToWideChar(CP_UTF8, 0, library, -1, wlibrary, 255) == 0 ) return false; if ( flags & DYNAMIC_LINK_LOAD ) tmp_handle = LoadPackagedLibrary( wlibrary, 0 ); - if (tmp_handle != NULL){ + if (tmp_handle != nullptr){ return resolve_symbols(tmp_handle, descriptors, required); }else{ return false; @@ -175,8 +173,8 @@ OPEN_INTERNAL_NAMESPACE current one, it is the directory tbb.dll loaded from. Example: - Let us assume "tbb.dll" is located in "c:\program files\common\intel\" directory, e. g. - absolute path of tbb library is "c:\program files\common\intel\tbb.dll". Absolute path for + Let us assume "tbb.dll" is located in "c:\program files\common\intel\" directory, e.g. + absolute path of the library is "c:\program files\common\intel\tbb.dll". Absolute path for "tbbmalloc.dll" would be "c:\program files\common\intel\tbbmalloc.dll". Absolute path for "malloc\tbbmalloc.dll" would be "c:\program files\common\intel\malloc\tbbmalloc.dll". */ @@ -188,80 +186,30 @@ OPEN_INTERNAL_NAMESPACE // the constructor is called. #define MAX_LOADED_MODULES 8 // The number of maximum possible modules which can be loaded -#if __USE_TBB_ATOMICS - typedef ::tbb::atomic<size_t> atomic_incrementer; - void init_atomic_incrementer( atomic_incrementer & ) {} - - static void atomic_once( void( *func ) (void), tbb::atomic< tbb::internal::do_once_state > &once_state ) { - tbb::internal::atomic_do_once( func, once_state ); - } - #define ATOMIC_ONCE_DECL( var ) tbb::atomic< tbb::internal::do_once_state > var -#else - static void pthread_assert( int error_code, const char* msg ) { - LIBRARY_ASSERT( error_code == 0, msg ); - } - - class atomic_incrementer { - size_t my_val; - pthread_spinlock_t my_lock; - public: - void init() { - my_val = 0; - pthread_assert( pthread_spin_init( &my_lock, PTHREAD_PROCESS_PRIVATE ), "pthread_spin_init failed" ); - } - size_t operator++(int) { - pthread_assert( pthread_spin_lock( &my_lock ), "pthread_spin_lock failed" ); - size_t prev_val = my_val++; - pthread_assert( pthread_spin_unlock( &my_lock ), "pthread_spin_unlock failed" ); - return prev_val; - } - operator size_t() { - pthread_assert( pthread_spin_lock( &my_lock ), "pthread_spin_lock failed" ); - size_t val = my_val; - pthread_assert( pthread_spin_unlock( &my_lock ), "pthread_spin_unlock failed" ); - return val; - } - ~atomic_incrementer() { - pthread_assert( pthread_spin_destroy( &my_lock ), "pthread_spin_destroy failed" ); - } - }; + using atomic_incrementer = std::atomic<std::size_t>; - void init_atomic_incrementer( atomic_incrementer &r ) { - r.init(); - } - - static void atomic_once( void( *func ) (), pthread_once_t &once_state ) { - pthread_assert( pthread_once( &once_state, func ), "pthread_once failed" ); - } - #define ATOMIC_ONCE_DECL( var ) pthread_once_t var = PTHREAD_ONCE_INIT -#endif /* __USE_TBB_ATOMICS */ - - struct handles_t { + static struct handles_t { atomic_incrementer my_size; dynamic_link_handle my_handles[MAX_LOADED_MODULES]; - void init() { - init_atomic_incrementer( my_size ); - } - void add(const dynamic_link_handle &handle) { - const size_t ind = my_size++; - LIBRARY_ASSERT( ind < MAX_LOADED_MODULES, "Too many modules are loaded" ); + const std::size_t ind = my_size++; + __TBB_ASSERT_EX( ind < MAX_LOADED_MODULES, "Too many modules are loaded" ); my_handles[ind] = handle; } void free() { - const size_t size = my_size; - for (size_t i=0; i<size; ++i) + const std::size_t size = my_size; + for (std::size_t i=0; i<size; ++i) dynamic_unlink( my_handles[i] ); } } handles; - ATOMIC_ONCE_DECL( init_dl_data_state ); + static std::once_flag init_dl_data_state; static struct ap_data_t { char _path[PATH_MAX+1]; - size_t _len; + std::size_t _len; } ap_data; static void init_ap_data() { @@ -290,14 +238,14 @@ OPEN_INTERNAL_NAMESPACE return; } // Find the position of the last backslash. - char *backslash = strrchr( ap_data._path, '\\' ); + char *backslash = std::strrchr( ap_data._path, '\\' ); if ( !backslash ) { // Backslash not found. - LIBRARY_ASSERT( backslash!=NULL, "Unbelievable."); + __TBB_ASSERT_EX( backslash != nullptr, "Unbelievable."); return; } - LIBRARY_ASSERT( backslash >= ap_data._path, "Unbelievable."); - ap_data._len = (size_t)(backslash - ap_data._path) + 1; + __TBB_ASSERT_EX( backslash >= ap_data._path, "Unbelievable."); + ap_data._len = (std::size_t)(backslash - ap_data._path) + 1; *(backslash+1) = 0; #else // Get the library path @@ -308,17 +256,17 @@ OPEN_INTERNAL_NAMESPACE DYNAMIC_LINK_WARNING( dl_sys_fail, "dladdr", err ); return; } else { - LIBRARY_ASSERT( dlinfo.dli_fname!=NULL, "Unbelievable." ); + __TBB_ASSERT_EX( dlinfo.dli_fname!=nullptr, "Unbelievable." ); } - char const *slash = strrchr( dlinfo.dli_fname, '/' ); - size_t fname_len=0; + char const *slash = std::strrchr( dlinfo.dli_fname, '/' ); + std::size_t fname_len=0; if ( slash ) { - LIBRARY_ASSERT( slash >= dlinfo.dli_fname, "Unbelievable."); - fname_len = (size_t)(slash - dlinfo.dli_fname) + 1; + __TBB_ASSERT_EX( slash >= dlinfo.dli_fname, "Unbelievable."); + fname_len = (std::size_t)(slash - dlinfo.dli_fname) + 1; } - size_t rc; + std::size_t rc; if ( dlinfo.dli_fname[0]=='/' ) { // The library path is absolute rc = 0; @@ -329,26 +277,25 @@ OPEN_INTERNAL_NAMESPACE DYNAMIC_LINK_WARNING( dl_buff_too_small ); return; } - ap_data._len = strlen( ap_data._path ); + ap_data._len = std::strlen( ap_data._path ); ap_data._path[ap_data._len++]='/'; rc = ap_data._len; } if ( fname_len>0 ) { + ap_data._len += fname_len; if ( ap_data._len>PATH_MAX ) { DYNAMIC_LINK_WARNING( dl_buff_too_small ); ap_data._len=0; return; } - strncpy( ap_data._path+rc, dlinfo.dli_fname, fname_len ); - ap_data._len += fname_len; + std::strncpy( ap_data._path+rc, dlinfo.dli_fname, fname_len ); ap_data._path[ap_data._len]=0; } #endif /* _WIN32 */ } static void init_dl_data() { - handles.init(); init_ap_data(); } @@ -362,27 +309,28 @@ OPEN_INTERNAL_NAMESPACE in len -- Size of buffer. ret -- 0 -- Error occurred. > len -- Buffer too short, required size returned. - otherwise -- Ok, number of characters (not counting terminating null) written to - buffer. + otherwise -- Ok, number of characters (incl. terminating null) written to buffer. */ - static size_t abs_path( char const * name, char * path, size_t len ) { - if ( !ap_data._len ) + static std::size_t abs_path( char const * name, char * path, std::size_t len ) { + if ( ap_data._len == 0 ) return 0; - size_t name_len = strlen( name ); - size_t full_len = name_len+ap_data._len; + std::size_t name_len = std::strlen( name ); + std::size_t full_len = name_len+ap_data._len; if ( full_len < len ) { - strncpy( path, ap_data._path, ap_data._len ); - memcpy( path+ap_data._len, name, name_len ); - path[full_len] = 0; + __TBB_ASSERT( ap_data._path[ap_data._len] == 0, nullptr); + __TBB_ASSERT( std::strlen(ap_data._path) == ap_data._len, nullptr); + std::strncpy( path, ap_data._path, ap_data._len + 1 ); + __TBB_ASSERT( path[ap_data._len] == 0, nullptr); + std::strncat( path, name, len - ap_data._len ); + __TBB_ASSERT( std::strlen(path) == full_len, nullptr); } - return full_len; + return full_len+1; // +1 for null character } #endif // __TBB_DYNAMIC_LOAD_ENABLED - void init_dynamic_link_data() { #if __TBB_DYNAMIC_LOAD_ENABLED - atomic_once( &init_dl_data, init_dl_data_state ); + std::call_once( init_dl_data_state, init_dl_data ); #endif } @@ -398,19 +346,19 @@ OPEN_INTERNAL_NAMESPACE #endif #if __TBB_WEAK_SYMBOLS_PRESENT - static bool weak_symbol_link( const dynamic_link_descriptor descriptors[], size_t required ) + static bool weak_symbol_link( const dynamic_link_descriptor descriptors[], std::size_t required ) { // Check if the required entries are present in what was loaded into our process. - for ( size_t k = 0; k < required; ++k ) + for ( std::size_t k = 0; k < required; ++k ) if ( !descriptors[k].ptr ) return false; // Commit the entry points. - for ( size_t k = 0; k < required; ++k ) + for ( std::size_t k = 0; k < required; ++k ) *descriptors[k].handler = (pointer_to_handler) descriptors[k].ptr; return true; } #else - static bool weak_symbol_link( const dynamic_link_descriptor[], size_t ) { + static bool weak_symbol_link( const dynamic_link_descriptor[], std::size_t ) { return false; } #endif /* __TBB_WEAK_SYMBOLS_PRESENT */ @@ -430,79 +378,30 @@ OPEN_INTERNAL_NAMESPACE #endif } -#if !_WIN32 -#if __TBB_DYNAMIC_LOAD_ENABLED - static dynamic_link_handle pin_symbols( dynamic_link_descriptor desc, const dynamic_link_descriptor* descriptors, size_t required ) { - // It is supposed that all symbols are from the only one library - // The library has been loaded by another module and contains at least one requested symbol. - // But after we obtained the symbol the library can be unloaded by another thread - // invalidating our symbol. Therefore we need to pin the library in memory. - dynamic_link_handle library_handle = 0; - Dl_info info; - // Get library's name from earlier found symbol - if ( dladdr( (void*)*desc.handler, &info ) ) { - // Pin the library - library_handle = dlopen( info.dli_fname, RTLD_LAZY ); - if ( library_handle ) { - // If original library was unloaded before we pinned it - // and then another module loaded in its place, the earlier - // found symbol would become invalid. So revalidate them. - if ( !resolve_symbols( library_handle, descriptors, required ) ) { - // Wrong library. - dynamic_unlink(library_handle); - library_handle = 0; - } - } else { - char const * err = dlerror(); - DYNAMIC_LINK_WARNING( dl_lib_not_found, info.dli_fname, err ); - } - } - // else the library has been unloaded by another thread - return library_handle; - } -#endif /* __TBB_DYNAMIC_LOAD_ENABLED */ -#endif /* !_WIN32 */ - - static dynamic_link_handle global_symbols_link( const char* library, const dynamic_link_descriptor descriptors[], size_t required ) { - ::tbb::internal::suppress_unused_warning( library ); - dynamic_link_handle library_handle; + static dynamic_link_handle global_symbols_link( const char* library, const dynamic_link_descriptor descriptors[], std::size_t required ) { + dynamic_link_handle library_handle{}; #if _WIN32 - if ( GetModuleHandleEx( 0, library, &library_handle ) ) { - if ( resolve_symbols( library_handle, descriptors, required ) ) - return library_handle; - else - FreeLibrary( library_handle ); - } + auto res = GetModuleHandleEx(0, library, &library_handle); + __TBB_ASSERT_EX((res && library_handle) || (!res && !library_handle), nullptr); #else /* _WIN32 */ #if !__TBB_DYNAMIC_LOAD_ENABLED /* only __TBB_WEAK_SYMBOLS_PRESENT is defined */ if ( !dlopen ) return 0; #endif /* !__TBB_DYNAMIC_LOAD_ENABLED */ - library_handle = dlopen( NULL, RTLD_LAZY ); - #if !__ANDROID__ - // On Android dlopen( NULL ) returns NULL if it is called during dynamic module initialization. - LIBRARY_ASSERT( library_handle, "The handle for the main program is NULL" ); - #endif - #if __TBB_DYNAMIC_LOAD_ENABLED - // Check existence of the first symbol only, then use it to find the library and load all necessary symbols. - pointer_to_handler handler; - dynamic_link_descriptor desc; - desc.name = descriptors[0].name; - desc.handler = &handler; - if ( resolve_symbols( library_handle, &desc, 1 ) ) { - dynamic_unlink( library_handle ); - return pin_symbols( desc, descriptors, required ); - } - #else /* only __TBB_WEAK_SYMBOLS_PRESENT is defined */ - if ( resolve_symbols( library_handle, descriptors, required ) ) - return library_handle; - #endif - dynamic_unlink( library_handle ); + // RTLD_GLOBAL - to guarantee that old TBB will find the loaded library + // RTLD_NOLOAD - not to load the library without the full path + library_handle = dlopen(library, RTLD_LAZY | RTLD_GLOBAL | RTLD_NOLOAD); #endif /* _WIN32 */ - return 0; + if (library_handle) { + if (!resolve_symbols(library_handle, descriptors, required)) { + dynamic_unlink(library_handle); + library_handle = nullptr; + } + } + return library_handle; } static void save_library_handle( dynamic_link_handle src, dynamic_link_handle *dst ) { - LIBRARY_ASSERT( src, "The library handle to store must be non-zero" ); + __TBB_ASSERT_EX( src, "The library handle to store must be non-zero" ); if ( dst ) *dst = src; #if __TBB_DYNAMIC_LOAD_ENABLED @@ -511,49 +410,74 @@ OPEN_INTERNAL_NAMESPACE #endif /* __TBB_DYNAMIC_LOAD_ENABLED */ } - dynamic_link_handle dynamic_load( const char* library, const dynamic_link_descriptor descriptors[], size_t required ) { - ::tbb::internal::suppress_unused_warning( library, descriptors, required ); - #if __TBB_DYNAMIC_LOAD_ENABLED +#if !_WIN32 + int loading_flags(bool local_binding) { + int flags = RTLD_NOW; + if (local_binding) { + flags = flags | RTLD_LOCAL; +#if (__linux__ && __GLIBC__) && !__TBB_USE_SANITIZERS + if( !GetBoolEnvironmentVariable("TBB_ENABLE_SANITIZERS") ) { + flags = flags | RTLD_DEEPBIND; + } +#endif + } else { + flags = flags | RTLD_GLOBAL; + } + return flags; + } +#endif - size_t const len = PATH_MAX + 1; - char path[ len ]; - size_t rc = abs_path( library, path, len ); - if ( 0 < rc && rc < len ) { + dynamic_link_handle dynamic_load( const char* library, const dynamic_link_descriptor descriptors[], std::size_t required, bool local_binding ) { + ::tbb::detail::suppress_unused_warning( library, descriptors, required, local_binding ); +#if __TBB_DYNAMIC_LOAD_ENABLED + std::size_t const len = PATH_MAX + 1; + char path[ len ]; + std::size_t rc = abs_path( library, path, len ); + if ( 0 < rc && rc <= len ) { #if _WIN32 - // Prevent Windows from displaying silly message boxes if it fails to load library - // (e.g. because of MS runtime problems - one of those crazy manifest related ones) - UINT prev_mode = SetErrorMode (SEM_FAILCRITICALERRORS); + // Prevent Windows from displaying silly message boxes if it fails to load library + // (e.g. because of MS runtime problems - one of those crazy manifest related ones) + UINT prev_mode = SetErrorMode (SEM_FAILCRITICALERRORS); #endif /* _WIN32 */ - dynamic_link_handle library_handle = dlopen( path, RTLD_LAZY ); + // The second argument (loading_flags) is ignored on Windows + dynamic_link_handle library_handle = dlopen( path, loading_flags(local_binding) ); #if _WIN32 - SetErrorMode (prev_mode); + SetErrorMode (prev_mode); #endif /* _WIN32 */ - if( library_handle ) { - if( !resolve_symbols( library_handle, descriptors, required ) ) { - // The loaded library does not contain all the expected entry points - dynamic_unlink( library_handle ); - library_handle = NULL; - } - } else - DYNAMIC_LINK_WARNING( dl_lib_not_found, path, dlerror() ); - return library_handle; - } else if ( rc>=len ) - DYNAMIC_LINK_WARNING( dl_buff_too_small ); - // rc == 0 means failing of init_ap_data so the warning has already been issued. + if( library_handle ) { + if( !resolve_symbols( library_handle, descriptors, required ) ) { + // The loaded library does not contain all the expected entry points + dynamic_unlink( library_handle ); + library_handle = nullptr; + } + } else + DYNAMIC_LINK_WARNING( dl_lib_not_found, path, dlerror() ); + return library_handle; + } else if ( rc>len ) + DYNAMIC_LINK_WARNING( dl_buff_too_small ); + // rc == 0 means failing of init_ap_data so the warning has already been issued. - #endif /* __TBB_DYNAMIC_LOAD_ENABLED */ - return 0; +#endif /* __TBB_DYNAMIC_LOAD_ENABLED */ + return nullptr; } - bool dynamic_link( const char* library, const dynamic_link_descriptor descriptors[], size_t required, dynamic_link_handle *handle, int flags ) { + bool dynamic_link( const char* library, const dynamic_link_descriptor descriptors[], std::size_t required, dynamic_link_handle *handle, int flags ) { init_dynamic_link_data(); // TODO: May global_symbols_link find weak symbols? - dynamic_link_handle library_handle = ( flags & DYNAMIC_LINK_GLOBAL ) ? global_symbols_link( library, descriptors, required ) : 0; + dynamic_link_handle library_handle = ( flags & DYNAMIC_LINK_GLOBAL ) ? global_symbols_link( library, descriptors, required ) : nullptr; +#if defined(_MSC_VER) && _MSC_VER <= 1900 +// #pragma warning (push) +// MSVC 2015 warning: 'int': forcing value to bool 'true' or 'false' +// #pragma warning (disable: 4800) +#endif if ( !library_handle && ( flags & DYNAMIC_LINK_LOAD ) ) - library_handle = dynamic_load( library, descriptors, required ); + library_handle = dynamic_load( library, descriptors, required, flags & DYNAMIC_LINK_LOCAL ); +#if defined(_MSC_VER) && _MSC_VER <= 1900 +// #pragma warning (pop) +#endif if ( !library_handle && ( flags & DYNAMIC_LINK_WEAK ) ) return weak_symbol_link( descriptors, required ); @@ -566,7 +490,7 @@ OPEN_INTERNAL_NAMESPACE #endif /*__TBB_WIN8UI_SUPPORT*/ #else /* __TBB_WEAK_SYMBOLS_PRESENT || __TBB_DYNAMIC_LOAD_ENABLED */ - bool dynamic_link( const char*, const dynamic_link_descriptor*, size_t, dynamic_link_handle *handle, int ) { + bool dynamic_link( const char*, const dynamic_link_descriptor*, std::size_t, dynamic_link_handle *handle, int ) { if ( handle ) *handle=0; return false; @@ -575,4 +499,6 @@ OPEN_INTERNAL_NAMESPACE void dynamic_unlink_all() {} #endif /* __TBB_WEAK_SYMBOLS_PRESENT || __TBB_DYNAMIC_LOAD_ENABLED */ -CLOSE_INTERNAL_NAMESPACE +} // namespace r1 +} // namespace detail +} // namespace tbb diff --git a/src/tbb/src/tbb/dynamic_link.h b/src/tbb/src/tbb/dynamic_link.h index cff511806..f07750b66 100644 --- a/src/tbb/src/tbb/dynamic_link.h +++ b/src/tbb/src/tbb/dynamic_link.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2022 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_dynamic_link @@ -23,29 +19,23 @@ // Support for dynamic loading entry points from other shared libraries. -#include "tbb/tbb_stddef.h" +#include "oneapi/tbb/detail/_config.h" -#ifdef LIBRARY_ASSERT - #undef __TBB_ASSERT - #define __TBB_ASSERT(x,y) LIBRARY_ASSERT(x,y) -#else - #define LIBRARY_ASSERT(x,y) __TBB_ASSERT_EX(x,y) -#endif /* !LIBRARY_ASSERT */ +#include <atomic> +#include <mutex> /** By default, symbols declared and defined here go into namespace tbb::internal. To put them in other namespace, define macros OPEN_INTERNAL_NAMESPACE and CLOSE_INTERNAL_NAMESPACE to override the following default definitions. **/ -#ifndef OPEN_INTERNAL_NAMESPACE -#define OPEN_INTERNAL_NAMESPACE namespace tbb { namespace internal { -#define CLOSE_INTERNAL_NAMESPACE }} -#endif /* OPEN_INTERNAL_NAMESPACE */ -#include <stddef.h> -#if _WIN32 -#include "tbb/machine/windows_api.h" +#include <cstddef> +#ifdef _WIN32 +#include <windows.h> #endif /* _WIN32 */ -OPEN_INTERNAL_NAMESPACE +namespace tbb { +namespace detail { +namespace r1 { //! Type definition for a pointer to a void somefunc(void) typedef void (*pointer_to_handler)(); @@ -55,8 +45,10 @@ typedef void (*pointer_to_handler)(); // prevent warnings from some compilers (g++ 4.1) #if __TBB_WEAK_SYMBOLS_PRESENT #define DLD(s,h) {#s, (pointer_to_handler*)(void*)(&h), (pointer_to_handler)&s} +#define DLD_NOWEAK(s,h) {#s, (pointer_to_handler*)(void*)(&h), nullptr} #else #define DLD(s,h) {#s, (pointer_to_handler*)(void*)(&h)} +#define DLD_NOWEAK(s,h) DLD(s,h) #endif /* __TBB_WEAK_SYMBOLS_PRESENT */ //! Association between a handler name and location of pointer to it. struct dynamic_link_descriptor { @@ -71,15 +63,18 @@ struct dynamic_link_descriptor { }; #if _WIN32 -typedef HMODULE dynamic_link_handle; +using dynamic_link_handle = HMODULE; #else -typedef void* dynamic_link_handle; +using dynamic_link_handle = void*; #endif /* _WIN32 */ -const int DYNAMIC_LINK_GLOBAL = 0x01; -const int DYNAMIC_LINK_LOAD = 0x02; -const int DYNAMIC_LINK_WEAK = 0x04; -const int DYNAMIC_LINK_ALL = DYNAMIC_LINK_GLOBAL | DYNAMIC_LINK_LOAD | DYNAMIC_LINK_WEAK; +const int DYNAMIC_LINK_GLOBAL = 0x01; +const int DYNAMIC_LINK_LOAD = 0x02; +const int DYNAMIC_LINK_WEAK = 0x04; +const int DYNAMIC_LINK_LOCAL = 0x08; + +const int DYNAMIC_LINK_LOCAL_BINDING = DYNAMIC_LINK_LOCAL | DYNAMIC_LINK_LOAD; +const int DYNAMIC_LINK_DEFAULT = DYNAMIC_LINK_GLOBAL | DYNAMIC_LINK_LOAD | DYNAMIC_LINK_WEAK; //! Fill in dynamically linked handlers. /** 'library' is the name of the requested library. It should not contain a full @@ -99,9 +94,9 @@ const int DYNAMIC_LINK_ALL = DYNAMIC_LINK_GLOBAL | DYNAMIC_LINK_LOAD | DYNAMI **/ bool dynamic_link( const char* library, const dynamic_link_descriptor descriptors[], - size_t required, - dynamic_link_handle* handle = 0, - int flags = DYNAMIC_LINK_ALL ); + std::size_t required, + dynamic_link_handle* handle = nullptr, + int flags = DYNAMIC_LINK_DEFAULT ); void dynamic_unlink( dynamic_link_handle handle ); @@ -116,6 +111,8 @@ enum dynamic_link_error_t { dl_buff_too_small // none }; // dynamic_link_error_t -CLOSE_INTERNAL_NAMESPACE +} // namespace r1 +} // namespace detail +} // namespace tbb #endif /* __TBB_dynamic_link */ diff --git a/src/tbb/src/tbb/environment.h b/src/tbb/src/tbb/environment.h new file mode 100644 index 000000000..eac6f2023 --- /dev/null +++ b/src/tbb/src/tbb/environment.h @@ -0,0 +1,81 @@ +/* + Copyright (c) 2018-2022 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_tbb_environment_H +#define __TBB_tbb_environment_H + +#include <cstdlib> +#include <cstring> +#include <cerrno> +#include <cctype> + +namespace tbb { +namespace detail { +namespace r1 { + +#if __TBB_WIN8UI_SUPPORT +static inline bool GetBoolEnvironmentVariable( const char * ) { + return false; +} + +static inline long GetIntegralEnvironmentVariable( const char * ) { + return -1; +} +#else /* __TBB_WIN8UI_SUPPORT */ +static inline bool GetBoolEnvironmentVariable( const char * name ) { + if ( const char* s = std::getenv(name) ) { + // The result is defined as true only if the environment variable contains + // no characters except one '1' character and an arbitrary number of spaces + // (including the absence of spaces). + size_t index = std::strspn(s, " "); + if (s[index] != '1') return false; + index++; + // Memory access after incrementing is safe, since the getenv() returns a + // null-terminated string, and even if the character getting by index is '1', + // and this character is the end of string, after incrementing we will get + // an index of character, that contains '\0' + index += std::strspn(&s[index], " "); + return !s[index]; + } + return false; +} + +static inline long GetIntegralEnvironmentVariable( const char * name ) { + if ( const char* s = std::getenv(name) ) { + char* end = nullptr; + errno = 0; + long value = std::strtol(s, &end, 10); + + // We have exceeded the range, value is negative or string is incovertable + if ( errno == ERANGE || value < 0 || end==s ) { + return -1; + } + for ( ; *end != '\0'; end++ ) { + if ( !std::isspace(*end) ) { + return -1; + } + } + return value; + } + return -1; +} +#endif /* __TBB_WIN8UI_SUPPORT */ + +} // namespace r1 +} // namespace detail +} // namespace tbb + +#endif // __TBB_tbb_environment_H diff --git a/src/tbb/src/tbb/exception.cpp b/src/tbb/src/tbb/exception.cpp new file mode 100644 index 000000000..efc9b2a4d --- /dev/null +++ b/src/tbb/src/tbb/exception.cpp @@ -0,0 +1,166 @@ +/* + Copyright (c) 2005-2022 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "oneapi/tbb/detail/_exception.h" +#include "oneapi/tbb/detail/_assert.h" +#include "oneapi/tbb/detail/_template_helpers.h" + +#include <cstring> +#include <cstdio> +#include <stdexcept> // std::runtime_error +#include <new> +#include <stdexcept> + +#define __TBB_STD_RETHROW_EXCEPTION_POSSIBLY_BROKEN \ + (__GLIBCXX__ && __TBB_GLIBCXX_VERSION>=40700 && __TBB_GLIBCXX_VERSION<60000 && TBB_USE_EXCEPTIONS) + +#if __TBB_STD_RETHROW_EXCEPTION_POSSIBLY_BROKEN +// GCC ABI declarations necessary for a workaround +#include <cxxabi.h> +#endif + +namespace tbb { +namespace detail { +namespace r1 { + +const char* bad_last_alloc::what() const noexcept(true) { return "bad allocation in previous or concurrent attempt"; } +const char* user_abort::what() const noexcept(true) { return "User-initiated abort has terminated this operation"; } +const char* missing_wait::what() const noexcept(true) { return "wait() was not called on the structured_task_group"; } + +#if TBB_USE_EXCEPTIONS + template <typename F> + /*[[noreturn]]*/ void do_throw_noexcept(F throw_func) noexcept { + throw_func(); + } + + /*[[noreturn]]*/ void do_throw_noexcept(void (*throw_func)()) noexcept { + throw_func(); +#if __GNUC__ == 7 + // In release, GCC 7 loses noexcept attribute during tail call optimization. + // The following statement prevents tail call optimization. + volatile bool reach_this_point = true; + suppress_unused_warning(reach_this_point); +#endif + } + + bool terminate_on_exception(); // defined in global_control.cpp and ipc_server.cpp + + template <typename F> + /*[[noreturn]]*/ void do_throw(F throw_func) { + if (terminate_on_exception()) { + do_throw_noexcept(throw_func); + } + throw_func(); + } + + #define DO_THROW(exc, init_args) do_throw( []{ throw exc init_args; } ); +#else /* !TBB_USE_EXCEPTIONS */ + #define PRINT_ERROR_AND_ABORT(exc_name, msg) \ + std::fprintf (stderr, "Exception %s with message %s would have been thrown, " \ + "if exception handling had not been disabled. Aborting.\n", exc_name, msg); \ + std::fflush(stderr); \ + std::abort(); + #define DO_THROW(exc, init_args) PRINT_ERROR_AND_ABORT(#exc, #init_args) +#endif /* !TBB_USE_EXCEPTIONS */ + +void throw_exception ( exception_id eid ) { + switch ( eid ) { + case exception_id::bad_alloc: DO_THROW(std::bad_alloc, ()); break; + case exception_id::bad_last_alloc: DO_THROW(bad_last_alloc, ()); break; + case exception_id::user_abort: DO_THROW( user_abort, () ); break; + case exception_id::nonpositive_step: DO_THROW(std::invalid_argument, ("Step must be positive") ); break; + case exception_id::out_of_range: DO_THROW(std::out_of_range, ("Index out of requested size range")); break; + case exception_id::reservation_length_error: DO_THROW(std::length_error, ("Attempt to exceed implementation defined length limits")); break; + case exception_id::missing_wait: DO_THROW(missing_wait, ()); break; + case exception_id::invalid_load_factor: DO_THROW(std::out_of_range, ("Invalid hash load factor")); break; + case exception_id::invalid_key: DO_THROW(std::out_of_range, ("invalid key")); break; + case exception_id::bad_tagged_msg_cast: DO_THROW(std::runtime_error, ("Illegal tagged_msg cast")); break; + case exception_id::unsafe_wait: DO_THROW(unsafe_wait, ("Unsafe to wait further")); break; + default: __TBB_ASSERT ( false, "Unknown exception ID" ); + } + __TBB_ASSERT(false, "Unreachable code"); +} + +/* The "what" should be fairly short, not more than about 128 characters. + Because we control all the call sites to handle_perror, it is pointless + to bullet-proof it for very long strings. + + Design note: ADR put this routine off to the side in tbb_misc.cpp instead of + Task.cpp because the throw generates a pathetic lot of code, and ADR wanted + this large chunk of code to be placed on a cold page. */ +void handle_perror( int error_code, const char* what ) { + const int BUF_SIZE = 255; + char buf[BUF_SIZE + 1] = { 0 }; + std::strncat(buf, what, BUF_SIZE); + std::size_t buf_len = std::strlen(buf); + if (error_code) { + std::strncat(buf, ": ", BUF_SIZE - buf_len); + buf_len = std::strlen(buf); + std::strncat(buf, std::strerror(error_code), BUF_SIZE - buf_len); + buf_len = std::strlen(buf); + } + __TBB_ASSERT(buf_len <= BUF_SIZE && buf[buf_len] == 0, nullptr); +#if TBB_USE_EXCEPTIONS + do_throw([&buf] { throw std::runtime_error(buf); }); +#else + PRINT_ERROR_AND_ABORT( "runtime_error", buf); +#endif /* !TBB_USE_EXCEPTIONS */ +} + +#if __TBB_STD_RETHROW_EXCEPTION_POSSIBLY_BROKEN +// Runtime detection and workaround for the GCC bug 62258. +// The problem is that std::rethrow_exception() does not increment a counter +// of active exceptions, causing std::uncaught_exception() to return a wrong value. +// The code is created after, and roughly reflects, the workaround +// at https://gcc.gnu.org/bugzilla/attachment.cgi?id=34683 + +void fix_broken_rethrow() { + struct gcc_eh_data { + void * caughtExceptions; + unsigned int uncaughtExceptions; + }; + gcc_eh_data* eh_data = punned_cast<gcc_eh_data*>( abi::__cxa_get_globals() ); + ++eh_data->uncaughtExceptions; +} + +bool gcc_rethrow_exception_broken() { + bool is_broken; + __TBB_ASSERT( !std::uncaught_exception(), + "gcc_rethrow_exception_broken() must not be called when an exception is active" ); + try { + // Throw, catch, and rethrow an exception + try { + throw __TBB_GLIBCXX_VERSION; + } catch(...) { + std::rethrow_exception( std::current_exception() ); + } + } catch(...) { + // Check the bug presence + is_broken = std::uncaught_exception(); + } + if( is_broken ) fix_broken_rethrow(); + __TBB_ASSERT( !std::uncaught_exception(), nullptr); + return is_broken; +} +#else +void fix_broken_rethrow() {} +bool gcc_rethrow_exception_broken() { return false; } +#endif /* __TBB_STD_RETHROW_EXCEPTION_POSSIBLY_BROKEN */ + +} // namespace r1 +} // namespace detail +} // namespace tbb + diff --git a/src/tbb/src/tbb/global_control.cpp b/src/tbb/src/tbb/global_control.cpp new file mode 100644 index 000000000..f45c66b87 --- /dev/null +++ b/src/tbb/src/tbb/global_control.cpp @@ -0,0 +1,285 @@ +/* + Copyright (c) 2005-2024 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "oneapi/tbb/detail/_config.h" +#include "oneapi/tbb/detail/_template_helpers.h" + +#include "oneapi/tbb/cache_aligned_allocator.h" +#include "oneapi/tbb/global_control.h" +#include "oneapi/tbb/tbb_allocator.h" +#include "oneapi/tbb/spin_mutex.h" + +#include "governor.h" +#include "threading_control.h" +#include "market.h" +#include "misc.h" + +#include <atomic> +#include <set> + +namespace tbb { +namespace detail { +namespace r1 { + +//! Comparator for a set of global_control objects +struct control_storage_comparator { + bool operator()(const d1::global_control* lhs, const d1::global_control* rhs) const; +}; + +class control_storage { + friend struct global_control_impl; + friend std::size_t global_control_active_value(int); + friend void global_control_lock(); + friend void global_control_unlock(); + friend std::size_t global_control_active_value_unsafe(d1::global_control::parameter); +protected: + std::size_t my_active_value{0}; + std::set<d1::global_control*, control_storage_comparator, tbb_allocator<d1::global_control*>> my_list{}; + spin_mutex my_list_mutex{}; +public: + virtual ~control_storage() = default; + virtual std::size_t default_value() const = 0; + virtual void apply_active(std::size_t new_active) { + my_active_value = new_active; + } + virtual bool is_first_arg_preferred(std::size_t a, std::size_t b) const { + return a>b; // prefer max by default + } + virtual std::size_t active_value() { + spin_mutex::scoped_lock lock(my_list_mutex); // protect my_list.empty() call + return !my_list.empty() ? my_active_value : default_value(); + } + + std::size_t active_value_unsafe() { + return !my_list.empty() ? my_active_value : default_value(); + } +}; + +class alignas(max_nfs_size) allowed_parallelism_control : public control_storage { + std::size_t default_value() const override { + return max(1U, governor::default_num_threads()); + } + bool is_first_arg_preferred(std::size_t a, std::size_t b) const override { + return a<b; // prefer min allowed parallelism + } + void apply_active(std::size_t new_active) override { + control_storage::apply_active(new_active); + __TBB_ASSERT(my_active_value >= 1, nullptr); + // -1 to take external thread into account + threading_control::set_active_num_workers(my_active_value - 1); + } + std::size_t active_value() override { + spin_mutex::scoped_lock lock(my_list_mutex); // protect my_list.empty() call + if (my_list.empty()) { + return default_value(); + } + + // non-zero, if market is active + const std::size_t workers = threading_control::max_num_workers(); + // We can't exceed market's maximal number of workers. + // +1 to take external thread into account + return workers ? min(workers + 1, my_active_value) : my_active_value; + } +}; + +class alignas(max_nfs_size) stack_size_control : public control_storage { + std::size_t default_value() const override { +#if _WIN32_WINNT >= 0x0602 /* _WIN32_WINNT_WIN8 */ + static auto ThreadStackSizeDefault = [] { + ULONG_PTR hi, lo; + GetCurrentThreadStackLimits(&lo, &hi); + return hi - lo; + }(); + return ThreadStackSizeDefault; +#elif defined(EMSCRIPTEN) + return __TBB_EMSCRIPTEN_STACK_SIZE; +#else + return ThreadStackSize; +#endif + } + void apply_active(std::size_t new_active) override { + control_storage::apply_active(new_active); +#if __TBB_WIN8UI_SUPPORT && (_WIN32_WINNT < 0x0A00) + __TBB_ASSERT( false, "For Windows 8 Store* apps we must not set stack size" ); +#endif + } +}; + +class alignas(max_nfs_size) terminate_on_exception_control : public control_storage { + std::size_t default_value() const override { + return 0; + } +}; + +class alignas(max_nfs_size) lifetime_control : public control_storage { + bool is_first_arg_preferred(std::size_t, std::size_t) const override { + return false; // not interested + } + std::size_t default_value() const override { + return 0; + } + void apply_active(std::size_t new_active) override { + if (new_active == 1) { + // reserve the market reference + threading_control::register_lifetime_control(); + } else if (new_active == 0) { // new_active == 0 + threading_control::unregister_lifetime_control(/*blocking_terminate*/ false); + } + control_storage::apply_active(new_active); + } +}; + +static control_storage* controls[] = {nullptr, nullptr, nullptr, nullptr}; + +void global_control_acquire() { + controls[0] = new (cache_aligned_allocate(sizeof(allowed_parallelism_control))) allowed_parallelism_control{}; + controls[1] = new (cache_aligned_allocate(sizeof(stack_size_control))) stack_size_control{}; + controls[2] = new (cache_aligned_allocate(sizeof(terminate_on_exception_control))) terminate_on_exception_control{}; + controls[3] = new (cache_aligned_allocate(sizeof(lifetime_control))) lifetime_control{}; +} + +void global_control_release() { + for (auto& ptr : controls) { + ptr->~control_storage(); + cache_aligned_deallocate(ptr); + ptr = nullptr; + } +} + +void global_control_lock() { + for (auto& ctl : controls) { + ctl->my_list_mutex.lock(); + } +} + +void global_control_unlock() { + int N = std::distance(std::begin(controls), std::end(controls)); + for (int i = N - 1; i >= 0; --i) { + controls[i]->my_list_mutex.unlock(); + } +} + +std::size_t global_control_active_value_unsafe(d1::global_control::parameter param) { + __TBB_ASSERT_RELEASE(param < d1::global_control::parameter_max, nullptr); + return controls[param]->active_value_unsafe(); +} + +//! Comparator for a set of global_control objects +inline bool control_storage_comparator::operator()(const d1::global_control* lhs, const d1::global_control* rhs) const { + __TBB_ASSERT_RELEASE(lhs->my_param < d1::global_control::parameter_max , nullptr); + return lhs->my_value < rhs->my_value || (lhs->my_value == rhs->my_value && lhs < rhs); +} + +bool terminate_on_exception() { + return d1::global_control::active_value(d1::global_control::terminate_on_exception) == 1; +} + +struct global_control_impl { +private: + static bool erase_if_present(control_storage* const c, d1::global_control& gc) { + auto it = c->my_list.find(&gc); + if (it != c->my_list.end()) { + c->my_list.erase(it); + return true; + } + return false; + } + +public: + + static void create(d1::global_control& gc) { + __TBB_ASSERT_RELEASE(gc.my_param < d1::global_control::parameter_max, nullptr); + control_storage* const c = controls[gc.my_param]; + + spin_mutex::scoped_lock lock(c->my_list_mutex); + if (c->my_list.empty() || c->is_first_arg_preferred(gc.my_value, c->my_active_value)) { + // to guarantee that apply_active() is called with current active value, + // calls it here and in internal_destroy() under my_list_mutex + c->apply_active(gc.my_value); + } + c->my_list.insert(&gc); + } + + static void destroy(d1::global_control& gc) { + __TBB_ASSERT_RELEASE(gc.my_param < d1::global_control::parameter_max, nullptr); + control_storage* const c = controls[gc.my_param]; + // Concurrent reading and changing global parameter is possible. + spin_mutex::scoped_lock lock(c->my_list_mutex); + __TBB_ASSERT(gc.my_param == d1::global_control::scheduler_handle || !c->my_list.empty(), nullptr); + std::size_t new_active = (std::size_t)(-1), old_active = c->my_active_value; + + if (!erase_if_present(c, gc)) { + __TBB_ASSERT(gc.my_param == d1::global_control::scheduler_handle , nullptr); + return; + } + if (c->my_list.empty()) { + __TBB_ASSERT(new_active == (std::size_t) - 1, nullptr); + new_active = c->default_value(); + } else { + new_active = (*c->my_list.begin())->my_value; + } + if (new_active != old_active) { + c->apply_active(new_active); + } + } + + static bool remove_and_check_if_empty(d1::global_control& gc) { + __TBB_ASSERT_RELEASE(gc.my_param < d1::global_control::parameter_max, nullptr); + control_storage* const c = controls[gc.my_param]; + + spin_mutex::scoped_lock lock(c->my_list_mutex); + __TBB_ASSERT(!c->my_list.empty(), nullptr); + erase_if_present(c, gc); + return c->my_list.empty(); + } +#if TBB_USE_ASSERT + static bool is_present(d1::global_control& gc) { + __TBB_ASSERT_RELEASE(gc.my_param < d1::global_control::parameter_max, nullptr); + control_storage* const c = controls[gc.my_param]; + + spin_mutex::scoped_lock lock(c->my_list_mutex); + auto it = c->my_list.find(&gc); + if (it != c->my_list.end()) { + return true; + } + return false; + } +#endif // TBB_USE_ASSERT +}; + +void __TBB_EXPORTED_FUNC create(d1::global_control& gc) { + global_control_impl::create(gc); +} +void __TBB_EXPORTED_FUNC destroy(d1::global_control& gc) { + global_control_impl::destroy(gc); +} + +bool remove_and_check_if_empty(d1::global_control& gc) { + return global_control_impl::remove_and_check_if_empty(gc); +} +#if TBB_USE_ASSERT +bool is_present(d1::global_control& gc) { + return global_control_impl::is_present(gc); +} +#endif // TBB_USE_ASSERT +std::size_t __TBB_EXPORTED_FUNC global_control_active_value(int param) { + __TBB_ASSERT_RELEASE(param < d1::global_control::parameter_max, nullptr); + return controls[param]->active_value(); +} + +} // namespace r1 +} // namespace detail +} // namespace tbb diff --git a/src/tbb/src/tbb/governor.cpp b/src/tbb/src/tbb/governor.cpp index b492e8ac7..218a2bc53 100644 --- a/src/tbb/src/tbb/governor.cpp +++ b/src/tbb/src/tbb/governor.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2024 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,89 +12,101 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ -#include <stdio.h> -#include <stdlib.h> #include "governor.h" -#include "tbb_main.h" -#include "scheduler.h" +#include "threading_control.h" +#include "main.h" +#include "thread_data.h" #include "market.h" #include "arena.h" - -#include "tbb/task_scheduler_init.h" - #include "dynamic_link.h" +#include "concurrent_monitor.h" +#include "thread_dispatcher.h" -namespace tbb { -namespace internal { +#include "oneapi/tbb/task_group.h" +#include "oneapi/tbb/global_control.h" +#include "oneapi/tbb/tbb_allocator.h" +#include "oneapi/tbb/info.h" -//------------------------------------------------------------------------ -// governor -//------------------------------------------------------------------------ +#include "task_dispatcher.h" -#if __TBB_SURVIVE_THREAD_SWITCH -// Support for interoperability with Intel(R) Cilk(TM) Plus. +#include <cstdio> +#include <cstdlib> +#include <cstring> +#include <atomic> +#include <algorithm> -#if _WIN32 -#define CILKLIB_NAME "cilkrts20.dll" -#else -#define CILKLIB_NAME "libcilkrts.so" +#ifdef EMSCRIPTEN +#include <emscripten/stack.h> #endif -//! Handler for interoperation with cilkrts library. -static __cilk_tbb_retcode (*watch_stack_handler)(struct __cilk_tbb_unwatch_thunk* u, - struct __cilk_tbb_stack_op_thunk o); +namespace tbb { +namespace detail { +namespace r1 { -//! Table describing how to link the handlers. -static const dynamic_link_descriptor CilkLinkTable[] = { - { "__cilkrts_watch_stack", (pointer_to_handler*)(void*)(&watch_stack_handler) } -}; +#if TBB_USE_ASSERT +std::atomic<int> the_observer_proxy_count; +#endif /* TBB_USE_ASSERT */ -static atomic<do_once_state> cilkrts_load_state; +void clear_address_waiter_table(); +void global_control_acquire(); +void global_control_release(); -bool initialize_cilk_interop() { - // Pinning can fail. This is a normal situation, and means that the current - // thread does not use cilkrts and consequently does not need interop. - return dynamic_link( CILKLIB_NAME, CilkLinkTable, 1, /*handle=*/0, DYNAMIC_LINK_GLOBAL ); -} -#endif /* __TBB_SURVIVE_THREAD_SWITCH */ +//! global_control.cpp contains definition +bool remove_and_check_if_empty(d1::global_control& gc); +bool is_present(d1::global_control& gc); namespace rml { - tbb_server* make_private_server( tbb_client& client ); +tbb_server* make_private_server( tbb_client& client ); +} // namespace rml + +namespace system_topology { + void destroy(); } +//------------------------------------------------------------------------ +// governor +//------------------------------------------------------------------------ + void governor::acquire_resources () { -#if USE_PTHREAD + global_control_acquire(); +#if __TBB_USE_POSIX int status = theTLS.create(auto_terminate); #else int status = theTLS.create(); #endif if( status ) handle_perror(status, "TBB failed to initialize task scheduler TLS\n"); - is_speculation_enabled = cpu_has_speculation(); + detect_cpu_features(cpu_features); + is_rethrow_broken = gcc_rethrow_exception_broken(); } void governor::release_resources () { theRMLServerFactory.close(); destroy_process_mask(); -#if TBB_USE_ASSERT - if( __TBB_InitOnce::initialization_done() && theTLS.get() ) - runtime_warning( "TBB is unloaded while tbb::task_scheduler_init object is alive?" ); -#endif + + __TBB_ASSERT(!(__TBB_InitOnce::initialization_done() && theTLS.get()), "TBB is unloaded while thread data still alive?"); + int status = theTLS.destroy(); if( status ) - runtime_warning("failed to destroy task scheduler TLS: %s", strerror(status)); + runtime_warning("failed to destroy task scheduler TLS: %s", std::strerror(status)); + clear_address_waiter_table(); + +#if TBB_USE_ASSERT + if (the_observer_proxy_count != 0) { + runtime_warning("Leaked %ld observer_proxy objects\n", long(the_observer_proxy_count)); + } +#endif /* TBB_USE_ASSERT */ + + system_topology::destroy(); dynamic_unlink_all(); + global_control_release(); } rml::tbb_server* governor::create_rml_server ( rml::tbb_client& client ) { - rml::tbb_server* server = NULL; + rml::tbb_server* server = nullptr; if( !UsePrivateRML ) { ::rml::factory::status_type status = theRMLServerFactory.make_server( server, client ); if( status != ::rml::factory::st_success ) { @@ -103,277 +115,492 @@ rml::tbb_server* governor::create_rml_server ( rml::tbb_client& client ) { } } if ( !server ) { - __TBB_ASSERT( UsePrivateRML, NULL ); + __TBB_ASSERT( UsePrivateRML, nullptr); server = rml::make_private_server( client ); } __TBB_ASSERT( server, "Failed to create RML server" ); return server; } +void governor::one_time_init() { + if ( !__TBB_InitOnce::initialization_done() ) { + DoOneTimeInitialization(); + } +} + +bool governor::does_client_join_workers(const rml::tbb_client &client) { + return ((const thread_dispatcher&)client).must_join_workers(); +} -uintptr_t governor::tls_value_of( generic_scheduler* s ) { - __TBB_ASSERT( (uintptr_t(s)&1) == 0, "Bad pointer to the scheduler" ); - // LSB marks the scheduler initialized with arena - return uintptr_t(s) | uintptr_t((s && (s->my_arena || s->is_worker()))? 1 : 0); +/* + There is no portable way to get stack base address in Posix, however the modern + Linux versions provide pthread_attr_np API that can be used to obtain thread's + stack size and base address. Unfortunately even this function does not provide + enough information for the main thread on IA-64 architecture (RSE spill area + and memory stack are allocated as two separate discontinuous chunks of memory), + and there is no portable way to discern the main and the secondary threads. + Thus for macOS* and IA-64 architecture for Linux* OS we use the TBB worker stack size for + all threads and use the current stack top as the stack base. This simplified + approach is based on the following assumptions: + 1) If the default stack size is insufficient for the user app needs, the + required amount will be explicitly specified by the user at the point of the + TBB scheduler initialization (as an argument to tbb::task_scheduler_init + constructor). + 2) When an external thread initializes the scheduler, it has enough space on its + stack. Here "enough" means "at least as much as worker threads have". + 3) If the user app strives to conserve the memory by cutting stack size, it + should do this for TBB workers too (as in the #1). +*/ +static std::uintptr_t get_stack_base(std::size_t stack_size) { + // Stacks are growing top-down. Highest address is called "stack base", + // and the lowest is "stack limit". +#if __TBB_USE_WINAPI + suppress_unused_warning(stack_size); + NT_TIB* pteb = (NT_TIB*)NtCurrentTeb(); + __TBB_ASSERT(&pteb < pteb->StackBase && &pteb > pteb->StackLimit, "invalid stack info in TEB"); + return reinterpret_cast<std::uintptr_t>(pteb->StackBase); +#elif defined(EMSCRIPTEN) + suppress_unused_warning(stack_size); + return reinterpret_cast<std::uintptr_t>(emscripten_stack_get_base()); +#else + // There is no portable way to get stack base address in Posix, so we use + // non-portable method (on all modern Linux) or the simplified approach + // based on the common sense assumptions. The most important assumption + // is that the main thread's stack size is not less than that of other threads. + + // Points to the lowest addressable byte of a stack. + void* stack_limit = nullptr; +#if __linux__ && !__bg__ + size_t np_stack_size = 0; + pthread_attr_t np_attr_stack; + if (0 == pthread_getattr_np(pthread_self(), &np_attr_stack)) { + if (0 == pthread_attr_getstack(&np_attr_stack, &stack_limit, &np_stack_size)) { + __TBB_ASSERT( &stack_limit > stack_limit, "stack size must be positive" ); + } + pthread_attr_destroy(&np_attr_stack); + } +#endif /* __linux__ */ + std::uintptr_t stack_base{}; + if (stack_limit) { + stack_base = reinterpret_cast<std::uintptr_t>(stack_limit) + stack_size; + } else { + // Use an anchor as a base stack address. + int anchor{}; + stack_base = reinterpret_cast<std::uintptr_t>(&anchor); + } + return stack_base; +#endif /* __TBB_USE_WINAPI */ } -void governor::assume_scheduler( generic_scheduler* s ) { - theTLS.set( tls_value_of(s) ); +#if (_WIN32||_WIN64) && !__TBB_DYNAMIC_LOAD_ENABLED +static void register_external_thread_destructor() { + struct thread_destructor { + ~thread_destructor() { + governor::terminate_external_thread(); + } + }; + // ~thread_destructor() will be call during the calling thread termination + static thread_local thread_destructor thr_destructor; } +#endif // (_WIN32||_WIN64) && !__TBB_DYNAMIC_LOAD_ENABLED -bool governor::is_set( generic_scheduler* s ) { - return theTLS.get() == tls_value_of(s); +void governor::init_external_thread() { + one_time_init(); + // Create new scheduler instance with arena + int num_slots = default_num_threads(); + // TODO_REVAMP: support an external thread without an implicit arena + int num_reserved_slots = 1; + unsigned arena_priority_level = 1; // corresponds to tbb::task_arena::priority::normal + std::size_t stack_size = 0; + threading_control* thr_control = threading_control::register_public_reference(); + arena& a = arena::create(thr_control, num_slots, num_reserved_slots, arena_priority_level); + // External thread always occupies the first slot + thread_data& td = *new(cache_aligned_allocate(sizeof(thread_data))) thread_data(0, false); + td.attach_arena(a, /*slot index*/ 0); + __TBB_ASSERT(td.my_inbox.is_idle_state(false), nullptr); + + stack_size = a.my_threading_control->worker_stack_size(); + std::uintptr_t stack_base = get_stack_base(stack_size); + task_dispatcher& task_disp = td.my_arena_slot->default_task_dispatcher(); + td.enter_task_dispatcher(task_disp, calculate_stealing_threshold(stack_base, stack_size)); + + td.my_arena_slot->occupy(); + thr_control->register_thread(td); + set_thread_data(td); +#if (_WIN32||_WIN64) && !__TBB_DYNAMIC_LOAD_ENABLED + // The external thread destructor is called from dllMain but it is not available with a static build. + // Therefore, we need to register the current thread to call the destructor during thread termination. + register_external_thread_destructor(); +#endif } -void governor::sign_on(generic_scheduler* s) { - __TBB_ASSERT( is_set(NULL) && s, NULL ); - assume_scheduler( s ); -#if __TBB_SURVIVE_THREAD_SWITCH - if( watch_stack_handler ) { - __cilk_tbb_stack_op_thunk o; - o.routine = &stack_op_handler; - o.data = s; - if( (*watch_stack_handler)(&s->my_cilk_unwatch_thunk, o) ) { - // Failed to register with cilkrts, make sure we are clean - s->my_cilk_unwatch_thunk.routine = NULL; +void governor::auto_terminate(void* tls) { + __TBB_ASSERT(get_thread_data_if_initialized() == nullptr || + get_thread_data_if_initialized() == tls, nullptr); + if (tls) { + thread_data* td = static_cast<thread_data*>(tls); + + auto clear_tls = [td] { + td->~thread_data(); + cache_aligned_deallocate(td); + clear_thread_data(); + }; + + // Only external thread can be inside an arena during termination. + if (td->my_arena_slot) { + arena* a = td->my_arena; + threading_control* thr_control = a->my_threading_control; + + // If the TLS slot is already cleared by OS or underlying concurrency + // runtime, restore its value to properly clean up arena + if (!is_thread_data_set(td)) { + set_thread_data(*td); + } + + a->my_observers.notify_exit_observers(td->my_last_observer, td->my_is_worker); + + td->leave_task_dispatcher(); + td->my_arena_slot->release(); + // Release an arena + a->on_thread_leaving(arena::ref_external); + + thr_control->unregister_thread(*td); + + // The tls should be cleared before market::release because + // market can destroy the tls key if we keep the last reference + clear_tls(); + + // If there was an associated arena, it added a public market reference + thr_control->unregister_public_reference(/* blocking terminate =*/ false); + } else { + clear_tls(); } -#if TBB_USE_ASSERT - else - s->my_cilk_state = generic_scheduler::cs_running; -#endif /* TBB_USE_ASSERT */ } -#endif /* __TBB_SURVIVE_THREAD_SWITCH */ - __TBB_ASSERT( is_set(s), NULL ); + __TBB_ASSERT(get_thread_data_if_initialized() == nullptr, nullptr); } -void governor::sign_off(generic_scheduler* s) { - suppress_unused_warning(s); - __TBB_ASSERT( is_set(s), "attempt to unregister a wrong scheduler instance" ); - assume_scheduler(NULL); -#if __TBB_SURVIVE_THREAD_SWITCH - __cilk_tbb_unwatch_thunk &ut = s->my_cilk_unwatch_thunk; - if ( ut.routine ) - (*ut.routine)(ut.data); -#endif /* __TBB_SURVIVE_THREAD_SWITCH */ +void governor::initialize_rml_factory () { + ::rml::factory::status_type res = theRMLServerFactory.open(); + UsePrivateRML = res != ::rml::factory::st_success; } -void governor::one_time_init() { - if( !__TBB_InitOnce::initialization_done() ) - DoOneTimeInitializations(); -#if __TBB_SURVIVE_THREAD_SWITCH - atomic_do_once( &initialize_cilk_interop, cilkrts_load_state ); -#endif /* __TBB_SURVIVE_THREAD_SWITCH */ +void __TBB_EXPORTED_FUNC get(d1::task_scheduler_handle& handle) { + handle.m_ctl = new(allocate_memory(sizeof(global_control))) global_control(global_control::scheduler_handle, 1); } -generic_scheduler* governor::init_scheduler_weak() { - one_time_init(); - __TBB_ASSERT( is_set(NULL), "TLS contains a scheduler?" ); - generic_scheduler* s = generic_scheduler::create_master( NULL ); // without arena - s->my_auto_initialized = true; - return s; +void release_impl(d1::task_scheduler_handle& handle) { + if (handle.m_ctl != nullptr) { + handle.m_ctl->~global_control(); + deallocate_memory(handle.m_ctl); + handle.m_ctl = nullptr; + } } -generic_scheduler* governor::init_scheduler( int num_threads, stack_size_type stack_size, bool auto_init ) { - one_time_init(); - if ( uintptr_t v = theTLS.get() ) { - generic_scheduler* s = tls_scheduler_of( v ); - if ( (v&1) == 0 ) { // TLS holds scheduler instance without arena - __TBB_ASSERT( s->my_ref_count == 1, "weakly initialized scheduler must have refcount equal to 1" ); - __TBB_ASSERT( !s->my_arena, "weakly initialized scheduler must have no arena" ); - __TBB_ASSERT( s->my_auto_initialized, "weakly initialized scheduler is supposed to be auto-initialized" ); - s->attach_arena( market::create_arena( default_num_threads(), 1, 0 ), 0, /*is_master*/true ); - __TBB_ASSERT( s->my_arena_index == 0, "Master thread must occupy the first slot in its arena" ); - s->my_arena_slot->my_scheduler = s; - s->my_arena->my_default_ctx = s->default_context(); // it also transfers implied ownership - // Mark the scheduler as fully initialized - assume_scheduler( s ); +bool finalize_impl(d1::task_scheduler_handle& handle) { + __TBB_ASSERT_RELEASE(handle, "trying to finalize with null handle"); + __TBB_ASSERT(is_present(*handle.m_ctl), "finalize or release was already called on this object"); + + bool ok = true; // ok if threading_control does not exist yet + if (threading_control::is_present()) { + thread_data* td = governor::get_thread_data_if_initialized(); + if (td) { + task_dispatcher* task_disp = td->my_task_dispatcher; + __TBB_ASSERT(task_disp, nullptr); + if (task_disp->m_properties.outermost && !td->my_is_worker) { // is not inside a parallel region + governor::auto_terminate(td); + } + } + + if (remove_and_check_if_empty(*handle.m_ctl)) { + ok = threading_control::unregister_lifetime_control(/*blocking_terminate*/ true); + } else { + ok = false; } - // Increment refcount only for explicit instances of task_scheduler_init. - if ( !auto_init ) s->my_ref_count += 1; - __TBB_ASSERT( s->my_arena, "scheduler is not initialized fully" ); - return s; - } - // Create new scheduler instance with arena - if( num_threads == task_scheduler_init::automatic ) - num_threads = default_num_threads(); - arena *a = market::create_arena( num_threads, 1, stack_size ); - generic_scheduler* s = generic_scheduler::create_master( a ); - __TBB_ASSERT(s, "Somehow a local scheduler creation for a master thread failed"); - __TBB_ASSERT( is_set(s), NULL ); - s->my_auto_initialized = auto_init; - return s; -} - -bool governor::terminate_scheduler( generic_scheduler* s, const task_scheduler_init* tsi_ptr, bool blocking ) { - bool ok = false; - __TBB_ASSERT( is_set(s), "Attempt to terminate non-local scheduler instance" ); - if (0 == --(s->my_ref_count)) { - ok = s->cleanup_master( blocking ); - __TBB_ASSERT( is_set(NULL), "cleanup_master has not cleared its TLS slot" ); } + return ok; } -void governor::auto_terminate(void* arg){ - generic_scheduler* s = tls_scheduler_of( uintptr_t(arg) ); // arg is equivalent to theTLS.get() - if( s && s->my_auto_initialized ) { - if( !--(s->my_ref_count) ) { - // If the TLS slot is already cleared by OS or underlying concurrency - // runtime, restore its value. - if( !is_set(s) ) - assume_scheduler(s); - s->cleanup_master( /*blocking_terminate=*/false ); - __TBB_ASSERT( is_set(NULL), "cleanup_master has not cleared its TLS slot" ); +bool __TBB_EXPORTED_FUNC finalize(d1::task_scheduler_handle& handle, std::intptr_t mode) { + if (mode == d1::release_nothrowing) { + release_impl(handle); + return true; + } else { + bool ok = finalize_impl(handle); + // TODO: it is unsafe when finalize is called concurrently and further library unload + release_impl(handle); + if (mode == d1::finalize_throwing && !ok) { + throw_exception(exception_id::unsafe_wait); } + return ok; } } -void governor::print_version_info () { - if ( UsePrivateRML ) - PrintExtraVersionInfo( "RML", "private" ); - else { - PrintExtraVersionInfo( "RML", "shared" ); - theRMLServerFactory.call_with_server_info( PrintRMLVersionInfo, (void*)"" ); - } -#if __TBB_SURVIVE_THREAD_SWITCH - if( watch_stack_handler ) - PrintExtraVersionInfo( "CILK", CILKLIB_NAME ); -#endif /* __TBB_SURVIVE_THREAD_SWITCH */ +#if __TBB_ARENA_BINDING + +#if __TBB_WEAK_SYMBOLS_PRESENT +#pragma weak __TBB_internal_initialize_system_topology +#pragma weak __TBB_internal_destroy_system_topology +#pragma weak __TBB_internal_allocate_binding_handler +#pragma weak __TBB_internal_deallocate_binding_handler +#pragma weak __TBB_internal_apply_affinity +#pragma weak __TBB_internal_restore_affinity +#pragma weak __TBB_internal_get_default_concurrency + +extern "C" { +void __TBB_internal_initialize_system_topology( + size_t groups_num, + int& numa_nodes_count, int*& numa_indexes_list, + int& core_types_count, int*& core_types_indexes_list +); +void __TBB_internal_destroy_system_topology( ); + +//TODO: consider renaming to `create_binding_handler` and `destroy_binding_handler` +binding_handler* __TBB_internal_allocate_binding_handler( int slot_num, int numa_id, int core_type_id, int max_threads_per_core ); +void __TBB_internal_deallocate_binding_handler( binding_handler* handler_ptr ); + +void __TBB_internal_apply_affinity( binding_handler* handler_ptr, int slot_num ); +void __TBB_internal_restore_affinity( binding_handler* handler_ptr, int slot_num ); + +int __TBB_internal_get_default_concurrency( int numa_id, int core_type_id, int max_threads_per_core ); } +#endif /* __TBB_WEAK_SYMBOLS_PRESENT */ + +// Stubs that will be used if TBBbind library is unavailable. +static void dummy_destroy_system_topology ( ) { } +static binding_handler* dummy_allocate_binding_handler ( int, int, int, int ) { return nullptr; } +static void dummy_deallocate_binding_handler ( binding_handler* ) { } +static void dummy_apply_affinity ( binding_handler*, int ) { } +static void dummy_restore_affinity ( binding_handler*, int ) { } +static int dummy_get_default_concurrency( int, int, int ) { return governor::default_num_threads(); } + +// Handlers for communication with TBBbind +static void (*initialize_system_topology_ptr)( + size_t groups_num, + int& numa_nodes_count, int*& numa_indexes_list, + int& core_types_count, int*& core_types_indexes_list +) = nullptr; +static void (*destroy_system_topology_ptr)( ) = dummy_destroy_system_topology; + +static binding_handler* (*allocate_binding_handler_ptr)( int slot_num, int numa_id, int core_type_id, int max_threads_per_core ) + = dummy_allocate_binding_handler; +static void (*deallocate_binding_handler_ptr)( binding_handler* handler_ptr ) + = dummy_deallocate_binding_handler; +static void (*apply_affinity_ptr)( binding_handler* handler_ptr, int slot_num ) + = dummy_apply_affinity; +static void (*restore_affinity_ptr)( binding_handler* handler_ptr, int slot_num ) + = dummy_restore_affinity; +int (*get_default_concurrency_ptr)( int numa_id, int core_type_id, int max_threads_per_core ) + = dummy_get_default_concurrency; + +#if _WIN32 || _WIN64 || __unix__ || __APPLE__ + +// Table describing how to link the handlers. +static const dynamic_link_descriptor TbbBindLinkTable[] = { + DLD(__TBB_internal_initialize_system_topology, initialize_system_topology_ptr), + DLD(__TBB_internal_destroy_system_topology, destroy_system_topology_ptr), +#if __TBB_CPUBIND_PRESENT + DLD(__TBB_internal_allocate_binding_handler, allocate_binding_handler_ptr), + DLD(__TBB_internal_deallocate_binding_handler, deallocate_binding_handler_ptr), + DLD(__TBB_internal_apply_affinity, apply_affinity_ptr), + DLD(__TBB_internal_restore_affinity, restore_affinity_ptr), +#endif + DLD(__TBB_internal_get_default_concurrency, get_default_concurrency_ptr) +}; -void governor::initialize_rml_factory () { - ::rml::factory::status_type res = theRMLServerFactory.open(); - UsePrivateRML = res != ::rml::factory::st_success; +static const unsigned LinkTableSize = sizeof(TbbBindLinkTable) / sizeof(dynamic_link_descriptor); + +#if TBB_USE_DEBUG +#define DEBUG_SUFFIX "_debug" +#else +#define DEBUG_SUFFIX +#endif /* TBB_USE_DEBUG */ + +#if _WIN32 || _WIN64 +#define LIBRARY_EXTENSION ".dll" +#define LIBRARY_PREFIX +#elif __APPLE__ +#define LIBRARY_EXTENSION __TBB_STRING(.3.dylib) +#define LIBRARY_PREFIX "lib" +#elif __unix__ +#define LIBRARY_EXTENSION __TBB_STRING(.so.3) +#define LIBRARY_PREFIX "lib" +#endif /* __unix__ */ + +#define TBBBIND_NAME LIBRARY_PREFIX "tbbbind" DEBUG_SUFFIX LIBRARY_EXTENSION +#define TBBBIND_2_0_NAME LIBRARY_PREFIX "tbbbind_2_0" DEBUG_SUFFIX LIBRARY_EXTENSION + +#define TBBBIND_2_5_NAME LIBRARY_PREFIX "tbbbind_2_5" DEBUG_SUFFIX LIBRARY_EXTENSION +#endif /* _WIN32 || _WIN64 || __unix__ */ + +// Representation of system hardware topology information on the TBB side. +// System topology may be initialized by third-party component (e.g. hwloc) +// or just filled in with default stubs. +namespace system_topology { + +constexpr int automatic = -1; + +static std::atomic<do_once_state> initialization_state; + +namespace { +int numa_nodes_count = 0; +int* numa_nodes_indexes = nullptr; + +int core_types_count = 0; +int* core_types_indexes = nullptr; + +const char* load_tbbbind_shared_object() { +#if _WIN32 || _WIN64 || __unix__ || __APPLE__ +#if _WIN32 && !_WIN64 + // For 32-bit Windows applications, process affinity masks can only support up to 32 logical CPUs. + SYSTEM_INFO si; + GetNativeSystemInfo(&si); + if (si.dwNumberOfProcessors > 32) return nullptr; +#endif /* _WIN32 && !_WIN64 */ + for (const auto& tbbbind_version : {TBBBIND_2_5_NAME, TBBBIND_2_0_NAME, TBBBIND_NAME}) { + if (dynamic_link(tbbbind_version, TbbBindLinkTable, LinkTableSize, nullptr, DYNAMIC_LINK_LOCAL_BINDING)) { + return tbbbind_version; + } + } +#endif /* _WIN32 || _WIN64 || __unix__ || __APPLE__ */ + return nullptr; } -#if __TBB_SURVIVE_THREAD_SWITCH -__cilk_tbb_retcode governor::stack_op_handler( __cilk_tbb_stack_op op, void* data ) { - __TBB_ASSERT(data,NULL); - generic_scheduler* s = static_cast<generic_scheduler*>(data); -#if TBB_USE_ASSERT - void* current = local_scheduler_if_initialized(); -#if _WIN32||_WIN64 - uintptr_t thread_id = GetCurrentThreadId(); +int processor_groups_num() { +#if _WIN32 + return NumberOfProcessorGroups(); #else - uintptr_t thread_id = uintptr_t(pthread_self()); + // Stub to improve code readability by reducing number of the compile-time conditions + return 1; #endif - -#endif /* TBB_USE_ASSERT */ - switch( op ) { - default: - __TBB_ASSERT( 0, "invalid op" ); - case CILK_TBB_STACK_ADOPT: { - __TBB_ASSERT( !current && s->my_cilk_state==generic_scheduler::cs_limbo || - current==s && s->my_cilk_state==generic_scheduler::cs_running, "invalid adoption" ); -#if TBB_USE_ASSERT - if( current==s ) - runtime_warning( "redundant adoption of %p by thread %p\n", s, (void*)thread_id ); - s->my_cilk_state = generic_scheduler::cs_running; -#endif /* TBB_USE_ASSERT */ - assume_scheduler( s ); - break; - } - case CILK_TBB_STACK_ORPHAN: { - __TBB_ASSERT( current==s && s->my_cilk_state==generic_scheduler::cs_running, "invalid orphaning" ); -#if TBB_USE_ASSERT - s->my_cilk_state = generic_scheduler::cs_limbo; -#endif /* TBB_USE_ASSERT */ - assume_scheduler(NULL); - break; - } - case CILK_TBB_STACK_RELEASE: { - __TBB_ASSERT( !current && s->my_cilk_state==generic_scheduler::cs_limbo || - current==s && s->my_cilk_state==generic_scheduler::cs_running, "invalid release" ); -#if TBB_USE_ASSERT - s->my_cilk_state = generic_scheduler::cs_freed; -#endif /* TBB_USE_ASSERT */ - s->my_cilk_unwatch_thunk.routine = NULL; - auto_terminate( s ); - } +} +} // internal namespace + +// Tries to load TBBbind library API, if success, gets NUMA topology information from it, +// in another case, fills NUMA topology by stubs. +void initialization_impl() { + governor::one_time_init(); + + if (const char* tbbbind_name = load_tbbbind_shared_object()) { + initialize_system_topology_ptr( + processor_groups_num(), + numa_nodes_count, numa_nodes_indexes, + core_types_count, core_types_indexes + ); + + PrintExtraVersionInfo("TBBBIND", tbbbind_name); + return; } - return 0; + + static int dummy_index = automatic; + + numa_nodes_count = 1; + numa_nodes_indexes = &dummy_index; + + core_types_count = 1; + core_types_indexes = &dummy_index; + + PrintExtraVersionInfo("TBBBIND", "UNAVAILABLE"); } -#endif /* __TBB_SURVIVE_THREAD_SWITCH */ -} // namespace internal +void initialize() { + atomic_do_once(initialization_impl, initialization_state); +} -//------------------------------------------------------------------------ -// task_scheduler_init -//------------------------------------------------------------------------ +void destroy() { + destroy_system_topology_ptr(); +} +} // namespace system_topology -using namespace internal; +binding_handler* construct_binding_handler(int slot_num, int numa_id, int core_type_id, int max_threads_per_core) { + system_topology::initialize(); + return allocate_binding_handler_ptr(slot_num, numa_id, core_type_id, max_threads_per_core); +} -/** Left out-of-line for the sake of the backward binary compatibility **/ -void task_scheduler_init::initialize( int number_of_threads ) { - initialize( number_of_threads, 0 ); +void destroy_binding_handler(binding_handler* handler_ptr) { + __TBB_ASSERT(deallocate_binding_handler_ptr, "tbbbind loading was not performed"); + deallocate_binding_handler_ptr(handler_ptr); } -void task_scheduler_init::initialize( int number_of_threads, stack_size_type thread_stack_size ) { -#if __TBB_TASK_GROUP_CONTEXT && TBB_USE_EXCEPTIONS - uintptr_t new_mode = thread_stack_size & propagation_mode_mask; -#endif - thread_stack_size &= ~(stack_size_type)propagation_mode_mask; - if( number_of_threads!=deferred ) { - __TBB_ASSERT_RELEASE( !my_scheduler, "task_scheduler_init already initialized" ); - __TBB_ASSERT_RELEASE( number_of_threads==automatic || number_of_threads > 0, - "number_of_threads for task_scheduler_init must be automatic or positive" ); - internal::generic_scheduler *s = governor::init_scheduler( number_of_threads, thread_stack_size, /*auto_init=*/false ); -#if __TBB_TASK_GROUP_CONTEXT && TBB_USE_EXCEPTIONS - if ( s->master_outermost_level() ) { - uintptr_t &vt = s->default_context()->my_version_and_traits; - uintptr_t prev_mode = vt & task_group_context::exact_exception ? propagation_mode_exact : 0; - vt = new_mode & propagation_mode_exact ? vt | task_group_context::exact_exception - : new_mode & propagation_mode_captured ? vt & ~task_group_context::exact_exception : vt; - // Use least significant bit of the scheduler pointer to store previous mode. - // This is necessary when components compiled with different compilers and/or - // TBB versions initialize the - my_scheduler = static_cast<scheduler*>((generic_scheduler*)((uintptr_t)s | prev_mode)); - } - else -#endif /* __TBB_TASK_GROUP_CONTEXT && TBB_USE_EXCEPTIONS */ - my_scheduler = s; - } else { - __TBB_ASSERT_RELEASE( !thread_stack_size, "deferred initialization ignores stack size setting" ); - } +void apply_affinity_mask(binding_handler* handler_ptr, int slot_index) { + __TBB_ASSERT(slot_index >= 0, "Negative thread index"); + __TBB_ASSERT(apply_affinity_ptr, "tbbbind loading was not performed"); + apply_affinity_ptr(handler_ptr, slot_index); } -bool task_scheduler_init::internal_terminate( bool blocking ) { -#if __TBB_TASK_GROUP_CONTEXT && TBB_USE_EXCEPTIONS - uintptr_t prev_mode = (uintptr_t)my_scheduler & propagation_mode_exact; - my_scheduler = (scheduler*)((uintptr_t)my_scheduler & ~(uintptr_t)propagation_mode_exact); -#endif /* __TBB_TASK_GROUP_CONTEXT && TBB_USE_EXCEPTIONS */ - generic_scheduler* s = static_cast<generic_scheduler*>(my_scheduler); - my_scheduler = NULL; - __TBB_ASSERT_RELEASE( s, "task_scheduler_init::terminate without corresponding task_scheduler_init::initialize()"); -#if __TBB_TASK_GROUP_CONTEXT && TBB_USE_EXCEPTIONS - if ( s->master_outermost_level() ) { - uintptr_t &vt = s->default_context()->my_version_and_traits; - vt = prev_mode & propagation_mode_exact ? vt | task_group_context::exact_exception - : vt & ~task_group_context::exact_exception; +void restore_affinity_mask(binding_handler* handler_ptr, int slot_index) { + __TBB_ASSERT(slot_index >= 0, "Negative thread index"); + __TBB_ASSERT(restore_affinity_ptr, "tbbbind loading was not performed"); + restore_affinity_ptr(handler_ptr, slot_index); +} + +unsigned __TBB_EXPORTED_FUNC numa_node_count() { + system_topology::initialize(); + return system_topology::numa_nodes_count; +} + +void __TBB_EXPORTED_FUNC fill_numa_indices(int* index_array) { + system_topology::initialize(); + std::memcpy(index_array, system_topology::numa_nodes_indexes, system_topology::numa_nodes_count * sizeof(int)); +} + +int __TBB_EXPORTED_FUNC numa_default_concurrency(int node_id) { + if (node_id >= 0) { + system_topology::initialize(); + int result = get_default_concurrency_ptr( + node_id, + /*core_type*/system_topology::automatic, + /*threads_per_core*/system_topology::automatic + ); + if (result > 0) return result; } -#endif /* __TBB_TASK_GROUP_CONTEXT && TBB_USE_EXCEPTIONS */ - return governor::terminate_scheduler(s, this, blocking); + return governor::default_num_threads(); } -void task_scheduler_init::terminate() { - internal_terminate(/*blocking_terminate=*/false); +unsigned __TBB_EXPORTED_FUNC core_type_count(intptr_t /*reserved*/) { + system_topology::initialize(); + return system_topology::core_types_count; } -#if __TBB_SUPPORTS_WORKERS_WAITING_IN_TERMINATE -bool task_scheduler_init::internal_blocking_terminate( bool throwing ) { - bool ok = internal_terminate( /*blocking_terminate=*/true ); -#if TBB_USE_EXCEPTIONS - if( throwing && !ok ) - throw_exception( eid_blocking_thread_join_impossible ); -#else - suppress_unused_warning( throwing ); -#endif - return ok; +void __TBB_EXPORTED_FUNC fill_core_type_indices(int* index_array, intptr_t /*reserved*/) { + system_topology::initialize(); + std::memcpy(index_array, system_topology::core_types_indexes, system_topology::core_types_count * sizeof(int)); +} + +void constraints_assertion(d1::constraints c) { + bool is_topology_initialized = system_topology::initialization_state == do_once_state::initialized; + __TBB_ASSERT_RELEASE(c.max_threads_per_core == system_topology::automatic || c.max_threads_per_core > 0, + "Wrong max_threads_per_core constraints field value."); + + auto numa_nodes_begin = system_topology::numa_nodes_indexes; + auto numa_nodes_end = system_topology::numa_nodes_indexes + system_topology::numa_nodes_count; + __TBB_ASSERT_RELEASE( + c.numa_id == system_topology::automatic || + (is_topology_initialized && std::find(numa_nodes_begin, numa_nodes_end, c.numa_id) != numa_nodes_end), + "The constraints::numa_id value is not known to the library. Use tbb::info::numa_nodes() to get the list of possible values."); + + int* core_types_begin = system_topology::core_types_indexes; + int* core_types_end = system_topology::core_types_indexes + system_topology::core_types_count; + __TBB_ASSERT_RELEASE(c.core_type == system_topology::automatic || + (is_topology_initialized && std::find(core_types_begin, core_types_end, c.core_type) != core_types_end), + "The constraints::core_type value is not known to the library. Use tbb::info::core_types() to get the list of possible values."); } -#endif // __TBB_SUPPORTS_WORKERS_WAITING_IN_TERMINATE -int task_scheduler_init::default_num_threads() { +int __TBB_EXPORTED_FUNC constraints_default_concurrency(const d1::constraints& c, intptr_t /*reserved*/) { + constraints_assertion(c); + + if (c.numa_id >= 0 || c.core_type >= 0 || c.max_threads_per_core > 0) { + system_topology::initialize(); + return get_default_concurrency_ptr(c.numa_id, c.core_type, c.max_threads_per_core); + } return governor::default_num_threads(); } +int __TBB_EXPORTED_FUNC constraints_threads_per_core(const d1::constraints&, intptr_t /*reserved*/) { + return system_topology::automatic; +} +#endif /* __TBB_ARENA_BINDING */ + +} // namespace r1 +} // namespace detail } // namespace tbb diff --git a/src/tbb/src/tbb/governor.h b/src/tbb/src/tbb/governor.h index 92f98bb41..573443d72 100644 --- a/src/tbb/src/tbb/governor.h +++ b/src/tbb/src/tbb/governor.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2024 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,35 +12,30 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef _TBB_governor_H #define _TBB_governor_H -#include "tbb/task_scheduler_init.h" -#include "../rml/include/rml_tbb.h" +#include "rml_tbb.h" -#include "tbb_misc.h" // for AvailableHwConcurrency +#include "misc.h" // for AvailableHwConcurrency #include "tls.h" -#if __TBB_SURVIVE_THREAD_SWITCH -#include "cilk-tbb-interop.h" -#endif /* __TBB_SURVIVE_THREAD_SWITCH */ - namespace tbb { -namespace internal { +namespace detail { +namespace r1 { class market; -class generic_scheduler; +class thread_data; class __TBB_InitOnce; -namespace rml { -class tbb_client; -} +#if __TBB_USE_ITT_NOTIFY +//! Defined in profiling.cpp +extern bool ITT_Present; +#endif + +typedef std::size_t stack_size_type; //------------------------------------------------------------------------ // Class governor @@ -52,20 +47,20 @@ class tbb_client; class governor { private: friend class __TBB_InitOnce; - friend class market; + friend class thread_dispatcher; + friend class threading_control_impl; + // TODO: consider using thread_local (measure performance and side effects) //! TLS for scheduler instances associated with individual threads - static basic_tls<uintptr_t> theTLS; - - //! Caches the maximal level of parallelism supported by the hardware - static unsigned DefaultNumberOfThreads; + static basic_tls<thread_data*> theTLS; + // TODO (TBB_REVAMP_TODO): reconsider constant names static rml::tbb_factory theRMLServerFactory; static bool UsePrivateRML; // Flags for runtime-specific conditions - static bool is_speculation_enabled; + static cpu_features_type cpu_features; static bool is_rethrow_broken; //! Create key for thread-local storage and initialize RML. @@ -76,88 +71,88 @@ class governor { static rml::tbb_server* create_rml_server ( rml::tbb_client& ); - //! The internal routine to undo automatic initialization. - /** The signature is written with void* so that the routine - can be the destructor argument to pthread_key_create. */ - static void auto_terminate(void* scheduler); - public: static unsigned default_num_threads () { - // No memory fence required, because at worst each invoking thread calls AvailableHwConcurrency once. - return DefaultNumberOfThreads ? DefaultNumberOfThreads : - DefaultNumberOfThreads = AvailableHwConcurrency(); + // Caches the maximal level of parallelism supported by the hardware + static unsigned num_threads = AvailableHwConcurrency(); + return num_threads; + } + static std::size_t default_page_size () { + // Caches the size of OS regular memory page + static std::size_t page_size = DefaultSystemPageSize(); + return page_size; } static void one_time_init(); - //! Processes scheduler initialization request (possibly nested) in a master thread + //! Processes scheduler initialization request (possibly nested) in an external thread /** If necessary creates new instance of arena and/or local scheduler. The auto_init argument specifies if the call is due to automatic initialization. **/ - static generic_scheduler* init_scheduler( int num_threads, stack_size_type stack_size, bool auto_init ); - - //! Automatic initialization of scheduler in a master thread with default settings without arena - static generic_scheduler* init_scheduler_weak(); - - //! Processes scheduler termination request (possibly nested) in a master thread - static bool terminate_scheduler( generic_scheduler* s, const task_scheduler_init *tsi_ptr, bool blocking ); - - //! Register TBB scheduler instance in thread-local storage. - static void sign_on( generic_scheduler* s ); + static void init_external_thread(); - //! Unregister TBB scheduler instance from thread-local storage. - static void sign_off( generic_scheduler* s ); - - //! Used to check validity of the local scheduler TLS contents. - static bool is_set( generic_scheduler* s ); - - //! Temporarily set TLS slot to the given scheduler - static void assume_scheduler( generic_scheduler* s ); + //! The routine to undo automatic initialization. + /** The signature is written with void* so that the routine + can be the destructor argument to pthread_key_create. */ + static void auto_terminate(void* tls); - //! Computes the value of the TLS - static uintptr_t tls_value_of( generic_scheduler* s ); + //! Obtain the thread-local instance of the thread data. + /** If the scheduler has not been initialized yet, initialization is done automatically. + Note that auto-initialized scheduler instance is destroyed only when its thread terminates. **/ + static thread_data* get_thread_data() { + thread_data* td = theTLS.get(); + if (td) { + return td; + } + init_external_thread(); + td = theTLS.get(); + __TBB_ASSERT(td, nullptr); + return td; + } - // TODO IDEA: refactor bit manipulations over pointer types to a class? - //! Converts TLS value to the scheduler pointer - static generic_scheduler* tls_scheduler_of( uintptr_t v ) { - return (generic_scheduler*)(v & ~uintptr_t(1)); + static void set_thread_data(thread_data& td) { + theTLS.set(&td); } - //! Obtain the thread-local instance of the TBB scheduler. - /** If the scheduler has not been initialized yet, initialization is done automatically. - Note that auto-initialized scheduler instance is destroyed only when its thread terminates. **/ - static generic_scheduler* local_scheduler () { - uintptr_t v = theTLS.get(); - return (v&1) ? tls_scheduler_of(v) : init_scheduler( task_scheduler_init::automatic, 0, /*auto_init=*/true ); + static void clear_thread_data() { + theTLS.set(nullptr); } - static generic_scheduler* local_scheduler_weak () { - uintptr_t v = theTLS.get(); - return v ? tls_scheduler_of(v) : init_scheduler_weak(); + static thread_data* get_thread_data_if_initialized () { + return theTLS.get(); } - static generic_scheduler* local_scheduler_if_initialized () { - return tls_scheduler_of( theTLS.get() ); + static bool is_thread_data_set(thread_data* td) { + return theTLS.get() == td; } //! Undo automatic initialization if necessary; call when a thread exits. - static void terminate_auto_initialized_scheduler() { - auto_terminate( local_scheduler_if_initialized() ); + static void terminate_external_thread() { + auto_terminate(get_thread_data_if_initialized()); } - static void print_version_info (); - static void initialize_rml_factory (); - static bool does_client_join_workers (const tbb::internal::rml::tbb_client &client); + static bool does_client_join_workers (const rml::tbb_client &client); -#if __TBB_SURVIVE_THREAD_SWITCH - static __cilk_tbb_retcode stack_op_handler( __cilk_tbb_stack_op op, void* ); -#endif /* __TBB_SURVIVE_THREAD_SWITCH */ + static bool speculation_enabled() { return cpu_features.rtm_enabled; } + +#if __TBB_WAITPKG_INTRINSICS_PRESENT + static bool wait_package_enabled() { return cpu_features.waitpkg_enabled; } +#endif + + static bool hybrid_cpu() { return cpu_features.hybrid; } - static bool speculation_enabled() { return is_speculation_enabled; } static bool rethrow_exception_broken() { return is_rethrow_broken; } + static bool is_itt_present() { +#if __TBB_USE_ITT_NOTIFY + return ITT_Present; +#else + return false; +#endif + } }; // class governor -} // namespace internal +} // namespace r1 +} // namespace detail } // namespace tbb #endif /* _TBB_governor_H */ diff --git a/src/tbb/src/tbb/ia32-masm/atomic_support.asm b/src/tbb/src/tbb/ia32-masm/atomic_support.asm deleted file mode 100644 index 3cb82b971..000000000 --- a/src/tbb/src/tbb/ia32-masm/atomic_support.asm +++ /dev/null @@ -1,188 +0,0 @@ -; Copyright (c) 2005-2017 Intel Corporation -; -; Licensed under the Apache License, Version 2.0 (the "License"); -; you may not use this file except in compliance with the License. -; You may obtain a copy of the License at -; -; http://www.apache.org/licenses/LICENSE-2.0 -; -; Unless required by applicable law or agreed to in writing, software -; distributed under the License is distributed on an "AS IS" BASIS, -; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -; See the License for the specific language governing permissions and -; limitations under the License. -; -; -; -; - -.686 -.model flat,c -.code - ALIGN 4 - PUBLIC c __TBB_machine_fetchadd1 -__TBB_machine_fetchadd1: - mov edx,4[esp] - mov eax,8[esp] - lock xadd [edx],al - ret -.code - ALIGN 4 - PUBLIC c __TBB_machine_fetchstore1 -__TBB_machine_fetchstore1: - mov edx,4[esp] - mov eax,8[esp] - lock xchg [edx],al - ret -.code - ALIGN 4 - PUBLIC c __TBB_machine_cmpswp1 -__TBB_machine_cmpswp1: - mov edx,4[esp] - mov ecx,8[esp] - mov eax,12[esp] - lock cmpxchg [edx],cl - ret -.code - ALIGN 4 - PUBLIC c __TBB_machine_fetchadd2 -__TBB_machine_fetchadd2: - mov edx,4[esp] - mov eax,8[esp] - lock xadd [edx],ax - ret -.code - ALIGN 4 - PUBLIC c __TBB_machine_fetchstore2 -__TBB_machine_fetchstore2: - mov edx,4[esp] - mov eax,8[esp] - lock xchg [edx],ax - ret -.code - ALIGN 4 - PUBLIC c __TBB_machine_cmpswp2 -__TBB_machine_cmpswp2: - mov edx,4[esp] - mov ecx,8[esp] - mov eax,12[esp] - lock cmpxchg [edx],cx - ret -.code - ALIGN 4 - PUBLIC c __TBB_machine_fetchadd4 -__TBB_machine_fetchadd4: - mov edx,4[esp] - mov eax,8[esp] - lock xadd [edx],eax - ret -.code - ALIGN 4 - PUBLIC c __TBB_machine_fetchstore4 -__TBB_machine_fetchstore4: - mov edx,4[esp] - mov eax,8[esp] - lock xchg [edx],eax - ret -.code - ALIGN 4 - PUBLIC c __TBB_machine_cmpswp4 -__TBB_machine_cmpswp4: - mov edx,4[esp] - mov ecx,8[esp] - mov eax,12[esp] - lock cmpxchg [edx],ecx - ret -.code - ALIGN 4 - PUBLIC c __TBB_machine_fetchadd8 -__TBB_machine_fetchadd8: - push ebx - push edi - mov edi,12[esp] - mov eax,[edi] - mov edx,4[edi] -__TBB_machine_fetchadd8_loop: - mov ebx,16[esp] - mov ecx,20[esp] - add ebx,eax - adc ecx,edx - lock cmpxchg8b qword ptr [edi] - jnz __TBB_machine_fetchadd8_loop - pop edi - pop ebx - ret -.code - ALIGN 4 - PUBLIC c __TBB_machine_fetchstore8 -__TBB_machine_fetchstore8: - push ebx - push edi - mov edi,12[esp] - mov ebx,16[esp] - mov ecx,20[esp] - mov eax,[edi] - mov edx,4[edi] -__TBB_machine_fetchstore8_loop: - lock cmpxchg8b qword ptr [edi] - jnz __TBB_machine_fetchstore8_loop - pop edi - pop ebx - ret -.code - ALIGN 4 - PUBLIC c __TBB_machine_cmpswp8 -__TBB_machine_cmpswp8: - push ebx - push edi - mov edi,12[esp] - mov ebx,16[esp] - mov ecx,20[esp] - mov eax,24[esp] - mov edx,28[esp] - lock cmpxchg8b qword ptr [edi] - pop edi - pop ebx - ret -.code - ALIGN 4 - PUBLIC c __TBB_machine_load8 -__TBB_machine_Load8: - ; If location is on stack, compiler may have failed to align it correctly, so we do dynamic check. - mov ecx,4[esp] - test ecx,7 - jne load_slow - ; Load within a cache line - sub esp,12 - fild qword ptr [ecx] - fistp qword ptr [esp] - mov eax,[esp] - mov edx,4[esp] - add esp,12 - ret -load_slow: - ; Load is misaligned. Use cmpxchg8b. - push ebx - push edi - mov edi,ecx - xor eax,eax - xor ebx,ebx - xor ecx,ecx - xor edx,edx - lock cmpxchg8b qword ptr [edi] - pop edi - pop ebx - ret -EXTRN __TBB_machine_store8_slow:PROC -.code - ALIGN 4 - PUBLIC c __TBB_machine_store8 -__TBB_machine_Store8: - ; If location is on stack, compiler may have failed to align it correctly, so we do dynamic check. - mov ecx,4[esp] - test ecx,7 - jne __TBB_machine_store8_slow ;; tail call to tbb_misc.cpp - fild qword ptr 8[esp] - fistp qword ptr [ecx] - ret -end diff --git a/src/tbb/src/tbb/ia32-masm/itsx.asm b/src/tbb/src/tbb/ia32-masm/itsx.asm deleted file mode 100644 index 10e08357f..000000000 --- a/src/tbb/src/tbb/ia32-masm/itsx.asm +++ /dev/null @@ -1,80 +0,0 @@ -; Copyright (c) 2005-2017 Intel Corporation -; -; Licensed under the Apache License, Version 2.0 (the "License"); -; you may not use this file except in compliance with the License. -; You may obtain a copy of the License at -; -; http://www.apache.org/licenses/LICENSE-2.0 -; -; Unless required by applicable law or agreed to in writing, software -; distributed under the License is distributed on an "AS IS" BASIS, -; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -; See the License for the specific language governing permissions and -; limitations under the License. -; -; -; -; - -.686 -.model flat,c -.code - ALIGN 4 - PUBLIC c __TBB_machine_try_lock_elided -__TBB_machine_try_lock_elided: - mov ecx, 4[esp] - xor eax, eax - mov al, 1 - BYTE 0F2H - xchg al, byte ptr [ecx] - xor al, 1 - ret -.code - ALIGN 4 - PUBLIC c __TBB_machine_unlock_elided -__TBB_machine_unlock_elided: - mov ecx, 4[esp] - BYTE 0F3H - mov byte ptr [ecx], 0 - ret -.code - ALIGN 4 - PUBLIC c __TBB_machine_begin_transaction -__TBB_machine_begin_transaction: - mov eax, -1 - BYTE 0C7H - BYTE 0F8H - BYTE 000H - BYTE 000H - BYTE 000H - BYTE 000H - ret -.code - ALIGN 4 - PUBLIC c __TBB_machine_end_transaction -__TBB_machine_end_transaction: - BYTE 00FH - BYTE 001H - BYTE 0D5H - ret -.code - ALIGN 4 - PUBLIC c __TBB_machine_transaction_conflict_abort -__TBB_machine_transaction_conflict_abort: - BYTE 0C6H - BYTE 0F8H - BYTE 0FFH ; 12.4.5 Abort argument: lock not free when tested - ret -.code - ALIGN 4 - PUBLIC c __TBB_machine_is_in_transaction -__TBB_machine_is_in_transaction: - xor eax, eax - BYTE 00FH - BYTE 001H - BYTE 0D6H - JZ rset - MOV al,1 -rset: - RET -end diff --git a/src/tbb/src/tbb/ia32-masm/lock_byte.asm b/src/tbb/src/tbb/ia32-masm/lock_byte.asm deleted file mode 100644 index daad51983..000000000 --- a/src/tbb/src/tbb/ia32-masm/lock_byte.asm +++ /dev/null @@ -1,38 +0,0 @@ -; Copyright (c) 2005-2017 Intel Corporation -; -; Licensed under the Apache License, Version 2.0 (the "License"); -; you may not use this file except in compliance with the License. -; You may obtain a copy of the License at -; -; http://www.apache.org/licenses/LICENSE-2.0 -; -; Unless required by applicable law or agreed to in writing, software -; distributed under the License is distributed on an "AS IS" BASIS, -; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -; See the License for the specific language governing permissions and -; limitations under the License. -; -; -; -; - -; DO NOT EDIT - AUTOMATICALLY GENERATED FROM .s FILE -.686 -.model flat,c -.code - ALIGN 4 - PUBLIC c __TBB_machine_trylockbyte -__TBB_machine_trylockbyte: - mov edx,4[esp] - mov al,[edx] - mov cl,1 - test al,1 - jnz __TBB_machine_trylockbyte_contended - lock cmpxchg [edx],cl - jne __TBB_machine_trylockbyte_contended - mov eax,1 - ret -__TBB_machine_trylockbyte_contended: - xor eax,eax - ret -end diff --git a/src/tbb/src/tbb/ia64-gas/atomic_support.s b/src/tbb/src/tbb/ia64-gas/atomic_support.s deleted file mode 100644 index cf800dbc6..000000000 --- a/src/tbb/src/tbb/ia64-gas/atomic_support.s +++ /dev/null @@ -1,670 +0,0 @@ -// Copyright (c) 2005-2017 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// -// -// - -// DO NOT EDIT - AUTOMATICALLY GENERATED FROM tools/generate_atomic/ipf_generate.sh -# 1 "<stdin>" -# 1 "<built-in>" -# 1 "<command line>" -# 1 "<stdin>" - - - - - - .section .text - .align 16 - - - .proc __TBB_machine_fetchadd1__TBB_full_fence# - .global __TBB_machine_fetchadd1__TBB_full_fence# -__TBB_machine_fetchadd1__TBB_full_fence: -{ - mf - br __TBB_machine_fetchadd1acquire -} - .endp __TBB_machine_fetchadd1__TBB_full_fence# - - .proc __TBB_machine_fetchadd1acquire# - .global __TBB_machine_fetchadd1acquire# -__TBB_machine_fetchadd1acquire: - - - - - - - - ld1 r9=[r32] -;; -Retry_1acquire: - mov ar.ccv=r9 - mov r8=r9; - add r10=r9,r33 -;; - cmpxchg1.acq r9=[r32],r10,ar.ccv -;; - cmp.ne p7,p0=r8,r9 - (p7) br.cond.dpnt Retry_1acquire - br.ret.sptk.many b0 -# 49 "<stdin>" - .endp __TBB_machine_fetchadd1acquire# -# 62 "<stdin>" - .section .text - .align 16 - .proc __TBB_machine_fetchstore1__TBB_full_fence# - .global __TBB_machine_fetchstore1__TBB_full_fence# -__TBB_machine_fetchstore1__TBB_full_fence: - mf -;; - xchg1 r8=[r32],r33 - br.ret.sptk.many b0 - .endp __TBB_machine_fetchstore1__TBB_full_fence# - - - .proc __TBB_machine_fetchstore1acquire# - .global __TBB_machine_fetchstore1acquire# -__TBB_machine_fetchstore1acquire: - xchg1 r8=[r32],r33 - br.ret.sptk.many b0 - .endp __TBB_machine_fetchstore1acquire# -# 88 "<stdin>" - .section .text - .align 16 - - - .proc __TBB_machine_cmpswp1__TBB_full_fence# - .global __TBB_machine_cmpswp1__TBB_full_fence# -__TBB_machine_cmpswp1__TBB_full_fence: -{ - mf - br __TBB_machine_cmpswp1acquire -} - .endp __TBB_machine_cmpswp1__TBB_full_fence# - - .proc __TBB_machine_cmpswp1acquire# - .global __TBB_machine_cmpswp1acquire# -__TBB_machine_cmpswp1acquire: - - zxt1 r34=r34 -;; - - mov ar.ccv=r34 -;; - cmpxchg1.acq r8=[r32],r33,ar.ccv - br.ret.sptk.many b0 - .endp __TBB_machine_cmpswp1acquire# -// DO NOT EDIT - AUTOMATICALLY GENERATED FROM tools/generate_atomic/ipf_generate.sh -# 1 "<stdin>" -# 1 "<built-in>" -# 1 "<command line>" -# 1 "<stdin>" - - - - - - .section .text - .align 16 - - - .proc __TBB_machine_fetchadd2__TBB_full_fence# - .global __TBB_machine_fetchadd2__TBB_full_fence# -__TBB_machine_fetchadd2__TBB_full_fence: -{ - mf - br __TBB_machine_fetchadd2acquire -} - .endp __TBB_machine_fetchadd2__TBB_full_fence# - - .proc __TBB_machine_fetchadd2acquire# - .global __TBB_machine_fetchadd2acquire# -__TBB_machine_fetchadd2acquire: - - - - - - - - ld2 r9=[r32] -;; -Retry_2acquire: - mov ar.ccv=r9 - mov r8=r9; - add r10=r9,r33 -;; - cmpxchg2.acq r9=[r32],r10,ar.ccv -;; - cmp.ne p7,p0=r8,r9 - (p7) br.cond.dpnt Retry_2acquire - br.ret.sptk.many b0 -# 49 "<stdin>" - .endp __TBB_machine_fetchadd2acquire# -# 62 "<stdin>" - .section .text - .align 16 - .proc __TBB_machine_fetchstore2__TBB_full_fence# - .global __TBB_machine_fetchstore2__TBB_full_fence# -__TBB_machine_fetchstore2__TBB_full_fence: - mf -;; - xchg2 r8=[r32],r33 - br.ret.sptk.many b0 - .endp __TBB_machine_fetchstore2__TBB_full_fence# - - - .proc __TBB_machine_fetchstore2acquire# - .global __TBB_machine_fetchstore2acquire# -__TBB_machine_fetchstore2acquire: - xchg2 r8=[r32],r33 - br.ret.sptk.many b0 - .endp __TBB_machine_fetchstore2acquire# -# 88 "<stdin>" - .section .text - .align 16 - - - .proc __TBB_machine_cmpswp2__TBB_full_fence# - .global __TBB_machine_cmpswp2__TBB_full_fence# -__TBB_machine_cmpswp2__TBB_full_fence: -{ - mf - br __TBB_machine_cmpswp2acquire -} - .endp __TBB_machine_cmpswp2__TBB_full_fence# - - .proc __TBB_machine_cmpswp2acquire# - .global __TBB_machine_cmpswp2acquire# -__TBB_machine_cmpswp2acquire: - - zxt2 r34=r34 -;; - - mov ar.ccv=r34 -;; - cmpxchg2.acq r8=[r32],r33,ar.ccv - br.ret.sptk.many b0 - .endp __TBB_machine_cmpswp2acquire# -// DO NOT EDIT - AUTOMATICALLY GENERATED FROM tools/generate_atomic/ipf_generate.sh -# 1 "<stdin>" -# 1 "<built-in>" -# 1 "<command line>" -# 1 "<stdin>" - - - - - - .section .text - .align 16 - - - .proc __TBB_machine_fetchadd4__TBB_full_fence# - .global __TBB_machine_fetchadd4__TBB_full_fence# -__TBB_machine_fetchadd4__TBB_full_fence: -{ - mf - br __TBB_machine_fetchadd4acquire -} - .endp __TBB_machine_fetchadd4__TBB_full_fence# - - .proc __TBB_machine_fetchadd4acquire# - .global __TBB_machine_fetchadd4acquire# -__TBB_machine_fetchadd4acquire: - - cmp.eq p6,p0=1,r33 - cmp.eq p8,p0=-1,r33 - (p6) br.cond.dptk Inc_4acquire - (p8) br.cond.dpnt Dec_4acquire -;; - - ld4 r9=[r32] -;; -Retry_4acquire: - mov ar.ccv=r9 - mov r8=r9; - add r10=r9,r33 -;; - cmpxchg4.acq r9=[r32],r10,ar.ccv -;; - cmp.ne p7,p0=r8,r9 - (p7) br.cond.dpnt Retry_4acquire - br.ret.sptk.many b0 - -Inc_4acquire: - fetchadd4.acq r8=[r32],1 - br.ret.sptk.many b0 -Dec_4acquire: - fetchadd4.acq r8=[r32],-1 - br.ret.sptk.many b0 - - .endp __TBB_machine_fetchadd4acquire# -# 62 "<stdin>" - .section .text - .align 16 - .proc __TBB_machine_fetchstore4__TBB_full_fence# - .global __TBB_machine_fetchstore4__TBB_full_fence# -__TBB_machine_fetchstore4__TBB_full_fence: - mf -;; - xchg4 r8=[r32],r33 - br.ret.sptk.many b0 - .endp __TBB_machine_fetchstore4__TBB_full_fence# - - - .proc __TBB_machine_fetchstore4acquire# - .global __TBB_machine_fetchstore4acquire# -__TBB_machine_fetchstore4acquire: - xchg4 r8=[r32],r33 - br.ret.sptk.many b0 - .endp __TBB_machine_fetchstore4acquire# -# 88 "<stdin>" - .section .text - .align 16 - - - .proc __TBB_machine_cmpswp4__TBB_full_fence# - .global __TBB_machine_cmpswp4__TBB_full_fence# -__TBB_machine_cmpswp4__TBB_full_fence: -{ - mf - br __TBB_machine_cmpswp4acquire -} - .endp __TBB_machine_cmpswp4__TBB_full_fence# - - .proc __TBB_machine_cmpswp4acquire# - .global __TBB_machine_cmpswp4acquire# -__TBB_machine_cmpswp4acquire: - - zxt4 r34=r34 -;; - - mov ar.ccv=r34 -;; - cmpxchg4.acq r8=[r32],r33,ar.ccv - br.ret.sptk.many b0 - .endp __TBB_machine_cmpswp4acquire# -// DO NOT EDIT - AUTOMATICALLY GENERATED FROM tools/generate_atomic/ipf_generate.sh -# 1 "<stdin>" -# 1 "<built-in>" -# 1 "<command line>" -# 1 "<stdin>" - - - - - - .section .text - .align 16 - - - .proc __TBB_machine_fetchadd8__TBB_full_fence# - .global __TBB_machine_fetchadd8__TBB_full_fence# -__TBB_machine_fetchadd8__TBB_full_fence: -{ - mf - br __TBB_machine_fetchadd8acquire -} - .endp __TBB_machine_fetchadd8__TBB_full_fence# - - .proc __TBB_machine_fetchadd8acquire# - .global __TBB_machine_fetchadd8acquire# -__TBB_machine_fetchadd8acquire: - - cmp.eq p6,p0=1,r33 - cmp.eq p8,p0=-1,r33 - (p6) br.cond.dptk Inc_8acquire - (p8) br.cond.dpnt Dec_8acquire -;; - - ld8 r9=[r32] -;; -Retry_8acquire: - mov ar.ccv=r9 - mov r8=r9; - add r10=r9,r33 -;; - cmpxchg8.acq r9=[r32],r10,ar.ccv -;; - cmp.ne p7,p0=r8,r9 - (p7) br.cond.dpnt Retry_8acquire - br.ret.sptk.many b0 - -Inc_8acquire: - fetchadd8.acq r8=[r32],1 - br.ret.sptk.many b0 -Dec_8acquire: - fetchadd8.acq r8=[r32],-1 - br.ret.sptk.many b0 - - .endp __TBB_machine_fetchadd8acquire# -# 62 "<stdin>" - .section .text - .align 16 - .proc __TBB_machine_fetchstore8__TBB_full_fence# - .global __TBB_machine_fetchstore8__TBB_full_fence# -__TBB_machine_fetchstore8__TBB_full_fence: - mf -;; - xchg8 r8=[r32],r33 - br.ret.sptk.many b0 - .endp __TBB_machine_fetchstore8__TBB_full_fence# - - - .proc __TBB_machine_fetchstore8acquire# - .global __TBB_machine_fetchstore8acquire# -__TBB_machine_fetchstore8acquire: - xchg8 r8=[r32],r33 - br.ret.sptk.many b0 - .endp __TBB_machine_fetchstore8acquire# -# 88 "<stdin>" - .section .text - .align 16 - - - .proc __TBB_machine_cmpswp8__TBB_full_fence# - .global __TBB_machine_cmpswp8__TBB_full_fence# -__TBB_machine_cmpswp8__TBB_full_fence: -{ - mf - br __TBB_machine_cmpswp8acquire -} - .endp __TBB_machine_cmpswp8__TBB_full_fence# - - .proc __TBB_machine_cmpswp8acquire# - .global __TBB_machine_cmpswp8acquire# -__TBB_machine_cmpswp8acquire: - - - - - mov ar.ccv=r34 -;; - cmpxchg8.acq r8=[r32],r33,ar.ccv - br.ret.sptk.many b0 - .endp __TBB_machine_cmpswp8acquire# -// DO NOT EDIT - AUTOMATICALLY GENERATED FROM tools/generate_atomic/ipf_generate.sh -# 1 "<stdin>" -# 1 "<built-in>" -# 1 "<command line>" -# 1 "<stdin>" - - - - - - .section .text - .align 16 -# 19 "<stdin>" - .proc __TBB_machine_fetchadd1release# - .global __TBB_machine_fetchadd1release# -__TBB_machine_fetchadd1release: - - - - - - - - ld1 r9=[r32] -;; -Retry_1release: - mov ar.ccv=r9 - mov r8=r9; - add r10=r9,r33 -;; - cmpxchg1.rel r9=[r32],r10,ar.ccv -;; - cmp.ne p7,p0=r8,r9 - (p7) br.cond.dpnt Retry_1release - br.ret.sptk.many b0 -# 49 "<stdin>" - .endp __TBB_machine_fetchadd1release# -# 62 "<stdin>" - .section .text - .align 16 - .proc __TBB_machine_fetchstore1release# - .global __TBB_machine_fetchstore1release# -__TBB_machine_fetchstore1release: - mf -;; - xchg1 r8=[r32],r33 - br.ret.sptk.many b0 - .endp __TBB_machine_fetchstore1release# -# 88 "<stdin>" - .section .text - .align 16 -# 101 "<stdin>" - .proc __TBB_machine_cmpswp1release# - .global __TBB_machine_cmpswp1release# -__TBB_machine_cmpswp1release: - - zxt1 r34=r34 -;; - - mov ar.ccv=r34 -;; - cmpxchg1.rel r8=[r32],r33,ar.ccv - br.ret.sptk.many b0 - .endp __TBB_machine_cmpswp1release# -// DO NOT EDIT - AUTOMATICALLY GENERATED FROM tools/generate_atomic/ipf_generate.sh -# 1 "<stdin>" -# 1 "<built-in>" -# 1 "<command line>" -# 1 "<stdin>" - - - - - - .section .text - .align 16 -# 19 "<stdin>" - .proc __TBB_machine_fetchadd2release# - .global __TBB_machine_fetchadd2release# -__TBB_machine_fetchadd2release: - - - - - - - - ld2 r9=[r32] -;; -Retry_2release: - mov ar.ccv=r9 - mov r8=r9; - add r10=r9,r33 -;; - cmpxchg2.rel r9=[r32],r10,ar.ccv -;; - cmp.ne p7,p0=r8,r9 - (p7) br.cond.dpnt Retry_2release - br.ret.sptk.many b0 -# 49 "<stdin>" - .endp __TBB_machine_fetchadd2release# -# 62 "<stdin>" - .section .text - .align 16 - .proc __TBB_machine_fetchstore2release# - .global __TBB_machine_fetchstore2release# -__TBB_machine_fetchstore2release: - mf -;; - xchg2 r8=[r32],r33 - br.ret.sptk.many b0 - .endp __TBB_machine_fetchstore2release# -# 88 "<stdin>" - .section .text - .align 16 -# 101 "<stdin>" - .proc __TBB_machine_cmpswp2release# - .global __TBB_machine_cmpswp2release# -__TBB_machine_cmpswp2release: - - zxt2 r34=r34 -;; - - mov ar.ccv=r34 -;; - cmpxchg2.rel r8=[r32],r33,ar.ccv - br.ret.sptk.many b0 - .endp __TBB_machine_cmpswp2release# -// DO NOT EDIT - AUTOMATICALLY GENERATED FROM tools/generate_atomic/ipf_generate.sh -# 1 "<stdin>" -# 1 "<built-in>" -# 1 "<command line>" -# 1 "<stdin>" - - - - - - .section .text - .align 16 -# 19 "<stdin>" - .proc __TBB_machine_fetchadd4release# - .global __TBB_machine_fetchadd4release# -__TBB_machine_fetchadd4release: - - cmp.eq p6,p0=1,r33 - cmp.eq p8,p0=-1,r33 - (p6) br.cond.dptk Inc_4release - (p8) br.cond.dpnt Dec_4release -;; - - ld4 r9=[r32] -;; -Retry_4release: - mov ar.ccv=r9 - mov r8=r9; - add r10=r9,r33 -;; - cmpxchg4.rel r9=[r32],r10,ar.ccv -;; - cmp.ne p7,p0=r8,r9 - (p7) br.cond.dpnt Retry_4release - br.ret.sptk.many b0 - -Inc_4release: - fetchadd4.rel r8=[r32],1 - br.ret.sptk.many b0 -Dec_4release: - fetchadd4.rel r8=[r32],-1 - br.ret.sptk.many b0 - - .endp __TBB_machine_fetchadd4release# -# 62 "<stdin>" - .section .text - .align 16 - .proc __TBB_machine_fetchstore4release# - .global __TBB_machine_fetchstore4release# -__TBB_machine_fetchstore4release: - mf -;; - xchg4 r8=[r32],r33 - br.ret.sptk.many b0 - .endp __TBB_machine_fetchstore4release# -# 88 "<stdin>" - .section .text - .align 16 -# 101 "<stdin>" - .proc __TBB_machine_cmpswp4release# - .global __TBB_machine_cmpswp4release# -__TBB_machine_cmpswp4release: - - zxt4 r34=r34 -;; - - mov ar.ccv=r34 -;; - cmpxchg4.rel r8=[r32],r33,ar.ccv - br.ret.sptk.many b0 - .endp __TBB_machine_cmpswp4release# -// DO NOT EDIT - AUTOMATICALLY GENERATED FROM tools/generate_atomic/ipf_generate.sh -# 1 "<stdin>" -# 1 "<built-in>" -# 1 "<command line>" -# 1 "<stdin>" - - - - - - .section .text - .align 16 -# 19 "<stdin>" - .proc __TBB_machine_fetchadd8release# - .global __TBB_machine_fetchadd8release# -__TBB_machine_fetchadd8release: - - cmp.eq p6,p0=1,r33 - cmp.eq p8,p0=-1,r33 - (p6) br.cond.dptk Inc_8release - (p8) br.cond.dpnt Dec_8release -;; - - ld8 r9=[r32] -;; -Retry_8release: - mov ar.ccv=r9 - mov r8=r9; - add r10=r9,r33 -;; - cmpxchg8.rel r9=[r32],r10,ar.ccv -;; - cmp.ne p7,p0=r8,r9 - (p7) br.cond.dpnt Retry_8release - br.ret.sptk.many b0 - -Inc_8release: - fetchadd8.rel r8=[r32],1 - br.ret.sptk.many b0 -Dec_8release: - fetchadd8.rel r8=[r32],-1 - br.ret.sptk.many b0 - - .endp __TBB_machine_fetchadd8release# -# 62 "<stdin>" - .section .text - .align 16 - .proc __TBB_machine_fetchstore8release# - .global __TBB_machine_fetchstore8release# -__TBB_machine_fetchstore8release: - mf -;; - xchg8 r8=[r32],r33 - br.ret.sptk.many b0 - .endp __TBB_machine_fetchstore8release# -# 88 "<stdin>" - .section .text - .align 16 -# 101 "<stdin>" - .proc __TBB_machine_cmpswp8release# - .global __TBB_machine_cmpswp8release# -__TBB_machine_cmpswp8release: - - - - - mov ar.ccv=r34 -;; - cmpxchg8.rel r8=[r32],r33,ar.ccv - br.ret.sptk.many b0 - .endp __TBB_machine_cmpswp8release# diff --git a/src/tbb/src/tbb/ia64-gas/ia64_misc.s b/src/tbb/src/tbb/ia64-gas/ia64_misc.s deleted file mode 100644 index b62ce393f..000000000 --- a/src/tbb/src/tbb/ia64-gas/ia64_misc.s +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright (c) 2005-2017 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// -// -// - - // RSE backing store pointer retrieval - .section .text - .align 16 - .proc __TBB_get_bsp# - .global __TBB_get_bsp# -__TBB_get_bsp: - mov r8=ar.bsp - br.ret.sptk.many b0 - .endp __TBB_get_bsp# - - .section .text - .align 16 - .proc __TBB_machine_load8_relaxed# - .global __TBB_machine_load8_relaxed# -__TBB_machine_load8_relaxed: - ld8 r8=[r32] - br.ret.sptk.many b0 - .endp __TBB_machine_load8_relaxed# - - .section .text - .align 16 - .proc __TBB_machine_store8_relaxed# - .global __TBB_machine_store8_relaxed# -__TBB_machine_store8_relaxed: - st8 [r32]=r33 - br.ret.sptk.many b0 - .endp __TBB_machine_store8_relaxed# - - .section .text - .align 16 - .proc __TBB_machine_load4_relaxed# - .global __TBB_machine_load4_relaxed# -__TBB_machine_load4_relaxed: - ld4 r8=[r32] - br.ret.sptk.many b0 - .endp __TBB_machine_load4_relaxed# - - .section .text - .align 16 - .proc __TBB_machine_store4_relaxed# - .global __TBB_machine_store4_relaxed# -__TBB_machine_store4_relaxed: - st4 [r32]=r33 - br.ret.sptk.many b0 - .endp __TBB_machine_store4_relaxed# - - .section .text - .align 16 - .proc __TBB_machine_load2_relaxed# - .global __TBB_machine_load2_relaxed# -__TBB_machine_load2_relaxed: - ld2 r8=[r32] - br.ret.sptk.many b0 - .endp __TBB_machine_load2_relaxed# - - .section .text - .align 16 - .proc __TBB_machine_store2_relaxed# - .global __TBB_machine_store2_relaxed# -__TBB_machine_store2_relaxed: - st2 [r32]=r33 - br.ret.sptk.many b0 - .endp __TBB_machine_store2_relaxed# - - .section .text - .align 16 - .proc __TBB_machine_load1_relaxed# - .global __TBB_machine_load1_relaxed# -__TBB_machine_load1_relaxed: - ld1 r8=[r32] - br.ret.sptk.many b0 - .endp __TBB_machine_load1_relaxed# - - .section .text - .align 16 - .proc __TBB_machine_store1_relaxed# - .global __TBB_machine_store1_relaxed# -__TBB_machine_store1_relaxed: - st1 [r32]=r33 - br.ret.sptk.many b0 - .endp __TBB_machine_store1_relaxed# diff --git a/src/tbb/src/tbb/ia64-gas/lock_byte.s b/src/tbb/src/tbb/ia64-gas/lock_byte.s deleted file mode 100644 index 551ac3303..000000000 --- a/src/tbb/src/tbb/ia64-gas/lock_byte.s +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright (c) 2005-2017 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// -// -// - - // Support for class TinyLock - .section .text - .align 16 - // unsigned int __TBB_machine_trylockbyte( byte& flag ); - // r32 = address of flag - .proc __TBB_machine_trylockbyte# - .global __TBB_machine_trylockbyte# -ADDRESS_OF_FLAG=r32 -RETCODE=r8 -FLAG=r9 -BUSY=r10 -SCRATCH=r11 -__TBB_machine_trylockbyte: - ld1.acq FLAG=[ADDRESS_OF_FLAG] - mov BUSY=1 - mov RETCODE=0 -;; - cmp.ne p6,p0=0,FLAG - mov ar.ccv=r0 -(p6) br.ret.sptk.many b0 -;; - cmpxchg1.acq SCRATCH=[ADDRESS_OF_FLAG],BUSY,ar.ccv // Try to acquire lock -;; - cmp.eq p6,p0=0,SCRATCH -;; -(p6) mov RETCODE=1 - br.ret.sptk.many b0 - .endp __TBB_machine_trylockbyte# diff --git a/src/tbb/src/tbb/ia64-gas/log2.s b/src/tbb/src/tbb/ia64-gas/log2.s deleted file mode 100644 index d4ad9e2bc..000000000 --- a/src/tbb/src/tbb/ia64-gas/log2.s +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright (c) 2005-2017 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// -// -// - - .section .text - .align 16 - // unsigned long __TBB_machine_lg( unsigned long x ); - // r32 = x - .proc __TBB_machine_lg# - .global __TBB_machine_lg# -__TBB_machine_lg: - shr r16=r32,1 // .x -;; - shr r17=r32,2 // ..x - or r32=r32,r16 // xx -;; - shr r16=r32,3 // ...xx - or r32=r32,r17 // xxx -;; - shr r17=r32,5 // .....xxx - or r32=r32,r16 // xxxxx -;; - shr r16=r32,8 // ........xxxxx - or r32=r32,r17 // xxxxxxxx -;; - shr r17=r32,13 - or r32=r32,r16 // 13x -;; - shr r16=r32,21 - or r32=r32,r17 // 21x -;; - shr r17=r32,34 - or r32=r32,r16 // 34x -;; - shr r16=r32,55 - or r32=r32,r17 // 55x -;; - or r32=r32,r16 // 64x -;; - popcnt r8=r32 -;; - add r8=-1,r8 - br.ret.sptk.many b0 - .endp __TBB_machine_lg# diff --git a/src/tbb/src/tbb/ia64-gas/pause.s b/src/tbb/src/tbb/ia64-gas/pause.s deleted file mode 100644 index a22b4203f..000000000 --- a/src/tbb/src/tbb/ia64-gas/pause.s +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright (c) 2005-2017 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// -// -// - - .section .text - .align 16 - // void __TBB_machine_pause( long count ); - // r32 = count - .proc __TBB_machine_pause# - .global __TBB_machine_pause# -count = r32 -__TBB_machine_pause: - hint.m 0 - add count=-1,count -;; - cmp.eq p6,p7=0,count -(p7) br.cond.dpnt __TBB_machine_pause -(p6) br.ret.sptk.many b0 - .endp __TBB_machine_pause# diff --git a/src/tbb/src/tbb/ibm_aix51/atomic_support.c b/src/tbb/src/tbb/ibm_aix51/atomic_support.c deleted file mode 100644 index 0aa08a1e6..000000000 --- a/src/tbb/src/tbb/ibm_aix51/atomic_support.c +++ /dev/null @@ -1,55 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#include <stdint.h> -#include <sys/atomic_op.h> - -/* This file must be compiled with gcc. The IBM compiler doesn't seem to - support inline assembly statements (October 2007). */ - -#ifdef __GNUC__ - -int32_t __TBB_machine_cas_32 (volatile void* ptr, int32_t value, int32_t comparand) { - __asm__ __volatile__ ("sync\n"); /* memory release operation */ - compare_and_swap ((atomic_p) ptr, &comparand, value); - __asm__ __volatile__ ("isync\n"); /* memory acquire operation */ - return comparand; -} - -int64_t __TBB_machine_cas_64 (volatile void* ptr, int64_t value, int64_t comparand) { - __asm__ __volatile__ ("sync\n"); /* memory release operation */ - compare_and_swaplp ((atomic_l) ptr, &comparand, value); - __asm__ __volatile__ ("isync\n"); /* memory acquire operation */ - return comparand; -} - -void __TBB_machine_flush () { - __asm__ __volatile__ ("sync\n"); -} - -void __TBB_machine_lwsync () { - __asm__ __volatile__ ("lwsync\n"); -} - -void __TBB_machine_isync () { - __asm__ __volatile__ ("isync\n"); -} - -#endif /* __GNUC__ */ diff --git a/src/tbb/src/tbb/index.html b/src/tbb/src/tbb/index.html deleted file mode 100644 index 9ba7f57ef..000000000 --- a/src/tbb/src/tbb/index.html +++ /dev/null @@ -1,31 +0,0 @@ -<HTML> -<BODY> - -<H2>Overview</H2> -This directory contains the source code of the TBB core components. - -<H2>Directories</H2> -<DL> -<DT><A HREF="tools_api">tools_api</A> -<DD>Source code of the interface components provided by the Intel® Parallel Studio tools. -<DT><A HREF="intel64-masm">intel64-masm</A> -<DD>Assembly code for the Intel® 64 architecture. -<DT><A HREF="ia32-masm">ia32-masm</A> -<DD>Assembly code for IA32 architecture. -<DT><A HREF="ia64-gas">ia64-gas</A> -<DD>Assembly code for IA-64 architecture. -<DT><A HREF="ibm_aix51">ibm_aix51</A> -<DD>Assembly code for AIX 5.1 port. -</DL> - -<HR> -<A HREF="../index.html">Up to parent directory</A> -<p></p> -Copyright © 2005-2017 Intel Corporation. All Rights Reserved. -<P></P> -Intel is a registered trademark or trademark of Intel Corporation -or its subsidiaries in the United States and other countries. -<p></p> -* Other names and brands may be claimed as the property of others. -</BODY> -</HTML> diff --git a/src/tbb/src/tbb/intel64-masm/atomic_support.asm b/src/tbb/src/tbb/intel64-masm/atomic_support.asm deleted file mode 100644 index d3c7f6984..000000000 --- a/src/tbb/src/tbb/intel64-masm/atomic_support.asm +++ /dev/null @@ -1,72 +0,0 @@ -; Copyright (c) 2005-2017 Intel Corporation -; -; Licensed under the Apache License, Version 2.0 (the "License"); -; you may not use this file except in compliance with the License. -; You may obtain a copy of the License at -; -; http://www.apache.org/licenses/LICENSE-2.0 -; -; Unless required by applicable law or agreed to in writing, software -; distributed under the License is distributed on an "AS IS" BASIS, -; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -; See the License for the specific language governing permissions and -; limitations under the License. -; -; -; -; - -; DO NOT EDIT - AUTOMATICALLY GENERATED FROM .s FILE -.code - ALIGN 8 - PUBLIC __TBB_machine_fetchadd1 -__TBB_machine_fetchadd1: - mov rax,rdx - lock xadd [rcx],al - ret -.code - ALIGN 8 - PUBLIC __TBB_machine_fetchstore1 -__TBB_machine_fetchstore1: - mov rax,rdx - lock xchg [rcx],al - ret -.code - ALIGN 8 - PUBLIC __TBB_machine_cmpswp1 -__TBB_machine_cmpswp1: - mov rax,r8 - lock cmpxchg [rcx],dl - ret -.code - ALIGN 8 - PUBLIC __TBB_machine_fetchadd2 -__TBB_machine_fetchadd2: - mov rax,rdx - lock xadd [rcx],ax - ret -.code - ALIGN 8 - PUBLIC __TBB_machine_fetchstore2 -__TBB_machine_fetchstore2: - mov rax,rdx - lock xchg [rcx],ax - ret -.code - ALIGN 8 - PUBLIC __TBB_machine_cmpswp2 -__TBB_machine_cmpswp2: - mov rax,r8 - lock cmpxchg [rcx],dx - ret -.code - ALIGN 8 - PUBLIC __TBB_machine_pause -__TBB_machine_pause: -L1: - dw 090f3H; pause - add ecx,-1 - jne L1 - ret -end - diff --git a/src/tbb/src/tbb/intel64-masm/intel64_misc.asm b/src/tbb/src/tbb/intel64-masm/intel64_misc.asm deleted file mode 100644 index 4fe751529..000000000 --- a/src/tbb/src/tbb/intel64-masm/intel64_misc.asm +++ /dev/null @@ -1,33 +0,0 @@ -; Copyright (c) 2005-2017 Intel Corporation -; -; Licensed under the Apache License, Version 2.0 (the "License"); -; you may not use this file except in compliance with the License. -; You may obtain a copy of the License at -; -; http://www.apache.org/licenses/LICENSE-2.0 -; -; Unless required by applicable law or agreed to in writing, software -; distributed under the License is distributed on an "AS IS" BASIS, -; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -; See the License for the specific language governing permissions and -; limitations under the License. -; -; -; -; - -.code - ALIGN 8 - PUBLIC __TBB_get_cpu_ctl_env -__TBB_get_cpu_ctl_env: - stmxcsr [rcx] - fstcw [rcx+4] - ret -.code - ALIGN 8 - PUBLIC __TBB_set_cpu_ctl_env -__TBB_set_cpu_ctl_env: - ldmxcsr [rcx] - fldcw [rcx+4] - ret -end diff --git a/src/tbb/src/tbb/intel64-masm/itsx.asm b/src/tbb/src/tbb/intel64-masm/itsx.asm deleted file mode 100644 index 1a5958fb2..000000000 --- a/src/tbb/src/tbb/intel64-masm/itsx.asm +++ /dev/null @@ -1,76 +0,0 @@ -; Copyright (c) 2005-2017 Intel Corporation -; -; Licensed under the Apache License, Version 2.0 (the "License"); -; you may not use this file except in compliance with the License. -; You may obtain a copy of the License at -; -; http://www.apache.org/licenses/LICENSE-2.0 -; -; Unless required by applicable law or agreed to in writing, software -; distributed under the License is distributed on an "AS IS" BASIS, -; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -; See the License for the specific language governing permissions and -; limitations under the License. -; -; -; -; - -.code - ALIGN 8 - PUBLIC __TBB_machine_try_lock_elided -__TBB_machine_try_lock_elided: - xor rax, rax - mov al, 1 - BYTE 0F2H - xchg al, byte ptr [rcx] - xor al, 1 - ret -.code - ALIGN 8 - PUBLIC __TBB_machine_unlock_elided -__TBB_machine_unlock_elided: - BYTE 0F3H - mov byte ptr [rcx], 0 - ret -.code - ALIGN 8 - PUBLIC __TBB_machine_begin_transaction -__TBB_machine_begin_transaction: - mov eax, -1 - BYTE 0C7H - BYTE 0F8H - BYTE 000H - BYTE 000H - BYTE 000H - BYTE 000H - ret -.code - ALIGN 8 - PUBLIC __TBB_machine_end_transaction -__TBB_machine_end_transaction: - BYTE 00FH - BYTE 001H - BYTE 0D5H - ret -.code - ALIGN 8 - PUBLIC __TBB_machine_transaction_conflict_abort -__TBB_machine_transaction_conflict_abort: - BYTE 0C6H - BYTE 0F8H - BYTE 0FFH ; 12.4.5 Abort argument: lock not free when tested - ret -.code - ALIGN 8 - PUBLIC __TBB_machine_is_in_transaction -__TBB_machine_is_in_transaction: - xor eax, eax - BYTE 00FH ; _xtest sets or clears ZF - BYTE 001H - BYTE 0D6H - jz rset - mov al,1 -rset: - ret -end diff --git a/src/tbb/src/tbb/intrusive_list.h b/src/tbb/src/tbb/intrusive_list.h index cb2190aff..d317f5554 100644 --- a/src/tbb/src/tbb/intrusive_list.h +++ b/src/tbb/src/tbb/intrusive_list.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,32 +12,18 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef _TBB_intrusive_list_H #define _TBB_intrusive_list_H -#include "tbb/tbb_stddef.h" +#include "oneapi/tbb/detail/_intrusive_list_node.h" namespace tbb { -namespace internal { - -//! Data structure to be inherited by the types that can form intrusive lists. -/** Intrusive list is formed by means of the member_intrusive_list<T> template class. - Note that type T must derive from intrusive_list_node either publicly or - declare instantiation member_intrusive_list<T> as a friend. - This class implements a limited subset of std::list interface. **/ -struct intrusive_list_node { - intrusive_list_node *my_prev_node, - *my_next_node; -#if TBB_USE_ASSERT - intrusive_list_node () { my_prev_node = my_next_node = this; } -#endif /* TBB_USE_ASSERT */ -}; +namespace detail { +namespace r1 { + +using d1::intrusive_list_node; //! List of element of type T, where T is derived from intrusive_list_node /** The class is not thread safe. **/ @@ -47,75 +33,76 @@ class intrusive_list_base { intrusive_list_node my_head; //! Number of list elements - size_t my_size; + std::size_t my_size; static intrusive_list_node& node ( T& item ) { return List::node(item); } static T& item ( intrusive_list_node* node ) { return List::item(node); } - template<class Iterator> - class iterator_impl { - Iterator& self () { return *static_cast<Iterator*>(this); } + static const T& item( const intrusive_list_node* node ) { return List::item(node); } - //! Node the iterator points to at the moment - intrusive_list_node *my_pos; + template <typename DereferenceType> + class iterator_impl { + static_assert(std::is_same<DereferenceType, T>::value || + std::is_same<DereferenceType, const T>::value, + "Incorrect DereferenceType in iterator_impl"); - protected: - iterator_impl (intrusive_list_node* pos ) - : my_pos(pos) - {} - - T& item () const { - return intrusive_list_base::item(my_pos); - } + using pointer_type = typename std::conditional<std::is_same<DereferenceType, T>::value, + intrusive_list_node*, + const intrusive_list_node*>::type; public: - iterator_impl () : my_pos(NULL) {} + iterator_impl() : my_pos(nullptr) {} - Iterator& operator = ( const Iterator& it ) { - return my_pos = it.my_pos; + iterator_impl( pointer_type pos ) : my_pos(pos) {} + + iterator_impl& operator++() { + my_pos = my_pos->my_next_node; + return *this; } - Iterator& operator = ( const T& val ) { - return my_pos = &node(val); + iterator_impl operator++( int ) { + iterator_impl it(*this); + ++*this; + return it; } - bool operator == ( const Iterator& it ) const { - return my_pos == it.my_pos; + iterator_impl& operator--() { + my_pos = my_pos->my_prev_node; + return *this; } - bool operator != ( const Iterator& it ) const { - return my_pos != it.my_pos; + iterator_impl operator--( int ) { + iterator_impl it(*this); + --*this; + return it; } - Iterator& operator++ () { - my_pos = my_pos->my_next_node; - return self(); + bool operator==( const iterator_impl& rhs ) const { + return my_pos == rhs.my_pos; } - Iterator& operator-- () { - my_pos = my_pos->my_prev_node; - return self(); + bool operator!=( const iterator_impl& rhs ) const { + return my_pos != rhs.my_pos; } - Iterator operator++ ( int ) { - Iterator result = self(); - ++(*this); - return result; + DereferenceType& operator*() const { + return intrusive_list_base::item(my_pos); } - Iterator operator-- ( int ) { - Iterator result = self(); - --(*this); - return result; + DereferenceType* operator->() const { + return &intrusive_list_base::item(my_pos); } - }; // intrusive_list_base::iterator_impl + private: + // Node the iterator points to at the moment + pointer_type my_pos; + }; // class iterator_impl void assert_ok () const { __TBB_ASSERT( (my_head.my_prev_node == &my_head && !my_size) || (my_head.my_next_node != &my_head && my_size >0), "intrusive_list_base corrupted" ); #if TBB_USE_ASSERT >= 2 - size_t i = 0; + std::size_t i = 0; for ( intrusive_list_node *n = my_head.my_next_node; n != &my_head; n = n->my_next_node ) ++i; __TBB_ASSERT( my_size == i, "Wrong size" ); @@ -123,31 +110,8 @@ class intrusive_list_base { } public: - class iterator : public iterator_impl<iterator> { - template <class U, class V> friend class intrusive_list_base; - public: - iterator (intrusive_list_node* pos ) - : iterator_impl<iterator>(pos ) - {} - iterator () {} - - T* operator-> () const { return &this->item(); } - - T& operator* () const { return this->item(); } - }; // class iterator - - class const_iterator : public iterator_impl<const_iterator> { - template <class U, class V> friend class intrusive_list_base; - public: - const_iterator (const intrusive_list_node* pos ) - : iterator_impl<const_iterator>(const_cast<intrusive_list_node*>(pos) ) - {} - const_iterator () {} - - const T* operator-> () const { return &this->item(); } - - const T& operator* () const { return this->item(); } - }; // class iterator + using iterator = iterator_impl<T>; + using const_iterator = iterator_impl<const T>; intrusive_list_base () : my_size(0) { my_head.my_prev_node = &my_head; @@ -156,7 +120,7 @@ class intrusive_list_base { bool empty () const { return my_head.my_next_node == &my_head; } - size_t size () const { return my_size; } + std::size_t size () const { return my_size; } iterator begin () { return iterator(my_head.my_next_node); } @@ -199,6 +163,21 @@ class intrusive_list_base { }; // intrusive_list_base +#if __TBB_TODO +// With standard compliant compilers memptr_intrusive_list could be named simply intrusive_list, +// and inheritance based intrusive_list version would become its partial specialization. +// Here are the corresponding declarations: + +struct dummy_intrusive_list_item { intrusive_list_node my_node; }; + +template <class T, class U = dummy_intrusive_list_item, intrusive_list_node U::*NodePtr = &dummy_intrusive_list_item::my_node> +class intrusive_list : public intrusive_list_base<intrusive_list<T, U, NodePtr>, T>; + +template <class T> +class intrusive_list<T, dummy_intrusive_list_item, &dummy_intrusive_list_item::my_node> + : public intrusive_list_base<intrusive_list<T>, T>; + +#endif /* __TBB_TODO */ //! Double linked list of items of type T containing a member of type intrusive_list_node. /** NodePtr is a member pointer to the node data field. Class U is either T or @@ -223,6 +202,11 @@ class memptr_intrusive_list : public intrusive_list_base<memptr_intrusive_list<T // __TBB_offsetof implementation breaks operations with normal member names. return *reinterpret_cast<T*>((char*)node - ((ptrdiff_t)&(reinterpret_cast<T*>(0x1000)->*NodePtr) - 0x1000)); } + + static const T& item( const intrusive_list_node* node ) { + return item(const_cast<intrusive_list_node*>(node)); + } + }; // intrusive_list<T, U, NodePtr> //! Double linked list of items of type T that is derived from intrusive_list_node class. @@ -238,9 +222,12 @@ class intrusive_list : public intrusive_list_base<intrusive_list<T>, T> static intrusive_list_node& node ( T& val ) { return val; } static T& item ( intrusive_list_node* node ) { return *static_cast<T*>(node); } + + static const T& item( const intrusive_list_node* node ) { return *static_cast<const T*>(node); } }; // intrusive_list<T> -} // namespace internal +} // namespace r1 +} // namespace detail } // namespace tbb #endif /* _TBB_intrusive_list_H */ diff --git a/src/tbb/src/tbb/itt_notify.cpp b/src/tbb/src/tbb/itt_notify.cpp index 670c84d9a..eda5e6ad5 100644 --- a/src/tbb/src/tbb/itt_notify.cpp +++ b/src/tbb/src/tbb/itt_notify.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2022 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,13 +12,9 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ -#if DO_ITT_NOTIFY +#if __TBB_USE_ITT_NOTIFY #if _WIN32||_WIN64 #ifndef UNICODE @@ -47,52 +43,27 @@ extern "C" void MallocInitializeITT(); #include "tools_api/ittnotify_static.c" namespace tbb { -namespace internal { +namespace detail { +namespace r1 { + +/** This extra proxy method is necessary since __itt_init_lib is declared as static **/ int __TBB_load_ittnotify() { #if !(_WIN32||_WIN64) // tool_api crashes without dlopen, check that it's present. Common case // for lack of dlopen is static binaries, i.e. ones build with -static. - if (dlopen == NULL) + if (dlopen == nullptr) return 0; #endif - return __itt_init_ittlib(NULL, // groups for: + return __itt_init_ittlib(nullptr, // groups for: (__itt_group_id)(__itt_group_sync // prepare/cancel/acquired/releasing | __itt_group_thread // name threads | __itt_group_stitch // stack stitching -#if __TBB_CPF_BUILD | __itt_group_structure -#endif )); } -}} // namespaces - -#endif /* DO_ITT_NOTIFY */ - -#define __TBB_NO_IMPLICIT_LINKAGE 1 -#include "itt_notify.h" - -namespace tbb { - -#if DO_ITT_NOTIFY - const tchar - *SyncType_GlobalLock = _T("TbbGlobalLock"), - *SyncType_Scheduler = _T("%Constant") - ; - const tchar - *SyncObj_SchedulerInitialization = _T("TbbSchedulerInitialization"), - *SyncObj_SchedulersList = _T("TbbSchedulersList"), - *SyncObj_WorkerLifeCycleMgmt = _T("TBB Scheduler"), - *SyncObj_TaskStealingLoop = _T("TBB Scheduler"), - *SyncObj_WorkerTaskPool = _T("TBB Scheduler"), - *SyncObj_MasterTaskPool = _T("TBB Scheduler"), - *SyncObj_TaskPoolSpinning = _T("TBB Scheduler"), - *SyncObj_Mailbox = _T("TBB Scheduler"), - *SyncObj_TaskReturnList = _T("TBB Scheduler"), - *SyncObj_TaskStream = _T("TBB Scheduler"), - *SyncObj_ContextsList = _T("TBB Scheduler") - ; -#endif /* DO_ITT_NOTIFY */ - +} //namespace r1 +} //namespace detail } // namespace tbb +#endif /* __TBB_USE_ITT_NOTIFY */ diff --git a/src/tbb/src/tbb/itt_notify.h b/src/tbb/src/tbb/itt_notify.h index 529437c81..48ddc5cae 100644 --- a/src/tbb/src/tbb/itt_notify.h +++ b/src/tbb/src/tbb/itt_notify.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2022 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,18 +12,14 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef _TBB_ITT_NOTIFY #define _TBB_ITT_NOTIFY -#include "tbb/tbb_stddef.h" +#include "oneapi/tbb/detail/_config.h" -#if DO_ITT_NOTIFY +#if __TBB_USE_ITT_NOTIFY #if _WIN32||_WIN64 #ifndef UNICODE @@ -38,91 +34,84 @@ #include "tools_api/ittnotify.h" #include "tools_api/legacy/ittnotify.h" extern "C" void __itt_fini_ittlib(void); +extern "C" void __itt_release_resources(void); #if _WIN32||_WIN64 #undef _T - #undef __itt_event_create - #define __itt_event_create __itt_event_createA #endif /* WIN */ - -#endif /* DO_ITT_NOTIFY */ +#endif /* __TBB_USE_ITT_NOTIFY */ #if !ITT_CALLER_NULL #define ITT_CALLER_NULL ((__itt_caller)0) #endif namespace tbb { +namespace detail { +namespace r1 { + //! Unicode support -#if (_WIN32||_WIN64) && !__MINGW32__ +#if (_WIN32||_WIN64) //! Unicode character type. Always wchar_t on Windows. /** We do not use typedefs from Windows TCHAR family to keep consistence of TBB coding style. **/ - typedef wchar_t tchar; + using tchar = wchar_t; //! Standard Windows macro to markup the string literals. #define _T(string_literal) L ## string_literal #else /* !WIN */ - typedef char tchar; + using tchar = char; //! Standard Windows style macro to markup the string literals. #define _T(string_literal) string_literal #endif /* !WIN */ -} // namespace tbb -#if DO_ITT_NOTIFY -namespace tbb { - //! Display names of internal synchronization types - extern const tchar - *SyncType_GlobalLock, - *SyncType_Scheduler; - //! Display names of internal synchronization components/scenarios - extern const tchar - *SyncObj_SchedulerInitialization, - *SyncObj_SchedulersList, - *SyncObj_WorkerLifeCycleMgmt, - *SyncObj_TaskStealingLoop, - *SyncObj_WorkerTaskPool, - *SyncObj_MasterTaskPool, - *SyncObj_TaskPoolSpinning, - *SyncObj_Mailbox, - *SyncObj_TaskReturnList, - *SyncObj_TaskStream, - *SyncObj_ContextsList - ; - - namespace internal { - void __TBB_EXPORTED_FUNC itt_set_sync_name_v3( void* obj, const tchar* name); - - } // namespace internal - -} // namespace tbb +//! Display names of internal synchronization types +extern const tchar + *SyncType_Scheduler; +//! Display names of internal synchronization components/scenarios +extern const tchar + *SyncObj_ContextsList + ; +#if __TBB_USE_ITT_NOTIFY // const_cast<void*>() is necessary to cast off volatility -#define ITT_NOTIFY(name,obj) __itt_notify_##name(const_cast<void*>(static_cast<volatile void*>(obj))) +#define ITT_NOTIFY(name,obj) __itt_##name(const_cast<void*>(static_cast<volatile void*>(obj))) #define ITT_THREAD_SET_NAME(name) __itt_thread_set_name(name) #define ITT_FINI_ITTLIB() __itt_fini_ittlib() +#define ITT_RELEASE_RESOURCES() __itt_release_resources() #define ITT_SYNC_CREATE(obj, type, name) __itt_sync_create((void*)(obj), type, name, 2) -#define ITT_SYNC_RENAME(obj, name) __itt_sync_rename(obj, name) #define ITT_STACK_CREATE(obj) obj = __itt_stack_caller_create() -#if __TBB_TASK_GROUP_CONTEXT -#define ITT_STACK(precond, name, obj) (precond) ? __itt_stack_##name(obj) : ((void)0); -#else -#define ITT_STACK(precond, name, obj) ((void)0) -#endif /* !__TBB_TASK_GROUP_CONTEXT */ +#define ITT_STACK_DESTROY(obj) (obj!=nullptr) ? __itt_stack_caller_destroy(static_cast<__itt_caller>(obj)) : ((void)0) +#define ITT_CALLEE_ENTER(cond, t, obj) if(cond) {\ + __itt_stack_callee_enter(static_cast<__itt_caller>(obj));\ + __itt_sync_acquired(t);\ + } +#define ITT_CALLEE_LEAVE(cond, obj) (cond) ? __itt_stack_callee_leave(static_cast<__itt_caller>(obj)) : ((void)0) + +#define ITT_TASK_GROUP(obj,name,parent) r1::itt_make_task_group(d1::ITT_DOMAIN_MAIN,(void*)(obj),ALGORITHM,(void*)(parent),(parent!=nullptr) ? ALGORITHM : FLOW_NULL,name) +#define ITT_TASK_BEGIN(obj,name,id) r1::itt_task_begin(d1::ITT_DOMAIN_MAIN,(void*)(id),ALGORITHM,(void*)(obj),ALGORITHM,name) +#define ITT_TASK_END r1::itt_task_end(d1::ITT_DOMAIN_MAIN) + -#else /* !DO_ITT_NOTIFY */ +#else /* !__TBB_USE_ITT_NOTIFY */ #define ITT_NOTIFY(name,obj) ((void)0) #define ITT_THREAD_SET_NAME(name) ((void)0) #define ITT_FINI_ITTLIB() ((void)0) +#define ITT_RELEASE_RESOURCES() ((void)0) #define ITT_SYNC_CREATE(obj, type, name) ((void)0) -#define ITT_SYNC_RENAME(obj, name) ((void)0) #define ITT_STACK_CREATE(obj) ((void)0) -#define ITT_STACK(precond, name, obj) ((void)0) +#define ITT_STACK_DESTROY(obj) ((void)0) +#define ITT_CALLEE_ENTER(cond, t, obj) ((void)0) +#define ITT_CALLEE_LEAVE(cond, obj) ((void)0) +#define ITT_TASK_GROUP(type,name,parent) ((void)0) +#define ITT_TASK_BEGIN(type,name,id) ((void)0) +#define ITT_TASK_END ((void)0) -#endif /* !DO_ITT_NOTIFY */ +#endif /* !__TBB_USE_ITT_NOTIFY */ -namespace tbb { -namespace internal { int __TBB_load_ittnotify(); -}} + +} // namespace r1 +} // namespace detail +} // namespace tbb #endif /* _TBB_ITT_NOTIFY */ diff --git a/src/tbb/src/tbb/lin32-tbb-export.def b/src/tbb/src/tbb/lin32-tbb-export.def deleted file mode 100644 index d1f7c9c46..000000000 --- a/src/tbb/src/tbb/lin32-tbb-export.def +++ /dev/null @@ -1,49 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -{ -global: - -#define __TBB_SYMBOL( sym ) sym; -#include "lin32-tbb-export.lst" - -local: - -/* TBB symbols */ -*3tbb*; -*__TBB*; - -/* ITT symbols */ -__itt_*; - -/* Intel Compiler (libirc) symbols */ -__intel_*; -_intel_*; -get_memcpy_largest_cachelinesize; -get_memcpy_largest_cache_size; -get_mem_ops_method; -init_mem_ops_method; -irc__get_msg; -irc__print; -override_mem_ops_method; -set_memcpy_largest_cachelinesize; -set_memcpy_largest_cache_size; - -}; diff --git a/src/tbb/src/tbb/lin32-tbb-export.lst b/src/tbb/src/tbb/lin32-tbb-export.lst deleted file mode 100644 index 10f272662..000000000 --- a/src/tbb/src/tbb/lin32-tbb-export.lst +++ /dev/null @@ -1,404 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#include "tbb/tbb_config.h" - -/* cache_aligned_allocator.cpp */ -__TBB_SYMBOL( _ZN3tbb8internal12NFS_AllocateEjjPv ) -__TBB_SYMBOL( _ZN3tbb8internal15NFS_GetLineSizeEv ) -__TBB_SYMBOL( _ZN3tbb8internal8NFS_FreeEPv ) -__TBB_SYMBOL( _ZN3tbb8internal23allocate_via_handler_v3Ej ) -__TBB_SYMBOL( _ZN3tbb8internal25deallocate_via_handler_v3EPv ) -__TBB_SYMBOL( _ZN3tbb8internal17is_malloc_used_v3Ev ) - -/* task.cpp v3 */ -__TBB_SYMBOL( _ZN3tbb4task13note_affinityEt ) -__TBB_SYMBOL( _ZN3tbb4task22internal_set_ref_countEi ) -__TBB_SYMBOL( _ZN3tbb4task28internal_decrement_ref_countEv ) -__TBB_SYMBOL( _ZN3tbb4task22spawn_and_wait_for_allERNS_9task_listE ) -__TBB_SYMBOL( _ZN3tbb4task4selfEv ) -__TBB_SYMBOL( _ZN3tbb10interface58internal9task_base7destroyERNS_4taskE ) -__TBB_SYMBOL( _ZNK3tbb4task26is_owned_by_current_threadEv ) -__TBB_SYMBOL( _ZN3tbb8internal19allocate_root_proxy4freeERNS_4taskE ) -__TBB_SYMBOL( _ZN3tbb8internal19allocate_root_proxy8allocateEj ) -__TBB_SYMBOL( _ZN3tbb8internal28affinity_partitioner_base_v36resizeEj ) -__TBB_SYMBOL( _ZNK3tbb8internal20allocate_child_proxy4freeERNS_4taskE ) -__TBB_SYMBOL( _ZNK3tbb8internal20allocate_child_proxy8allocateEj ) -__TBB_SYMBOL( _ZNK3tbb8internal27allocate_continuation_proxy4freeERNS_4taskE ) -__TBB_SYMBOL( _ZNK3tbb8internal27allocate_continuation_proxy8allocateEj ) -__TBB_SYMBOL( _ZNK3tbb8internal34allocate_additional_child_of_proxy4freeERNS_4taskE ) -__TBB_SYMBOL( _ZNK3tbb8internal34allocate_additional_child_of_proxy8allocateEj ) -__TBB_SYMBOL( _ZTIN3tbb4taskE ) -__TBB_SYMBOL( _ZTSN3tbb4taskE ) -__TBB_SYMBOL( _ZTVN3tbb4taskE ) -__TBB_SYMBOL( _ZN3tbb19task_scheduler_init19default_num_threadsEv ) -__TBB_SYMBOL( _ZN3tbb19task_scheduler_init10initializeEij ) -__TBB_SYMBOL( _ZN3tbb19task_scheduler_init10initializeEi ) -__TBB_SYMBOL( _ZN3tbb19task_scheduler_init9terminateEv ) -__TBB_SYMBOL( _ZN3tbb19task_scheduler_init27internal_blocking_terminateEb ) -#if __TBB_SCHEDULER_OBSERVER -__TBB_SYMBOL( _ZN3tbb8internal26task_scheduler_observer_v37observeEb ) -#endif /* __TBB_SCHEDULER_OBSERVER */ -__TBB_SYMBOL( _ZN3tbb10empty_task7executeEv ) -__TBB_SYMBOL( _ZN3tbb10empty_taskD0Ev ) -__TBB_SYMBOL( _ZN3tbb10empty_taskD1Ev ) -__TBB_SYMBOL( _ZTIN3tbb10empty_taskE ) -__TBB_SYMBOL( _ZTSN3tbb10empty_taskE ) -__TBB_SYMBOL( _ZTVN3tbb10empty_taskE ) - -/* arena.cpp */ -__TBB_SYMBOL( _ZN3tbb10interface78internal15task_arena_base19internal_initializeEv ) -__TBB_SYMBOL( _ZN3tbb10interface78internal15task_arena_base18internal_terminateEv ) -__TBB_SYMBOL( _ZN3tbb10interface78internal15task_arena_base15internal_attachEv ) -__TBB_SYMBOL( _ZNK3tbb10interface78internal15task_arena_base16internal_enqueueERNS_4taskEi ) -__TBB_SYMBOL( _ZNK3tbb10interface78internal15task_arena_base16internal_executeERNS1_13delegate_baseE ) -__TBB_SYMBOL( _ZNK3tbb10interface78internal15task_arena_base13internal_waitEv ) -__TBB_SYMBOL( _ZN3tbb10interface78internal15task_arena_base21internal_current_slotEv ) -__TBB_SYMBOL( _ZN3tbb10interface78internal15task_arena_base24internal_max_concurrencyEPKNS0_10task_arenaE ) -#if __TBB_TASK_ISOLATION -__TBB_SYMBOL( _ZN3tbb10interface78internal20isolate_within_arenaERNS1_13delegate_baseEi ) -#endif /* __TBB_TASK_ISOLATION */ - -#if !TBB_NO_LEGACY -/* task_v2.cpp */ -__TBB_SYMBOL( _ZN3tbb4task7destroyERS0_ ) -#endif /* !TBB_NO_LEGACY */ - -/* Exception handling in task scheduler */ -#if __TBB_TASK_GROUP_CONTEXT -__TBB_SYMBOL( _ZNK3tbb8internal32allocate_root_with_context_proxy8allocateEj ) -__TBB_SYMBOL( _ZNK3tbb8internal32allocate_root_with_context_proxy4freeERNS_4taskE ) -__TBB_SYMBOL( _ZN3tbb4task12change_groupERNS_18task_group_contextE ) -__TBB_SYMBOL( _ZNK3tbb18task_group_context28is_group_execution_cancelledEv ) -__TBB_SYMBOL( _ZN3tbb18task_group_context22cancel_group_executionEv ) -__TBB_SYMBOL( _ZN3tbb18task_group_context26register_pending_exceptionEv ) -__TBB_SYMBOL( _ZN3tbb18task_group_context5resetEv ) -__TBB_SYMBOL( _ZN3tbb18task_group_context19capture_fp_settingsEv ) -__TBB_SYMBOL( _ZN3tbb18task_group_context4initEv ) -__TBB_SYMBOL( _ZN3tbb18task_group_contextD1Ev ) -__TBB_SYMBOL( _ZN3tbb18task_group_contextD2Ev ) -#if __TBB_TASK_PRIORITY -__TBB_SYMBOL( _ZN3tbb18task_group_context12set_priorityENS_10priority_tE ) -__TBB_SYMBOL( _ZNK3tbb18task_group_context8priorityEv ) -#endif /* __TBB_TASK_PRIORITY */ -__TBB_SYMBOL( _ZNK3tbb18captured_exception4nameEv ) -__TBB_SYMBOL( _ZNK3tbb18captured_exception4whatEv ) -__TBB_SYMBOL( _ZN3tbb18captured_exception10throw_selfEv ) -__TBB_SYMBOL( _ZN3tbb18captured_exception3setEPKcS2_ ) -__TBB_SYMBOL( _ZN3tbb18captured_exception4moveEv ) -__TBB_SYMBOL( _ZN3tbb18captured_exception5clearEv ) -__TBB_SYMBOL( _ZN3tbb18captured_exception7destroyEv ) -__TBB_SYMBOL( _ZN3tbb18captured_exception8allocateEPKcS2_ ) -__TBB_SYMBOL( _ZN3tbb18captured_exceptionD0Ev ) -__TBB_SYMBOL( _ZN3tbb18captured_exceptionD1Ev ) -__TBB_SYMBOL( _ZN3tbb18captured_exceptionD2Ev ) -__TBB_SYMBOL( _ZTIN3tbb18captured_exceptionE ) -__TBB_SYMBOL( _ZTSN3tbb18captured_exceptionE ) -__TBB_SYMBOL( _ZTVN3tbb18captured_exceptionE ) -__TBB_SYMBOL( _ZN3tbb13tbb_exceptionD2Ev ) -__TBB_SYMBOL( _ZTIN3tbb13tbb_exceptionE ) -__TBB_SYMBOL( _ZTSN3tbb13tbb_exceptionE ) -__TBB_SYMBOL( _ZTVN3tbb13tbb_exceptionE ) -#endif /* __TBB_TASK_GROUP_CONTEXT */ - -/* Symbols for exceptions thrown from TBB */ -__TBB_SYMBOL( _ZN3tbb8internal33throw_bad_last_alloc_exception_v4Ev ) -__TBB_SYMBOL( _ZN3tbb8internal18throw_exception_v4ENS0_12exception_idE ) -__TBB_SYMBOL( _ZN3tbb14bad_last_allocD0Ev ) -__TBB_SYMBOL( _ZN3tbb14bad_last_allocD1Ev ) -__TBB_SYMBOL( _ZNK3tbb14bad_last_alloc4whatEv ) -__TBB_SYMBOL( _ZTIN3tbb14bad_last_allocE ) -__TBB_SYMBOL( _ZTSN3tbb14bad_last_allocE ) -__TBB_SYMBOL( _ZTVN3tbb14bad_last_allocE ) -__TBB_SYMBOL( _ZN3tbb12missing_waitD0Ev ) -__TBB_SYMBOL( _ZN3tbb12missing_waitD1Ev ) -__TBB_SYMBOL( _ZNK3tbb12missing_wait4whatEv ) -__TBB_SYMBOL( _ZTIN3tbb12missing_waitE ) -__TBB_SYMBOL( _ZTSN3tbb12missing_waitE ) -__TBB_SYMBOL( _ZTVN3tbb12missing_waitE ) -__TBB_SYMBOL( _ZN3tbb27invalid_multiple_schedulingD0Ev ) -__TBB_SYMBOL( _ZN3tbb27invalid_multiple_schedulingD1Ev ) -__TBB_SYMBOL( _ZNK3tbb27invalid_multiple_scheduling4whatEv ) -__TBB_SYMBOL( _ZTIN3tbb27invalid_multiple_schedulingE ) -__TBB_SYMBOL( _ZTSN3tbb27invalid_multiple_schedulingE ) -__TBB_SYMBOL( _ZTVN3tbb27invalid_multiple_schedulingE ) -__TBB_SYMBOL( _ZN3tbb13improper_lockD0Ev ) -__TBB_SYMBOL( _ZN3tbb13improper_lockD1Ev ) -__TBB_SYMBOL( _ZNK3tbb13improper_lock4whatEv ) -__TBB_SYMBOL( _ZTIN3tbb13improper_lockE ) -__TBB_SYMBOL( _ZTSN3tbb13improper_lockE ) -__TBB_SYMBOL( _ZTVN3tbb13improper_lockE ) -__TBB_SYMBOL( _ZN3tbb10user_abortD0Ev ) -__TBB_SYMBOL( _ZN3tbb10user_abortD1Ev ) -__TBB_SYMBOL( _ZNK3tbb10user_abort4whatEv ) -__TBB_SYMBOL( _ZTIN3tbb10user_abortE ) -__TBB_SYMBOL( _ZTSN3tbb10user_abortE ) -__TBB_SYMBOL( _ZTVN3tbb10user_abortE ) - -/* tbb_misc.cpp */ -__TBB_SYMBOL( _ZN3tbb17assertion_failureEPKciS1_S1_ ) -__TBB_SYMBOL( _ZN3tbb21set_assertion_handlerEPFvPKciS1_S1_E ) -__TBB_SYMBOL( _ZN3tbb8internal36get_initial_auto_partitioner_divisorEv ) -__TBB_SYMBOL( _ZN3tbb8internal13handle_perrorEiPKc ) -__TBB_SYMBOL( _ZN3tbb8internal15runtime_warningEPKcz ) -#if __TBB_x86_32 -__TBB_SYMBOL( __TBB_machine_store8_slow_perf_warning ) -__TBB_SYMBOL( __TBB_machine_store8_slow ) -#endif -__TBB_SYMBOL( TBB_runtime_interface_version ) - -/* tbb_main.cpp */ -__TBB_SYMBOL( _ZN3tbb8internal32itt_load_pointer_with_acquire_v3EPKv ) -__TBB_SYMBOL( _ZN3tbb8internal33itt_store_pointer_with_release_v3EPvS1_ ) -__TBB_SYMBOL( _ZN3tbb8internal18call_itt_notify_v5EiPv ) -__TBB_SYMBOL( _ZN3tbb8internal20itt_set_sync_name_v3EPvPKc ) -__TBB_SYMBOL( _ZN3tbb8internal19itt_load_pointer_v3EPKv ) -#if __TBB_ITT_STRUCTURE_API -__TBB_SYMBOL( _ZN3tbb8internal22itt_make_task_group_v7ENS0_15itt_domain_enumEPvyS2_yNS0_12string_indexE ) -__TBB_SYMBOL( _ZN3tbb8internal23itt_metadata_str_add_v7ENS0_15itt_domain_enumEPvyNS0_12string_indexEPKc ) -__TBB_SYMBOL( _ZN3tbb8internal19itt_relation_add_v7ENS0_15itt_domain_enumEPvyNS0_12itt_relationES2_y ) -__TBB_SYMBOL( _ZN3tbb8internal17itt_task_begin_v7ENS0_15itt_domain_enumEPvyS2_yNS0_12string_indexE ) -__TBB_SYMBOL( _ZN3tbb8internal15itt_task_end_v7ENS0_15itt_domain_enumE ) -__TBB_SYMBOL( _ZN3tbb8internal19itt_region_begin_v9ENS0_15itt_domain_enumEPvyS2_yNS0_12string_indexE ) -__TBB_SYMBOL( _ZN3tbb8internal17itt_region_end_v9ENS0_15itt_domain_enumEPvy ) -#endif - -/* pipeline.cpp */ -__TBB_SYMBOL( _ZTIN3tbb6filterE ) -__TBB_SYMBOL( _ZTSN3tbb6filterE ) -__TBB_SYMBOL( _ZTVN3tbb6filterE ) -__TBB_SYMBOL( _ZN3tbb6filterD2Ev ) -__TBB_SYMBOL( _ZN3tbb8pipeline10add_filterERNS_6filterE ) -__TBB_SYMBOL( _ZN3tbb8pipeline12inject_tokenERNS_4taskE ) -__TBB_SYMBOL( _ZN3tbb8pipeline13remove_filterERNS_6filterE ) -__TBB_SYMBOL( _ZN3tbb8pipeline3runEj ) -#if __TBB_TASK_GROUP_CONTEXT -__TBB_SYMBOL( _ZN3tbb8pipeline3runEjRNS_18task_group_contextE ) -#endif -__TBB_SYMBOL( _ZN3tbb8pipeline5clearEv ) -__TBB_SYMBOL( _ZN3tbb19thread_bound_filter12process_itemEv ) -__TBB_SYMBOL( _ZN3tbb19thread_bound_filter16try_process_itemEv ) -__TBB_SYMBOL( _ZTIN3tbb8pipelineE ) -__TBB_SYMBOL( _ZTSN3tbb8pipelineE ) -__TBB_SYMBOL( _ZTVN3tbb8pipelineE ) -__TBB_SYMBOL( _ZN3tbb8pipelineC1Ev ) -__TBB_SYMBOL( _ZN3tbb8pipelineC2Ev ) -__TBB_SYMBOL( _ZN3tbb8pipelineD0Ev ) -__TBB_SYMBOL( _ZN3tbb8pipelineD1Ev ) -__TBB_SYMBOL( _ZN3tbb8pipelineD2Ev ) -__TBB_SYMBOL( _ZN3tbb6filter16set_end_of_inputEv ) - -/* queuing_rw_mutex.cpp */ -__TBB_SYMBOL( _ZN3tbb16queuing_rw_mutex18internal_constructEv ) -__TBB_SYMBOL( _ZN3tbb16queuing_rw_mutex11scoped_lock17upgrade_to_writerEv ) -__TBB_SYMBOL( _ZN3tbb16queuing_rw_mutex11scoped_lock19downgrade_to_readerEv ) -__TBB_SYMBOL( _ZN3tbb16queuing_rw_mutex11scoped_lock7acquireERS0_b ) -__TBB_SYMBOL( _ZN3tbb16queuing_rw_mutex11scoped_lock7releaseEv ) -__TBB_SYMBOL( _ZN3tbb16queuing_rw_mutex11scoped_lock11try_acquireERS0_b ) - -/* reader_writer_lock.cpp */ -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock11scoped_lock16internal_destroyEv ) -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock11scoped_lock18internal_constructERS1_ ) -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock13try_lock_readEv ) -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock16scoped_lock_read16internal_destroyEv ) -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock16scoped_lock_read18internal_constructERS1_ ) -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock16internal_destroyEv ) -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock18internal_constructEv ) -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock4lockEv ) -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock6unlockEv ) -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock8try_lockEv ) -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock9lock_readEv ) - -#if !TBB_NO_LEGACY -/* spin_rw_mutex.cpp v2 */ -__TBB_SYMBOL( _ZN3tbb13spin_rw_mutex16internal_upgradeEPS0_ ) -__TBB_SYMBOL( _ZN3tbb13spin_rw_mutex22internal_itt_releasingEPS0_ ) -__TBB_SYMBOL( _ZN3tbb13spin_rw_mutex23internal_acquire_readerEPS0_ ) -__TBB_SYMBOL( _ZN3tbb13spin_rw_mutex23internal_acquire_writerEPS0_ ) -__TBB_SYMBOL( _ZN3tbb13spin_rw_mutex18internal_downgradeEPS0_ ) -__TBB_SYMBOL( _ZN3tbb13spin_rw_mutex23internal_release_readerEPS0_ ) -__TBB_SYMBOL( _ZN3tbb13spin_rw_mutex23internal_release_writerEPS0_ ) -__TBB_SYMBOL( _ZN3tbb13spin_rw_mutex27internal_try_acquire_readerEPS0_ ) -__TBB_SYMBOL( _ZN3tbb13spin_rw_mutex27internal_try_acquire_writerEPS0_ ) -#endif - -/* spin_rw_mutex v3 */ -__TBB_SYMBOL( _ZN3tbb16spin_rw_mutex_v318internal_constructEv ) -__TBB_SYMBOL( _ZN3tbb16spin_rw_mutex_v316internal_upgradeEv ) -__TBB_SYMBOL( _ZN3tbb16spin_rw_mutex_v318internal_downgradeEv ) -__TBB_SYMBOL( _ZN3tbb16spin_rw_mutex_v323internal_acquire_readerEv ) -__TBB_SYMBOL( _ZN3tbb16spin_rw_mutex_v323internal_acquire_writerEv ) -__TBB_SYMBOL( _ZN3tbb16spin_rw_mutex_v323internal_release_readerEv ) -__TBB_SYMBOL( _ZN3tbb16spin_rw_mutex_v323internal_release_writerEv ) -__TBB_SYMBOL( _ZN3tbb16spin_rw_mutex_v327internal_try_acquire_readerEv ) -__TBB_SYMBOL( _ZN3tbb16spin_rw_mutex_v327internal_try_acquire_writerEv ) - -// x86_rtm_rw_mutex.cpp -#if __TBB_TSX_AVAILABLE -__TBB_SYMBOL( _ZN3tbb10interface88internal16x86_rtm_rw_mutex16internal_releaseERNS2_11scoped_lockE ) -__TBB_SYMBOL( _ZN3tbb10interface88internal16x86_rtm_rw_mutex18internal_constructEv ) -__TBB_SYMBOL( _ZN3tbb10interface88internal16x86_rtm_rw_mutex23internal_acquire_readerERNS2_11scoped_lockEb ) -__TBB_SYMBOL( _ZN3tbb10interface88internal16x86_rtm_rw_mutex23internal_acquire_writerERNS2_11scoped_lockEb ) -__TBB_SYMBOL( _ZN3tbb10interface88internal16x86_rtm_rw_mutex16internal_upgradeERNS2_11scoped_lockE ) -__TBB_SYMBOL( _ZN3tbb10interface88internal16x86_rtm_rw_mutex18internal_downgradeERNS2_11scoped_lockE ) -__TBB_SYMBOL( _ZN3tbb10interface88internal16x86_rtm_rw_mutex27internal_try_acquire_writerERNS2_11scoped_lockE ) -#endif - -/* spin_mutex.cpp */ -__TBB_SYMBOL( _ZN3tbb10spin_mutex18internal_constructEv ) -__TBB_SYMBOL( _ZN3tbb10spin_mutex11scoped_lock16internal_acquireERS0_ ) -__TBB_SYMBOL( _ZN3tbb10spin_mutex11scoped_lock16internal_releaseEv ) -__TBB_SYMBOL( _ZN3tbb10spin_mutex11scoped_lock20internal_try_acquireERS0_ ) - -/* mutex.cpp */ -__TBB_SYMBOL( _ZN3tbb5mutex11scoped_lock16internal_acquireERS0_ ) -__TBB_SYMBOL( _ZN3tbb5mutex11scoped_lock16internal_releaseEv ) -__TBB_SYMBOL( _ZN3tbb5mutex11scoped_lock20internal_try_acquireERS0_ ) -__TBB_SYMBOL( _ZN3tbb5mutex16internal_destroyEv ) -__TBB_SYMBOL( _ZN3tbb5mutex18internal_constructEv ) - -/* recursive_mutex.cpp */ -__TBB_SYMBOL( _ZN3tbb15recursive_mutex11scoped_lock16internal_acquireERS0_ ) -__TBB_SYMBOL( _ZN3tbb15recursive_mutex11scoped_lock16internal_releaseEv ) -__TBB_SYMBOL( _ZN3tbb15recursive_mutex11scoped_lock20internal_try_acquireERS0_ ) -__TBB_SYMBOL( _ZN3tbb15recursive_mutex16internal_destroyEv ) -__TBB_SYMBOL( _ZN3tbb15recursive_mutex18internal_constructEv ) - -/* QueuingMutex.cpp */ -__TBB_SYMBOL( _ZN3tbb13queuing_mutex18internal_constructEv ) -__TBB_SYMBOL( _ZN3tbb13queuing_mutex11scoped_lock7acquireERS0_ ) -__TBB_SYMBOL( _ZN3tbb13queuing_mutex11scoped_lock7releaseEv ) -__TBB_SYMBOL( _ZN3tbb13queuing_mutex11scoped_lock11try_acquireERS0_ ) - -/* critical_section.cpp */ -__TBB_SYMBOL( _ZN3tbb8internal19critical_section_v418internal_constructEv ) - -#if !TBB_NO_LEGACY -/* concurrent_hash_map */ -__TBB_SYMBOL( _ZNK3tbb8internal21hash_map_segment_base23internal_grow_predicateEv ) - -/* concurrent_queue.cpp v2 */ -__TBB_SYMBOL( _ZN3tbb8internal21concurrent_queue_base12internal_popEPv ) -__TBB_SYMBOL( _ZN3tbb8internal21concurrent_queue_base13internal_pushEPKv ) -__TBB_SYMBOL( _ZN3tbb8internal21concurrent_queue_base21internal_set_capacityEij ) -__TBB_SYMBOL( _ZN3tbb8internal21concurrent_queue_base23internal_pop_if_presentEPv ) -__TBB_SYMBOL( _ZN3tbb8internal21concurrent_queue_base25internal_push_if_not_fullEPKv ) -__TBB_SYMBOL( _ZN3tbb8internal21concurrent_queue_baseC2Ej ) -__TBB_SYMBOL( _ZN3tbb8internal21concurrent_queue_baseD2Ev ) -__TBB_SYMBOL( _ZTIN3tbb8internal21concurrent_queue_baseE ) -__TBB_SYMBOL( _ZTSN3tbb8internal21concurrent_queue_baseE ) -__TBB_SYMBOL( _ZTVN3tbb8internal21concurrent_queue_baseE ) -__TBB_SYMBOL( _ZN3tbb8internal30concurrent_queue_iterator_base6assignERKS1_ ) -__TBB_SYMBOL( _ZN3tbb8internal30concurrent_queue_iterator_base7advanceEv ) -__TBB_SYMBOL( _ZN3tbb8internal30concurrent_queue_iterator_baseC2ERKNS0_21concurrent_queue_baseE ) -__TBB_SYMBOL( _ZN3tbb8internal30concurrent_queue_iterator_baseD2Ev ) -__TBB_SYMBOL( _ZNK3tbb8internal21concurrent_queue_base13internal_sizeEv ) -#endif - -/* concurrent_queue v3 */ -/* constructors */ -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v3C2Ej ) -__TBB_SYMBOL( _ZN3tbb8internal33concurrent_queue_iterator_base_v3C2ERKNS0_24concurrent_queue_base_v3E ) -__TBB_SYMBOL( _ZN3tbb8internal33concurrent_queue_iterator_base_v3C2ERKNS0_24concurrent_queue_base_v3Ej ) -/* destructors */ -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v3D2Ev ) -__TBB_SYMBOL( _ZN3tbb8internal33concurrent_queue_iterator_base_v3D2Ev ) -/* typeinfo */ -__TBB_SYMBOL( _ZTIN3tbb8internal24concurrent_queue_base_v3E ) -__TBB_SYMBOL( _ZTSN3tbb8internal24concurrent_queue_base_v3E ) -/* vtable */ -__TBB_SYMBOL( _ZTVN3tbb8internal24concurrent_queue_base_v3E ) -/* methods */ -__TBB_SYMBOL( _ZN3tbb8internal33concurrent_queue_iterator_base_v37advanceEv ) -__TBB_SYMBOL( _ZN3tbb8internal33concurrent_queue_iterator_base_v36assignERKS1_ ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v313internal_pushEPKv ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v818internal_push_moveEPKv ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v325internal_push_if_not_fullEPKv ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v830internal_push_move_if_not_fullEPKv ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v312internal_popEPv ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v323internal_pop_if_presentEPv ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v314internal_abortEv ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v321internal_set_capacityEij ) -__TBB_SYMBOL( _ZNK3tbb8internal24concurrent_queue_base_v313internal_sizeEv ) -__TBB_SYMBOL( _ZNK3tbb8internal24concurrent_queue_base_v314internal_emptyEv ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v321internal_finish_clearEv ) -__TBB_SYMBOL( _ZNK3tbb8internal24concurrent_queue_base_v324internal_throw_exceptionEv ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v36assignERKS1_ ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v812move_contentERS1_ ) - -#if !TBB_NO_LEGACY -/* concurrent_vector.cpp v2 */ -__TBB_SYMBOL( _ZN3tbb8internal22concurrent_vector_base13internal_copyERKS1_jPFvPvPKvjE ) -__TBB_SYMBOL( _ZN3tbb8internal22concurrent_vector_base14internal_clearEPFvPvjEb ) -__TBB_SYMBOL( _ZN3tbb8internal22concurrent_vector_base15internal_assignERKS1_jPFvPvjEPFvS4_PKvjESA_ ) -__TBB_SYMBOL( _ZN3tbb8internal22concurrent_vector_base16internal_grow_byEjjPFvPvjE ) -__TBB_SYMBOL( _ZN3tbb8internal22concurrent_vector_base16internal_reserveEjjj ) -__TBB_SYMBOL( _ZN3tbb8internal22concurrent_vector_base18internal_push_backEjRj ) -__TBB_SYMBOL( _ZN3tbb8internal22concurrent_vector_base25internal_grow_to_at_leastEjjPFvPvjE ) -__TBB_SYMBOL( _ZNK3tbb8internal22concurrent_vector_base17internal_capacityEv ) -#endif - -/* concurrent_vector v3 */ -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v313internal_copyERKS1_jPFvPvPKvjE ) -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v314internal_clearEPFvPvjE ) -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v315internal_assignERKS1_jPFvPvjEPFvS4_PKvjESA_ ) -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v316internal_grow_byEjjPFvPvPKvjES4_ ) -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v316internal_reserveEjjj ) -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v318internal_push_backEjRj ) -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v325internal_grow_to_at_leastEjjPFvPvPKvjES4_ ) -__TBB_SYMBOL( _ZNK3tbb8internal25concurrent_vector_base_v317internal_capacityEv ) -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v316internal_compactEjPvPFvS2_jEPFvS2_PKvjE ) -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v313internal_swapERS1_ ) -__TBB_SYMBOL( _ZNK3tbb8internal25concurrent_vector_base_v324internal_throw_exceptionEj ) -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v3D2Ev ) -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v315internal_resizeEjjjPKvPFvPvjEPFvS4_S3_jE ) -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v337internal_grow_to_at_least_with_resultEjjPFvPvPKvjES4_ ) - -/* tbb_thread */ -#if __MINGW32__ -__TBB_SYMBOL( _ZN3tbb8internal13tbb_thread_v314internal_startEPFjPvES2_ ) -#else -__TBB_SYMBOL( _ZN3tbb8internal13tbb_thread_v314internal_startEPFPvS2_ES2_ ) -#endif -__TBB_SYMBOL( _ZN3tbb8internal13tbb_thread_v320hardware_concurrencyEv ) -__TBB_SYMBOL( _ZN3tbb8internal13tbb_thread_v34joinEv ) -__TBB_SYMBOL( _ZN3tbb8internal13tbb_thread_v36detachEv ) -__TBB_SYMBOL( _ZN3tbb8internal15free_closure_v3EPv ) -__TBB_SYMBOL( _ZN3tbb8internal15thread_sleep_v3ERKNS_10tick_count10interval_tE ) -__TBB_SYMBOL( _ZN3tbb8internal15thread_yield_v3Ev ) -__TBB_SYMBOL( _ZN3tbb8internal16thread_get_id_v3Ev ) -__TBB_SYMBOL( _ZN3tbb8internal19allocate_closure_v3Ej ) -__TBB_SYMBOL( _ZN3tbb8internal7move_v3ERNS0_13tbb_thread_v3ES2_ ) - -#if __MINGW32__ -/* condition_variable */ -__TBB_SYMBOL( _ZN3tbb10interface58internal32internal_condition_variable_waitERNS1_14condvar_impl_tEPNS_5mutexEPKNS_10tick_count10interval_tE ) -__TBB_SYMBOL( _ZN3tbb10interface58internal35internal_destroy_condition_variableERNS1_14condvar_impl_tE ) -__TBB_SYMBOL( _ZN3tbb10interface58internal38internal_condition_variable_notify_allERNS1_14condvar_impl_tE ) -__TBB_SYMBOL( _ZN3tbb10interface58internal38internal_condition_variable_notify_oneERNS1_14condvar_impl_tE ) -__TBB_SYMBOL( _ZN3tbb10interface58internal38internal_initialize_condition_variableERNS1_14condvar_impl_tE ) -#endif - -// global parameter -__TBB_SYMBOL( _ZN3tbb10interface914global_control12active_valueEi ) -__TBB_SYMBOL( _ZN3tbb10interface914global_control15internal_createEv ) -__TBB_SYMBOL( _ZN3tbb10interface914global_control16internal_destroyEv ) - -#undef __TBB_SYMBOL diff --git a/src/tbb/src/tbb/lin64-tbb-export.lst b/src/tbb/src/tbb/lin64-tbb-export.lst deleted file mode 100644 index f05e68b7c..000000000 --- a/src/tbb/src/tbb/lin64-tbb-export.lst +++ /dev/null @@ -1,385 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#include "tbb/tbb_config.h" - -/* cache_aligned_allocator.cpp */ -__TBB_SYMBOL( _ZN3tbb8internal12NFS_AllocateEmmPv ) -__TBB_SYMBOL( _ZN3tbb8internal15NFS_GetLineSizeEv ) -__TBB_SYMBOL( _ZN3tbb8internal8NFS_FreeEPv ) -__TBB_SYMBOL( _ZN3tbb8internal23allocate_via_handler_v3Em ) -__TBB_SYMBOL( _ZN3tbb8internal25deallocate_via_handler_v3EPv ) -__TBB_SYMBOL( _ZN3tbb8internal17is_malloc_used_v3Ev ) - -/* task.cpp v3 */ -__TBB_SYMBOL( _ZN3tbb4task13note_affinityEt ) -__TBB_SYMBOL( _ZN3tbb4task22internal_set_ref_countEi ) -__TBB_SYMBOL( _ZN3tbb4task28internal_decrement_ref_countEv ) -__TBB_SYMBOL( _ZN3tbb4task22spawn_and_wait_for_allERNS_9task_listE ) -__TBB_SYMBOL( _ZN3tbb4task4selfEv ) -__TBB_SYMBOL( _ZN3tbb10interface58internal9task_base7destroyERNS_4taskE ) -__TBB_SYMBOL( _ZNK3tbb4task26is_owned_by_current_threadEv ) -__TBB_SYMBOL( _ZN3tbb8internal19allocate_root_proxy4freeERNS_4taskE ) -__TBB_SYMBOL( _ZN3tbb8internal19allocate_root_proxy8allocateEm ) -__TBB_SYMBOL( _ZN3tbb8internal28affinity_partitioner_base_v36resizeEj ) -__TBB_SYMBOL( _ZNK3tbb8internal20allocate_child_proxy4freeERNS_4taskE ) -__TBB_SYMBOL( _ZNK3tbb8internal20allocate_child_proxy8allocateEm ) -__TBB_SYMBOL( _ZNK3tbb8internal27allocate_continuation_proxy4freeERNS_4taskE ) -__TBB_SYMBOL( _ZNK3tbb8internal27allocate_continuation_proxy8allocateEm ) -__TBB_SYMBOL( _ZNK3tbb8internal34allocate_additional_child_of_proxy4freeERNS_4taskE ) -__TBB_SYMBOL( _ZNK3tbb8internal34allocate_additional_child_of_proxy8allocateEm ) -__TBB_SYMBOL( _ZTIN3tbb4taskE ) -__TBB_SYMBOL( _ZTSN3tbb4taskE ) -__TBB_SYMBOL( _ZTVN3tbb4taskE ) -__TBB_SYMBOL( _ZN3tbb19task_scheduler_init19default_num_threadsEv ) -__TBB_SYMBOL( _ZN3tbb19task_scheduler_init10initializeEim ) -__TBB_SYMBOL( _ZN3tbb19task_scheduler_init10initializeEi ) -__TBB_SYMBOL( _ZN3tbb19task_scheduler_init9terminateEv ) -__TBB_SYMBOL( _ZN3tbb19task_scheduler_init27internal_blocking_terminateEb ) -#if __TBB_SCHEDULER_OBSERVER -__TBB_SYMBOL( _ZN3tbb8internal26task_scheduler_observer_v37observeEb ) -#endif /* __TBB_SCHEDULER_OBSERVER */ -__TBB_SYMBOL( _ZN3tbb10empty_task7executeEv ) -__TBB_SYMBOL( _ZN3tbb10empty_taskD0Ev ) -__TBB_SYMBOL( _ZN3tbb10empty_taskD1Ev ) -__TBB_SYMBOL( _ZTIN3tbb10empty_taskE ) -__TBB_SYMBOL( _ZTSN3tbb10empty_taskE ) -__TBB_SYMBOL( _ZTVN3tbb10empty_taskE ) - -/* arena.cpp */ -__TBB_SYMBOL( _ZN3tbb10interface78internal15task_arena_base19internal_initializeEv ) -__TBB_SYMBOL( _ZN3tbb10interface78internal15task_arena_base18internal_terminateEv ) -__TBB_SYMBOL( _ZN3tbb10interface78internal15task_arena_base15internal_attachEv ) -__TBB_SYMBOL( _ZNK3tbb10interface78internal15task_arena_base16internal_enqueueERNS_4taskEl ) -__TBB_SYMBOL( _ZNK3tbb10interface78internal15task_arena_base16internal_executeERNS1_13delegate_baseE ) -__TBB_SYMBOL( _ZNK3tbb10interface78internal15task_arena_base13internal_waitEv ) -__TBB_SYMBOL( _ZN3tbb10interface78internal15task_arena_base21internal_current_slotEv ) -__TBB_SYMBOL( _ZN3tbb10interface78internal15task_arena_base24internal_max_concurrencyEPKNS0_10task_arenaE ) -#if __TBB_TASK_ISOLATION -__TBB_SYMBOL( _ZN3tbb10interface78internal20isolate_within_arenaERNS1_13delegate_baseEl ) -#endif - -#if !TBB_NO_LEGACY -/* task_v2.cpp */ -__TBB_SYMBOL( _ZN3tbb4task7destroyERS0_ ) -#endif /* !TBB_NO_LEGACY */ - -/* Exception handling in task scheduler */ -#if __TBB_TASK_GROUP_CONTEXT -__TBB_SYMBOL( _ZNK3tbb8internal32allocate_root_with_context_proxy8allocateEm ) -__TBB_SYMBOL( _ZNK3tbb8internal32allocate_root_with_context_proxy4freeERNS_4taskE ) -__TBB_SYMBOL( _ZN3tbb4task12change_groupERNS_18task_group_contextE ) -__TBB_SYMBOL( _ZNK3tbb18task_group_context28is_group_execution_cancelledEv ) -__TBB_SYMBOL( _ZN3tbb18task_group_context22cancel_group_executionEv ) -__TBB_SYMBOL( _ZN3tbb18task_group_context26register_pending_exceptionEv ) -__TBB_SYMBOL( _ZN3tbb18task_group_context5resetEv ) -__TBB_SYMBOL( _ZN3tbb18task_group_context19capture_fp_settingsEv ) -__TBB_SYMBOL( _ZN3tbb18task_group_context4initEv ) -__TBB_SYMBOL( _ZN3tbb18task_group_contextD1Ev ) -__TBB_SYMBOL( _ZN3tbb18task_group_contextD2Ev ) -#if __TBB_TASK_PRIORITY -__TBB_SYMBOL( _ZN3tbb18task_group_context12set_priorityENS_10priority_tE ) -__TBB_SYMBOL( _ZNK3tbb18task_group_context8priorityEv ) -#endif /* __TBB_TASK_PRIORITY */ -__TBB_SYMBOL( _ZNK3tbb18captured_exception4nameEv ) -__TBB_SYMBOL( _ZNK3tbb18captured_exception4whatEv ) -__TBB_SYMBOL( _ZN3tbb18captured_exception10throw_selfEv ) -__TBB_SYMBOL( _ZN3tbb18captured_exception3setEPKcS2_ ) -__TBB_SYMBOL( _ZN3tbb18captured_exception4moveEv ) -__TBB_SYMBOL( _ZN3tbb18captured_exception5clearEv ) -__TBB_SYMBOL( _ZN3tbb18captured_exception7destroyEv ) -__TBB_SYMBOL( _ZN3tbb18captured_exception8allocateEPKcS2_ ) -__TBB_SYMBOL( _ZN3tbb18captured_exceptionD0Ev ) -__TBB_SYMBOL( _ZN3tbb18captured_exceptionD1Ev ) -__TBB_SYMBOL( _ZN3tbb18captured_exceptionD2Ev ) -__TBB_SYMBOL( _ZTIN3tbb18captured_exceptionE ) -__TBB_SYMBOL( _ZTSN3tbb18captured_exceptionE ) -__TBB_SYMBOL( _ZTVN3tbb18captured_exceptionE ) -__TBB_SYMBOL( _ZN3tbb13tbb_exceptionD2Ev ) -__TBB_SYMBOL( _ZTIN3tbb13tbb_exceptionE ) -__TBB_SYMBOL( _ZTSN3tbb13tbb_exceptionE ) -__TBB_SYMBOL( _ZTVN3tbb13tbb_exceptionE ) -#endif /* __TBB_TASK_GROUP_CONTEXT */ - -/* Symbols for exceptions thrown from TBB */ -__TBB_SYMBOL( _ZN3tbb8internal33throw_bad_last_alloc_exception_v4Ev ) -__TBB_SYMBOL( _ZN3tbb8internal18throw_exception_v4ENS0_12exception_idE ) -__TBB_SYMBOL( _ZN3tbb14bad_last_allocD0Ev ) -__TBB_SYMBOL( _ZN3tbb14bad_last_allocD1Ev ) -__TBB_SYMBOL( _ZNK3tbb14bad_last_alloc4whatEv ) -__TBB_SYMBOL( _ZTIN3tbb14bad_last_allocE ) -__TBB_SYMBOL( _ZTSN3tbb14bad_last_allocE ) -__TBB_SYMBOL( _ZTVN3tbb14bad_last_allocE ) -__TBB_SYMBOL( _ZN3tbb12missing_waitD0Ev ) -__TBB_SYMBOL( _ZN3tbb12missing_waitD1Ev ) -__TBB_SYMBOL( _ZNK3tbb12missing_wait4whatEv ) -__TBB_SYMBOL( _ZTIN3tbb12missing_waitE ) -__TBB_SYMBOL( _ZTSN3tbb12missing_waitE ) -__TBB_SYMBOL( _ZTVN3tbb12missing_waitE ) -__TBB_SYMBOL( _ZN3tbb27invalid_multiple_schedulingD0Ev ) -__TBB_SYMBOL( _ZN3tbb27invalid_multiple_schedulingD1Ev ) -__TBB_SYMBOL( _ZNK3tbb27invalid_multiple_scheduling4whatEv ) -__TBB_SYMBOL( _ZTIN3tbb27invalid_multiple_schedulingE ) -__TBB_SYMBOL( _ZTSN3tbb27invalid_multiple_schedulingE ) -__TBB_SYMBOL( _ZTVN3tbb27invalid_multiple_schedulingE ) -__TBB_SYMBOL( _ZN3tbb13improper_lockD0Ev ) -__TBB_SYMBOL( _ZN3tbb13improper_lockD1Ev ) -__TBB_SYMBOL( _ZNK3tbb13improper_lock4whatEv ) -__TBB_SYMBOL( _ZTIN3tbb13improper_lockE ) -__TBB_SYMBOL( _ZTSN3tbb13improper_lockE ) -__TBB_SYMBOL( _ZTVN3tbb13improper_lockE ) -__TBB_SYMBOL( _ZN3tbb10user_abortD0Ev ) -__TBB_SYMBOL( _ZN3tbb10user_abortD1Ev ) -__TBB_SYMBOL( _ZNK3tbb10user_abort4whatEv ) -__TBB_SYMBOL( _ZTIN3tbb10user_abortE ) -__TBB_SYMBOL( _ZTSN3tbb10user_abortE ) -__TBB_SYMBOL( _ZTVN3tbb10user_abortE ) -/* tbb_misc.cpp */ -__TBB_SYMBOL( _ZN3tbb17assertion_failureEPKciS1_S1_ ) -__TBB_SYMBOL( _ZN3tbb21set_assertion_handlerEPFvPKciS1_S1_E ) -__TBB_SYMBOL( _ZN3tbb8internal36get_initial_auto_partitioner_divisorEv ) -__TBB_SYMBOL( _ZN3tbb8internal13handle_perrorEiPKc ) -__TBB_SYMBOL( _ZN3tbb8internal15runtime_warningEPKcz ) -__TBB_SYMBOL( TBB_runtime_interface_version ) - -/* tbb_main.cpp */ -__TBB_SYMBOL( _ZN3tbb8internal32itt_load_pointer_with_acquire_v3EPKv ) -__TBB_SYMBOL( _ZN3tbb8internal33itt_store_pointer_with_release_v3EPvS1_ ) -__TBB_SYMBOL( _ZN3tbb8internal18call_itt_notify_v5EiPv ) -__TBB_SYMBOL( _ZN3tbb8internal20itt_set_sync_name_v3EPvPKc ) -__TBB_SYMBOL( _ZN3tbb8internal19itt_load_pointer_v3EPKv ) -#if __TBB_ITT_STRUCTURE_API -__TBB_SYMBOL( _ZN3tbb8internal23itt_metadata_str_add_v7ENS0_15itt_domain_enumEPvyNS0_12string_indexEPKc ) -__TBB_SYMBOL( _ZN3tbb8internal22itt_make_task_group_v7ENS0_15itt_domain_enumEPvyS2_yNS0_12string_indexE ) -__TBB_SYMBOL( _ZN3tbb8internal17itt_task_begin_v7ENS0_15itt_domain_enumEPvyS2_yNS0_12string_indexE ) -__TBB_SYMBOL( _ZN3tbb8internal19itt_relation_add_v7ENS0_15itt_domain_enumEPvyNS0_12itt_relationES2_y ) -__TBB_SYMBOL( _ZN3tbb8internal15itt_task_end_v7ENS0_15itt_domain_enumE ) -__TBB_SYMBOL( _ZN3tbb8internal19itt_region_begin_v9ENS0_15itt_domain_enumEPvyS2_yNS0_12string_indexE ) -__TBB_SYMBOL( _ZN3tbb8internal17itt_region_end_v9ENS0_15itt_domain_enumEPvy ) -#endif - -/* pipeline.cpp */ -__TBB_SYMBOL( _ZTIN3tbb6filterE ) -__TBB_SYMBOL( _ZTSN3tbb6filterE ) -__TBB_SYMBOL( _ZTVN3tbb6filterE ) -__TBB_SYMBOL( _ZN3tbb6filterD2Ev ) -__TBB_SYMBOL( _ZN3tbb8pipeline10add_filterERNS_6filterE ) -__TBB_SYMBOL( _ZN3tbb8pipeline12inject_tokenERNS_4taskE ) -__TBB_SYMBOL( _ZN3tbb8pipeline13remove_filterERNS_6filterE ) -__TBB_SYMBOL( _ZN3tbb8pipeline3runEm ) -#if __TBB_TASK_GROUP_CONTEXT -__TBB_SYMBOL( _ZN3tbb8pipeline3runEmRNS_18task_group_contextE ) -#endif -__TBB_SYMBOL( _ZN3tbb8pipeline5clearEv ) -__TBB_SYMBOL( _ZN3tbb19thread_bound_filter12process_itemEv ) -__TBB_SYMBOL( _ZN3tbb19thread_bound_filter16try_process_itemEv ) -__TBB_SYMBOL( _ZTIN3tbb8pipelineE ) -__TBB_SYMBOL( _ZTSN3tbb8pipelineE ) -__TBB_SYMBOL( _ZTVN3tbb8pipelineE ) -__TBB_SYMBOL( _ZN3tbb8pipelineC1Ev ) -__TBB_SYMBOL( _ZN3tbb8pipelineC2Ev ) -__TBB_SYMBOL( _ZN3tbb8pipelineD0Ev ) -__TBB_SYMBOL( _ZN3tbb8pipelineD1Ev ) -__TBB_SYMBOL( _ZN3tbb8pipelineD2Ev ) -__TBB_SYMBOL( _ZN3tbb6filter16set_end_of_inputEv ) - -/* queuing_rw_mutex.cpp */ -__TBB_SYMBOL( _ZN3tbb16queuing_rw_mutex18internal_constructEv ) -__TBB_SYMBOL( _ZN3tbb16queuing_rw_mutex11scoped_lock17upgrade_to_writerEv ) -__TBB_SYMBOL( _ZN3tbb16queuing_rw_mutex11scoped_lock19downgrade_to_readerEv ) -__TBB_SYMBOL( _ZN3tbb16queuing_rw_mutex11scoped_lock7acquireERS0_b ) -__TBB_SYMBOL( _ZN3tbb16queuing_rw_mutex11scoped_lock7releaseEv ) -__TBB_SYMBOL( _ZN3tbb16queuing_rw_mutex11scoped_lock11try_acquireERS0_b ) - -/* reader_writer_lock.cpp */ -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock11scoped_lock16internal_destroyEv ) -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock11scoped_lock18internal_constructERS1_ ) -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock13try_lock_readEv ) -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock16scoped_lock_read16internal_destroyEv ) -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock16scoped_lock_read18internal_constructERS1_ ) -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock16internal_destroyEv ) -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock18internal_constructEv ) -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock4lockEv ) -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock6unlockEv ) -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock8try_lockEv ) -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock9lock_readEv ) - -#if !TBB_NO_LEGACY -/* spin_rw_mutex.cpp v2 */ -__TBB_SYMBOL( _ZN3tbb13spin_rw_mutex16internal_upgradeEPS0_ ) -__TBB_SYMBOL( _ZN3tbb13spin_rw_mutex22internal_itt_releasingEPS0_ ) -__TBB_SYMBOL( _ZN3tbb13spin_rw_mutex23internal_acquire_readerEPS0_ ) -__TBB_SYMBOL( _ZN3tbb13spin_rw_mutex23internal_acquire_writerEPS0_ ) -__TBB_SYMBOL( _ZN3tbb13spin_rw_mutex18internal_downgradeEPS0_ ) -__TBB_SYMBOL( _ZN3tbb13spin_rw_mutex23internal_release_readerEPS0_ ) -__TBB_SYMBOL( _ZN3tbb13spin_rw_mutex23internal_release_writerEPS0_ ) -__TBB_SYMBOL( _ZN3tbb13spin_rw_mutex27internal_try_acquire_readerEPS0_ ) -__TBB_SYMBOL( _ZN3tbb13spin_rw_mutex27internal_try_acquire_writerEPS0_ ) -#endif - -// x86_rtm_rw_mutex.cpp -#if __TBB_TSX_AVAILABLE -__TBB_SYMBOL( _ZN3tbb10interface88internal16x86_rtm_rw_mutex18internal_constructEv ) -__TBB_SYMBOL( _ZN3tbb10interface88internal16x86_rtm_rw_mutex23internal_acquire_writerERNS2_11scoped_lockEb ) -__TBB_SYMBOL( _ZN3tbb10interface88internal16x86_rtm_rw_mutex27internal_try_acquire_writerERNS2_11scoped_lockE ) -__TBB_SYMBOL( _ZN3tbb10interface88internal16x86_rtm_rw_mutex23internal_acquire_readerERNS2_11scoped_lockEb ) -__TBB_SYMBOL( _ZN3tbb10interface88internal16x86_rtm_rw_mutex16internal_releaseERNS2_11scoped_lockE ) -__TBB_SYMBOL( _ZN3tbb10interface88internal16x86_rtm_rw_mutex16internal_upgradeERNS2_11scoped_lockE ) -__TBB_SYMBOL( _ZN3tbb10interface88internal16x86_rtm_rw_mutex18internal_downgradeERNS2_11scoped_lockE ) -#endif - -/* spin_rw_mutex v3 */ -__TBB_SYMBOL( _ZN3tbb16spin_rw_mutex_v318internal_constructEv ) -__TBB_SYMBOL( _ZN3tbb16spin_rw_mutex_v316internal_upgradeEv ) -__TBB_SYMBOL( _ZN3tbb16spin_rw_mutex_v318internal_downgradeEv ) -__TBB_SYMBOL( _ZN3tbb16spin_rw_mutex_v323internal_acquire_readerEv ) -__TBB_SYMBOL( _ZN3tbb16spin_rw_mutex_v323internal_acquire_writerEv ) -__TBB_SYMBOL( _ZN3tbb16spin_rw_mutex_v323internal_release_readerEv ) -__TBB_SYMBOL( _ZN3tbb16spin_rw_mutex_v323internal_release_writerEv ) -__TBB_SYMBOL( _ZN3tbb16spin_rw_mutex_v327internal_try_acquire_readerEv ) -__TBB_SYMBOL( _ZN3tbb16spin_rw_mutex_v327internal_try_acquire_writerEv ) - -/* spin_mutex.cpp */ -__TBB_SYMBOL( _ZN3tbb10spin_mutex11scoped_lock16internal_acquireERS0_ ) -__TBB_SYMBOL( _ZN3tbb10spin_mutex11scoped_lock16internal_releaseEv ) -__TBB_SYMBOL( _ZN3tbb10spin_mutex11scoped_lock20internal_try_acquireERS0_ ) -__TBB_SYMBOL( _ZN3tbb10spin_mutex18internal_constructEv ) - -/* mutex.cpp */ -__TBB_SYMBOL( _ZN3tbb5mutex11scoped_lock16internal_acquireERS0_ ) -__TBB_SYMBOL( _ZN3tbb5mutex11scoped_lock16internal_releaseEv ) -__TBB_SYMBOL( _ZN3tbb5mutex11scoped_lock20internal_try_acquireERS0_ ) -__TBB_SYMBOL( _ZN3tbb5mutex16internal_destroyEv ) -__TBB_SYMBOL( _ZN3tbb5mutex18internal_constructEv ) - -/* recursive_mutex.cpp */ -__TBB_SYMBOL( _ZN3tbb15recursive_mutex11scoped_lock16internal_acquireERS0_ ) -__TBB_SYMBOL( _ZN3tbb15recursive_mutex11scoped_lock16internal_releaseEv ) -__TBB_SYMBOL( _ZN3tbb15recursive_mutex11scoped_lock20internal_try_acquireERS0_ ) -__TBB_SYMBOL( _ZN3tbb15recursive_mutex16internal_destroyEv ) -__TBB_SYMBOL( _ZN3tbb15recursive_mutex18internal_constructEv ) - -/* QueuingMutex.cpp */ -__TBB_SYMBOL( _ZN3tbb13queuing_mutex18internal_constructEv ) -__TBB_SYMBOL( _ZN3tbb13queuing_mutex11scoped_lock7acquireERS0_ ) -__TBB_SYMBOL( _ZN3tbb13queuing_mutex11scoped_lock7releaseEv ) -__TBB_SYMBOL( _ZN3tbb13queuing_mutex11scoped_lock11try_acquireERS0_ ) - -/* critical_section.cpp */ -__TBB_SYMBOL( _ZN3tbb8internal19critical_section_v418internal_constructEv ) - -#if !TBB_NO_LEGACY -/* concurrent_hash_map */ -__TBB_SYMBOL( _ZNK3tbb8internal21hash_map_segment_base23internal_grow_predicateEv ) - -/* concurrent_queue.cpp v2 */ -__TBB_SYMBOL( _ZN3tbb8internal21concurrent_queue_base12internal_popEPv ) -__TBB_SYMBOL( _ZN3tbb8internal21concurrent_queue_base13internal_pushEPKv ) -__TBB_SYMBOL( _ZN3tbb8internal21concurrent_queue_base21internal_set_capacityElm ) -__TBB_SYMBOL( _ZN3tbb8internal21concurrent_queue_base23internal_pop_if_presentEPv ) -__TBB_SYMBOL( _ZN3tbb8internal21concurrent_queue_base25internal_push_if_not_fullEPKv ) -__TBB_SYMBOL( _ZN3tbb8internal21concurrent_queue_baseC2Em ) -__TBB_SYMBOL( _ZN3tbb8internal21concurrent_queue_baseD2Ev ) -__TBB_SYMBOL( _ZTIN3tbb8internal21concurrent_queue_baseE ) -__TBB_SYMBOL( _ZTSN3tbb8internal21concurrent_queue_baseE ) -__TBB_SYMBOL( _ZTVN3tbb8internal21concurrent_queue_baseE ) -__TBB_SYMBOL( _ZN3tbb8internal30concurrent_queue_iterator_base6assignERKS1_ ) -__TBB_SYMBOL( _ZN3tbb8internal30concurrent_queue_iterator_base7advanceEv ) -__TBB_SYMBOL( _ZN3tbb8internal30concurrent_queue_iterator_baseC2ERKNS0_21concurrent_queue_baseE ) -__TBB_SYMBOL( _ZN3tbb8internal30concurrent_queue_iterator_baseD2Ev ) -__TBB_SYMBOL( _ZNK3tbb8internal21concurrent_queue_base13internal_sizeEv ) -#endif - -/* concurrent_queue v3 */ -/* constructors */ -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v3C2Em ) -__TBB_SYMBOL( _ZN3tbb8internal33concurrent_queue_iterator_base_v3C2ERKNS0_24concurrent_queue_base_v3E ) -__TBB_SYMBOL( _ZN3tbb8internal33concurrent_queue_iterator_base_v3C2ERKNS0_24concurrent_queue_base_v3Em ) -/* destructors */ -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v3D2Ev ) -__TBB_SYMBOL( _ZN3tbb8internal33concurrent_queue_iterator_base_v3D2Ev ) -/* typeinfo */ -__TBB_SYMBOL( _ZTIN3tbb8internal24concurrent_queue_base_v3E ) -__TBB_SYMBOL( _ZTSN3tbb8internal24concurrent_queue_base_v3E ) -/* vtable */ -__TBB_SYMBOL( _ZTVN3tbb8internal24concurrent_queue_base_v3E ) -/* methods */ -__TBB_SYMBOL( _ZN3tbb8internal33concurrent_queue_iterator_base_v36assignERKS1_ ) -__TBB_SYMBOL( _ZN3tbb8internal33concurrent_queue_iterator_base_v37advanceEv ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v313internal_pushEPKv ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v818internal_push_moveEPKv ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v325internal_push_if_not_fullEPKv ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v830internal_push_move_if_not_fullEPKv ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v312internal_popEPv ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v323internal_pop_if_presentEPv ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v314internal_abortEv ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v321internal_finish_clearEv ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v321internal_set_capacityElm ) -__TBB_SYMBOL( _ZNK3tbb8internal24concurrent_queue_base_v313internal_sizeEv ) -__TBB_SYMBOL( _ZNK3tbb8internal24concurrent_queue_base_v314internal_emptyEv ) -__TBB_SYMBOL( _ZNK3tbb8internal24concurrent_queue_base_v324internal_throw_exceptionEv ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v36assignERKS1_ ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v812move_contentERS1_ ) - -#if !TBB_NO_LEGACY -/* concurrent_vector.cpp v2 */ -__TBB_SYMBOL( _ZN3tbb8internal22concurrent_vector_base13internal_copyERKS1_mPFvPvPKvmE ) -__TBB_SYMBOL( _ZN3tbb8internal22concurrent_vector_base14internal_clearEPFvPvmEb ) -__TBB_SYMBOL( _ZN3tbb8internal22concurrent_vector_base15internal_assignERKS1_mPFvPvmEPFvS4_PKvmESA_ ) -__TBB_SYMBOL( _ZN3tbb8internal22concurrent_vector_base16internal_grow_byEmmPFvPvmE ) -__TBB_SYMBOL( _ZN3tbb8internal22concurrent_vector_base16internal_reserveEmmm ) -__TBB_SYMBOL( _ZN3tbb8internal22concurrent_vector_base18internal_push_backEmRm ) -__TBB_SYMBOL( _ZN3tbb8internal22concurrent_vector_base25internal_grow_to_at_leastEmmPFvPvmE ) -__TBB_SYMBOL( _ZNK3tbb8internal22concurrent_vector_base17internal_capacityEv ) -#endif - -/* concurrent_vector v3 */ -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v313internal_copyERKS1_mPFvPvPKvmE ) -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v314internal_clearEPFvPvmE ) -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v315internal_assignERKS1_mPFvPvmEPFvS4_PKvmESA_ ) -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v316internal_grow_byEmmPFvPvPKvmES4_ ) -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v316internal_reserveEmmm ) -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v318internal_push_backEmRm ) -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v325internal_grow_to_at_leastEmmPFvPvPKvmES4_ ) -__TBB_SYMBOL( _ZNK3tbb8internal25concurrent_vector_base_v317internal_capacityEv ) -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v316internal_compactEmPvPFvS2_mEPFvS2_PKvmE ) -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v313internal_swapERS1_ ) -__TBB_SYMBOL( _ZNK3tbb8internal25concurrent_vector_base_v324internal_throw_exceptionEm ) -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v3D2Ev ) -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v315internal_resizeEmmmPKvPFvPvmEPFvS4_S3_mE ) -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v337internal_grow_to_at_least_with_resultEmmPFvPvPKvmES4_ ) - -/* tbb_thread */ -__TBB_SYMBOL( _ZN3tbb8internal13tbb_thread_v320hardware_concurrencyEv ) -__TBB_SYMBOL( _ZN3tbb8internal13tbb_thread_v36detachEv ) -__TBB_SYMBOL( _ZN3tbb8internal16thread_get_id_v3Ev ) -__TBB_SYMBOL( _ZN3tbb8internal15free_closure_v3EPv ) -__TBB_SYMBOL( _ZN3tbb8internal13tbb_thread_v34joinEv ) -__TBB_SYMBOL( _ZN3tbb8internal13tbb_thread_v314internal_startEPFPvS2_ES2_ ) -__TBB_SYMBOL( _ZN3tbb8internal19allocate_closure_v3Em ) -__TBB_SYMBOL( _ZN3tbb8internal7move_v3ERNS0_13tbb_thread_v3ES2_ ) -__TBB_SYMBOL( _ZN3tbb8internal15thread_yield_v3Ev ) -__TBB_SYMBOL( _ZN3tbb8internal15thread_sleep_v3ERKNS_10tick_count10interval_tE ) - -/* global_parameter */ -__TBB_SYMBOL( _ZN3tbb10interface914global_control12active_valueEi ) -__TBB_SYMBOL( _ZN3tbb10interface914global_control15internal_createEv ) -__TBB_SYMBOL( _ZN3tbb10interface914global_control16internal_destroyEv ) -#undef __TBB_SYMBOL diff --git a/src/tbb/src/tbb/lin64ipf-tbb-export.def b/src/tbb/src/tbb/lin64ipf-tbb-export.def deleted file mode 100644 index c2111c1ef..000000000 --- a/src/tbb/src/tbb/lin64ipf-tbb-export.def +++ /dev/null @@ -1,48 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -{ -global: - -#define __TBB_SYMBOL( sym ) sym; -#include "lin64ipf-tbb-export.lst" - -local: - -/* TBB symbols */ -*3tbb*; -*__TBB*; - -/* ITT symbols */ -__itt_*; - -/* Intel Compiler (libirc) symbols */ -__intel_*; -_intel_*; -?0_memcopyA; -?0_memcopyDu; -?0_memcpyD; -?1__memcpy; -?1__memmove; -?1__serial_memmove; -memcpy; -memset; - -}; diff --git a/src/tbb/src/tbb/lin64ipf-tbb-export.lst b/src/tbb/src/tbb/lin64ipf-tbb-export.lst deleted file mode 100644 index 49f8cee72..000000000 --- a/src/tbb/src/tbb/lin64ipf-tbb-export.lst +++ /dev/null @@ -1,417 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#include "tbb/tbb_config.h" - -/* cache_aligned_allocator.cpp */ -__TBB_SYMBOL( _ZN3tbb8internal12NFS_AllocateEmmPv ) -__TBB_SYMBOL( _ZN3tbb8internal15NFS_GetLineSizeEv ) -__TBB_SYMBOL( _ZN3tbb8internal8NFS_FreeEPv ) -__TBB_SYMBOL( _ZN3tbb8internal23allocate_via_handler_v3Em ) -__TBB_SYMBOL( _ZN3tbb8internal25deallocate_via_handler_v3EPv ) -__TBB_SYMBOL( _ZN3tbb8internal17is_malloc_used_v3Ev ) - -/* task.cpp v3 */ -__TBB_SYMBOL( _ZN3tbb4task13note_affinityEt ) -__TBB_SYMBOL( _ZN3tbb4task22internal_set_ref_countEi ) -__TBB_SYMBOL( _ZN3tbb4task28internal_decrement_ref_countEv ) -__TBB_SYMBOL( _ZN3tbb4task22spawn_and_wait_for_allERNS_9task_listE ) -__TBB_SYMBOL( _ZN3tbb4task4selfEv ) -__TBB_SYMBOL( _ZN3tbb10interface58internal9task_base7destroyERNS_4taskE ) -__TBB_SYMBOL( _ZNK3tbb4task26is_owned_by_current_threadEv ) -__TBB_SYMBOL( _ZN3tbb8internal19allocate_root_proxy4freeERNS_4taskE ) -__TBB_SYMBOL( _ZN3tbb8internal19allocate_root_proxy8allocateEm ) -__TBB_SYMBOL( _ZN3tbb8internal28affinity_partitioner_base_v36resizeEj ) -__TBB_SYMBOL( _ZNK3tbb8internal20allocate_child_proxy4freeERNS_4taskE ) -__TBB_SYMBOL( _ZNK3tbb8internal20allocate_child_proxy8allocateEm ) -__TBB_SYMBOL( _ZNK3tbb8internal27allocate_continuation_proxy4freeERNS_4taskE ) -__TBB_SYMBOL( _ZNK3tbb8internal27allocate_continuation_proxy8allocateEm ) -__TBB_SYMBOL( _ZNK3tbb8internal34allocate_additional_child_of_proxy4freeERNS_4taskE ) -__TBB_SYMBOL( _ZNK3tbb8internal34allocate_additional_child_of_proxy8allocateEm ) -__TBB_SYMBOL( _ZTIN3tbb4taskE ) -__TBB_SYMBOL( _ZTSN3tbb4taskE ) -__TBB_SYMBOL( _ZTVN3tbb4taskE ) -__TBB_SYMBOL( _ZN3tbb19task_scheduler_init19default_num_threadsEv ) -__TBB_SYMBOL( _ZN3tbb19task_scheduler_init10initializeEim ) -__TBB_SYMBOL( _ZN3tbb19task_scheduler_init10initializeEi ) -__TBB_SYMBOL( _ZN3tbb19task_scheduler_init9terminateEv ) -__TBB_SYMBOL( _ZN3tbb19task_scheduler_init27internal_blocking_terminateEb ) -#if __TBB_SCHEDULER_OBSERVER -__TBB_SYMBOL( _ZN3tbb8internal26task_scheduler_observer_v37observeEb ) -#endif /* __TBB_SCHEDULER_OBSERVER */ -__TBB_SYMBOL( _ZN3tbb10empty_task7executeEv ) -__TBB_SYMBOL( _ZN3tbb10empty_taskD0Ev ) -__TBB_SYMBOL( _ZN3tbb10empty_taskD1Ev ) -__TBB_SYMBOL( _ZTIN3tbb10empty_taskE ) -__TBB_SYMBOL( _ZTSN3tbb10empty_taskE ) -__TBB_SYMBOL( _ZTVN3tbb10empty_taskE ) - -/* arena.cpp */ -__TBB_SYMBOL( _ZN3tbb10interface78internal15task_arena_base19internal_initializeEv ) -__TBB_SYMBOL( _ZN3tbb10interface78internal15task_arena_base18internal_terminateEv ) -__TBB_SYMBOL( _ZN3tbb10interface78internal15task_arena_base15internal_attachEv ) -__TBB_SYMBOL( _ZNK3tbb10interface78internal15task_arena_base16internal_enqueueERNS_4taskEl ) -__TBB_SYMBOL( _ZNK3tbb10interface78internal15task_arena_base16internal_executeERNS1_13delegate_baseE ) -__TBB_SYMBOL( _ZNK3tbb10interface78internal15task_arena_base13internal_waitEv ) -__TBB_SYMBOL( _ZN3tbb10interface78internal15task_arena_base21internal_current_slotEv ) -__TBB_SYMBOL( _ZN3tbb10interface78internal15task_arena_base24internal_max_concurrencyEPKNS0_10task_arenaE ) -#if __TBB_TASK_ISOLATION -__TBB_SYMBOL( _ZN3tbb10interface78internal20isolate_within_arenaERNS1_13delegate_baseEl ) -#endif - -#if !TBB_NO_LEGACY -/* task_v2.cpp */ -__TBB_SYMBOL( _ZN3tbb4task7destroyERS0_ ) -#endif /* !TBB_NO_LEGACY */ - -/* Exception handling in task scheduler */ -#if __TBB_TASK_GROUP_CONTEXT -__TBB_SYMBOL( _ZNK3tbb8internal32allocate_root_with_context_proxy8allocateEm ) -__TBB_SYMBOL( _ZNK3tbb8internal32allocate_root_with_context_proxy4freeERNS_4taskE ) -__TBB_SYMBOL( _ZN3tbb4task12change_groupERNS_18task_group_contextE ) -__TBB_SYMBOL( _ZNK3tbb18task_group_context28is_group_execution_cancelledEv ) -__TBB_SYMBOL( _ZN3tbb18task_group_context22cancel_group_executionEv ) -__TBB_SYMBOL( _ZN3tbb18task_group_context26register_pending_exceptionEv ) -__TBB_SYMBOL( _ZN3tbb18task_group_context5resetEv ) -__TBB_SYMBOL( _ZN3tbb18task_group_context19capture_fp_settingsEv ) -__TBB_SYMBOL( _ZN3tbb18task_group_context4initEv ) -__TBB_SYMBOL( _ZN3tbb18task_group_contextD1Ev ) -__TBB_SYMBOL( _ZN3tbb18task_group_contextD2Ev ) -#if __TBB_TASK_PRIORITY -__TBB_SYMBOL( _ZN3tbb18task_group_context12set_priorityENS_10priority_tE ) -__TBB_SYMBOL( _ZNK3tbb18task_group_context8priorityEv ) -#endif /* __TBB_TASK_PRIORITY */ -__TBB_SYMBOL( _ZNK3tbb18captured_exception4nameEv ) -__TBB_SYMBOL( _ZNK3tbb18captured_exception4whatEv ) -__TBB_SYMBOL( _ZN3tbb18captured_exception10throw_selfEv ) -__TBB_SYMBOL( _ZN3tbb18captured_exception3setEPKcS2_ ) -__TBB_SYMBOL( _ZN3tbb18captured_exception4moveEv ) -__TBB_SYMBOL( _ZN3tbb18captured_exception5clearEv ) -__TBB_SYMBOL( _ZN3tbb18captured_exception7destroyEv ) -__TBB_SYMBOL( _ZN3tbb18captured_exception8allocateEPKcS2_ ) -__TBB_SYMBOL( _ZN3tbb18captured_exceptionD0Ev ) -__TBB_SYMBOL( _ZN3tbb18captured_exceptionD1Ev ) -__TBB_SYMBOL( _ZN3tbb18captured_exceptionD2Ev ) -__TBB_SYMBOL( _ZTIN3tbb18captured_exceptionE ) -__TBB_SYMBOL( _ZTSN3tbb18captured_exceptionE ) -__TBB_SYMBOL( _ZTVN3tbb18captured_exceptionE ) -__TBB_SYMBOL( _ZN3tbb13tbb_exceptionD2Ev ) -__TBB_SYMBOL( _ZTIN3tbb13tbb_exceptionE ) -__TBB_SYMBOL( _ZTSN3tbb13tbb_exceptionE ) -__TBB_SYMBOL( _ZTVN3tbb13tbb_exceptionE ) -#endif /* __TBB_TASK_GROUP_CONTEXT */ - -/* Symbols for exceptions thrown from TBB */ -__TBB_SYMBOL( _ZN3tbb8internal33throw_bad_last_alloc_exception_v4Ev ) -__TBB_SYMBOL( _ZN3tbb8internal18throw_exception_v4ENS0_12exception_idE ) -__TBB_SYMBOL( _ZN3tbb14bad_last_allocD0Ev ) -__TBB_SYMBOL( _ZN3tbb14bad_last_allocD1Ev ) -__TBB_SYMBOL( _ZNK3tbb14bad_last_alloc4whatEv ) -__TBB_SYMBOL( _ZTIN3tbb14bad_last_allocE ) -__TBB_SYMBOL( _ZTSN3tbb14bad_last_allocE ) -__TBB_SYMBOL( _ZTVN3tbb14bad_last_allocE ) -__TBB_SYMBOL( _ZN3tbb12missing_waitD0Ev ) -__TBB_SYMBOL( _ZN3tbb12missing_waitD1Ev ) -__TBB_SYMBOL( _ZNK3tbb12missing_wait4whatEv ) -__TBB_SYMBOL( _ZTIN3tbb12missing_waitE ) -__TBB_SYMBOL( _ZTSN3tbb12missing_waitE ) -__TBB_SYMBOL( _ZTVN3tbb12missing_waitE ) -__TBB_SYMBOL( _ZN3tbb27invalid_multiple_schedulingD0Ev ) -__TBB_SYMBOL( _ZN3tbb27invalid_multiple_schedulingD1Ev ) -__TBB_SYMBOL( _ZNK3tbb27invalid_multiple_scheduling4whatEv ) -__TBB_SYMBOL( _ZTIN3tbb27invalid_multiple_schedulingE ) -__TBB_SYMBOL( _ZTSN3tbb27invalid_multiple_schedulingE ) -__TBB_SYMBOL( _ZTVN3tbb27invalid_multiple_schedulingE ) -__TBB_SYMBOL( _ZN3tbb13improper_lockD0Ev ) -__TBB_SYMBOL( _ZN3tbb13improper_lockD1Ev ) -__TBB_SYMBOL( _ZNK3tbb13improper_lock4whatEv ) -__TBB_SYMBOL( _ZTIN3tbb13improper_lockE ) -__TBB_SYMBOL( _ZTSN3tbb13improper_lockE ) -__TBB_SYMBOL( _ZTVN3tbb13improper_lockE ) -__TBB_SYMBOL( _ZN3tbb10user_abortD0Ev ) -__TBB_SYMBOL( _ZN3tbb10user_abortD1Ev ) -__TBB_SYMBOL( _ZNK3tbb10user_abort4whatEv ) -__TBB_SYMBOL( _ZTIN3tbb10user_abortE ) -__TBB_SYMBOL( _ZTSN3tbb10user_abortE ) -__TBB_SYMBOL( _ZTVN3tbb10user_abortE ) - -/* tbb_misc.cpp */ -__TBB_SYMBOL( _ZN3tbb17assertion_failureEPKciS1_S1_ ) -__TBB_SYMBOL( _ZN3tbb21set_assertion_handlerEPFvPKciS1_S1_E ) -__TBB_SYMBOL( _ZN3tbb8internal36get_initial_auto_partitioner_divisorEv ) -__TBB_SYMBOL( _ZN3tbb8internal13handle_perrorEiPKc ) -__TBB_SYMBOL( _ZN3tbb8internal15runtime_warningEPKcz ) -__TBB_SYMBOL( TBB_runtime_interface_version ) - -/* tbb_main.cpp */ -__TBB_SYMBOL( _ZN3tbb8internal32itt_load_pointer_with_acquire_v3EPKv ) -__TBB_SYMBOL( _ZN3tbb8internal33itt_store_pointer_with_release_v3EPvS1_ ) -__TBB_SYMBOL( _ZN3tbb8internal18call_itt_notify_v5EiPv ) -__TBB_SYMBOL( _ZN3tbb8internal20itt_set_sync_name_v3EPvPKc ) -__TBB_SYMBOL( _ZN3tbb8internal19itt_load_pointer_v3EPKv ) - -/* pipeline.cpp */ -__TBB_SYMBOL( _ZTIN3tbb6filterE ) -__TBB_SYMBOL( _ZTSN3tbb6filterE ) -__TBB_SYMBOL( _ZTVN3tbb6filterE ) -__TBB_SYMBOL( _ZN3tbb6filterD2Ev ) -__TBB_SYMBOL( _ZN3tbb8pipeline10add_filterERNS_6filterE ) -__TBB_SYMBOL( _ZN3tbb8pipeline12inject_tokenERNS_4taskE ) -__TBB_SYMBOL( _ZN3tbb8pipeline13remove_filterERNS_6filterE ) -__TBB_SYMBOL( _ZN3tbb8pipeline3runEm ) -#if __TBB_TASK_GROUP_CONTEXT -__TBB_SYMBOL( _ZN3tbb8pipeline3runEmRNS_18task_group_contextE ) -#endif -__TBB_SYMBOL( _ZN3tbb8pipeline5clearEv ) -__TBB_SYMBOL( _ZN3tbb19thread_bound_filter12process_itemEv ) -__TBB_SYMBOL( _ZN3tbb19thread_bound_filter16try_process_itemEv ) -__TBB_SYMBOL( _ZTIN3tbb8pipelineE ) -__TBB_SYMBOL( _ZTSN3tbb8pipelineE ) -__TBB_SYMBOL( _ZTVN3tbb8pipelineE ) -__TBB_SYMBOL( _ZN3tbb8pipelineC1Ev ) -__TBB_SYMBOL( _ZN3tbb8pipelineC2Ev ) -__TBB_SYMBOL( _ZN3tbb8pipelineD0Ev ) -__TBB_SYMBOL( _ZN3tbb8pipelineD1Ev ) -__TBB_SYMBOL( _ZN3tbb8pipelineD2Ev ) -__TBB_SYMBOL( _ZN3tbb6filter16set_end_of_inputEv ) - -/* queuing_rw_mutex.cpp */ -__TBB_SYMBOL( _ZN3tbb16queuing_rw_mutex18internal_constructEv ) -__TBB_SYMBOL( _ZN3tbb16queuing_rw_mutex11scoped_lock17upgrade_to_writerEv ) -__TBB_SYMBOL( _ZN3tbb16queuing_rw_mutex11scoped_lock19downgrade_to_readerEv ) -__TBB_SYMBOL( _ZN3tbb16queuing_rw_mutex11scoped_lock7acquireERS0_b ) -__TBB_SYMBOL( _ZN3tbb16queuing_rw_mutex11scoped_lock7releaseEv ) -__TBB_SYMBOL( _ZN3tbb16queuing_rw_mutex11scoped_lock11try_acquireERS0_b ) - -/* reader_writer_lock.cpp */ -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock11scoped_lock16internal_destroyEv ) -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock11scoped_lock18internal_constructERS1_ ) -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock13try_lock_readEv ) -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock16scoped_lock_read16internal_destroyEv ) -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock16scoped_lock_read18internal_constructERS1_ ) -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock16internal_destroyEv ) -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock18internal_constructEv ) -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock4lockEv ) -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock6unlockEv ) -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock8try_lockEv ) -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock9lock_readEv ) - -#if !TBB_NO_LEGACY -/* spin_rw_mutex.cpp v2 */ -__TBB_SYMBOL( _ZN3tbb13spin_rw_mutex16internal_upgradeEPS0_ ) -__TBB_SYMBOL( _ZN3tbb13spin_rw_mutex22internal_itt_releasingEPS0_ ) -__TBB_SYMBOL( _ZN3tbb13spin_rw_mutex23internal_acquire_readerEPS0_ ) -__TBB_SYMBOL( _ZN3tbb13spin_rw_mutex23internal_acquire_writerEPS0_ ) -__TBB_SYMBOL( _ZN3tbb13spin_rw_mutex18internal_downgradeEPS0_ ) -__TBB_SYMBOL( _ZN3tbb13spin_rw_mutex23internal_release_readerEPS0_ ) -__TBB_SYMBOL( _ZN3tbb13spin_rw_mutex23internal_release_writerEPS0_ ) -__TBB_SYMBOL( _ZN3tbb13spin_rw_mutex27internal_try_acquire_readerEPS0_ ) -__TBB_SYMBOL( _ZN3tbb13spin_rw_mutex27internal_try_acquire_writerEPS0_ ) -#endif - -/* spin_rw_mutex v3 */ -__TBB_SYMBOL( _ZN3tbb16spin_rw_mutex_v318internal_constructEv ) -__TBB_SYMBOL( _ZN3tbb16spin_rw_mutex_v316internal_upgradeEv ) -__TBB_SYMBOL( _ZN3tbb16spin_rw_mutex_v318internal_downgradeEv ) -__TBB_SYMBOL( _ZN3tbb16spin_rw_mutex_v323internal_acquire_readerEv ) -__TBB_SYMBOL( _ZN3tbb16spin_rw_mutex_v323internal_acquire_writerEv ) -__TBB_SYMBOL( _ZN3tbb16spin_rw_mutex_v323internal_release_readerEv ) -__TBB_SYMBOL( _ZN3tbb16spin_rw_mutex_v323internal_release_writerEv ) -__TBB_SYMBOL( _ZN3tbb16spin_rw_mutex_v327internal_try_acquire_readerEv ) -__TBB_SYMBOL( _ZN3tbb16spin_rw_mutex_v327internal_try_acquire_writerEv ) - -/* spin_mutex.cpp */ -__TBB_SYMBOL( _ZN3tbb10spin_mutex18internal_constructEv ) -__TBB_SYMBOL( _ZN3tbb10spin_mutex11scoped_lock16internal_acquireERS0_ ) -__TBB_SYMBOL( _ZN3tbb10spin_mutex11scoped_lock16internal_releaseEv ) -__TBB_SYMBOL( _ZN3tbb10spin_mutex11scoped_lock20internal_try_acquireERS0_ ) - -/* mutex.cpp */ -__TBB_SYMBOL( _ZN3tbb5mutex11scoped_lock16internal_acquireERS0_ ) -__TBB_SYMBOL( _ZN3tbb5mutex11scoped_lock16internal_releaseEv ) -__TBB_SYMBOL( _ZN3tbb5mutex11scoped_lock20internal_try_acquireERS0_ ) -__TBB_SYMBOL( _ZN3tbb5mutex16internal_destroyEv ) -__TBB_SYMBOL( _ZN3tbb5mutex18internal_constructEv ) - -/* recursive_mutex.cpp */ -__TBB_SYMBOL( _ZN3tbb15recursive_mutex11scoped_lock16internal_acquireERS0_ ) -__TBB_SYMBOL( _ZN3tbb15recursive_mutex11scoped_lock16internal_releaseEv ) -__TBB_SYMBOL( _ZN3tbb15recursive_mutex11scoped_lock20internal_try_acquireERS0_ ) -__TBB_SYMBOL( _ZN3tbb15recursive_mutex16internal_destroyEv ) -__TBB_SYMBOL( _ZN3tbb15recursive_mutex18internal_constructEv ) - -/* QueuingMutex.cpp */ -__TBB_SYMBOL( _ZN3tbb13queuing_mutex18internal_constructEv ) -__TBB_SYMBOL( _ZN3tbb13queuing_mutex11scoped_lock7acquireERS0_ ) -__TBB_SYMBOL( _ZN3tbb13queuing_mutex11scoped_lock7releaseEv ) -__TBB_SYMBOL( _ZN3tbb13queuing_mutex11scoped_lock11try_acquireERS0_ ) - -/* critical_section.cpp */ -__TBB_SYMBOL( _ZN3tbb8internal19critical_section_v418internal_constructEv ) - -#if !TBB_NO_LEGACY -/* concurrent_hash_map */ -__TBB_SYMBOL( _ZNK3tbb8internal21hash_map_segment_base23internal_grow_predicateEv ) - -/* concurrent_queue.cpp v2 */ -__TBB_SYMBOL( _ZN3tbb8internal21concurrent_queue_base12internal_popEPv ) -__TBB_SYMBOL( _ZN3tbb8internal21concurrent_queue_base13internal_pushEPKv ) -__TBB_SYMBOL( _ZN3tbb8internal21concurrent_queue_base21internal_set_capacityElm ) -__TBB_SYMBOL( _ZN3tbb8internal21concurrent_queue_base23internal_pop_if_presentEPv ) -__TBB_SYMBOL( _ZN3tbb8internal21concurrent_queue_base25internal_push_if_not_fullEPKv ) -__TBB_SYMBOL( _ZN3tbb8internal21concurrent_queue_baseC2Em ) -__TBB_SYMBOL( _ZN3tbb8internal21concurrent_queue_baseD2Ev ) -__TBB_SYMBOL( _ZTIN3tbb8internal21concurrent_queue_baseE ) -__TBB_SYMBOL( _ZTSN3tbb8internal21concurrent_queue_baseE ) -__TBB_SYMBOL( _ZTVN3tbb8internal21concurrent_queue_baseE ) -__TBB_SYMBOL( _ZN3tbb8internal30concurrent_queue_iterator_base6assignERKS1_ ) -__TBB_SYMBOL( _ZN3tbb8internal30concurrent_queue_iterator_base7advanceEv ) -__TBB_SYMBOL( _ZN3tbb8internal30concurrent_queue_iterator_baseC2ERKNS0_21concurrent_queue_baseE ) -__TBB_SYMBOL( _ZN3tbb8internal30concurrent_queue_iterator_baseD2Ev ) -__TBB_SYMBOL( _ZNK3tbb8internal21concurrent_queue_base13internal_sizeEv ) -#endif - -/* concurrent_queue v3 */ -/* constructors */ -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v3C2Em ) -__TBB_SYMBOL( _ZN3tbb8internal33concurrent_queue_iterator_base_v3C2ERKNS0_24concurrent_queue_base_v3E ) -__TBB_SYMBOL( _ZN3tbb8internal33concurrent_queue_iterator_base_v3C2ERKNS0_24concurrent_queue_base_v3Em ) -/* destructors */ -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v3D2Ev ) -__TBB_SYMBOL( _ZN3tbb8internal33concurrent_queue_iterator_base_v3D2Ev ) -/* typeinfo */ -__TBB_SYMBOL( _ZTIN3tbb8internal24concurrent_queue_base_v3E ) -__TBB_SYMBOL( _ZTSN3tbb8internal24concurrent_queue_base_v3E ) -/* vtable */ -__TBB_SYMBOL( _ZTVN3tbb8internal24concurrent_queue_base_v3E ) -/* methods */ -__TBB_SYMBOL( _ZN3tbb8internal33concurrent_queue_iterator_base_v36assignERKS1_ ) -__TBB_SYMBOL( _ZN3tbb8internal33concurrent_queue_iterator_base_v37advanceEv ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v313internal_pushEPKv ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v818internal_push_moveEPKv ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v325internal_push_if_not_fullEPKv ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v830internal_push_move_if_not_fullEPKv ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v312internal_popEPv ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v323internal_pop_if_presentEPv ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v314internal_abortEv ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v321internal_finish_clearEv ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v321internal_set_capacityElm ) -__TBB_SYMBOL( _ZNK3tbb8internal24concurrent_queue_base_v313internal_sizeEv ) -__TBB_SYMBOL( _ZNK3tbb8internal24concurrent_queue_base_v314internal_emptyEv ) -__TBB_SYMBOL( _ZNK3tbb8internal24concurrent_queue_base_v324internal_throw_exceptionEv ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v36assignERKS1_ ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v812move_contentERS1_ ) - -#if !TBB_NO_LEGACY -/* concurrent_vector.cpp v2 */ -__TBB_SYMBOL( _ZN3tbb8internal22concurrent_vector_base13internal_copyERKS1_mPFvPvPKvmE ) -__TBB_SYMBOL( _ZN3tbb8internal22concurrent_vector_base14internal_clearEPFvPvmEb ) -__TBB_SYMBOL( _ZN3tbb8internal22concurrent_vector_base15internal_assignERKS1_mPFvPvmEPFvS4_PKvmESA_ ) -__TBB_SYMBOL( _ZN3tbb8internal22concurrent_vector_base16internal_grow_byEmmPFvPvmE ) -__TBB_SYMBOL( _ZN3tbb8internal22concurrent_vector_base16internal_reserveEmmm ) -__TBB_SYMBOL( _ZN3tbb8internal22concurrent_vector_base18internal_push_backEmRm ) -__TBB_SYMBOL( _ZN3tbb8internal22concurrent_vector_base25internal_grow_to_at_leastEmmPFvPvmE ) -__TBB_SYMBOL( _ZNK3tbb8internal22concurrent_vector_base17internal_capacityEv ) -#endif - -/* concurrent_vector v3 */ -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v313internal_copyERKS1_mPFvPvPKvmE ) -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v314internal_clearEPFvPvmE ) -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v315internal_assignERKS1_mPFvPvmEPFvS4_PKvmESA_ ) -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v316internal_grow_byEmmPFvPvPKvmES4_ ) -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v316internal_reserveEmmm ) -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v318internal_push_backEmRm ) -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v325internal_grow_to_at_leastEmmPFvPvPKvmES4_ ) -__TBB_SYMBOL( _ZNK3tbb8internal25concurrent_vector_base_v317internal_capacityEv ) -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v316internal_compactEmPvPFvS2_mEPFvS2_PKvmE ) -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v313internal_swapERS1_ ) -__TBB_SYMBOL( _ZNK3tbb8internal25concurrent_vector_base_v324internal_throw_exceptionEm ) -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v3D2Ev ) -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v315internal_resizeEmmmPKvPFvPvmEPFvS4_S3_mE ) -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v337internal_grow_to_at_least_with_resultEmmPFvPvPKvmES4_ ) - -/* tbb_thread */ -__TBB_SYMBOL( _ZN3tbb8internal13tbb_thread_v320hardware_concurrencyEv ) -__TBB_SYMBOL( _ZN3tbb8internal13tbb_thread_v36detachEv ) -__TBB_SYMBOL( _ZN3tbb8internal16thread_get_id_v3Ev ) -__TBB_SYMBOL( _ZN3tbb8internal15free_closure_v3EPv ) -__TBB_SYMBOL( _ZN3tbb8internal13tbb_thread_v34joinEv ) -__TBB_SYMBOL( _ZN3tbb8internal13tbb_thread_v314internal_startEPFPvS2_ES2_ ) -__TBB_SYMBOL( _ZN3tbb8internal19allocate_closure_v3Em ) -__TBB_SYMBOL( _ZN3tbb8internal7move_v3ERNS0_13tbb_thread_v3ES2_ ) -__TBB_SYMBOL( _ZN3tbb8internal15thread_yield_v3Ev ) -__TBB_SYMBOL( _ZN3tbb8internal15thread_sleep_v3ERKNS_10tick_count10interval_tE ) - -/* asm functions */ -__TBB_SYMBOL( __TBB_machine_fetchadd1__TBB_full_fence ) -__TBB_SYMBOL( __TBB_machine_fetchadd2__TBB_full_fence ) -__TBB_SYMBOL( __TBB_machine_fetchadd4__TBB_full_fence ) -__TBB_SYMBOL( __TBB_machine_fetchadd8__TBB_full_fence ) -__TBB_SYMBOL( __TBB_machine_fetchstore1__TBB_full_fence ) -__TBB_SYMBOL( __TBB_machine_fetchstore2__TBB_full_fence ) -__TBB_SYMBOL( __TBB_machine_fetchstore4__TBB_full_fence ) -__TBB_SYMBOL( __TBB_machine_fetchstore8__TBB_full_fence ) -__TBB_SYMBOL( __TBB_machine_fetchadd1acquire ) -__TBB_SYMBOL( __TBB_machine_fetchadd1release ) -__TBB_SYMBOL( __TBB_machine_fetchadd2acquire ) -__TBB_SYMBOL( __TBB_machine_fetchadd2release ) -__TBB_SYMBOL( __TBB_machine_fetchadd4acquire ) -__TBB_SYMBOL( __TBB_machine_fetchadd4release ) -__TBB_SYMBOL( __TBB_machine_fetchadd8acquire ) -__TBB_SYMBOL( __TBB_machine_fetchadd8release ) -__TBB_SYMBOL( __TBB_machine_fetchstore1acquire ) -__TBB_SYMBOL( __TBB_machine_fetchstore1release ) -__TBB_SYMBOL( __TBB_machine_fetchstore2acquire ) -__TBB_SYMBOL( __TBB_machine_fetchstore2release ) -__TBB_SYMBOL( __TBB_machine_fetchstore4acquire ) -__TBB_SYMBOL( __TBB_machine_fetchstore4release ) -__TBB_SYMBOL( __TBB_machine_fetchstore8acquire ) -__TBB_SYMBOL( __TBB_machine_fetchstore8release ) -__TBB_SYMBOL( __TBB_machine_cmpswp1acquire ) -__TBB_SYMBOL( __TBB_machine_cmpswp1release ) -__TBB_SYMBOL( __TBB_machine_cmpswp1__TBB_full_fence ) -__TBB_SYMBOL( __TBB_machine_cmpswp2acquire ) -__TBB_SYMBOL( __TBB_machine_cmpswp2release ) -__TBB_SYMBOL( __TBB_machine_cmpswp2__TBB_full_fence ) -__TBB_SYMBOL( __TBB_machine_cmpswp4acquire ) -__TBB_SYMBOL( __TBB_machine_cmpswp4release ) -__TBB_SYMBOL( __TBB_machine_cmpswp4__TBB_full_fence ) -__TBB_SYMBOL( __TBB_machine_cmpswp8acquire ) -__TBB_SYMBOL( __TBB_machine_cmpswp8release ) -__TBB_SYMBOL( __TBB_machine_cmpswp8__TBB_full_fence ) -__TBB_SYMBOL( __TBB_machine_lg ) -__TBB_SYMBOL( __TBB_machine_lockbyte ) -__TBB_SYMBOL( __TBB_machine_pause ) -__TBB_SYMBOL( __TBB_machine_trylockbyte ) -__TBB_SYMBOL( __TBB_machine_load8_relaxed ) -__TBB_SYMBOL( __TBB_machine_store8_relaxed ) -__TBB_SYMBOL( __TBB_machine_load4_relaxed ) -__TBB_SYMBOL( __TBB_machine_store4_relaxed ) -__TBB_SYMBOL( __TBB_machine_load2_relaxed ) -__TBB_SYMBOL( __TBB_machine_store2_relaxed ) -__TBB_SYMBOL( __TBB_machine_load1_relaxed ) -__TBB_SYMBOL( __TBB_machine_store1_relaxed ) - -/* global parameter */ -__TBB_SYMBOL( _ZN3tbb10interface914global_control12active_valueEi ) -__TBB_SYMBOL( _ZN3tbb10interface914global_control15internal_createEv ) -__TBB_SYMBOL( _ZN3tbb10interface914global_control16internal_destroyEv ) - -#undef __TBB_SYMBOL diff --git a/src/tbb/src/tbb/mac32-tbb-export.lst b/src/tbb/src/tbb/mac32-tbb-export.lst deleted file mode 100644 index 8ce79f2de..000000000 --- a/src/tbb/src/tbb/mac32-tbb-export.lst +++ /dev/null @@ -1,403 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#include "tbb/tbb_config.h" - -/* - Sometimes macOS* requires leading underscore (e. g. in export list file), but sometimes not - (e. g. when searching symbol in a dynamic library via dlsym()). Symbols in this file SHOULD - be listed WITHOUT one leading underscore. __TBB_SYMBOL macro should add underscore when - necessary, depending on the indended usage. -*/ - -// cache_aligned_allocator.cpp -__TBB_SYMBOL( _ZN3tbb8internal12NFS_AllocateEmmPv ) -__TBB_SYMBOL( _ZN3tbb8internal15NFS_GetLineSizeEv ) -__TBB_SYMBOL( _ZN3tbb8internal8NFS_FreeEPv ) -__TBB_SYMBOL( _ZN3tbb8internal23allocate_via_handler_v3Em ) -__TBB_SYMBOL( _ZN3tbb8internal25deallocate_via_handler_v3EPv ) -__TBB_SYMBOL( _ZN3tbb8internal17is_malloc_used_v3Ev ) - -// task.cpp v3 -__TBB_SYMBOL( _ZN3tbb4task13note_affinityEt ) -__TBB_SYMBOL( _ZN3tbb4task22internal_set_ref_countEi ) -__TBB_SYMBOL( _ZN3tbb4task28internal_decrement_ref_countEv ) -__TBB_SYMBOL( _ZN3tbb4task22spawn_and_wait_for_allERNS_9task_listE ) -__TBB_SYMBOL( _ZN3tbb4task4selfEv ) -__TBB_SYMBOL( _ZN3tbb10interface58internal9task_base7destroyERNS_4taskE ) -__TBB_SYMBOL( _ZNK3tbb4task26is_owned_by_current_threadEv ) -__TBB_SYMBOL( _ZN3tbb8internal19allocate_root_proxy4freeERNS_4taskE ) -__TBB_SYMBOL( _ZN3tbb8internal19allocate_root_proxy8allocateEm ) -__TBB_SYMBOL( _ZN3tbb8internal28affinity_partitioner_base_v36resizeEj ) -__TBB_SYMBOL( _ZN3tbb8internal36get_initial_auto_partitioner_divisorEv ) -__TBB_SYMBOL( _ZNK3tbb8internal20allocate_child_proxy4freeERNS_4taskE ) -__TBB_SYMBOL( _ZNK3tbb8internal20allocate_child_proxy8allocateEm ) -__TBB_SYMBOL( _ZNK3tbb8internal27allocate_continuation_proxy4freeERNS_4taskE ) -__TBB_SYMBOL( _ZNK3tbb8internal27allocate_continuation_proxy8allocateEm ) -__TBB_SYMBOL( _ZNK3tbb8internal34allocate_additional_child_of_proxy4freeERNS_4taskE ) -__TBB_SYMBOL( _ZNK3tbb8internal34allocate_additional_child_of_proxy8allocateEm ) -__TBB_SYMBOL( _ZTIN3tbb4taskE ) -__TBB_SYMBOL( _ZTSN3tbb4taskE ) -__TBB_SYMBOL( _ZTVN3tbb4taskE ) -__TBB_SYMBOL( _ZN3tbb19task_scheduler_init19default_num_threadsEv ) -__TBB_SYMBOL( _ZN3tbb19task_scheduler_init10initializeEim ) -__TBB_SYMBOL( _ZN3tbb19task_scheduler_init10initializeEi ) -__TBB_SYMBOL( _ZN3tbb19task_scheduler_init9terminateEv ) -__TBB_SYMBOL( _ZN3tbb19task_scheduler_init27internal_blocking_terminateEb ) -#if __TBB_SCHEDULER_OBSERVER -__TBB_SYMBOL( _ZN3tbb8internal26task_scheduler_observer_v37observeEb ) -#endif /* __TBB_SCHEDULER_OBSERVER */ -__TBB_SYMBOL( _ZN3tbb10empty_task7executeEv ) -__TBB_SYMBOL( _ZN3tbb10empty_taskD0Ev ) -__TBB_SYMBOL( _ZN3tbb10empty_taskD1Ev ) -__TBB_SYMBOL( _ZTIN3tbb10empty_taskE ) -__TBB_SYMBOL( _ZTSN3tbb10empty_taskE ) -__TBB_SYMBOL( _ZTVN3tbb10empty_taskE ) - -/* arena.cpp */ -__TBB_SYMBOL( _ZN3tbb10interface78internal15task_arena_base19internal_initializeEv ) -__TBB_SYMBOL( _ZN3tbb10interface78internal15task_arena_base18internal_terminateEv ) -__TBB_SYMBOL( _ZN3tbb10interface78internal15task_arena_base15internal_attachEv ) -__TBB_SYMBOL( _ZNK3tbb10interface78internal15task_arena_base16internal_enqueueERNS_4taskEl ) -__TBB_SYMBOL( _ZNK3tbb10interface78internal15task_arena_base16internal_executeERNS1_13delegate_baseE ) -__TBB_SYMBOL( _ZNK3tbb10interface78internal15task_arena_base13internal_waitEv ) -__TBB_SYMBOL( _ZN3tbb10interface78internal15task_arena_base21internal_current_slotEv ) -__TBB_SYMBOL( _ZN3tbb10interface78internal15task_arena_base24internal_max_concurrencyEPKNS0_10task_arenaE ) -#if __TBB_TASK_ISOLATION -__TBB_SYMBOL( _ZN3tbb10interface78internal20isolate_within_arenaERNS1_13delegate_baseEl ) -#endif /* __TBB_TASK_ISOLATION */ - -#if !TBB_NO_LEGACY -// task_v2.cpp -__TBB_SYMBOL( _ZN3tbb4task7destroyERS0_ ) -#endif - -// Exception handling in task scheduler -#if __TBB_TASK_GROUP_CONTEXT -__TBB_SYMBOL( _ZNK3tbb8internal32allocate_root_with_context_proxy8allocateEm ) -__TBB_SYMBOL( _ZNK3tbb8internal32allocate_root_with_context_proxy4freeERNS_4taskE ) -__TBB_SYMBOL( _ZN3tbb4task12change_groupERNS_18task_group_contextE ) -__TBB_SYMBOL( _ZNK3tbb18task_group_context28is_group_execution_cancelledEv ) -__TBB_SYMBOL( _ZN3tbb18task_group_context22cancel_group_executionEv ) -__TBB_SYMBOL( _ZN3tbb18task_group_context26register_pending_exceptionEv ) -__TBB_SYMBOL( _ZN3tbb18task_group_context5resetEv ) -__TBB_SYMBOL( _ZN3tbb18task_group_context19capture_fp_settingsEv ) -__TBB_SYMBOL( _ZN3tbb18task_group_context4initEv ) -__TBB_SYMBOL( _ZN3tbb18task_group_contextD1Ev ) -__TBB_SYMBOL( _ZN3tbb18task_group_contextD2Ev ) -#if __TBB_TASK_PRIORITY -__TBB_SYMBOL( _ZN3tbb18task_group_context12set_priorityENS_10priority_tE ) -__TBB_SYMBOL( _ZNK3tbb18task_group_context8priorityEv ) -#endif /* __TBB_TASK_PRIORITY */ -__TBB_SYMBOL( _ZNK3tbb18captured_exception4nameEv ) -__TBB_SYMBOL( _ZNK3tbb18captured_exception4whatEv ) -__TBB_SYMBOL( _ZN3tbb18captured_exception10throw_selfEv ) -__TBB_SYMBOL( _ZN3tbb18captured_exception3setEPKcS2_ ) -__TBB_SYMBOL( _ZN3tbb18captured_exception4moveEv ) -__TBB_SYMBOL( _ZN3tbb18captured_exception5clearEv ) -__TBB_SYMBOL( _ZN3tbb18captured_exception7destroyEv ) -__TBB_SYMBOL( _ZN3tbb18captured_exception8allocateEPKcS2_ ) -__TBB_SYMBOL( _ZN3tbb18captured_exceptionD0Ev ) -__TBB_SYMBOL( _ZN3tbb18captured_exceptionD1Ev ) -__TBB_SYMBOL( _ZN3tbb18captured_exceptionD2Ev ) -__TBB_SYMBOL( _ZTIN3tbb18captured_exceptionE ) -__TBB_SYMBOL( _ZTSN3tbb18captured_exceptionE ) -__TBB_SYMBOL( _ZTVN3tbb18captured_exceptionE ) -__TBB_SYMBOL( _ZTIN3tbb13tbb_exceptionE ) -__TBB_SYMBOL( _ZTSN3tbb13tbb_exceptionE ) -__TBB_SYMBOL( _ZTVN3tbb13tbb_exceptionE ) -#endif /* __TBB_TASK_GROUP_CONTEXT */ - -// Symbols for exceptions thrown from TBB -__TBB_SYMBOL( _ZN3tbb8internal33throw_bad_last_alloc_exception_v4Ev ) -__TBB_SYMBOL( _ZN3tbb8internal18throw_exception_v4ENS0_12exception_idE ) -__TBB_SYMBOL( _ZNSt13runtime_errorD1Ev ) -__TBB_SYMBOL( _ZTISt13runtime_error ) -__TBB_SYMBOL( _ZTSSt13runtime_error ) -__TBB_SYMBOL( _ZNSt16invalid_argumentD1Ev ) -__TBB_SYMBOL( _ZTISt16invalid_argument ) -__TBB_SYMBOL( _ZTSSt16invalid_argument ) -__TBB_SYMBOL( _ZNSt11range_errorD1Ev ) -__TBB_SYMBOL( _ZTISt11range_error ) -__TBB_SYMBOL( _ZTSSt11range_error ) -__TBB_SYMBOL( _ZNSt12length_errorD1Ev ) -__TBB_SYMBOL( _ZTISt12length_error ) -__TBB_SYMBOL( _ZTSSt12length_error ) -__TBB_SYMBOL( _ZNSt12out_of_rangeD1Ev ) -__TBB_SYMBOL( _ZTISt12out_of_range ) -__TBB_SYMBOL( _ZTSSt12out_of_range ) -__TBB_SYMBOL( _ZN3tbb14bad_last_allocD0Ev ) -__TBB_SYMBOL( _ZN3tbb14bad_last_allocD1Ev ) -__TBB_SYMBOL( _ZNK3tbb14bad_last_alloc4whatEv ) -__TBB_SYMBOL( _ZTIN3tbb14bad_last_allocE ) -__TBB_SYMBOL( _ZTSN3tbb14bad_last_allocE ) -__TBB_SYMBOL( _ZTVN3tbb14bad_last_allocE ) -__TBB_SYMBOL( _ZN3tbb12missing_waitD0Ev ) -__TBB_SYMBOL( _ZN3tbb12missing_waitD1Ev ) -__TBB_SYMBOL( _ZNK3tbb12missing_wait4whatEv ) -__TBB_SYMBOL( _ZTIN3tbb12missing_waitE ) -__TBB_SYMBOL( _ZTSN3tbb12missing_waitE ) -__TBB_SYMBOL( _ZTVN3tbb12missing_waitE ) -__TBB_SYMBOL( _ZN3tbb27invalid_multiple_schedulingD0Ev ) -__TBB_SYMBOL( _ZN3tbb27invalid_multiple_schedulingD1Ev ) -__TBB_SYMBOL( _ZNK3tbb27invalid_multiple_scheduling4whatEv ) -__TBB_SYMBOL( _ZTIN3tbb27invalid_multiple_schedulingE ) -__TBB_SYMBOL( _ZTSN3tbb27invalid_multiple_schedulingE ) -__TBB_SYMBOL( _ZTVN3tbb27invalid_multiple_schedulingE ) -__TBB_SYMBOL( _ZN3tbb13improper_lockD0Ev ) -__TBB_SYMBOL( _ZN3tbb13improper_lockD1Ev ) -__TBB_SYMBOL( _ZNK3tbb13improper_lock4whatEv ) -__TBB_SYMBOL( _ZTIN3tbb13improper_lockE ) -__TBB_SYMBOL( _ZTSN3tbb13improper_lockE ) -__TBB_SYMBOL( _ZTVN3tbb13improper_lockE ) -__TBB_SYMBOL( _ZN3tbb10user_abortD0Ev ) -__TBB_SYMBOL( _ZN3tbb10user_abortD1Ev ) -__TBB_SYMBOL( _ZNK3tbb10user_abort4whatEv ) -__TBB_SYMBOL( _ZTIN3tbb10user_abortE ) -__TBB_SYMBOL( _ZTSN3tbb10user_abortE ) -__TBB_SYMBOL( _ZTVN3tbb10user_abortE ) - -// tbb_misc.cpp -__TBB_SYMBOL( _ZN3tbb17assertion_failureEPKciS1_S1_ ) -__TBB_SYMBOL( _ZN3tbb21set_assertion_handlerEPFvPKciS1_S1_E ) -__TBB_SYMBOL( _ZN3tbb8internal13handle_perrorEiPKc ) -__TBB_SYMBOL( _ZN3tbb8internal15runtime_warningEPKcz ) -#if __TBB_x86_32 -__TBB_SYMBOL( __TBB_machine_store8_slow_perf_warning ) -__TBB_SYMBOL( __TBB_machine_store8_slow ) -#endif -__TBB_SYMBOL( TBB_runtime_interface_version ) - -// tbb_main.cpp -__TBB_SYMBOL( _ZN3tbb8internal32itt_load_pointer_with_acquire_v3EPKv ) -__TBB_SYMBOL( _ZN3tbb8internal33itt_store_pointer_with_release_v3EPvS1_ ) -__TBB_SYMBOL( _ZN3tbb8internal18call_itt_notify_v5EiPv ) -__TBB_SYMBOL( _ZN3tbb8internal19itt_load_pointer_v3EPKv ) -__TBB_SYMBOL( _ZN3tbb8internal20itt_set_sync_name_v3EPvPKc ) - -// pipeline.cpp -__TBB_SYMBOL( _ZTIN3tbb6filterE ) -__TBB_SYMBOL( _ZTSN3tbb6filterE ) -__TBB_SYMBOL( _ZTVN3tbb6filterE ) -__TBB_SYMBOL( _ZN3tbb6filterD2Ev ) -__TBB_SYMBOL( _ZN3tbb8pipeline10add_filterERNS_6filterE ) -__TBB_SYMBOL( _ZN3tbb8pipeline12inject_tokenERNS_4taskE ) -__TBB_SYMBOL( _ZN3tbb8pipeline13remove_filterERNS_6filterE ) -__TBB_SYMBOL( _ZN3tbb8pipeline3runEm ) -#if __TBB_TASK_GROUP_CONTEXT -__TBB_SYMBOL( _ZN3tbb8pipeline3runEmRNS_18task_group_contextE ) -#endif -__TBB_SYMBOL( _ZN3tbb8pipeline5clearEv ) -__TBB_SYMBOL( _ZN3tbb19thread_bound_filter12process_itemEv ) -__TBB_SYMBOL( _ZN3tbb19thread_bound_filter16try_process_itemEv ) -__TBB_SYMBOL( _ZN3tbb8pipelineC1Ev ) -__TBB_SYMBOL( _ZN3tbb8pipelineC2Ev ) -__TBB_SYMBOL( _ZN3tbb8pipelineD0Ev ) -__TBB_SYMBOL( _ZN3tbb8pipelineD1Ev ) -__TBB_SYMBOL( _ZN3tbb8pipelineD2Ev ) -__TBB_SYMBOL( _ZTIN3tbb8pipelineE ) -__TBB_SYMBOL( _ZTSN3tbb8pipelineE ) -__TBB_SYMBOL( _ZTVN3tbb8pipelineE ) -__TBB_SYMBOL( _ZN3tbb6filter16set_end_of_inputEv ) - -// queuing_rw_mutex.cpp -__TBB_SYMBOL( _ZN3tbb16queuing_rw_mutex11scoped_lock17upgrade_to_writerEv ) -__TBB_SYMBOL( _ZN3tbb16queuing_rw_mutex11scoped_lock19downgrade_to_readerEv ) -__TBB_SYMBOL( _ZN3tbb16queuing_rw_mutex11scoped_lock7acquireERS0_b ) -__TBB_SYMBOL( _ZN3tbb16queuing_rw_mutex11scoped_lock7releaseEv ) -__TBB_SYMBOL( _ZN3tbb16queuing_rw_mutex11scoped_lock11try_acquireERS0_b ) -__TBB_SYMBOL( _ZN3tbb16queuing_rw_mutex18internal_constructEv ) - -// reader_writer_lock.cpp -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock11scoped_lock16internal_destroyEv ) -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock11scoped_lock18internal_constructERS1_ ) -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock13try_lock_readEv ) -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock16scoped_lock_read16internal_destroyEv ) -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock16scoped_lock_read18internal_constructERS1_ ) -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock16internal_destroyEv ) -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock18internal_constructEv ) -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock4lockEv ) -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock6unlockEv ) -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock8try_lockEv ) -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock9lock_readEv ) - -#if !TBB_NO_LEGACY -// spin_rw_mutex.cpp v2 -__TBB_SYMBOL( _ZN3tbb13spin_rw_mutex16internal_upgradeEPS0_ ) -__TBB_SYMBOL( _ZN3tbb13spin_rw_mutex22internal_itt_releasingEPS0_ ) -__TBB_SYMBOL( _ZN3tbb13spin_rw_mutex23internal_acquire_readerEPS0_ ) -__TBB_SYMBOL( _ZN3tbb13spin_rw_mutex23internal_acquire_writerEPS0_ ) -__TBB_SYMBOL( _ZN3tbb13spin_rw_mutex18internal_downgradeEPS0_ ) -__TBB_SYMBOL( _ZN3tbb13spin_rw_mutex23internal_release_readerEPS0_ ) -__TBB_SYMBOL( _ZN3tbb13spin_rw_mutex23internal_release_writerEPS0_ ) -__TBB_SYMBOL( _ZN3tbb13spin_rw_mutex27internal_try_acquire_readerEPS0_ ) -__TBB_SYMBOL( _ZN3tbb13spin_rw_mutex27internal_try_acquire_writerEPS0_ ) -#endif - -// spin_rw_mutex v3 -__TBB_SYMBOL( _ZN3tbb16spin_rw_mutex_v316internal_upgradeEv ) -__TBB_SYMBOL( _ZN3tbb16spin_rw_mutex_v318internal_downgradeEv ) -__TBB_SYMBOL( _ZN3tbb16spin_rw_mutex_v323internal_acquire_readerEv ) -__TBB_SYMBOL( _ZN3tbb16spin_rw_mutex_v323internal_acquire_writerEv ) -__TBB_SYMBOL( _ZN3tbb16spin_rw_mutex_v323internal_release_readerEv ) -__TBB_SYMBOL( _ZN3tbb16spin_rw_mutex_v323internal_release_writerEv ) -__TBB_SYMBOL( _ZN3tbb16spin_rw_mutex_v327internal_try_acquire_readerEv ) -__TBB_SYMBOL( _ZN3tbb16spin_rw_mutex_v327internal_try_acquire_writerEv ) -__TBB_SYMBOL( _ZN3tbb16spin_rw_mutex_v318internal_constructEv ) - -// x86_rtm_rw_mutex.cpp -#if __TBB_TSX_AVAILABLE -__TBB_SYMBOL( _ZN3tbb10interface88internal16x86_rtm_rw_mutex16internal_releaseERNS2_11scoped_lockE ) -__TBB_SYMBOL( _ZN3tbb10interface88internal16x86_rtm_rw_mutex16internal_upgradeERNS2_11scoped_lockE ) -__TBB_SYMBOL( _ZN3tbb10interface88internal16x86_rtm_rw_mutex18internal_constructEv ) -__TBB_SYMBOL( _ZN3tbb10interface88internal16x86_rtm_rw_mutex18internal_downgradeERNS2_11scoped_lockE ) -__TBB_SYMBOL( _ZN3tbb10interface88internal16x86_rtm_rw_mutex23internal_acquire_readerERNS2_11scoped_lockEb ) -__TBB_SYMBOL( _ZN3tbb10interface88internal16x86_rtm_rw_mutex23internal_acquire_writerERNS2_11scoped_lockEb ) -__TBB_SYMBOL( _ZN3tbb10interface88internal16x86_rtm_rw_mutex27internal_try_acquire_writerERNS2_11scoped_lockE ) -#endif - -// spin_mutex.cpp -__TBB_SYMBOL( _ZN3tbb10spin_mutex11scoped_lock16internal_acquireERS0_ ) -__TBB_SYMBOL( _ZN3tbb10spin_mutex11scoped_lock16internal_releaseEv ) -__TBB_SYMBOL( _ZN3tbb10spin_mutex11scoped_lock20internal_try_acquireERS0_ ) -__TBB_SYMBOL( _ZN3tbb10spin_mutex18internal_constructEv ) - -// mutex.cpp -__TBB_SYMBOL( _ZN3tbb5mutex11scoped_lock16internal_acquireERS0_ ) -__TBB_SYMBOL( _ZN3tbb5mutex11scoped_lock16internal_releaseEv ) -__TBB_SYMBOL( _ZN3tbb5mutex11scoped_lock20internal_try_acquireERS0_ ) -__TBB_SYMBOL( _ZN3tbb5mutex16internal_destroyEv ) -__TBB_SYMBOL( _ZN3tbb5mutex18internal_constructEv ) - -// recursive_mutex.cpp -__TBB_SYMBOL( _ZN3tbb15recursive_mutex11scoped_lock16internal_acquireERS0_ ) -__TBB_SYMBOL( _ZN3tbb15recursive_mutex11scoped_lock16internal_releaseEv ) -__TBB_SYMBOL( _ZN3tbb15recursive_mutex11scoped_lock20internal_try_acquireERS0_ ) -__TBB_SYMBOL( _ZN3tbb15recursive_mutex16internal_destroyEv ) -__TBB_SYMBOL( _ZN3tbb15recursive_mutex18internal_constructEv ) - -// queuing_mutex.cpp -__TBB_SYMBOL( _ZN3tbb13queuing_mutex11scoped_lock7acquireERS0_ ) -__TBB_SYMBOL( _ZN3tbb13queuing_mutex11scoped_lock7releaseEv ) -__TBB_SYMBOL( _ZN3tbb13queuing_mutex11scoped_lock11try_acquireERS0_ ) -__TBB_SYMBOL( _ZN3tbb13queuing_mutex18internal_constructEv ) - -// critical_section.cpp -__TBB_SYMBOL( _ZN3tbb8internal19critical_section_v418internal_constructEv ) - -#if !TBB_NO_LEGACY -// concurrent_hash_map -__TBB_SYMBOL( _ZNK3tbb8internal21hash_map_segment_base23internal_grow_predicateEv ) - -// concurrent_queue.cpp v2 -__TBB_SYMBOL( _ZN3tbb8internal21concurrent_queue_base12internal_popEPv ) -__TBB_SYMBOL( _ZN3tbb8internal21concurrent_queue_base13internal_pushEPKv ) -__TBB_SYMBOL( _ZN3tbb8internal21concurrent_queue_base21internal_set_capacityEim ) -__TBB_SYMBOL( _ZN3tbb8internal21concurrent_queue_base23internal_pop_if_presentEPv ) -__TBB_SYMBOL( _ZN3tbb8internal21concurrent_queue_base25internal_push_if_not_fullEPKv ) -__TBB_SYMBOL( _ZN3tbb8internal21concurrent_queue_baseC2Em ) -__TBB_SYMBOL( _ZN3tbb8internal21concurrent_queue_baseD2Ev ) -__TBB_SYMBOL( _ZTIN3tbb8internal21concurrent_queue_baseE ) -__TBB_SYMBOL( _ZTSN3tbb8internal21concurrent_queue_baseE ) -__TBB_SYMBOL( _ZTVN3tbb8internal21concurrent_queue_baseE ) -__TBB_SYMBOL( _ZN3tbb8internal30concurrent_queue_iterator_base6assignERKS1_ ) -__TBB_SYMBOL( _ZN3tbb8internal30concurrent_queue_iterator_base7advanceEv ) -__TBB_SYMBOL( _ZN3tbb8internal30concurrent_queue_iterator_baseC2ERKNS0_21concurrent_queue_baseE ) -__TBB_SYMBOL( _ZN3tbb8internal30concurrent_queue_iterator_baseD2Ev ) -__TBB_SYMBOL( _ZNK3tbb8internal21concurrent_queue_base13internal_sizeEv ) -#endif - -// concurrent_queue v3 -// constructors -__TBB_SYMBOL( _ZN3tbb8internal33concurrent_queue_iterator_base_v3C2ERKNS0_24concurrent_queue_base_v3E ) -__TBB_SYMBOL( _ZN3tbb8internal33concurrent_queue_iterator_base_v3C2ERKNS0_24concurrent_queue_base_v3Em ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v3C2Em ) -// destructors -__TBB_SYMBOL( _ZN3tbb8internal33concurrent_queue_iterator_base_v3D2Ev ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v3D2Ev ) -// typeinfo -__TBB_SYMBOL( _ZTIN3tbb8internal24concurrent_queue_base_v3E ) -__TBB_SYMBOL( _ZTSN3tbb8internal24concurrent_queue_base_v3E ) -// vtable -__TBB_SYMBOL( _ZTVN3tbb8internal24concurrent_queue_base_v3E ) -// methods -__TBB_SYMBOL( _ZN3tbb8internal33concurrent_queue_iterator_base_v37advanceEv ) -__TBB_SYMBOL( _ZN3tbb8internal33concurrent_queue_iterator_base_v36assignERKS1_ ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v313internal_pushEPKv ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v818internal_push_moveEPKv ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v325internal_push_if_not_fullEPKv ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v830internal_push_move_if_not_fullEPKv ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v312internal_popEPv ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v323internal_pop_if_presentEPv ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v314internal_abortEv ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v321internal_set_capacityEim ) -__TBB_SYMBOL( _ZNK3tbb8internal24concurrent_queue_base_v313internal_sizeEv ) -__TBB_SYMBOL( _ZNK3tbb8internal24concurrent_queue_base_v314internal_emptyEv ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v321internal_finish_clearEv ) -__TBB_SYMBOL( _ZNK3tbb8internal24concurrent_queue_base_v324internal_throw_exceptionEv ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v36assignERKS1_ ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v812move_contentERS1_ ) - -#if !TBB_NO_LEGACY -// concurrent_vector.cpp v2 -__TBB_SYMBOL( _ZN3tbb8internal22concurrent_vector_base13internal_copyERKS1_mPFvPvPKvmE ) -__TBB_SYMBOL( _ZN3tbb8internal22concurrent_vector_base14internal_clearEPFvPvmEb ) -__TBB_SYMBOL( _ZN3tbb8internal22concurrent_vector_base15internal_assignERKS1_mPFvPvmEPFvS4_PKvmESA_ ) -__TBB_SYMBOL( _ZN3tbb8internal22concurrent_vector_base16internal_grow_byEmmPFvPvmE ) -__TBB_SYMBOL( _ZN3tbb8internal22concurrent_vector_base16internal_reserveEmmm ) -__TBB_SYMBOL( _ZN3tbb8internal22concurrent_vector_base18internal_push_backEmRm ) -__TBB_SYMBOL( _ZN3tbb8internal22concurrent_vector_base25internal_grow_to_at_leastEmmPFvPvmE ) -__TBB_SYMBOL( _ZNK3tbb8internal22concurrent_vector_base17internal_capacityEv ) -#endif - -// concurrent_vector v3 -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v313internal_copyERKS1_mPFvPvPKvmE ) -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v314internal_clearEPFvPvmE ) -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v315internal_assignERKS1_mPFvPvmEPFvS4_PKvmESA_ ) -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v316internal_grow_byEmmPFvPvPKvmES4_ ) -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v316internal_reserveEmmm ) -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v318internal_push_backEmRm ) -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v325internal_grow_to_at_leastEmmPFvPvPKvmES4_ ) -__TBB_SYMBOL( _ZNK3tbb8internal25concurrent_vector_base_v317internal_capacityEv ) -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v316internal_compactEmPvPFvS2_mEPFvS2_PKvmE ) -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v313internal_swapERS1_ ) -__TBB_SYMBOL( _ZNK3tbb8internal25concurrent_vector_base_v324internal_throw_exceptionEm ) -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v3D2Ev ) -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v315internal_resizeEmmmPKvPFvPvmEPFvS4_S3_mE ) -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v337internal_grow_to_at_least_with_resultEmmPFvPvPKvmES4_ ) - -// tbb_thread -__TBB_SYMBOL( _ZN3tbb8internal13tbb_thread_v314internal_startEPFPvS2_ES2_ ) -__TBB_SYMBOL( _ZN3tbb8internal13tbb_thread_v320hardware_concurrencyEv ) -__TBB_SYMBOL( _ZN3tbb8internal13tbb_thread_v34joinEv ) -__TBB_SYMBOL( _ZN3tbb8internal13tbb_thread_v36detachEv ) -__TBB_SYMBOL( _ZN3tbb8internal15free_closure_v3EPv ) -__TBB_SYMBOL( _ZN3tbb8internal15thread_sleep_v3ERKNS_10tick_count10interval_tE ) -__TBB_SYMBOL( _ZN3tbb8internal15thread_yield_v3Ev ) -__TBB_SYMBOL( _ZN3tbb8internal16thread_get_id_v3Ev ) -__TBB_SYMBOL( _ZN3tbb8internal19allocate_closure_v3Em ) -__TBB_SYMBOL( _ZN3tbb8internal7move_v3ERNS0_13tbb_thread_v3ES2_ ) - -// global parameter -__TBB_SYMBOL( _ZN3tbb10interface914global_control12active_valueEi ) -__TBB_SYMBOL( _ZN3tbb10interface914global_control15internal_createEv ) -__TBB_SYMBOL( _ZN3tbb10interface914global_control16internal_destroyEv ) - -#undef __TBB_SYMBOL diff --git a/src/tbb/src/tbb/mac64-tbb-export.lst b/src/tbb/src/tbb/mac64-tbb-export.lst deleted file mode 100644 index 67d3676d0..000000000 --- a/src/tbb/src/tbb/mac64-tbb-export.lst +++ /dev/null @@ -1,400 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#include "tbb/tbb_config.h" - -/* - Sometimes macOS* requires leading underscore (e. g. in export list file), but sometimes not - (e. g. when searching symbol in a dynamic library via dlsym()). Symbols in this file SHOULD - be listed WITHOUT one leading underscore. __TBB_SYMBOL macro should add underscore when - necessary, depending on the indended usage. -*/ - -// cache_aligned_allocator.cpp -__TBB_SYMBOL( _ZN3tbb8internal12NFS_AllocateEmmPv ) -__TBB_SYMBOL( _ZN3tbb8internal15NFS_GetLineSizeEv ) -__TBB_SYMBOL( _ZN3tbb8internal8NFS_FreeEPv ) -__TBB_SYMBOL( _ZN3tbb8internal23allocate_via_handler_v3Em ) -__TBB_SYMBOL( _ZN3tbb8internal25deallocate_via_handler_v3EPv ) -__TBB_SYMBOL( _ZN3tbb8internal17is_malloc_used_v3Ev ) - -// task.cpp v3 -__TBB_SYMBOL( _ZN3tbb4task13note_affinityEt ) -__TBB_SYMBOL( _ZN3tbb4task22internal_set_ref_countEi ) -__TBB_SYMBOL( _ZN3tbb4task28internal_decrement_ref_countEv ) -__TBB_SYMBOL( _ZN3tbb4task22spawn_and_wait_for_allERNS_9task_listE ) -__TBB_SYMBOL( _ZN3tbb4task4selfEv ) -__TBB_SYMBOL( _ZN3tbb10interface58internal9task_base7destroyERNS_4taskE ) -__TBB_SYMBOL( _ZNK3tbb4task26is_owned_by_current_threadEv ) -__TBB_SYMBOL( _ZN3tbb8internal19allocate_root_proxy4freeERNS_4taskE ) -__TBB_SYMBOL( _ZN3tbb8internal19allocate_root_proxy8allocateEm ) -__TBB_SYMBOL( _ZN3tbb8internal28affinity_partitioner_base_v36resizeEj ) -__TBB_SYMBOL( _ZN3tbb8internal36get_initial_auto_partitioner_divisorEv ) -__TBB_SYMBOL( _ZNK3tbb8internal20allocate_child_proxy4freeERNS_4taskE ) -__TBB_SYMBOL( _ZNK3tbb8internal20allocate_child_proxy8allocateEm ) -__TBB_SYMBOL( _ZNK3tbb8internal27allocate_continuation_proxy4freeERNS_4taskE ) -__TBB_SYMBOL( _ZNK3tbb8internal27allocate_continuation_proxy8allocateEm ) -__TBB_SYMBOL( _ZNK3tbb8internal34allocate_additional_child_of_proxy4freeERNS_4taskE ) -__TBB_SYMBOL( _ZNK3tbb8internal34allocate_additional_child_of_proxy8allocateEm ) -__TBB_SYMBOL( _ZTIN3tbb4taskE ) -__TBB_SYMBOL( _ZTSN3tbb4taskE ) -__TBB_SYMBOL( _ZTVN3tbb4taskE ) -__TBB_SYMBOL( _ZN3tbb19task_scheduler_init19default_num_threadsEv ) -__TBB_SYMBOL( _ZN3tbb19task_scheduler_init10initializeEim ) -__TBB_SYMBOL( _ZN3tbb19task_scheduler_init10initializeEi ) -__TBB_SYMBOL( _ZN3tbb19task_scheduler_init9terminateEv ) -__TBB_SYMBOL( _ZN3tbb19task_scheduler_init27internal_blocking_terminateEb ) -#if __TBB_SCHEDULER_OBSERVER -__TBB_SYMBOL( _ZN3tbb8internal26task_scheduler_observer_v37observeEb ) -#endif /* __TBB_SCHEDULER_OBSERVER */ -__TBB_SYMBOL( _ZN3tbb10empty_task7executeEv ) -__TBB_SYMBOL( _ZN3tbb10empty_taskD0Ev ) -__TBB_SYMBOL( _ZN3tbb10empty_taskD1Ev ) -__TBB_SYMBOL( _ZTIN3tbb10empty_taskE ) -__TBB_SYMBOL( _ZTSN3tbb10empty_taskE ) -__TBB_SYMBOL( _ZTVN3tbb10empty_taskE ) - -/* arena.cpp */ -__TBB_SYMBOL( _ZN3tbb10interface78internal15task_arena_base19internal_initializeEv ) -__TBB_SYMBOL( _ZN3tbb10interface78internal15task_arena_base18internal_terminateEv ) -__TBB_SYMBOL( _ZN3tbb10interface78internal15task_arena_base15internal_attachEv ) -__TBB_SYMBOL( _ZNK3tbb10interface78internal15task_arena_base16internal_enqueueERNS_4taskEl ) -__TBB_SYMBOL( _ZNK3tbb10interface78internal15task_arena_base16internal_executeERNS1_13delegate_baseE ) -__TBB_SYMBOL( _ZNK3tbb10interface78internal15task_arena_base13internal_waitEv ) -__TBB_SYMBOL( _ZN3tbb10interface78internal15task_arena_base21internal_current_slotEv ) -__TBB_SYMBOL( _ZN3tbb10interface78internal15task_arena_base24internal_max_concurrencyEPKNS0_10task_arenaE ) -#if __TBB_TASK_ISOLATION -__TBB_SYMBOL( _ZN3tbb10interface78internal20isolate_within_arenaERNS1_13delegate_baseEl ) -#endif /* __TBB_TASK_ISOLATION */ - -#if !TBB_NO_LEGACY -// task_v2.cpp -__TBB_SYMBOL( _ZN3tbb4task7destroyERS0_ ) -#endif - -// Exception handling in task scheduler -#if __TBB_TASK_GROUP_CONTEXT -__TBB_SYMBOL( _ZNK3tbb8internal32allocate_root_with_context_proxy8allocateEm ) -__TBB_SYMBOL( _ZNK3tbb8internal32allocate_root_with_context_proxy4freeERNS_4taskE ) -__TBB_SYMBOL( _ZN3tbb4task12change_groupERNS_18task_group_contextE ) -__TBB_SYMBOL( _ZNK3tbb18task_group_context28is_group_execution_cancelledEv ) -__TBB_SYMBOL( _ZN3tbb18task_group_context22cancel_group_executionEv ) -__TBB_SYMBOL( _ZN3tbb18task_group_context26register_pending_exceptionEv ) -__TBB_SYMBOL( _ZN3tbb18task_group_context5resetEv ) -__TBB_SYMBOL( _ZN3tbb18task_group_context19capture_fp_settingsEv ) -__TBB_SYMBOL( _ZN3tbb18task_group_context4initEv ) -__TBB_SYMBOL( _ZN3tbb18task_group_contextD1Ev ) -__TBB_SYMBOL( _ZN3tbb18task_group_contextD2Ev ) -#if __TBB_TASK_PRIORITY -__TBB_SYMBOL( _ZN3tbb18task_group_context12set_priorityENS_10priority_tE ) -__TBB_SYMBOL( _ZNK3tbb18task_group_context8priorityEv ) -#endif /* __TBB_TASK_PRIORITY */ -__TBB_SYMBOL( _ZNK3tbb18captured_exception4nameEv ) -__TBB_SYMBOL( _ZNK3tbb18captured_exception4whatEv ) -__TBB_SYMBOL( _ZN3tbb18captured_exception10throw_selfEv ) -__TBB_SYMBOL( _ZN3tbb18captured_exception3setEPKcS2_ ) -__TBB_SYMBOL( _ZN3tbb18captured_exception4moveEv ) -__TBB_SYMBOL( _ZN3tbb18captured_exception5clearEv ) -__TBB_SYMBOL( _ZN3tbb18captured_exception7destroyEv ) -__TBB_SYMBOL( _ZN3tbb18captured_exception8allocateEPKcS2_ ) -__TBB_SYMBOL( _ZN3tbb18captured_exceptionD0Ev ) -__TBB_SYMBOL( _ZN3tbb18captured_exceptionD1Ev ) -__TBB_SYMBOL( _ZN3tbb18captured_exceptionD2Ev ) -__TBB_SYMBOL( _ZTIN3tbb18captured_exceptionE ) -__TBB_SYMBOL( _ZTSN3tbb18captured_exceptionE ) -__TBB_SYMBOL( _ZTVN3tbb18captured_exceptionE ) -__TBB_SYMBOL( _ZTIN3tbb13tbb_exceptionE ) -__TBB_SYMBOL( _ZTSN3tbb13tbb_exceptionE ) -__TBB_SYMBOL( _ZTVN3tbb13tbb_exceptionE ) -#endif /* __TBB_TASK_GROUP_CONTEXT */ - -// Symbols for exceptions thrown from TBB -__TBB_SYMBOL( _ZN3tbb8internal33throw_bad_last_alloc_exception_v4Ev ) -__TBB_SYMBOL( _ZN3tbb8internal18throw_exception_v4ENS0_12exception_idE ) -__TBB_SYMBOL( _ZNSt13runtime_errorD1Ev ) -__TBB_SYMBOL( _ZTISt13runtime_error ) -__TBB_SYMBOL( _ZTSSt13runtime_error ) -__TBB_SYMBOL( _ZNSt16invalid_argumentD1Ev ) -__TBB_SYMBOL( _ZTISt16invalid_argument ) -__TBB_SYMBOL( _ZTSSt16invalid_argument ) -__TBB_SYMBOL( _ZNSt11range_errorD1Ev ) -__TBB_SYMBOL( _ZTISt11range_error ) -__TBB_SYMBOL( _ZTSSt11range_error ) -__TBB_SYMBOL( _ZNSt12length_errorD1Ev ) -__TBB_SYMBOL( _ZTISt12length_error ) -__TBB_SYMBOL( _ZTSSt12length_error ) -__TBB_SYMBOL( _ZNSt12out_of_rangeD1Ev ) -__TBB_SYMBOL( _ZTISt12out_of_range ) -__TBB_SYMBOL( _ZTSSt12out_of_range ) -__TBB_SYMBOL( _ZN3tbb14bad_last_allocD0Ev ) -__TBB_SYMBOL( _ZN3tbb14bad_last_allocD1Ev ) -__TBB_SYMBOL( _ZNK3tbb14bad_last_alloc4whatEv ) -__TBB_SYMBOL( _ZTIN3tbb14bad_last_allocE ) -__TBB_SYMBOL( _ZTSN3tbb14bad_last_allocE ) -__TBB_SYMBOL( _ZTVN3tbb14bad_last_allocE ) -__TBB_SYMBOL( _ZN3tbb12missing_waitD0Ev ) -__TBB_SYMBOL( _ZN3tbb12missing_waitD1Ev ) -__TBB_SYMBOL( _ZNK3tbb12missing_wait4whatEv ) -__TBB_SYMBOL( _ZTIN3tbb12missing_waitE ) -__TBB_SYMBOL( _ZTSN3tbb12missing_waitE ) -__TBB_SYMBOL( _ZTVN3tbb12missing_waitE ) -__TBB_SYMBOL( _ZN3tbb27invalid_multiple_schedulingD0Ev ) -__TBB_SYMBOL( _ZN3tbb27invalid_multiple_schedulingD1Ev ) -__TBB_SYMBOL( _ZNK3tbb27invalid_multiple_scheduling4whatEv ) -__TBB_SYMBOL( _ZTIN3tbb27invalid_multiple_schedulingE ) -__TBB_SYMBOL( _ZTSN3tbb27invalid_multiple_schedulingE ) -__TBB_SYMBOL( _ZTVN3tbb27invalid_multiple_schedulingE ) -__TBB_SYMBOL( _ZN3tbb13improper_lockD0Ev ) -__TBB_SYMBOL( _ZN3tbb13improper_lockD1Ev ) -__TBB_SYMBOL( _ZNK3tbb13improper_lock4whatEv ) -__TBB_SYMBOL( _ZTIN3tbb13improper_lockE ) -__TBB_SYMBOL( _ZTSN3tbb13improper_lockE ) -__TBB_SYMBOL( _ZTVN3tbb13improper_lockE ) -__TBB_SYMBOL( _ZN3tbb10user_abortD0Ev ) -__TBB_SYMBOL( _ZN3tbb10user_abortD1Ev ) -__TBB_SYMBOL( _ZNK3tbb10user_abort4whatEv ) -__TBB_SYMBOL( _ZTIN3tbb10user_abortE ) -__TBB_SYMBOL( _ZTSN3tbb10user_abortE ) -__TBB_SYMBOL( _ZTVN3tbb10user_abortE ) - - -// tbb_misc.cpp -__TBB_SYMBOL( _ZN3tbb17assertion_failureEPKciS1_S1_ ) -__TBB_SYMBOL( _ZN3tbb21set_assertion_handlerEPFvPKciS1_S1_E ) -__TBB_SYMBOL( _ZN3tbb8internal13handle_perrorEiPKc ) -__TBB_SYMBOL( _ZN3tbb8internal15runtime_warningEPKcz ) -__TBB_SYMBOL( TBB_runtime_interface_version ) - -// tbb_main.cpp -__TBB_SYMBOL( _ZN3tbb8internal32itt_load_pointer_with_acquire_v3EPKv ) -__TBB_SYMBOL( _ZN3tbb8internal33itt_store_pointer_with_release_v3EPvS1_ ) -__TBB_SYMBOL( _ZN3tbb8internal18call_itt_notify_v5EiPv ) -__TBB_SYMBOL( _ZN3tbb8internal19itt_load_pointer_v3EPKv ) -__TBB_SYMBOL( _ZN3tbb8internal20itt_set_sync_name_v3EPvPKc ) - -// pipeline.cpp -__TBB_SYMBOL( _ZTIN3tbb6filterE ) -__TBB_SYMBOL( _ZTSN3tbb6filterE ) -__TBB_SYMBOL( _ZTVN3tbb6filterE ) -__TBB_SYMBOL( _ZN3tbb6filterD2Ev ) -__TBB_SYMBOL( _ZN3tbb8pipeline10add_filterERNS_6filterE ) -__TBB_SYMBOL( _ZN3tbb8pipeline12inject_tokenERNS_4taskE ) -__TBB_SYMBOL( _ZN3tbb8pipeline13remove_filterERNS_6filterE ) -__TBB_SYMBOL( _ZN3tbb8pipeline3runEm ) -#if __TBB_TASK_GROUP_CONTEXT -__TBB_SYMBOL( _ZN3tbb8pipeline3runEmRNS_18task_group_contextE ) -#endif -__TBB_SYMBOL( _ZN3tbb8pipeline5clearEv ) -__TBB_SYMBOL( _ZN3tbb19thread_bound_filter12process_itemEv ) -__TBB_SYMBOL( _ZN3tbb19thread_bound_filter16try_process_itemEv ) -__TBB_SYMBOL( _ZN3tbb8pipelineC1Ev ) -__TBB_SYMBOL( _ZN3tbb8pipelineC2Ev ) -__TBB_SYMBOL( _ZN3tbb8pipelineD0Ev ) -__TBB_SYMBOL( _ZN3tbb8pipelineD1Ev ) -__TBB_SYMBOL( _ZN3tbb8pipelineD2Ev ) -__TBB_SYMBOL( _ZTIN3tbb8pipelineE ) -__TBB_SYMBOL( _ZTSN3tbb8pipelineE ) -__TBB_SYMBOL( _ZTVN3tbb8pipelineE ) -__TBB_SYMBOL( _ZN3tbb6filter16set_end_of_inputEv ) - -// queuing_rw_mutex.cpp -__TBB_SYMBOL( _ZN3tbb16queuing_rw_mutex11scoped_lock17upgrade_to_writerEv ) -__TBB_SYMBOL( _ZN3tbb16queuing_rw_mutex11scoped_lock19downgrade_to_readerEv ) -__TBB_SYMBOL( _ZN3tbb16queuing_rw_mutex11scoped_lock7acquireERS0_b ) -__TBB_SYMBOL( _ZN3tbb16queuing_rw_mutex11scoped_lock7releaseEv ) -__TBB_SYMBOL( _ZN3tbb16queuing_rw_mutex11scoped_lock11try_acquireERS0_b ) -__TBB_SYMBOL( _ZN3tbb16queuing_rw_mutex18internal_constructEv ) - -// reader_writer_lock.cpp -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock11scoped_lock16internal_destroyEv ) -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock11scoped_lock18internal_constructERS1_ ) -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock13try_lock_readEv ) -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock16scoped_lock_read16internal_destroyEv ) -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock16scoped_lock_read18internal_constructERS1_ ) -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock16internal_destroyEv ) -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock18internal_constructEv ) -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock4lockEv ) -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock6unlockEv ) -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock8try_lockEv ) -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock9lock_readEv ) - -#if !TBB_NO_LEGACY -// spin_rw_mutex.cpp v2 -__TBB_SYMBOL( _ZN3tbb13spin_rw_mutex16internal_upgradeEPS0_ ) -__TBB_SYMBOL( _ZN3tbb13spin_rw_mutex22internal_itt_releasingEPS0_ ) -__TBB_SYMBOL( _ZN3tbb13spin_rw_mutex23internal_acquire_readerEPS0_ ) -__TBB_SYMBOL( _ZN3tbb13spin_rw_mutex23internal_acquire_writerEPS0_ ) -__TBB_SYMBOL( _ZN3tbb13spin_rw_mutex18internal_downgradeEPS0_ ) -__TBB_SYMBOL( _ZN3tbb13spin_rw_mutex23internal_release_readerEPS0_ ) -__TBB_SYMBOL( _ZN3tbb13spin_rw_mutex23internal_release_writerEPS0_ ) -__TBB_SYMBOL( _ZN3tbb13spin_rw_mutex27internal_try_acquire_readerEPS0_ ) -__TBB_SYMBOL( _ZN3tbb13spin_rw_mutex27internal_try_acquire_writerEPS0_ ) -#endif - -// spin_rw_mutex v3 -__TBB_SYMBOL( _ZN3tbb16spin_rw_mutex_v316internal_upgradeEv ) -__TBB_SYMBOL( _ZN3tbb16spin_rw_mutex_v318internal_downgradeEv ) -__TBB_SYMBOL( _ZN3tbb16spin_rw_mutex_v323internal_acquire_readerEv ) -__TBB_SYMBOL( _ZN3tbb16spin_rw_mutex_v323internal_acquire_writerEv ) -__TBB_SYMBOL( _ZN3tbb16spin_rw_mutex_v323internal_release_readerEv ) -__TBB_SYMBOL( _ZN3tbb16spin_rw_mutex_v323internal_release_writerEv ) -__TBB_SYMBOL( _ZN3tbb16spin_rw_mutex_v327internal_try_acquire_readerEv ) -__TBB_SYMBOL( _ZN3tbb16spin_rw_mutex_v327internal_try_acquire_writerEv ) -__TBB_SYMBOL( _ZN3tbb16spin_rw_mutex_v318internal_constructEv ) - -// x86_rtm_rw_mutex.cpp -#if __TBB_TSX_AVAILABLE -__TBB_SYMBOL( _ZN3tbb10interface88internal16x86_rtm_rw_mutex16internal_releaseERNS2_11scoped_lockE ) -__TBB_SYMBOL( _ZN3tbb10interface88internal16x86_rtm_rw_mutex16internal_upgradeERNS2_11scoped_lockE ) -__TBB_SYMBOL( _ZN3tbb10interface88internal16x86_rtm_rw_mutex18internal_constructEv ) -__TBB_SYMBOL( _ZN3tbb10interface88internal16x86_rtm_rw_mutex18internal_downgradeERNS2_11scoped_lockE ) -__TBB_SYMBOL( _ZN3tbb10interface88internal16x86_rtm_rw_mutex23internal_acquire_readerERNS2_11scoped_lockEb ) -__TBB_SYMBOL( _ZN3tbb10interface88internal16x86_rtm_rw_mutex23internal_acquire_writerERNS2_11scoped_lockEb ) -__TBB_SYMBOL( _ZN3tbb10interface88internal16x86_rtm_rw_mutex27internal_try_acquire_writerERNS2_11scoped_lockE ) -#endif - -// spin_mutex.cpp -__TBB_SYMBOL( _ZN3tbb10spin_mutex11scoped_lock16internal_acquireERS0_ ) -__TBB_SYMBOL( _ZN3tbb10spin_mutex11scoped_lock16internal_releaseEv ) -__TBB_SYMBOL( _ZN3tbb10spin_mutex11scoped_lock20internal_try_acquireERS0_ ) -__TBB_SYMBOL( _ZN3tbb10spin_mutex18internal_constructEv ) - -// mutex.cpp -__TBB_SYMBOL( _ZN3tbb5mutex11scoped_lock16internal_acquireERS0_ ) -__TBB_SYMBOL( _ZN3tbb5mutex11scoped_lock16internal_releaseEv ) -__TBB_SYMBOL( _ZN3tbb5mutex11scoped_lock20internal_try_acquireERS0_ ) -__TBB_SYMBOL( _ZN3tbb5mutex16internal_destroyEv ) -__TBB_SYMBOL( _ZN3tbb5mutex18internal_constructEv ) - -// recursive_mutex.cpp -__TBB_SYMBOL( _ZN3tbb15recursive_mutex11scoped_lock16internal_acquireERS0_ ) -__TBB_SYMBOL( _ZN3tbb15recursive_mutex11scoped_lock16internal_releaseEv ) -__TBB_SYMBOL( _ZN3tbb15recursive_mutex11scoped_lock20internal_try_acquireERS0_ ) -__TBB_SYMBOL( _ZN3tbb15recursive_mutex16internal_destroyEv ) -__TBB_SYMBOL( _ZN3tbb15recursive_mutex18internal_constructEv ) - -// queuing_mutex.cpp -__TBB_SYMBOL( _ZN3tbb13queuing_mutex11scoped_lock7acquireERS0_ ) -__TBB_SYMBOL( _ZN3tbb13queuing_mutex11scoped_lock7releaseEv ) -__TBB_SYMBOL( _ZN3tbb13queuing_mutex11scoped_lock11try_acquireERS0_ ) -__TBB_SYMBOL( _ZN3tbb13queuing_mutex18internal_constructEv ) - -// critical_section.cpp -__TBB_SYMBOL( _ZN3tbb8internal19critical_section_v418internal_constructEv ) - -#if !TBB_NO_LEGACY -// concurrent_hash_map -__TBB_SYMBOL( _ZNK3tbb8internal21hash_map_segment_base23internal_grow_predicateEv ) - -// concurrent_queue.cpp v2 -__TBB_SYMBOL( _ZN3tbb8internal21concurrent_queue_base12internal_popEPv ) -__TBB_SYMBOL( _ZN3tbb8internal21concurrent_queue_base13internal_pushEPKv ) -__TBB_SYMBOL( _ZN3tbb8internal21concurrent_queue_base21internal_set_capacityElm ) -__TBB_SYMBOL( _ZN3tbb8internal21concurrent_queue_base23internal_pop_if_presentEPv ) -__TBB_SYMBOL( _ZN3tbb8internal21concurrent_queue_base25internal_push_if_not_fullEPKv ) -__TBB_SYMBOL( _ZN3tbb8internal21concurrent_queue_baseC2Em ) -__TBB_SYMBOL( _ZN3tbb8internal21concurrent_queue_baseD2Ev ) -__TBB_SYMBOL( _ZTIN3tbb8internal21concurrent_queue_baseE ) -__TBB_SYMBOL( _ZTSN3tbb8internal21concurrent_queue_baseE ) -__TBB_SYMBOL( _ZTVN3tbb8internal21concurrent_queue_baseE ) -__TBB_SYMBOL( _ZN3tbb8internal30concurrent_queue_iterator_base6assignERKS1_ ) -__TBB_SYMBOL( _ZN3tbb8internal30concurrent_queue_iterator_base7advanceEv ) -__TBB_SYMBOL( _ZN3tbb8internal30concurrent_queue_iterator_baseC2ERKNS0_21concurrent_queue_baseE ) -__TBB_SYMBOL( _ZN3tbb8internal30concurrent_queue_iterator_baseD2Ev ) -__TBB_SYMBOL( _ZNK3tbb8internal21concurrent_queue_base13internal_sizeEv ) -#endif - -// concurrent_queue v3 -// constructors -__TBB_SYMBOL( _ZN3tbb8internal33concurrent_queue_iterator_base_v3C2ERKNS0_24concurrent_queue_base_v3E ) -__TBB_SYMBOL( _ZN3tbb8internal33concurrent_queue_iterator_base_v3C2ERKNS0_24concurrent_queue_base_v3Em ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v3C2Em ) -// destructors -__TBB_SYMBOL( _ZN3tbb8internal33concurrent_queue_iterator_base_v3D2Ev ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v3D2Ev ) -// typeinfo -__TBB_SYMBOL( _ZTIN3tbb8internal24concurrent_queue_base_v3E ) -__TBB_SYMBOL( _ZTSN3tbb8internal24concurrent_queue_base_v3E ) -// vtable -__TBB_SYMBOL( _ZTVN3tbb8internal24concurrent_queue_base_v3E ) -// methods -__TBB_SYMBOL( _ZN3tbb8internal33concurrent_queue_iterator_base_v36assignERKS1_ ) -__TBB_SYMBOL( _ZN3tbb8internal33concurrent_queue_iterator_base_v37advanceEv ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v313internal_pushEPKv ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v818internal_push_moveEPKv ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v325internal_push_if_not_fullEPKv ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v830internal_push_move_if_not_fullEPKv ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v312internal_popEPv ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v323internal_pop_if_presentEPv ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v314internal_abortEv ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v321internal_finish_clearEv ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v321internal_set_capacityElm ) -__TBB_SYMBOL( _ZNK3tbb8internal24concurrent_queue_base_v313internal_sizeEv ) -__TBB_SYMBOL( _ZNK3tbb8internal24concurrent_queue_base_v314internal_emptyEv ) -__TBB_SYMBOL( _ZNK3tbb8internal24concurrent_queue_base_v324internal_throw_exceptionEv ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v36assignERKS1_ ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v812move_contentERS1_ ) - -#if !TBB_NO_LEGACY -// concurrent_vector.cpp v2 -__TBB_SYMBOL( _ZN3tbb8internal22concurrent_vector_base13internal_copyERKS1_mPFvPvPKvmE ) -__TBB_SYMBOL( _ZN3tbb8internal22concurrent_vector_base14internal_clearEPFvPvmEb ) -__TBB_SYMBOL( _ZN3tbb8internal22concurrent_vector_base15internal_assignERKS1_mPFvPvmEPFvS4_PKvmESA_ ) -__TBB_SYMBOL( _ZN3tbb8internal22concurrent_vector_base16internal_grow_byEmmPFvPvmE ) -__TBB_SYMBOL( _ZN3tbb8internal22concurrent_vector_base16internal_reserveEmmm ) -__TBB_SYMBOL( _ZN3tbb8internal22concurrent_vector_base18internal_push_backEmRm ) -__TBB_SYMBOL( _ZN3tbb8internal22concurrent_vector_base25internal_grow_to_at_leastEmmPFvPvmE ) -__TBB_SYMBOL( _ZNK3tbb8internal22concurrent_vector_base17internal_capacityEv ) -#endif - -// concurrent_vector v3 -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v313internal_copyERKS1_mPFvPvPKvmE ) -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v314internal_clearEPFvPvmE ) -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v315internal_assignERKS1_mPFvPvmEPFvS4_PKvmESA_ ) -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v316internal_grow_byEmmPFvPvPKvmES4_ ) -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v316internal_reserveEmmm ) -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v318internal_push_backEmRm ) -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v325internal_grow_to_at_leastEmmPFvPvPKvmES4_ ) -__TBB_SYMBOL( _ZNK3tbb8internal25concurrent_vector_base_v317internal_capacityEv ) -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v316internal_compactEmPvPFvS2_mEPFvS2_PKvmE ) -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v313internal_swapERS1_ ) -__TBB_SYMBOL( _ZNK3tbb8internal25concurrent_vector_base_v324internal_throw_exceptionEm ) -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v3D2Ev ) -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v315internal_resizeEmmmPKvPFvPvmEPFvS4_S3_mE ) -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v337internal_grow_to_at_least_with_resultEmmPFvPvPKvmES4_ ) - -// tbb_thread -__TBB_SYMBOL( _ZN3tbb8internal13tbb_thread_v320hardware_concurrencyEv ) -__TBB_SYMBOL( _ZN3tbb8internal13tbb_thread_v36detachEv ) -__TBB_SYMBOL( _ZN3tbb8internal16thread_get_id_v3Ev ) -__TBB_SYMBOL( _ZN3tbb8internal15free_closure_v3EPv ) -__TBB_SYMBOL( _ZN3tbb8internal13tbb_thread_v34joinEv ) -__TBB_SYMBOL( _ZN3tbb8internal13tbb_thread_v314internal_startEPFPvS2_ES2_ ) -__TBB_SYMBOL( _ZN3tbb8internal19allocate_closure_v3Em ) -__TBB_SYMBOL( _ZN3tbb8internal7move_v3ERNS0_13tbb_thread_v3ES2_ ) -__TBB_SYMBOL( _ZN3tbb8internal15thread_yield_v3Ev ) -__TBB_SYMBOL( _ZN3tbb8internal15thread_sleep_v3ERKNS_10tick_count10interval_tE ) - -// global parameter -__TBB_SYMBOL( _ZN3tbb10interface914global_control12active_valueEi ) -__TBB_SYMBOL( _ZN3tbb10interface914global_control15internal_createEv ) -__TBB_SYMBOL( _ZN3tbb10interface914global_control16internal_destroyEv ) - -#undef __TBB_SYMBOL diff --git a/src/tbb/src/tbb/mailbox.h b/src/tbb/src/tbb/mailbox.h index f651589e0..d9166c121 100644 --- a/src/tbb/src/tbb/mailbox.h +++ b/src/tbb/src/tbb/mailbox.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,25 +12,23 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef _TBB_mailbox_H #define _TBB_mailbox_H -#include "tbb/tbb_stddef.h" -#include "tbb/cache_aligned_allocator.h" +#include "oneapi/tbb/cache_aligned_allocator.h" +#include "oneapi/tbb/detail/_small_object_pool.h" #include "scheduler_common.h" -#include "tbb/atomic.h" + +#include <atomic> namespace tbb { -namespace internal { +namespace detail { +namespace r1 { -struct task_proxy : public task { +struct task_proxy : public d1::task { static const intptr_t pool_bit = 1<<0; static const intptr_t mailbox_bit = 1<<1; static const intptr_t location_mask = pool_bit | mailbox_bit; @@ -38,29 +36,34 @@ struct task_proxy : public task { Two low-order bits mean: 1 = proxy is/was/will be in task pool 2 = proxy is/was/will be in mailbox */ - intptr_t task_and_tag; + std::atomic<intptr_t> task_and_tag; //! Pointer to next task_proxy in a mailbox - task_proxy *__TBB_atomic next_in_mailbox; + std::atomic<task_proxy*> next_in_mailbox; //! Mailbox to which this was mailed. mail_outbox* outbox; + //! Task affinity id which is referenced + d1::slot_id slot; + + d1::small_object_allocator allocator; + //! True if the proxy is stored both in its sender's pool and in the destination mailbox. static bool is_shared ( intptr_t tat ) { return (tat & location_mask) == location_mask; } - //! Returns a pointer to the encapsulated task or NULL. + //! Returns a pointer to the encapsulated task or nullptr. static task* task_ptr ( intptr_t tat ) { return (task*)(tat & ~location_mask); } - //! Returns a pointer to the encapsulated task or NULL, and frees proxy if necessary. + //! Returns a pointer to the encapsulated task or nullptr, and frees proxy if necessary. template<intptr_t from_bit> inline task* extract_task () { - __TBB_ASSERT( prefix().extra_state == es_task_proxy, "Normal task misinterpreted as a proxy?" ); - intptr_t tat = __TBB_load_with_acquire(task_and_tag); + // __TBB_ASSERT( prefix().extra_state == es_task_proxy, "Normal task misinterpreted as a proxy?" ); + intptr_t tat = task_and_tag.load(std::memory_order_acquire); __TBB_ASSERT( tat == from_bit || (is_shared(tat) && task_ptr(tat)), "Proxy's tag cannot specify both locations if the proxy " "was retrieved from one of its original locations" ); @@ -69,71 +72,83 @@ struct task_proxy : public task { // Attempt to transition the proxy to the "empty" state with // cleaner_bit specifying entity responsible for its eventual freeing. // Explicit cast to void* is to work around a seeming ICC 11.1 bug. - if ( as_atomic(task_and_tag).compare_and_swap(cleaner_bit, tat) == tat ) { + if ( task_and_tag.compare_exchange_strong(tat, cleaner_bit) ) { // Successfully grabbed the task, and left new owner with the job of freeing the proxy return task_ptr(tat); } } // Proxied task has already been claimed from another proxy location. - __TBB_ASSERT( task_and_tag == from_bit, "Empty proxy cannot contain non-zero task pointer" ); - return NULL; + __TBB_ASSERT( task_and_tag.load(std::memory_order_relaxed) == from_bit, "Empty proxy cannot contain non-zero task pointer" ); + return nullptr; + } + + task* execute(d1::execution_data&) override { + __TBB_ASSERT_RELEASE(false, nullptr); + return nullptr; + } + task* cancel(d1::execution_data&) override { + __TBB_ASSERT_RELEASE(false, nullptr); + return nullptr; } }; // struct task_proxy //! Internal representation of mail_outbox, without padding. class unpadded_mail_outbox { protected: - typedef task_proxy*__TBB_atomic proxy_ptr; + typedef std::atomic<task_proxy*> atomic_proxy_ptr; - //! Pointer to first task_proxy in mailbox, or NULL if box is empty. - proxy_ptr my_first; + //! Pointer to first task_proxy in mailbox, or nullptr if box is empty. + atomic_proxy_ptr my_first; - //! Pointer to pointer that will point to next item in the queue. Never NULL. - proxy_ptr* __TBB_atomic my_last; + //! Pointer to pointer that will point to next item in the queue. Never nullptr. + std::atomic<atomic_proxy_ptr*> my_last; //! Owner of mailbox is not executing a task, and has drained its own task pool. - bool my_is_idle; + std::atomic<bool> my_is_idle; }; +// TODO: - consider moving to arena slot //! Class representing where mail is put. /** Padded to occupy a cache line. */ class mail_outbox : padded<unpadded_mail_outbox> { - task_proxy* internal_pop( __TBB_ISOLATION_EXPR(isolation_tag isolation) ) { - task_proxy* curr = __TBB_load_relaxed( my_first ); + task_proxy* internal_pop( isolation_type isolation ) { + task_proxy* curr = my_first.load(std::memory_order_acquire); if ( !curr ) - return NULL; - task_proxy **prev_ptr = &my_first; -#if __TBB_TASK_ISOLATION + return nullptr; + atomic_proxy_ptr* prev_ptr = &my_first; if ( isolation != no_isolation ) { - while ( curr->prefix().isolation != isolation ) { + while ( task_accessor::isolation(*curr) != isolation ) { prev_ptr = &curr->next_in_mailbox; - curr = curr->next_in_mailbox; + // The next_in_mailbox should be read with acquire to guarantee (*curr) consistency. + curr = curr->next_in_mailbox.load(std::memory_order_acquire); if ( !curr ) - return NULL; + return nullptr; } } -#endif /* __TBB_TASK_ISOLATION */ - __TBB_control_consistency_helper(); // on my_first // There is a first item in the mailbox. See if there is a second. - if ( task_proxy* second = curr->next_in_mailbox ) { + // The next_in_mailbox should be read with acquire to guarantee (*second) consistency. + if ( task_proxy* second = curr->next_in_mailbox.load(std::memory_order_acquire) ) { // There are at least two items, so first item can be popped easily. - *prev_ptr = second; + prev_ptr->store(second, std::memory_order_relaxed); } else { - // There is only one item. Some care is required to pop it. - *prev_ptr = NULL; - if ( as_atomic( my_last ).compare_and_swap( prev_ptr, &curr->next_in_mailbox ) == &curr->next_in_mailbox ) { + // There is only one item. Some care is required to pop it. + + prev_ptr->store(nullptr, std::memory_order_relaxed); + atomic_proxy_ptr* expected = &curr->next_in_mailbox; + if ( my_last.compare_exchange_strong( expected, prev_ptr ) ) { // Successfully transitioned mailbox from having one item to having none. - __TBB_ASSERT( !curr->next_in_mailbox, NULL ); + __TBB_ASSERT( !curr->next_in_mailbox.load(std::memory_order_relaxed), nullptr); } else { // Some other thread updated my_last but has not filled in first->next_in_mailbox // Wait until first item points to second item. atomic_backoff backoff; - while ( !(second = curr->next_in_mailbox) ) backoff.pause(); - *prev_ptr = second; + // The next_in_mailbox should be read with acquire to guarantee (*second) consistency. + while ( !(second = curr->next_in_mailbox.load(std::memory_order_acquire)) ) backoff.pause(); + prev_ptr->store( second, std::memory_order_relaxed); } } - __TBB_ASSERT( curr, NULL ); + assert_pointer_valid(curr); return curr; } public: @@ -142,17 +157,19 @@ class mail_outbox : padded<unpadded_mail_outbox> { //! Push task_proxy onto the mailbox queue of another thread. /** Implementation is wait-free. */ void push( task_proxy* t ) { - __TBB_ASSERT(t, NULL); - t->next_in_mailbox = NULL; - proxy_ptr * const link = (proxy_ptr *)__TBB_FetchAndStoreW(&my_last,(intptr_t)&t->next_in_mailbox); - // No release fence required for the next store, because there are no memory operations - // between the previous fully fenced atomic operation and the store. - __TBB_store_relaxed(*link, t); + assert_pointer_valid(t); + t->next_in_mailbox.store(nullptr, std::memory_order_relaxed); + atomic_proxy_ptr* const link = my_last.exchange(&t->next_in_mailbox); + // Logically, the release fence is not required because the exchange above provides the + // release-acquire semantic that guarantees that (*t) will be consistent when another thread + // loads the link atomic. However, C++11 memory model guarantees consistency of(*t) only + // when the same atomic is used for synchronization. + link->store(t, std::memory_order_release); } //! Return true if mailbox is empty bool empty() { - return __TBB_load_relaxed(my_first) == NULL; + return my_first.load(std::memory_order_relaxed) == nullptr; } //! Construct *this as a mailbox from zeroed memory. @@ -160,28 +177,26 @@ class mail_outbox : padded<unpadded_mail_outbox> { This method is provided instead of a full constructor since we know the object will be constructed in zeroed memory. */ void construct() { - __TBB_ASSERT( sizeof(*this)==NFS_MaxLineSize, NULL ); - __TBB_ASSERT( !my_first, NULL ); - __TBB_ASSERT( !my_last, NULL ); - __TBB_ASSERT( !my_is_idle, NULL ); - my_last=&my_first; + __TBB_ASSERT( sizeof(*this)==max_nfs_size, nullptr ); + __TBB_ASSERT( !my_first.load(std::memory_order_relaxed), nullptr ); + __TBB_ASSERT( !my_last.load(std::memory_order_relaxed), nullptr ); + __TBB_ASSERT( !my_is_idle.load(std::memory_order_relaxed), nullptr ); + my_last = &my_first; suppress_unused_warning(pad); } //! Drain the mailbox - intptr_t drain() { - intptr_t k = 0; + void drain() { // No fences here because other threads have already quit. - for( ; task_proxy* t = my_first; ++k ) { - my_first = t->next_in_mailbox; - NFS_Free((char*)t - task_prefix_reservation_size); + for( ; task_proxy* t = my_first; ) { + my_first.store(t->next_in_mailbox, std::memory_order_relaxed); + t->allocator.delete_object(t); } - return k; } //! True if thread that owns this mailbox is looking for work. bool recipient_is_idle() { - return my_is_idle; + return my_is_idle.load(std::memory_order_relaxed); } }; // class mail_outbox @@ -191,7 +206,7 @@ class mail_inbox { mail_outbox* my_putter; public: //! Construct unattached inbox - mail_inbox() : my_putter(NULL) {} + mail_inbox() : my_putter(nullptr) {} //! Attach inbox to a corresponding outbox. void attach( mail_outbox& putter ) { @@ -200,11 +215,11 @@ class mail_inbox { //! Detach inbox from its outbox void detach() { __TBB_ASSERT(my_putter,"not attached"); - my_putter = NULL; + my_putter = nullptr; } - //! Get next piece of mail, or NULL if mailbox is empty. - task_proxy* pop( __TBB_ISOLATION_EXPR( isolation_tag isolation ) ) { - return my_putter->internal_pop( __TBB_ISOLATION_EXPR( isolation ) ); + //! Get next piece of mail, or nullptr if mailbox is empty. + task_proxy* pop( isolation_type isolation ) { + return my_putter->internal_pop( isolation ); } //! Return true if mailbox is empty bool empty() { @@ -214,22 +229,18 @@ class mail_inbox { /** Raises assertion failure if mailbox is redundantly marked as not idle. */ void set_is_idle( bool value ) { if( my_putter ) { - __TBB_ASSERT( my_putter->my_is_idle || value, "attempt to redundantly mark mailbox as not idle" ); - my_putter->my_is_idle = value; + __TBB_ASSERT( my_putter->my_is_idle.load(std::memory_order_relaxed) || value, "attempt to redundantly mark mailbox as not idle" ); + my_putter->my_is_idle.store(value, std::memory_order_relaxed); } } //! Indicate whether thread that reads this mailbox is idle. bool is_idle_state ( bool value ) const { - return !my_putter || my_putter->my_is_idle == value; + return !my_putter || my_putter->my_is_idle.load(std::memory_order_relaxed) == value; } - -#if DO_ITT_NOTIFY - //! Get pointer to corresponding outbox used for ITT_NOTIFY calls. - void* outbox() const {return my_putter;} -#endif /* DO_ITT_NOTIFY */ }; // class mail_inbox -} // namespace internal +} // namespace r1 +} // namespace detail } // namespace tbb #endif /* _TBB_mailbox_H */ diff --git a/src/tbb/src/tbb/main.cpp b/src/tbb/src/tbb/main.cpp new file mode 100644 index 000000000..f43c33f5b --- /dev/null +++ b/src/tbb/src/tbb/main.cpp @@ -0,0 +1,161 @@ +/* + Copyright (c) 2005-2024 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "oneapi/tbb/detail/_config.h" + +#include "main.h" +#include "governor.h" +#include "threading_control.h" +#include "environment.h" +#include "market.h" +#include "tcm_adaptor.h" +#include "misc.h" +#include "itt_notify.h" + +namespace tbb { +namespace detail { +namespace r1 { + +//------------------------------------------------------------------------ +// Begin shared data layout. +// The following global data items are mostly read-only after initialization. +//------------------------------------------------------------------------ + +//------------------------------------------------------------------------ +// governor data +basic_tls<thread_data*> governor::theTLS; +rml::tbb_factory governor::theRMLServerFactory; +bool governor::UsePrivateRML; +bool governor::is_rethrow_broken; + +//------------------------------------------------------------------------ +// threading_control data +threading_control* threading_control::g_threading_control; +threading_control::global_mutex_type threading_control::g_threading_control_mutex; + +//------------------------------------------------------------------------ +// context propagation data +context_state_propagation_mutex_type the_context_state_propagation_mutex; +std::atomic<uintptr_t> the_context_state_propagation_epoch{}; + +//------------------------------------------------------------------------ +// One time initialization data + +//! Counter of references to global shared resources such as TLS. +std::atomic<int> __TBB_InitOnce::count{}; + +std::atomic_flag __TBB_InitOnce::InitializationLock = ATOMIC_FLAG_INIT; + +//! Flag that is set to true after one-time initializations are done. +std::atomic<bool> __TBB_InitOnce::InitializationDone{}; + +#if __TBB_USE_ITT_NOTIFY +//! Defined in profiling.cpp +extern bool ITT_Present; +void ITT_DoUnsafeOneTimeInitialization(); +#endif + +#if !(_WIN32||_WIN64) || __TBB_SOURCE_DIRECTLY_INCLUDED +static __TBB_InitOnce __TBB_InitOnceHiddenInstance; +#endif + +//------------------------------------------------------------------------ +// __TBB_InitOnce +//------------------------------------------------------------------------ + +void __TBB_InitOnce::add_ref() { + if (++count == 1) { + governor::acquire_resources(); + tcm_adaptor::initialize(); + } +} + +void __TBB_InitOnce::remove_ref() { + int k = --count; + __TBB_ASSERT(k>=0,"removed __TBB_InitOnce ref that was not added?"); + if( k==0 ) { + governor::release_resources(); + ITT_FINI_ITTLIB(); + ITT_RELEASE_RESOURCES(); + } +} + +//------------------------------------------------------------------------ +// One-time Initializations +//------------------------------------------------------------------------ + +//! Defined in cache_aligned_allocator.cpp +void initialize_cache_aligned_allocator(); + +//! Performs thread-safe lazy one-time general TBB initialization. +void DoOneTimeInitialization() { + __TBB_InitOnce::lock(); + // No fence required for load of InitializationDone, because we are inside a critical section. + if( !__TBB_InitOnce::InitializationDone ) { + __TBB_InitOnce::add_ref(); + if( GetBoolEnvironmentVariable("TBB_VERSION") ) { + PrintVersion(); + tcm_adaptor::print_version(); + } + bool itt_present = false; +#if __TBB_USE_ITT_NOTIFY + ITT_DoUnsafeOneTimeInitialization(); + itt_present = ITT_Present; +#endif /* __TBB_USE_ITT_NOTIFY */ + initialize_cache_aligned_allocator(); + governor::initialize_rml_factory(); + // Force processor groups support detection + governor::default_num_threads(); + // Force OS regular page size detection + governor::default_page_size(); + PrintExtraVersionInfo( "TOOLS SUPPORT", itt_present ? "enabled" : "disabled" ); + __TBB_InitOnce::InitializationDone = true; + } + __TBB_InitOnce::unlock(); +} + +#if (_WIN32||_WIN64) && !__TBB_SOURCE_DIRECTLY_INCLUDED +//! Windows "DllMain" that handles startup and shutdown of dynamic library. +extern "C" bool WINAPI DllMain( HANDLE /*hinstDLL*/, DWORD reason, LPVOID lpvReserved ) { + switch( reason ) { + case DLL_PROCESS_ATTACH: + __TBB_InitOnce::add_ref(); + break; + case DLL_PROCESS_DETACH: + // Since THREAD_DETACH is not called for the main thread, call auto-termination + // here as well - but not during process shutdown (due to risk of a deadlock). + if ( lpvReserved == nullptr ) { // library unload + governor::terminate_external_thread(); + } + __TBB_InitOnce::remove_ref(); + // It is assumed that InitializationDone is not set after DLL_PROCESS_DETACH, + // and thus no race on InitializationDone is possible. + if ( __TBB_InitOnce::initialization_done() ) { + // Remove reference that we added in DoOneTimeInitialization. + __TBB_InitOnce::remove_ref(); + } + break; + case DLL_THREAD_DETACH: + governor::terminate_external_thread(); + break; + } + return true; +} +#endif /* (_WIN32||_WIN64) && !__TBB_SOURCE_DIRECTLY_INCLUDED */ + +} // namespace r1 +} // namespace detail +} // namespace tbb diff --git a/src/tbb/src/tbb/tbb_main.h b/src/tbb/src/tbb/main.h similarity index 69% rename from src/tbb/src/tbb/tbb_main.h rename to src/tbb/src/tbb/main.h index 24d77d112..c6f54bb47 100644 --- a/src/tbb/src/tbb/tbb_main.h +++ b/src/tbb/src/tbb/main.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,35 +12,34 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ -#ifndef _TBB_tbb_main_H -#define _TBB_tbb_main_H +#ifndef _TBB_main_H +#define _TBB_main_H -#include "tbb/atomic.h" +#include "governor.h" -namespace tbb { +#include <atomic> -namespace internal { +namespace tbb { +namespace detail { +namespace r1 { -void DoOneTimeInitializations (); +void DoOneTimeInitialization(); //------------------------------------------------------------------------ // __TBB_InitOnce //------------------------------------------------------------------------ +// TODO (TBB_REVAMP_TODO): consider better names //! Class that supports TBB initialization. /** It handles acquisition and release of global resources (e.g. TLS) during startup and shutdown, - as well as synchronization for DoOneTimeInitializations. */ + as well as synchronization for DoOneTimeInitialization. */ class __TBB_InitOnce { - friend void DoOneTimeInitializations(); - friend void ITT_DoUnsafeOneTimeInitialization (); + friend void DoOneTimeInitialization(); + friend void ITT_DoUnsafeOneTimeInitialization(); - static atomic<int> count; + static std::atomic<int> count; //! Platform specific code to acquire resources. static void acquire_resources(); @@ -49,21 +48,24 @@ class __TBB_InitOnce { static void release_resources(); //! Specifies if the one-time initializations has been done. - static bool InitializationDone; + static std::atomic<bool> InitializationDone; //! Global initialization lock /** Scenarios are possible when tools interop has to be initialized before the TBB itself. This imposes a requirement that the global initialization lock has to support valid static initialization, and does not issue any tool notifications in any build mode. **/ - static __TBB_atomic_flag InitializationLock; + static std::atomic_flag InitializationLock; public: - static void lock() { __TBB_LockByte( InitializationLock ); } + static void lock() { + tbb::detail::atomic_backoff backoff; + while( InitializationLock.test_and_set() ) backoff.pause(); + } - static void unlock() { __TBB_UnlockByte( InitializationLock ); } + static void unlock() { InitializationLock.clear(std::memory_order_release); } - static bool initialization_done() { return __TBB_load_with_acquire(InitializationDone); } + static bool initialization_done() { return InitializationDone.load(std::memory_order_acquire); } //! Add initial reference to resources. /** We assume that dynamic loading of the library prevents any other threads @@ -73,11 +75,12 @@ class __TBB_InitOnce { //! Remove the initial reference to resources. /** This is not necessarily the last reference if other threads are still running. **/ ~__TBB_InitOnce() { + governor::terminate_external_thread(); // TLS dtor not called for the main thread remove_ref(); // We assume that InitializationDone is not set after file-scope destructors // start running, and thus no race on InitializationDone is possible. - if( initialization_done() ) { - // Remove an extra reference that was added in DoOneTimeInitializations. + if ( initialization_done() ) { + // Remove an extra reference that was added in DoOneTimeInitialization. remove_ref(); } } @@ -86,11 +89,11 @@ class __TBB_InitOnce { //! Remove reference to resources. If last reference removed, release the resources. static void remove_ref(); -}; // class __TBB_InitOnce +}; // class __TBB_InitOnce -} // namespace internal - +} // namespace r1 +} // namespace detail } // namespace tbb -#endif /* _TBB_tbb_main_H */ +#endif /* _TBB_main_H */ diff --git a/src/tbb/src/tbb/market.cpp b/src/tbb/src/tbb/market.cpp index 3b6efc630..ae3fadd47 100644 --- a/src/tbb/src/tbb/market.cpp +++ b/src/tbb/src/tbb/market.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2023 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,856 +12,132 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. +*/ +#include "arena.h" +#include "market.h" +#include <algorithm> // std::find +namespace tbb { +namespace detail { +namespace r1 { -*/ - -#include "tbb/tbb_stddef.h" -#include "tbb/global_control.h" // global_control::active_value -#include "market.h" -#include "tbb_main.h" -#include "governor.h" -#include "scheduler.h" -#include "itt_notify.h" +class tbb_permit_manager_client : public pm_client { +public: + tbb_permit_manager_client(arena& a) : pm_client(a) {} -namespace tbb { -namespace internal { + void register_thread() override {} -void market::insert_arena_into_list ( arena& a ) { -#if __TBB_TASK_PRIORITY - arena_list_type &arenas = my_priority_levels[a.my_top_priority].arenas; - arena *&next = my_priority_levels[a.my_top_priority].next_arena; -#else /* !__TBB_TASK_PRIORITY */ - arena_list_type &arenas = my_arenas; - arena *&next = my_next_arena; -#endif /* !__TBB_TASK_PRIORITY */ - arenas.push_front( a ); - if ( arenas.size() == 1 ) - next = &*arenas.begin(); -} + void unregister_thread() override {} -void market::remove_arena_from_list ( arena& a ) { -#if __TBB_TASK_PRIORITY - arena_list_type &arenas = my_priority_levels[a.my_top_priority].arenas; - arena *&next = my_priority_levels[a.my_top_priority].next_arena; -#else /* !__TBB_TASK_PRIORITY */ - arena_list_type &arenas = my_arenas; - arena *&next = my_next_arena; -#endif /* !__TBB_TASK_PRIORITY */ - arena_list_type::iterator it = next; - __TBB_ASSERT( it != arenas.end(), NULL ); - if ( next == &a ) { - if ( ++it == arenas.end() && arenas.size() > 1 ) - it = arenas.begin(); - next = &*it; + void set_allotment(unsigned allotment) { + my_arena.set_allotment(allotment); } - arenas.remove( a ); -} +}; //------------------------------------------------------------------------ // market //------------------------------------------------------------------------ -market::market ( unsigned workers_soft_limit, unsigned workers_hard_limit, size_t stack_size ) - : my_num_workers_hard_limit(workers_hard_limit) - , my_num_workers_soft_limit(workers_soft_limit) -#if __TBB_TASK_PRIORITY - , my_global_top_priority(normalized_normal_priority) - , my_global_bottom_priority(normalized_normal_priority) -#endif /* __TBB_TASK_PRIORITY */ - , my_ref_count(1) - , my_stack_size(stack_size) - , my_workers_soft_limit_to_report(workers_soft_limit) -{ -#if __TBB_TASK_PRIORITY - __TBB_ASSERT( my_global_reload_epoch == 0, NULL ); - my_priority_levels[normalized_normal_priority].workers_available = my_num_workers_soft_limit; -#endif /* __TBB_TASK_PRIORITY */ - - // Once created RML server will start initializing workers that will need - // global market instance to get worker stack size - my_server = governor::create_rml_server( *this ); - __TBB_ASSERT( my_server, "Failed to create RML server" ); -} - -static unsigned calc_workers_soft_limit(unsigned workers_soft_limit, unsigned workers_hard_limit) { - if( int soft_limit = market::app_parallelism_limit() ) - workers_soft_limit = soft_limit-1; - else // if user set no limits (yet), use market's parameter - workers_soft_limit = max( governor::default_num_threads() - 1, workers_soft_limit ); - if( workers_soft_limit >= workers_hard_limit ) - workers_soft_limit = workers_hard_limit-1; - return workers_soft_limit; -} - -market& market::global_market ( bool is_public, unsigned workers_requested, size_t stack_size ) { - global_market_mutex_type::scoped_lock lock( theMarketMutex ); - market *m = theMarket; - if( m ) { - ++m->my_ref_count; - const unsigned old_public_count = is_public? m->my_public_ref_count++ : /*any non-zero value*/1; - lock.release(); - if( old_public_count==0 ) - set_active_num_workers( calc_workers_soft_limit(workers_requested, m->my_num_workers_hard_limit) ); - - // do not warn if default number of workers is requested - if( workers_requested != governor::default_num_threads()-1 ) { - __TBB_ASSERT( skip_soft_limit_warning > workers_requested, - "skip_soft_limit_warning must be larger than any valid workers_requested" ); - unsigned soft_limit_to_report = m->my_workers_soft_limit_to_report; - if( soft_limit_to_report < workers_requested ) { - runtime_warning( "The number of workers is currently limited to %u. " - "The request for %u workers is ignored. Further requests for more workers " - "will be silently ignored until the limit changes.\n", - soft_limit_to_report, workers_requested ); - // The race is possible when multiple threads report warnings. - // We are OK with that, as there are just multiple warnings. - internal::as_atomic(m->my_workers_soft_limit_to_report). - compare_and_swap(skip_soft_limit_warning, soft_limit_to_report); - } - - } - if( m->my_stack_size < stack_size ) - runtime_warning( "Thread stack size has been already set to %u. " - "The request for larger stack (%u) cannot be satisfied.\n", - m->my_stack_size, stack_size ); - } - else { - // TODO: A lot is done under theMarketMutex locked. Can anything be moved out? - if( stack_size == 0 ) - stack_size = global_control::active_value(global_control::thread_stack_size); - // Expecting that 4P is suitable for most applications. - // Limit to 2P for large thread number. - // TODO: ask RML for max concurrency and possibly correct hard_limit - const unsigned factor = governor::default_num_threads()<=128? 4 : 2; - // The requested number of threads is intentionally not considered in - // computation of the hard limit, in order to separate responsibilities - // and avoid complicated interactions between global_control and task_scheduler_init. - // The market guarantees that at least 256 threads might be created. - const unsigned workers_hard_limit = max(max(factor*governor::default_num_threads(), 256u), app_parallelism_limit()); - const unsigned workers_soft_limit = calc_workers_soft_limit(workers_requested, workers_hard_limit); - // Create the global market instance - size_t size = sizeof(market); -#if __TBB_TASK_GROUP_CONTEXT - __TBB_ASSERT( __TBB_offsetof(market, my_workers) + sizeof(generic_scheduler*) == sizeof(market), - "my_workers must be the last data field of the market class"); - size += sizeof(generic_scheduler*) * (workers_hard_limit - 1); -#endif /* __TBB_TASK_GROUP_CONTEXT */ - __TBB_InitOnce::add_ref(); - void* storage = NFS_Allocate(1, size, NULL); - memset( storage, 0, size ); - // Initialize and publish global market - m = new (storage) market( workers_soft_limit, workers_hard_limit, stack_size ); - if( is_public ) - m->my_public_ref_count = 1; - theMarket = m; - // This check relies on the fact that for shared RML default_concurrency==max_concurrency - if ( !governor::UsePrivateRML && m->my_server->default_concurrency() < workers_soft_limit ) - runtime_warning( "RML might limit the number of workers to %u while %u is requested.\n" - , m->my_server->default_concurrency(), workers_soft_limit ); - } - return *m; -} +market::market(unsigned workers_soft_limit) + : my_num_workers_soft_limit(workers_soft_limit) +{} -void market::destroy () { -#if __TBB_COUNT_TASK_NODES - if ( my_task_node_count ) - runtime_warning( "Leaked %ld task objects\n", (long)my_task_node_count ); -#endif /* __TBB_COUNT_TASK_NODES */ - this->market::~market(); // qualified to suppress warning - NFS_Free( this ); - __TBB_InitOnce::remove_ref(); +pm_client* market::create_client(arena& a) { + return new (cache_aligned_allocate(sizeof(tbb_permit_manager_client))) tbb_permit_manager_client(a); } -bool market::release ( bool is_public, bool blocking_terminate ) { - __TBB_ASSERT( theMarket == this, "Global market instance was destroyed prematurely?" ); - bool do_release = false; - { - global_market_mutex_type::scoped_lock lock( theMarketMutex ); - if ( blocking_terminate ) { - __TBB_ASSERT( is_public, "Only an object with a public reference can request the blocking terminate" ); - while ( my_public_ref_count == 1 && my_ref_count > 1 ) { - lock.release(); - // To guarantee that request_close_connection() is called by the last master, we need to wait till all - // references are released. Re-read my_public_ref_count to limit waiting if new masters are created. - // Theoretically, new private references to the market can be added during waiting making it potentially - // endless. - // TODO: revise why the weak scheduler needs market's pointer and try to remove this wait. - // Note that the market should know about its schedulers for cancelation/exception/priority propagation, - // see e.g. task_group_context::cancel_group_execution() - while ( __TBB_load_with_acquire( my_public_ref_count ) == 1 && __TBB_load_with_acquire( my_ref_count ) > 1 ) - __TBB_Yield(); - lock.acquire( theMarketMutex ); - } - } - if ( is_public ) { - __TBB_ASSERT( theMarket == this, "Global market instance was destroyed prematurely?" ); - __TBB_ASSERT( my_public_ref_count, NULL ); - --my_public_ref_count; - } - if ( --my_ref_count == 0 ) { - __TBB_ASSERT( !my_public_ref_count, NULL ); - do_release = true; - theMarket = NULL; - } - } - if( do_release ) { - __TBB_ASSERT( !__TBB_load_with_acquire(my_public_ref_count), "No public references remain if we remove the market." ); - // inform RML that blocking termination is required - my_join_workers = blocking_terminate; - my_server->request_close_connection(); - return blocking_terminate; - } - return false; +void market::register_client(pm_client* c, d1::constraints&) { + mutex_type::scoped_lock lock(my_mutex); + my_clients[c->priority_level()].push_back(c); } -void market::set_active_num_workers ( unsigned soft_limit ) { - int old_requested=0, requested=0; - bool need_mandatory = false; - market *m; - +void market::unregister_and_destroy_client(pm_client& c) { { - global_market_mutex_type::scoped_lock lock( theMarketMutex ); - if ( !theMarket ) - return; // actual value will be used at market creation - m = theMarket; - ++m->my_ref_count; + mutex_type::scoped_lock lock(my_mutex); + auto& clients = my_clients[c.priority_level()]; + auto it = std::find(clients.begin(), clients.end(), &c); + __TBB_ASSERT(it != clients.end(), "Destroying of an unregistered client"); + clients.erase(it); } - // have my_ref_count for market, use it safely - { - arenas_list_mutex_type::scoped_lock lock( m->my_arenas_list_mutex ); - __TBB_ASSERT(soft_limit <= m->my_num_workers_hard_limit, NULL); - m->my_num_workers_soft_limit = soft_limit; - // report only once after new soft limit value is set - m->my_workers_soft_limit_to_report = soft_limit; -#if __TBB_ENQUEUE_ENFORCED_CONCURRENCY - // updates soft_limit to zero must be postponed - // while mandatory parallelism is enabled - if( !(m->my_mandatory_num_requested && !soft_limit) ) -#endif - { - const int demand = -#if __TBB_ENQUEUE_ENFORCED_CONCURRENCY - m->my_mandatory_num_requested? 0 : -#endif - m->my_total_demand; - requested = min(demand, (int)soft_limit); - old_requested = m->my_num_workers_requested; - m->my_num_workers_requested = requested; -#if __TBB_TASK_PRIORITY - m->my_priority_levels[m->my_global_top_priority].workers_available = soft_limit; - m->update_allotment( m->my_global_top_priority ); -#else - m->update_allotment(); -#endif - } -#if __TBB_ENQUEUE_ENFORCED_CONCURRENCY - if( !m->my_mandatory_num_requested && !soft_limit ) { - // enable mandatory concurrency, if enqueued tasks are found - // and zero soft_limit requested -#if __TBB_TASK_PRIORITY - for( int p = m->my_global_top_priority; p >= m->my_global_bottom_priority; --p ) { - priority_level_info &pl = m->my_priority_levels[p]; - arena_list_type &arenas = pl.arenas; -#else - const int p = 0; - arena_list_type &arenas = m->my_arenas; -#endif /* __TBB_TASK_PRIORITY */ - for( arena_list_type::iterator it = arenas.begin(); it != arenas.end(); ++it ) { - if( !it->my_task_stream.empty(p) ) { - // switch local_mandatory to global_mandatory unconditionally - if( m->mandatory_concurrency_enable_impl( &*it ) ) - need_mandatory = true; - } - } -#if __TBB_TASK_PRIORITY - } -#endif /* __TBB_TASK_PRIORITY */ - } -#endif /* __TBB_ENQUEUE_ENFORCED_CONCURRENCY */ - } - // adjust_job_count_estimate must be called outside of any locks - int delta = requested - old_requested; - if( need_mandatory ) ++delta; - if( delta!=0 ) - m->my_server->adjust_job_count_estimate( delta ); - // release internal market reference to match ++m->my_ref_count above - m->release( /*is_public=*/false, /*blocking_terminate=*/false ); -} - -bool governor::does_client_join_workers (const tbb::internal::rml::tbb_client &client) { - return ((const market&)client).must_join_workers(); -} - -arena* market::create_arena ( int num_slots, int num_reserved_slots, size_t stack_size ) { - __TBB_ASSERT( num_slots > 0, NULL ); - __TBB_ASSERT( num_reserved_slots <= num_slots, NULL ); - // Add public market reference for master thread/task_arena (that adds an internal reference in exchange). - market &m = global_market( /*is_public=*/true, num_slots-num_reserved_slots, stack_size ); - - arena& a = arena::allocate_arena( m, num_slots, num_reserved_slots ); - // Add newly created arena into the existing market's list. - arenas_list_mutex_type::scoped_lock lock(m.my_arenas_list_mutex); - m.insert_arena_into_list(a); - return &a; -} - -/** This method must be invoked under my_arenas_list_mutex. **/ -void market::detach_arena ( arena& a ) { - __TBB_ASSERT( theMarket == this, "Global market instance was destroyed prematurely?" ); - __TBB_ASSERT( !a.my_slots[0].my_scheduler, NULL ); - remove_arena_from_list(a); - if ( a.my_aba_epoch == my_arenas_aba_epoch ) - ++my_arenas_aba_epoch; -} - -void market::try_destroy_arena ( arena* a, uintptr_t aba_epoch ) { - bool locked = true; - __TBB_ASSERT( a, NULL ); - // we hold reference to the market, so it cannot be destroyed at any moment here - __TBB_ASSERT( this == theMarket, NULL ); - __TBB_ASSERT( my_ref_count!=0, NULL ); - my_arenas_list_mutex.lock(); - assert_market_valid(); -#if __TBB_TASK_PRIORITY - // scan all priority levels, not only in [my_global_bottom_priority;my_global_top_priority] - // range, because arena to be destoyed can have no outstanding request for workers - for ( int p = num_priority_levels-1; p >= 0; --p ) { - priority_level_info &pl = my_priority_levels[p]; - arena_list_type &my_arenas = pl.arenas; -#endif /* __TBB_TASK_PRIORITY */ - arena_list_type::iterator it = my_arenas.begin(); - for ( ; it != my_arenas.end(); ++it ) { - if ( a == &*it ) { - if ( it->my_aba_epoch == aba_epoch ) { - // Arena is alive - if ( !a->my_num_workers_requested && !a->my_references ) { - __TBB_ASSERT( !a->my_num_workers_allotted && (a->my_pool_state == arena::SNAPSHOT_EMPTY || !a->my_max_num_workers), "Inconsistent arena state" ); - // Arena is abandoned. Destroy it. - detach_arena( *a ); - my_arenas_list_mutex.unlock(); - locked = false; - a->free_arena(); - } - } - if (locked) - my_arenas_list_mutex.unlock(); - return; - } - } -#if __TBB_TASK_PRIORITY - } -#endif /* __TBB_TASK_PRIORITY */ - my_arenas_list_mutex.unlock(); + auto client = static_cast<tbb_permit_manager_client*>(&c); + client->~tbb_permit_manager_client(); + cache_aligned_deallocate(client); } -/** This method must be invoked under my_arenas_list_mutex. **/ -arena* market::arena_in_need ( arena_list_type &arenas, arena *&next ) { - if ( arenas.empty() ) - return NULL; - arena_list_type::iterator it = next; - __TBB_ASSERT( it != arenas.end(), NULL ); - do { - arena& a = *it; - if ( ++it == arenas.end() ) - it = arenas.begin(); - if( a.num_workers_active() < a.my_num_workers_allotted -#if __TBB_ENQUEUE_ENFORCED_CONCURRENCY - && !a.recall_by_mandatory_request() -#endif - ) { - a.my_references += arena::ref_worker; - as_atomic(next) = &*it; // a subject for innocent data race under the reader lock - // TODO: rework global round robin policy to local or random to avoid this write - return &a; - } - } while ( it != next ); - return NULL; -} +void market::update_allotment() { + int effective_soft_limit = my_mandatory_num_requested > 0 && my_num_workers_soft_limit == 0 ? 1 : my_num_workers_soft_limit; + int max_workers = min(my_total_demand, effective_soft_limit); + __TBB_ASSERT(max_workers >= 0, nullptr); -int market::update_allotment ( arena_list_type& arenas, int workers_demand, int max_workers ) { - __TBB_ASSERT( workers_demand, NULL ); - max_workers = min(workers_demand, max_workers); - int carry = 0; + int unassigned_workers = max_workers; int assigned = 0; - arena_list_type::iterator it = arenas.begin(); - for ( ; it != arenas.end(); ++it ) { - arena& a = *it; - if ( a.my_num_workers_requested <= 0 ) { - __TBB_ASSERT( !a.my_num_workers_allotted, NULL ); - continue; - } - int tmp = a.my_num_workers_requested * max_workers + carry; - int allotted = tmp / workers_demand; - carry = tmp % workers_demand; - // a.my_num_workers_requested may temporarily exceed a.my_max_num_workers - allotted = min( allotted, (int)a.my_max_num_workers ); -#if __TBB_ENQUEUE_ENFORCED_CONCURRENCY - if ( !allotted && a.must_have_concurrency() ) - allotted = 1; -#endif - a.my_num_workers_allotted = allotted; - assigned += allotted; - } -#if __TBB_ENQUEUE_ENFORCED_CONCURRENCY - __TBB_ASSERT( assigned <= workers_demand, NULL ); // weaker assertion due to enforced allotment -#else - __TBB_ASSERT( assigned <= max_workers, NULL ); -#endif - return assigned; -} - -#if __TBB_TASK_PRIORITY -inline void market::update_global_top_priority ( intptr_t newPriority ) { - GATHER_STATISTIC( ++governor::local_scheduler_if_initialized()->my_counters.market_prio_switches ); - my_global_top_priority = newPriority; - my_priority_levels[newPriority].workers_available = -#if __TBB_ENQUEUE_ENFORCED_CONCURRENCY - my_mandatory_num_requested && !my_num_workers_soft_limit ? 1 : -#endif - my_num_workers_soft_limit; - advance_global_reload_epoch(); -} - -inline void market::reset_global_priority () { - my_global_bottom_priority = normalized_normal_priority; - update_global_top_priority(normalized_normal_priority); -} + int carry = 0; + unsigned max_priority_level = num_priority_levels; + for (unsigned list_idx = 0; list_idx < num_priority_levels; ++list_idx ) { + int assigned_per_priority = min(my_priority_level_demand[list_idx], unassigned_workers); + unassigned_workers -= assigned_per_priority; + // We use reverse iterator there to serve last added clients first + for (auto it = my_clients[list_idx].rbegin(); it != my_clients[list_idx].rend(); ++it) { + tbb_permit_manager_client& client = static_cast<tbb_permit_manager_client&>(**it); + if (client.max_workers() == 0) { + client.set_allotment(0); + continue; + } -arena* market::arena_in_need ( arena* prev_arena ) -{ - suppress_unused_warning(prev_arena); - if( as_atomic(my_total_demand) <= 0 ) - return NULL; - arenas_list_mutex_type::scoped_lock lock(my_arenas_list_mutex, /*is_writer=*/false); - assert_market_valid(); - int p = my_global_top_priority; - arena *a = NULL; - do { - priority_level_info &pl = my_priority_levels[p]; - a = arena_in_need( pl.arenas, pl.next_arena ); - // TODO: When refactoring task priority code, take into consideration the - // __TBB_TRACK_PRIORITY_LEVEL_SATURATION sections from earlier versions of TBB - } while ( !a && --p >= my_global_bottom_priority ); - return a; -} + if (max_priority_level == num_priority_levels) { + max_priority_level = list_idx; + } -void market::update_allotment ( intptr_t highest_affected_priority ) { - intptr_t i = highest_affected_priority; - int available = my_priority_levels[i].workers_available; - for ( ; i >= my_global_bottom_priority; --i ) { - priority_level_info &pl = my_priority_levels[i]; - pl.workers_available = available; - if ( pl.workers_requested ) { - available -= update_allotment( pl.arenas, pl.workers_requested, available ); - if ( available < 0 ) { // TODO: assertion? - available = 0; - break; + int allotted = 0; + if (my_num_workers_soft_limit == 0) { + __TBB_ASSERT(max_workers == 0 || max_workers == 1, nullptr); + allotted = client.min_workers() > 0 && assigned < max_workers ? 1 : 0; + } else { + int tmp = client.max_workers() * assigned_per_priority + carry; + allotted = tmp / my_priority_level_demand[list_idx]; + carry = tmp % my_priority_level_demand[list_idx]; + __TBB_ASSERT(allotted <= client.max_workers(), nullptr); } + client.set_allotment(allotted); + client.set_top_priority(list_idx == max_priority_level); + assigned += allotted; } } - __TBB_ASSERT( i <= my_global_bottom_priority || !available, NULL ); - for ( --i; i >= my_global_bottom_priority; --i ) { - priority_level_info &pl = my_priority_levels[i]; - pl.workers_available = 0; - arena_list_type::iterator it = pl.arenas.begin(); - for ( ; it != pl.arenas.end(); ++it ) { - __TBB_ASSERT( it->my_num_workers_requested >= 0 || !it->my_num_workers_allotted, NULL ); -#if __TBB_ENQUEUE_ENFORCED_CONCURRENCY - it->my_num_workers_allotted = it->must_have_concurrency() ? 1 : 0; -#else - it->my_num_workers_allotted = 0; -#endif - } - } + __TBB_ASSERT(assigned == max_workers, nullptr); } -#endif /* __TBB_TASK_PRIORITY */ -#if __TBB_ENQUEUE_ENFORCED_CONCURRENCY -bool market::mandatory_concurrency_enable_impl ( arena *a, bool *enabled ) { - if( a->my_concurrency_mode==arena_base::cm_enforced_global ) { - if( enabled ) - *enabled = false; - return false; +void market::set_active_num_workers(int soft_limit) { + mutex_type::scoped_lock lock(my_mutex); + if (my_num_workers_soft_limit != soft_limit) { + my_num_workers_soft_limit = soft_limit; + update_allotment(); } - if( enabled ) - *enabled = true; - a->my_max_num_workers = 1; - a->my_concurrency_mode = arena_base::cm_enforced_global; -#if __TBB_TASK_PRIORITY - priority_level_info &pl = my_priority_levels[a->my_top_priority]; - pl.workers_requested++; - if( my_global_top_priority < a->my_top_priority ) { - my_global_top_priority = a->my_top_priority; - advance_global_reload_epoch(); - } -#endif - a->my_num_workers_requested++; - a->my_num_workers_allotted++; - if( 1 == ++my_mandatory_num_requested ) { - my_total_demand++; - return true; - } - return false; -} - -bool market::mandatory_concurrency_enable ( arena *a ) { - bool add_thread; - bool enabled; - { - arenas_list_mutex_type::scoped_lock lock(my_arenas_list_mutex); - add_thread = mandatory_concurrency_enable_impl(a, &enabled); - } - if( add_thread ) - my_server->adjust_job_count_estimate( 1 ); - return enabled; } -void market::mandatory_concurrency_disable ( arena *a ) { - bool remove_thread = false; - int delta_adjust_demand = 0; +void market::adjust_demand(pm_client& c, int mandatory_delta, int workers_delta) { + __TBB_ASSERT(-1 <= mandatory_delta && mandatory_delta <= 1, nullptr); + int delta{}; { - arenas_list_mutex_type::scoped_lock lock(my_arenas_list_mutex); - - if( a->my_concurrency_mode!=arena_base::cm_enforced_global ) - return; - __TBB_ASSERT( a->my_max_num_workers==1, NULL ); - a->my_max_num_workers = 0; -#if __TBB_TASK_PRIORITY - if ( a->my_top_priority != normalized_normal_priority ) { - update_arena_top_priority( *a, normalized_normal_priority ); - } - a->my_bottom_priority = normalized_normal_priority; -#endif - - int val = --my_mandatory_num_requested; - __TBB_ASSERT_EX( val >= 0, NULL ); - if( val == 0 ) { - my_total_demand--; - remove_thread = true; - } - a->my_num_workers_requested--; - if (a->my_num_workers_requested > 0) - delta_adjust_demand = a->my_num_workers_requested; - else - a->my_num_workers_allotted = 0; - -#if __TBB_TASK_PRIORITY - priority_level_info &pl = my_priority_levels[a->my_top_priority]; - pl.workers_requested--; - intptr_t p = my_global_top_priority; - for (; !my_priority_levels[p].workers_requested && p>0; p--) - ; - if( !p ) - reset_global_priority(); - else if( p!= my_global_top_priority ) - update_global_top_priority(p); -#endif - a->my_concurrency_mode = arena::cm_normal; - } - if( delta_adjust_demand ) - adjust_demand( *a, -delta_adjust_demand ); - if( remove_thread ) - my_server->adjust_job_count_estimate( -1 ); -} -#endif /* __TBB_ENQUEUE_ENFORCED_CONCURRENCY */ - -void market::adjust_demand ( arena& a, int delta ) { - __TBB_ASSERT( theMarket, "market instance was destroyed prematurely?" ); - if ( !delta ) - return; - my_arenas_list_mutex.lock(); - int prev_req = a.my_num_workers_requested; - a.my_num_workers_requested += delta; - if ( a.my_num_workers_requested <= 0 ) { -#if __TBB_ENQUEUE_ENFORCED_CONCURRENCY - // must not recall worker from arena with mandatory parallelism - if ( a.my_market->my_mandatory_num_requested && a.my_concurrency_mode!=arena_base::cm_normal ) - a.my_num_workers_allotted = 1; - else -#endif - a.my_num_workers_allotted = 0; - if ( prev_req <= 0 ) { - my_arenas_list_mutex.unlock(); - return; - } - delta = -prev_req; - } - else if ( prev_req < 0 ) { - delta = a.my_num_workers_requested; - } - my_total_demand += delta; -#if !__TBB_TASK_PRIORITY - update_allotment(); -#else /* !__TBB_TASK_PRIORITY */ - intptr_t p = a.my_top_priority; - priority_level_info &pl = my_priority_levels[p]; - pl.workers_requested += delta; - __TBB_ASSERT( pl.workers_requested >= 0, NULL ); - if ( a.my_num_workers_requested <= 0 ) { - if ( a.my_top_priority != normalized_normal_priority ) { - GATHER_STATISTIC( ++governor::local_scheduler_if_initialized()->my_counters.arena_prio_resets ); - update_arena_top_priority( a, normalized_normal_priority ); - } - a.my_bottom_priority = normalized_normal_priority; - } - if ( p == my_global_top_priority ) { - if ( !pl.workers_requested ) { - while ( --p >= my_global_bottom_priority && !my_priority_levels[p].workers_requested ) - continue; - if ( p < my_global_bottom_priority ) - reset_global_priority(); - else - update_global_top_priority(p); - } - update_allotment( my_global_top_priority ); - } - else if ( p > my_global_top_priority ) { - __TBB_ASSERT( pl.workers_requested > 0, NULL ); - // TODO: investigate if the following invariant is always valid - __TBB_ASSERT( a.my_num_workers_requested >= 0, NULL ); - update_global_top_priority(p); - a.my_num_workers_allotted = min( (int)my_num_workers_soft_limit, a.my_num_workers_requested ); -#if __TBB_ENQUEUE_ENFORCED_CONCURRENCY - // must not recall worker from arena with mandatory parallelism - if ( !a.my_num_workers_allotted && a.my_num_workers_requested - && a.my_market->my_mandatory_num_requested && a.my_concurrency_mode!=arena_base::cm_normal ) - a.my_num_workers_allotted = 1; -#endif - my_priority_levels[p - 1].workers_available = my_num_workers_soft_limit - a.my_num_workers_allotted; - update_allotment( p - 1 ); - } - else if ( p == my_global_bottom_priority ) { - if ( !pl.workers_requested ) { - while ( ++p <= my_global_top_priority && !my_priority_levels[p].workers_requested ) - continue; - if ( p > my_global_top_priority ) - reset_global_priority(); - else - my_global_bottom_priority = p; - } - else - update_allotment( p ); - } - else if ( p < my_global_bottom_priority ) { - int prev_bottom = my_global_bottom_priority; - my_global_bottom_priority = p; - update_allotment( prev_bottom ); - } - else { - __TBB_ASSERT( my_global_bottom_priority < p && p < my_global_top_priority, NULL ); - update_allotment( p ); - } - __TBB_ASSERT( my_global_top_priority >= a.my_top_priority || a.my_num_workers_requested<=0, NULL ); - assert_market_valid(); -#endif /* !__TBB_TASK_PRIORITY */ - if ( delta > 0 ) { - // can't overflow soft_limit, but remember values request by arenas in - // my_total_demand to not prematurely release workers to RML - if ( my_num_workers_requested+delta > (int)my_num_workers_soft_limit ) - delta = my_num_workers_soft_limit - my_num_workers_requested; - } else { - // the number of workers should not be decreased below my_total_demand - if ( my_num_workers_requested+delta < my_total_demand ) - delta = min(my_total_demand, (int)my_num_workers_soft_limit) - my_num_workers_requested; - } - my_num_workers_requested += delta; - __TBB_ASSERT( my_num_workers_requested <= (int)my_num_workers_soft_limit, NULL ); + mutex_type::scoped_lock lock(my_mutex); + // Update client's state + delta = c.update_request(mandatory_delta, workers_delta); - my_arenas_list_mutex.unlock(); - // Must be called outside of any locks - my_server->adjust_job_count_estimate( delta ); - GATHER_STATISTIC( governor::local_scheduler_if_initialized() ? ++governor::local_scheduler_if_initialized()->my_counters.gate_switches : 0 ); -} + // Update market's state + my_total_demand += delta; + my_priority_level_demand[c.priority_level()] += delta; + my_mandatory_num_requested += mandatory_delta; -void market::process( job& j ) { - generic_scheduler& s = static_cast<generic_scheduler&>(j); - arena *a = NULL; - __TBB_ASSERT( governor::is_set(&s), NULL ); - enum { - query_interval = 1000, - first_interval = 1 - }; - for(int i = first_interval; ; i--) { - while ( (a = arena_in_need(a)) ) - { - a->process(s); - i = first_interval; - } - // Workers leave market because there is no arena in need. It can happen earlier than - // adjust_job_count_estimate() decreases my_slack and RML can put this thread to sleep. - // It might result in a busy-loop checking for my_slack<0 and calling this method instantly. - // first_interval>0 and the yield refines this spinning. - if( i > 0 ) - __TBB_Yield(); - else -#if !__TBB_SLEEP_PERMISSION - break; -#else - { // i == 0 -#if __TBB_TASK_PRIORITY - arena_list_type &al = my_priority_levels[my_global_top_priority].arenas; -#else /* __TBB_TASK_PRIORITY */ - arena_list_type &al = my_arenas; -#endif /* __TBB_TASK_PRIORITY */ - if( al.empty() ) // races if any are innocent TODO: replace by an RML query interface - break; // no arenas left, perhaps going to shut down - if( the_global_observer_list.ask_permission_to_leave() ) - break; // go sleep - __TBB_Yield(); - i = query_interval; - } -#endif// !__TBB_SLEEP_PERMISSION - } - GATHER_STATISTIC( ++s.my_counters.market_roundtrips ); -} - -void market::cleanup( job& j ) { - __TBB_ASSERT( theMarket != this, NULL ); - generic_scheduler& s = static_cast<generic_scheduler&>(j); - generic_scheduler* mine = governor::local_scheduler_if_initialized(); - __TBB_ASSERT( !mine || mine->is_worker(), NULL ); - if( mine!=&s ) { - governor::assume_scheduler( &s ); - generic_scheduler::cleanup_worker( &s, mine!=NULL ); - governor::assume_scheduler( mine ); - } else { - generic_scheduler::cleanup_worker( &s, true ); - } -} - -void market::acknowledge_close_connection() { - destroy(); -} - -::rml::job* market::create_one_job() { - unsigned index = ++my_first_unused_worker_idx; - __TBB_ASSERT( index > 0, NULL ); - ITT_THREAD_SET_NAME(_T("TBB Worker Thread")); - // index serves as a hint decreasing conflicts between workers when they migrate between arenas - generic_scheduler* s = generic_scheduler::create_worker( *this, index ); -#if __TBB_TASK_GROUP_CONTEXT - __TBB_ASSERT( index <= my_num_workers_hard_limit, NULL ); - __TBB_ASSERT( !my_workers[index - 1], NULL ); - my_workers[index - 1] = s; -#endif /* __TBB_TASK_GROUP_CONTEXT */ - return s; -} - -#if __TBB_TASK_PRIORITY -void market::update_arena_top_priority ( arena& a, intptr_t new_priority ) { - GATHER_STATISTIC( ++governor::local_scheduler_if_initialized()->my_counters.arena_prio_switches ); - __TBB_ASSERT( a.my_top_priority != new_priority, NULL ); - priority_level_info &prev_level = my_priority_levels[a.my_top_priority], - &new_level = my_priority_levels[new_priority]; - remove_arena_from_list(a); - a.my_top_priority = new_priority; - insert_arena_into_list(a); - as_atomic( a.my_reload_epoch ).fetch_and_increment<tbb::release>(); // TODO: synch with global reload epoch in order to optimize usage of local reload epoch - prev_level.workers_requested -= a.my_num_workers_requested; - new_level.workers_requested += a.my_num_workers_requested; - __TBB_ASSERT( prev_level.workers_requested >= 0 && new_level.workers_requested >= 0, NULL ); -} - -bool market::lower_arena_priority ( arena& a, intptr_t new_priority, uintptr_t old_reload_epoch ) { - // TODO: replace the lock with a try_lock loop which performs a double check of the epoch - arenas_list_mutex_type::scoped_lock lock(my_arenas_list_mutex); - if ( a.my_reload_epoch != old_reload_epoch ) { - assert_market_valid(); - return false; - } - __TBB_ASSERT( a.my_top_priority > new_priority, NULL ); - __TBB_ASSERT( my_global_top_priority >= a.my_top_priority, NULL ); - - intptr_t p = a.my_top_priority; - update_arena_top_priority( a, new_priority ); - if ( a.my_num_workers_requested > 0 ) { - if ( my_global_bottom_priority > new_priority ) { - my_global_bottom_priority = new_priority; - } - if ( p == my_global_top_priority && !my_priority_levels[p].workers_requested ) { - // Global top level became empty - for ( --p; p>my_global_bottom_priority && !my_priority_levels[p].workers_requested; --p ) continue; - update_global_top_priority(p); - } - update_allotment( p ); - } - - __TBB_ASSERT( my_global_top_priority >= a.my_top_priority, NULL ); - assert_market_valid(); - return true; -} - -bool market::update_arena_priority ( arena& a, intptr_t new_priority ) { - // TODO: do not acquire this global lock while checking arena's state. - arenas_list_mutex_type::scoped_lock lock(my_arenas_list_mutex); - - tbb::internal::assert_priority_valid(new_priority); - __TBB_ASSERT( my_global_top_priority >= a.my_top_priority || a.my_num_workers_requested <= 0, NULL ); - assert_market_valid(); - if ( a.my_top_priority == new_priority ) { - return false; - } - else if ( a.my_top_priority > new_priority ) { - if ( a.my_bottom_priority > new_priority ) - a.my_bottom_priority = new_priority; - return false; - } - else if ( a.my_num_workers_requested <= 0 ) { - return false; - } - - __TBB_ASSERT( my_global_top_priority >= a.my_top_priority, NULL ); - - intptr_t p = a.my_top_priority; - intptr_t highest_affected_level = max(p, new_priority); - update_arena_top_priority( a, new_priority ); - - if ( my_global_top_priority < new_priority ) { - update_global_top_priority(new_priority); - } - else if ( my_global_top_priority == new_priority ) { - advance_global_reload_epoch(); - } - else { - __TBB_ASSERT( new_priority < my_global_top_priority, NULL ); - __TBB_ASSERT( new_priority > my_global_bottom_priority, NULL ); - if ( p == my_global_top_priority && !my_priority_levels[p].workers_requested ) { - // Global top level became empty - __TBB_ASSERT( my_global_bottom_priority < p, NULL ); - for ( --p; !my_priority_levels[p].workers_requested; --p ) continue; - __TBB_ASSERT( p >= new_priority, NULL ); - update_global_top_priority(p); - highest_affected_level = p; - } - } - if ( p == my_global_bottom_priority ) { - // Arena priority was increased from the global bottom level. - __TBB_ASSERT( p < new_priority, NULL ); - __TBB_ASSERT( new_priority <= my_global_top_priority, NULL ); - while ( my_global_bottom_priority < my_global_top_priority - && !my_priority_levels[my_global_bottom_priority].workers_requested ) - ++my_global_bottom_priority; - __TBB_ASSERT( my_global_bottom_priority <= new_priority, NULL ); -#if __TBB_ENQUEUE_ENFORCED_CONCURRENCY - const bool enforced_concurrency = my_mandatory_num_requested && a.must_have_concurrency(); -#else - const bool enforced_concurrency = false; -#endif - __TBB_ASSERT_EX( enforced_concurrency || my_priority_levels[my_global_bottom_priority].workers_requested > 0, NULL ); + update_allotment(); } - update_allotment( highest_affected_level ); - __TBB_ASSERT( my_global_top_priority >= a.my_top_priority, NULL ); - assert_market_valid(); - return true; + notify_thread_request(delta); } -#endif /* __TBB_TASK_PRIORITY */ -} // namespace internal +} // namespace r1 +} // namespace detail } // namespace tbb diff --git a/src/tbb/src/tbb/market.h b/src/tbb/src/tbb/market.h index 39d12d5e0..85532ff10 100644 --- a/src/tbb/src/tbb/market.h +++ b/src/tbb/src/tbb/market.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2023 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,361 +12,67 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef _TBB_market_H #define _TBB_market_H -#include "tbb/tbb_stddef.h" - -#include "scheduler_common.h" -#include "tbb/atomic.h" -#include "tbb/spin_rw_mutex.h" -#include "../rml/include/rml_tbb.h" +#include "oneapi/tbb/rw_mutex.h" +#include "oneapi/tbb/tbb_allocator.h" +#include "oneapi/tbb/task_arena.h" -#include "intrusive_list.h" +#include "permit_manager.h" +#include "pm_client.h" -#if defined(_MSC_VER) && defined(_Wp64) - // Workaround for overzealous compiler warnings in /Wp64 mode - #pragma warning (push) - #pragma warning (disable: 4244) -#endif +#include <atomic> +#include <vector> namespace tbb { +namespace detail { +namespace r1 { -class task_group_context; - -namespace internal { - -//------------------------------------------------------------------------ -// Class market -//------------------------------------------------------------------------ - -class market : no_copy, rml::tbb_client { - friend class generic_scheduler; - friend class arena; - friend class tbb::interface7::internal::task_arena_base; - template<typename SchedulerTraits> friend class custom_scheduler; - friend class tbb::task_group_context; -private: - friend void ITT_DoUnsafeOneTimeInitialization (); - - typedef intrusive_list<arena> arena_list_type; - typedef intrusive_list<generic_scheduler> scheduler_list_type; - - //! Currently active global market - static market* theMarket; - - typedef scheduler_mutex_type global_market_mutex_type; - - //! Mutex guarding creation/destruction of theMarket, insertions/deletions in my_arenas, and cancellation propagation - static global_market_mutex_type theMarketMutex; - - //! Lightweight mutex guarding accounting operations with arenas list - typedef spin_rw_mutex arenas_list_mutex_type; - arenas_list_mutex_type my_arenas_list_mutex; - - //! Pointer to the RML server object that services this TBB instance. - rml::tbb_server* my_server; - - //! Maximal number of workers allowed for use by the underlying resource manager - /** It can't be changed after market creation. **/ - unsigned my_num_workers_hard_limit; - - //! Current application-imposed limit on the number of workers (see set_active_num_workers()) - /** It can't be more than my_num_workers_hard_limit. **/ - unsigned my_num_workers_soft_limit; - - //! Number of workers currently requested from RML - int my_num_workers_requested; - - //! First unused index of worker - /** Used to assign indices to the new workers coming from RML, and busy part - of my_workers array. **/ - atomic<unsigned> my_first_unused_worker_idx; - - //! Number of workers that were requested by all arenas - int my_total_demand; - -#if __TBB_ENQUEUE_ENFORCED_CONCURRENCY - //! How many times mandatory concurrency was requested from the market - int my_mandatory_num_requested; -#endif - -#if __TBB_TASK_PRIORITY - //! Highest priority among active arenas in the market. - /** Arena priority level is its tasks highest priority (specified by arena's - my_top_priority member). - Arena is active when it has outstanding request for workers. Note that - inactive arena may have workers lingering there for some time. **/ - intptr_t my_global_top_priority; - - //! Lowest priority among active arenas in the market. - /** See also my_global_top_priority **/ - intptr_t my_global_bottom_priority; - - //! Tracks events that may bring tasks in offload areas to the top priority level. - /** Incremented when global top priority is decremented or a task group priority - is elevated to the current top level. **/ - uintptr_t my_global_reload_epoch; - - //! Information about arenas at a particular priority level - struct priority_level_info { - //! List of arenas at this priority level - arena_list_type arenas; - - //! The first arena to be checked when idle worker seeks for an arena to enter - /** The check happens in round-robin fashion. **/ - arena *next_arena; - - //! Total amount of workers requested by arenas at this priority level. - int workers_requested; - - //! Maximal amount of workers the market can tell off to this priority level. - int workers_available; - }; // struct priority_level_info - - //! Information about arenas at different priority levels - priority_level_info my_priority_levels[num_priority_levels]; - -#else /* !__TBB_TASK_PRIORITY */ - - //! List of registered arenas - arena_list_type my_arenas; - - //! The first arena to be checked when idle worker seeks for an arena to enter - /** The check happens in round-robin fashion. **/ - arena *my_next_arena; -#endif /* !__TBB_TASK_PRIORITY */ - - //! ABA prevention marker to assign to newly created arenas - uintptr_t my_arenas_aba_epoch; - - //! Reference count controlling market object lifetime - unsigned my_ref_count; - - //! Count of master threads attached - unsigned my_public_ref_count; - - //! Stack size of worker threads - size_t my_stack_size; - - //! Shutdown mode - bool my_join_workers; - - //! The value indicating that the soft limit warning is unnecessary - static const unsigned skip_soft_limit_warning = ~0U; - - //! Either workers soft limit to be reported via runtime_warning() or skip_soft_limit_warning - unsigned my_workers_soft_limit_to_report; -#if __TBB_COUNT_TASK_NODES - //! Net number of nodes that have been allocated from heap. - /** Updated each time a scheduler or arena is destroyed. */ - atomic<intptr_t> my_task_node_count; -#endif /* __TBB_COUNT_TASK_NODES */ - - //! Constructor - market ( unsigned workers_soft_limit, unsigned workers_hard_limit, size_t stack_size ); - - //! Factory method creating new market object - static market& global_market ( bool is_public, unsigned max_num_workers = 0, size_t stack_size = 0 ); - - //! Destroys and deallocates market object created by market::create() - void destroy (); - -#if __TBB_TASK_PRIORITY - //! Returns next arena that needs more workers, or NULL. - arena* arena_in_need ( arena* prev_arena ); - - //! Recalculates the number of workers assigned to each arena at and below the specified priority. - /** The actual number of workers servicing a particular arena may temporarily - deviate from the calculated value. **/ - void update_allotment ( intptr_t highest_affected_priority ); - - //! Changes arena's top priority and updates affected priority levels info in the market. - void update_arena_top_priority ( arena& a, intptr_t newPriority ); - - //! Changes market's global top priority and related settings. - inline void update_global_top_priority ( intptr_t newPriority ); - - //! Resets empty market's global top and bottom priority to the normal level. - inline void reset_global_priority (); - - inline void advance_global_reload_epoch () { - __TBB_store_with_release( my_global_reload_epoch, my_global_reload_epoch + 1 ); - } - - void assert_market_valid () const { - __TBB_ASSERT( (my_priority_levels[my_global_top_priority].workers_requested > 0 - && !my_priority_levels[my_global_top_priority].arenas.empty()) - || (my_global_top_priority == my_global_bottom_priority && - my_global_top_priority == normalized_normal_priority), NULL ); - } - -#else /* !__TBB_TASK_PRIORITY */ - - //! Recalculates the number of workers assigned to each arena in the list. - /** The actual number of workers servicing a particular arena may temporarily - deviate from the calculated value. **/ - void update_allotment () { - if ( my_total_demand ) - update_allotment( my_arenas, my_total_demand, (int)my_num_workers_soft_limit ); - } - - //! Returns next arena that needs more workers, or NULL. - arena* arena_in_need (arena*) { - if(__TBB_load_with_acquire(my_total_demand) <= 0) - return NULL; - arenas_list_mutex_type::scoped_lock lock(my_arenas_list_mutex, /*is_writer=*/false); - return arena_in_need(my_arenas, my_next_arena); - } - void assert_market_valid () const {} -#endif /* !__TBB_TASK_PRIORITY */ - - //////////////////////////////////////////////////////////////////////////////// - // Helpers to unify code branches dependent on priority feature presence - - void insert_arena_into_list ( arena& a ); - - void remove_arena_from_list ( arena& a ); - - arena* arena_in_need ( arena_list_type &arenas, arena *&next ); - - static int update_allotment ( arena_list_type& arenas, int total_demand, int max_workers ); - - - //////////////////////////////////////////////////////////////////////////////// - // Implementation of rml::tbb_client interface methods - - version_type version () const __TBB_override { return 0; } - - unsigned max_job_count () const __TBB_override { return my_num_workers_hard_limit; } - - size_t min_stack_size () const __TBB_override { return worker_stack_size(); } - - policy_type policy () const __TBB_override { return throughput; } - - job* create_one_job () __TBB_override; - - void cleanup( job& j ) __TBB_override; - - void acknowledge_close_connection () __TBB_override; - - void process( job& j ) __TBB_override; - +class market : public permit_manager { public: - //! Creates an arena object - /** If necessary, also creates global market instance, and boosts its ref count. - Each call to create_arena() must be matched by the call to arena::free_arena(). **/ - static arena* create_arena ( int num_slots, int num_reserved_slots, size_t stack_size ); - - //! Removes the arena from the market's list - void try_destroy_arena ( arena*, uintptr_t aba_epoch ); - - //! Removes the arena from the market's list - void detach_arena ( arena& ); - - //! Decrements market's refcount and destroys it in the end - bool release ( bool is_public, bool blocking_terminate ); - -#if __TBB_ENQUEUE_ENFORCED_CONCURRENCY - //! Imlpementation of mandatory concurrency enabling - bool mandatory_concurrency_enable_impl ( arena *a, bool *enabled = NULL ); - - //! Inform the master that there is an arena with mandatory concurrency - bool mandatory_concurrency_enable ( arena *a ); + market(unsigned soft_limit); - //! Inform the master that the arena is no more interested in mandatory concurrency - void mandatory_concurrency_disable ( arena *a ); -#endif /* __TBB_ENQUEUE_ENFORCED_CONCURRENCY */ + pm_client* create_client(arena& a) override; + void register_client(pm_client* client, d1::constraints&) override; + void unregister_and_destroy_client(pm_client& c) override; //! Request that arena's need in workers should be adjusted. - /** Concurrent invocations are possible only on behalf of different arenas. **/ - void adjust_demand ( arena&, int delta ); - - //! Used when RML asks for join mode during workers termination. - bool must_join_workers () const { return my_join_workers; } - - //! Returns the requested stack size of worker threads. - size_t worker_stack_size () const { return my_stack_size; } + void adjust_demand(pm_client&, int mandatory_delta, int workers_delta) override; //! Set number of active workers - static void set_active_num_workers( unsigned w ); - - //! Reports active parallelism level according to user's settings - static unsigned app_parallelism_limit(); - -#if _WIN32||_WIN64 - //! register master with the resource manager - void register_master( ::rml::server::execution_resource_t& rsc_handle ) { - __TBB_ASSERT( my_server, "RML server not defined?" ); - // the server may ignore registration and set master_exec_resource to NULL. - my_server->register_master( rsc_handle ); - } - - //! unregister master with the resource manager - void unregister_master( ::rml::server::execution_resource_t& rsc_handle ) const { - my_server->unregister_master( rsc_handle ); - } -#endif /* WIN */ - -#if __TBB_TASK_GROUP_CONTEXT - //! Finds all contexts affected by the state change and propagates the new state to them. - /** The propagation is relayed to the market because tasks created by one - master thread can be passed to and executed by other masters. This means - that context trees can span several arenas at once and thus state change - propagation cannot be generally localized to one arena only. **/ - template <typename T> - bool propagate_task_group_state ( T task_group_context::*mptr_state, task_group_context& src, T new_state ); -#endif /* __TBB_TASK_GROUP_CONTEXT */ + void set_active_num_workers(int soft_limit) override; +private: + //! Recalculates the number of workers assigned to each arena in the list. + void update_allotment(); -#if __TBB_TASK_PRIORITY - //! Lowers arena's priority is not higher than newPriority - /** Returns true if arena priority was actually elevated. **/ - bool lower_arena_priority ( arena& a, intptr_t new_priority, uintptr_t old_reload_epoch ); + //! Keys for the arena map array. The lower the value the higher priority of the arena list. + static constexpr unsigned num_priority_levels = d1::num_priority_levels; - //! Makes sure arena's priority is not lower than newPriority - /** Returns true if arena priority was elevated. Also updates arena's bottom - priority boundary if necessary. + using mutex_type = d1::rw_mutex; + mutex_type my_mutex; - This method is called whenever a user changes priority, because whether - it was hiked or sunk can be determined for sure only under the lock used - by this function. **/ - bool update_arena_priority ( arena& a, intptr_t new_priority ); -#endif /* __TBB_TASK_PRIORITY */ + //! Current application-imposed limit on the number of workers + int my_num_workers_soft_limit; -#if __TBB_COUNT_TASK_NODES - //! Net number of nodes that have been allocated from heap. - /** Updated each time a scheduler or arena is destroyed. */ - void update_task_node_count( intptr_t delta ) { my_task_node_count += delta; } -#endif /* __TBB_COUNT_TASK_NODES */ + //! Number of workers that were requested by all arenas on all priority levels + int my_total_demand{0}; -#if __TBB_TASK_GROUP_CONTEXT - //! List of registered master threads - scheduler_list_type my_masters; + //! Number of workers that were requested by arenas per single priority list item + int my_priority_level_demand[num_priority_levels] = {0}; - //! Array of pointers to the registered workers - /** Used by cancellation propagation mechanism. - Must be the last data member of the class market. **/ - generic_scheduler* my_workers[1]; -#endif /* __TBB_TASK_GROUP_CONTEXT */ + //! How many times mandatory concurrency was requested from the market + int my_mandatory_num_requested{0}; - static unsigned max_num_workers() { - global_market_mutex_type::scoped_lock lock( theMarketMutex ); - return theMarket? theMarket->my_num_workers_hard_limit : 0; - } + //! Per priority list of registered arenas + using clients_container_type = std::vector<pm_client*, tbb::tbb_allocator<pm_client*>>; + clients_container_type my_clients[num_priority_levels]; }; // class market -} // namespace internal +} // namespace r1 +} // namespace detail } // namespace tbb -#if defined(_MSC_VER) && defined(_Wp64) - // Workaround for overzealous compiler warnings in /Wp64 mode - #pragma warning (pop) -#endif // warning 4244 is back - #endif /* _TBB_market_H */ diff --git a/src/tbb/src/tbb/market_concurrent_monitor.h b/src/tbb/src/tbb/market_concurrent_monitor.h new file mode 100644 index 000000000..37927617b --- /dev/null +++ b/src/tbb/src/tbb/market_concurrent_monitor.h @@ -0,0 +1,116 @@ +/* + Copyright (c) 2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_market_concurrent_monitor_H +#define __TBB_market_concurrent_monitor_H + +#include "concurrent_monitor.h" +#include "scheduler_common.h" + +#include <atomic> + +namespace tbb { +namespace detail { +namespace r1 { + +struct market_context { + market_context() = default; + + market_context(std::uintptr_t first_addr, arena* a) : + my_uniq_addr(first_addr), my_arena_addr(a) + {} + + std::uintptr_t my_uniq_addr{0}; + arena* my_arena_addr{nullptr}; +}; + +#if __TBB_RESUMABLE_TASKS +class resume_node : public wait_node<market_context> { + using base_type = wait_node<market_context>; +public: + resume_node(market_context ctx, execution_data_ext& ed_ext, task_dispatcher& target) + : base_type(ctx), my_curr_dispatcher(ed_ext.task_disp), my_target_dispatcher(&target) + , my_suspend_point(my_curr_dispatcher->get_suspend_point()) + {} + + ~resume_node() override { + if (this->my_skipped_wakeup) { + spin_wait_until_eq(this->my_notify_calls, 1); + } + + poison_pointer(my_curr_dispatcher); + poison_pointer(my_target_dispatcher); + poison_pointer(my_suspend_point); + } + + void init() override { + base_type::init(); + } + + void wait() override { + my_curr_dispatcher->resume(*my_target_dispatcher); + __TBB_ASSERT(!this->my_is_in_list.load(std::memory_order_relaxed), "Still in the queue?"); + } + + void reset() override { + base_type::reset(); + spin_wait_until_eq(this->my_notify_calls, 1); + my_notify_calls.store(0, std::memory_order_relaxed); + } + + // notify is called (perhaps, concurrently) twice from: + // - concurrent_monitor::notify + // - post_resume_action::register_waiter + // The second notify is called after thread switches the stack + // (Because we can not call resume while the stack is occupied) + // We need calling resume only when both notifications are performed. + void notify() override { + if (++my_notify_calls == 2) { + r1::resume(my_suspend_point); + } + } + +private: + friend class thread_data; + friend struct suspend_point_type::resume_task; + task_dispatcher* my_curr_dispatcher; + task_dispatcher* my_target_dispatcher; + suspend_point_type* my_suspend_point; + std::atomic<int> my_notify_calls{0}; +}; +#endif // __TBB_RESUMABLE_TASKS + +class market_concurrent_monitor : public concurrent_monitor_base<market_context> { + using base_type = concurrent_monitor_base<market_context>; +public: + using base_type::base_type; + + ~market_concurrent_monitor() { + destroy(); + } + + /** per-thread descriptor for concurrent_monitor */ + using thread_context = sleep_node<market_context>; +#if __TBB_RESUMABLE_TASKS + using resume_context = resume_node; +#endif +}; + +} // namespace r1 +} // namespace detail +} // namespace tbb + +#endif // __TBB_market_concurrent_monitor_H diff --git a/src/tbb/src/tbb/misc.cpp b/src/tbb/src/tbb/misc.cpp new file mode 100644 index 000000000..115a5f388 --- /dev/null +++ b/src/tbb/src/tbb/misc.cpp @@ -0,0 +1,156 @@ +/* + Copyright (c) 2005-2024 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Source file for miscellaneous entities that are infrequently referenced by +// an executing program. + +#include "oneapi/tbb/detail/_exception.h" +#include "oneapi/tbb/detail/_machine.h" + +#include "oneapi/tbb/version.h" + +#include "misc.h" +#include "governor.h" +#include "assert_impl.h" // Out-of-line TBB assertion handling routines are instantiated here. +#include "concurrent_monitor_mutex.h" + +#include <cstdio> +#include <cstdlib> +#include <stdexcept> +#include <cstring> +#include <cstdarg> + +#if _WIN32||_WIN64 +#include <windows.h> +#endif + +#if !_WIN32 +#include <unistd.h> // sysconf(_SC_PAGESIZE) +#endif + +namespace tbb { +namespace detail { +namespace r1 { + +//------------------------------------------------------------------------ +// governor data +//------------------------------------------------------------------------ +cpu_features_type governor::cpu_features; + +//------------------------------------------------------------------------ +// concurrent_monitor_mutex data +//------------------------------------------------------------------------ +#if !__TBB_USE_FUTEX +std::mutex concurrent_monitor_mutex::my_init_mutex; +#endif + + +size_t DefaultSystemPageSize() { +#if _WIN32 + SYSTEM_INFO si; + GetSystemInfo(&si); + return si.dwPageSize; +#else + return sysconf(_SC_PAGESIZE); +#endif +} + +/** The leading "\0" is here so that applying "strings" to the binary delivers a clean result. */ +static const char VersionString[] = "\0" TBB_VERSION_STRINGS; + +static bool PrintVersionFlag = false; + +void PrintVersion() { + PrintVersionFlag = true; + std::fputs(VersionString+1,stderr); +} + +void PrintExtraVersionInfo( const char* category, const char* format, ... ) { + if( PrintVersionFlag ) { + char str[1024]; std::memset(str, 0, 1024); + va_list args; va_start(args, format); + // Note: correct vsnprintf definition obtained from tbb_assert_impl.h + std::vsnprintf( str, 1024-1, format, args); + va_end(args); + std::fprintf(stderr, "oneTBB: %s\t%s\n", category, str ); + } +} + +//! check for transaction support. +#if _MSC_VER +#include <intrin.h> // for __cpuid +#elif __APPLE__ +#include <sys/sysctl.h> +#endif + +#if __TBB_x86_32 || __TBB_x86_64 +void check_cpuid(int leaf, int sub_leaf, int registers[4]) { +#if _MSC_VER + __cpuidex(registers, leaf, sub_leaf); +#else + int reg_eax = 0; + int reg_ebx = 0; + int reg_ecx = 0; + int reg_edx = 0; +#if __TBB_x86_32 && __PIC__ + // On 32-bit systems with position-independent code GCC fails to work around the stuff in EBX + // register. We help it using backup and restore. + __asm__("mov %%ebx, %%esi\n\t" + "cpuid\n\t" + "xchg %%ebx, %%esi" + : "=a"(reg_eax), "=S"(reg_ebx), "=c"(reg_ecx), "=d"(reg_edx) + : "0"(leaf), "2"(sub_leaf) // read value from eax and ecx + ); +#else + __asm__("cpuid" + : "=a"(reg_eax), "=b"(reg_ebx), "=c"(reg_ecx), "=d"(reg_edx) + : "0"(leaf), "2"(sub_leaf) // read value from eax and ecx + ); +#endif + registers[0] = reg_eax; + registers[1] = reg_ebx; + registers[2] = reg_ecx; + registers[3] = reg_edx; +#endif +} +#endif + +void detect_cpu_features(cpu_features_type& cpu_features) { + suppress_unused_warning(cpu_features); +#if __TBB_x86_32 || __TBB_x86_64 + const int rtm_ebx_mask = 1 << 11; + const int waitpkg_ecx_mask = 1 << 5; + const int hybrid_edx_mask = 1 << 15; + int registers[4] = {0}; + + // Check RTM, WAITPKG, HYBRID + check_cpuid(7, 0, registers); + cpu_features.rtm_enabled = (registers[1] & rtm_ebx_mask) != 0; + cpu_features.waitpkg_enabled = (registers[2] & waitpkg_ecx_mask) != 0; + cpu_features.hybrid = (registers[3] & hybrid_edx_mask) != 0; +#elif __APPLE__ + // Check HYBRID (hw.nperflevels > 1) + uint64_t nperflevels = 0; + size_t nperflevels_size = sizeof(nperflevels); + if (!sysctlbyname("hw.nperflevels", &nperflevels, &nperflevels_size, nullptr, 0)) { + cpu_features.hybrid = (nperflevels > 1); + } +#endif +} + +} // namespace r1 +} // namespace detail +} // namespace tbb diff --git a/src/tbb/src/tbb/tbb_misc.h b/src/tbb/src/tbb/misc.h similarity index 60% rename from src/tbb/src/tbb/tbb_misc.h rename to src/tbb/src/tbb/misc.h index 8a05197e6..988c29b17 100644 --- a/src/tbb/src/tbb/tbb_misc.h +++ b/src/tbb/src/tbb/misc.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2024 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,26 +12,28 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef _TBB_tbb_misc_H #define _TBB_tbb_misc_H -#include "tbb/tbb_stddef.h" -#include "tbb/tbb_machine.h" -#include "tbb/atomic.h" // For atomic_xxx definitions +#include "oneapi/tbb/detail/_config.h" +#include "oneapi/tbb/detail/_assert.h" +#include "oneapi/tbb/detail/_utils.h" + +#if __TBB_ARENA_BINDING +#include "oneapi/tbb/info.h" +#endif /*__TBB_ARENA_BINDING*/ -#if __linux__ || __FreeBSD__ +#if __unix__ #include <sys/param.h> // __FreeBSD_version #if __FreeBSD_version >= 701000 #include <sys/cpuset.h> #endif #endif +#include <atomic> + // Does the operating system have a system call to pin a thread to a set of OS processors? #define __TBB_OS_AFFINITY_SYSCALL_PRESENT ((__linux__ && !__ANDROID__) || (__FreeBSD_version >= 701000)) // On IBM* Blue Gene* CNK nodes, the affinity API has restrictions that prevent its usability for TBB, @@ -39,17 +41,24 @@ #define __TBB_USE_OS_AFFINITY_SYSCALL (__TBB_OS_AFFINITY_SYSCALL_PRESENT && !__bg__) namespace tbb { -namespace internal { +namespace detail { +namespace r1 { -const size_t MByte = 1024*1024; +void runtime_warning(const char* format, ... ); -#if __TBB_WIN8UI_SUPPORT -// In Win8UI mode, TBB uses a thread creation API that does not allow to specify the stack size. -// Still, the thread stack size value, either explicit or default, is used by the scheduler. -// So here we set the default value to match the platform's default of 1MB. -const size_t ThreadStackSize = 1*MByte; +#if __TBB_ARENA_BINDING +class task_arena; +class task_scheduler_observer; +#endif /*__TBB_ARENA_BINDING*/ + +const std::size_t MByte = 1024*1024; + +#if __TBB_USE_WINAPI +// The Microsoft Documentation about Thread Stack Size states that +// "The default stack reservation size used by the linker is 1 MB" +const std::size_t ThreadStackSize = 1*MByte; #else -const size_t ThreadStackSize = (sizeof(uintptr_t) <= 4 ? 2 : 4 )*MByte; +const std::size_t ThreadStackSize = (sizeof(uintptr_t) <= 4 ? 2 : 4 )*MByte; #endif #ifndef __TBB_HardwareConcurrency @@ -65,13 +74,15 @@ inline int AvailableHwConcurrency() { } #endif /* __TBB_HardwareConcurrency */ - -#if _WIN32||_WIN64 +//! Returns OS regular memory page size +size_t DefaultSystemPageSize(); //! Returns number of processor groups in the current OS configuration. /** AvailableHwConcurrency must be called at least once before calling this method. **/ int NumberOfProcessorGroups(); +#if _WIN32||_WIN64 + //! Retrieves index of processor group containing processor with the given index int FindProcessorGroupIndex ( int processorIndex ); @@ -80,12 +91,6 @@ void MoveThreadIntoProcessorGroup( void* hThread, int groupIndex ); #endif /* _WIN32||_WIN64 */ -//! Throws std::runtime_error with what() returning error_code description prefixed with aux_info -void handle_win_error( int error_code ); - -//! True if environment variable with given name is set and not 0; otherwise false. -bool GetBoolEnvironmentVariable( const char * name ); - //! Prints TBB version information on stderr void PrintVersion(); @@ -126,20 +131,12 @@ template<int > struct int_to_type {}; // FastRandom //------------------------------------------------------------------------ -/** Defined in tbb_main.cpp **/ -unsigned GetPrime ( unsigned seed ); - //! A fast random number generator. /** Uses linear congruential method. */ class FastRandom { private: -#if __TBB_OLD_PRIMES_RNG - unsigned x, a; - static const unsigned c = 1; -#else unsigned x, c; static const unsigned a = 0x9e3779b1; // a big prime number -#endif //__TBB_OLD_PRIMES_RNG public: //! Get a random number. unsigned short get() { @@ -154,8 +151,7 @@ class FastRandom { } //! Construct a random number generator. FastRandom( void* unique_ptr ) { init(uintptr_t(unique_ptr)); } - FastRandom( uint32_t seed) { init(seed); } - FastRandom( uint64_t seed) { init(seed); } + template <typename T> void init( T seed ) { init(seed,int_to_type<sizeof(seed)>()); @@ -164,14 +160,9 @@ class FastRandom { init(uint32_t((seed>>32)+seed), int_to_type<4>()); } void init( uint32_t seed, int_to_type<4> ) { -#if __TBB_OLD_PRIMES_RNG - x = seed; - a = GetPrime( seed ); -#else // threads use different seeds for unique sequences c = (seed|1)*0xba5703f5; // c must be odd, shuffle by a prime number x = c^(seed>>1); // also shuffle x for the first get() invocation -#endif } }; @@ -181,63 +172,16 @@ class FastRandom { //! Atomically replaces value of dst with newValue if they satisfy condition of compare predicate /** Return value semantics is the same as for CAS. **/ -template<typename T1, typename T2, class Pred> -T1 atomic_update ( tbb::atomic<T1>& dst, T2 newValue, Pred compare ) { - T1 oldValue = dst; +template<typename T1, class Pred> +T1 atomic_update(std::atomic<T1>& dst, T1 newValue, Pred compare) { + T1 oldValue = dst.load(std::memory_order_acquire); while ( compare(oldValue, newValue) ) { - if ( dst.compare_and_swap((T1)newValue, oldValue) == oldValue ) + if ( dst.compare_exchange_strong(oldValue, newValue) ) break; - oldValue = dst; } return oldValue; } -//! One-time initialization states -enum do_once_state { - do_once_uninitialized = 0, ///< No execution attempts have been undertaken yet - do_once_pending, ///< A thread is executing associated do-once routine - do_once_executed, ///< Do-once routine has been executed - initialization_complete = do_once_executed ///< Convenience alias -}; - -//! One-time initialization function -/** /param initializer Pointer to function without arguments - The variant that returns bool is used for cases when initialization can fail - and it is OK to continue execution, but the state should be reset so that - the initialization attempt was repeated the next time. - /param state Shared state associated with initializer that specifies its - initialization state. Must be initially set to #uninitialized value - (e.g. by means of default static zero initialization). **/ -template <typename F> -void atomic_do_once ( const F& initializer, atomic<do_once_state>& state ) { - // tbb::atomic provides necessary acquire and release fences. - // The loop in the implementation is necessary to avoid race when thread T2 - // that arrived in the middle of initialization attempt by another thread T1 - // has just made initialization possible. - // In such a case T2 has to rely on T1 to initialize, but T1 may already be past - // the point where it can recognize the changed conditions. - while ( state != do_once_executed ) { - if( state == do_once_uninitialized ) { - if( state.compare_and_swap( do_once_pending, do_once_uninitialized ) == do_once_uninitialized ) { - run_initializer( initializer, state ); - break; - } - } - spin_wait_while_eq( state, do_once_pending ); - } -} - -// Run the initializer which can not fail -inline void run_initializer( void (*f)(), atomic<do_once_state>& state ) { - f(); - state = do_once_executed; -} - -// Run the initializer which can require repeated call -inline void run_initializer( bool (*f)(), atomic<do_once_state>& state ) { - state = f() ? do_once_executed : do_once_uninitialized; -} - #if __TBB_USE_OS_AFFINITY_SYSCALL #if __linux__ typedef cpu_set_t basic_mask_t; @@ -250,7 +194,7 @@ inline void run_initializer( bool (*f)(), atomic<do_once_state>& state ) { basic_mask_t* threadMask; int is_changed; public: - affinity_helper() : threadMask(NULL), is_changed(0) {} + affinity_helper() : threadMask(nullptr), is_changed(0) {} ~affinity_helper(); void protect_affinity_mask( bool restore_process_mask ); void dismiss(); @@ -260,16 +204,84 @@ inline void run_initializer( bool (*f)(), atomic<do_once_state>& state ) { class affinity_helper : no_copy { public: void protect_affinity_mask( bool ) {} - void dismiss() {} }; inline void destroy_process_mask(){} #endif /* __TBB_USE_OS_AFFINITY_SYSCALL */ -bool cpu_has_speculation(); -bool gcc_rethrow_exception_broken(); -void fix_broken_rethrow(); +struct cpu_features_type { + bool rtm_enabled{false}; + bool waitpkg_enabled{false}; + bool hybrid{false}; +}; + +void detect_cpu_features(cpu_features_type& cpu_features); + +#if __TBB_ARENA_BINDING +class binding_handler; + +binding_handler* construct_binding_handler(int slot_num, int numa_id, int core_type_id, int max_threads_per_core); +void destroy_binding_handler(binding_handler* handler_ptr); +void apply_affinity_mask(binding_handler* handler_ptr, int slot_num); +void restore_affinity_mask(binding_handler* handler_ptr, int slot_num); + +#endif /*__TBB_ARENA_BINDING*/ + +// RTM specific section +// abort code for mutexes that detect a conflict with another thread. +enum { + speculation_not_supported = 0x00, + speculation_transaction_aborted = 0x01, + speculation_can_retry = 0x02, + speculation_memadd_conflict = 0x04, + speculation_buffer_overflow = 0x08, + speculation_breakpoint_hit = 0x10, + speculation_nested_abort = 0x20, + speculation_xabort_mask = 0xFF000000, + speculation_xabort_shift = 24, + speculation_xabort_not_free = 0xFF, // The value (0xFF) below comes from the Intel(R) 64 and IA-32 Architectures Optimization Reference Manual 12.4.5 lock not free + speculation_successful_begin = 0xFFFFFFFF, + speculation_retry = speculation_transaction_aborted + | speculation_can_retry + | speculation_memadd_conflict +}; + +// We suppose that successful transactions are sequentially ordered and +// do not require additional memory fences around them. +// Technically it can be achieved only if xbegin has implicit +// acquire memory semantics an xend/xabort has release memory semantics on compiler and hardware level. +// See the article: https://arxiv.org/pdf/1710.04839.pdf +static inline unsigned int begin_transaction() { +#if __TBB_TSX_INTRINSICS_PRESENT + return _xbegin(); +#else + return speculation_not_supported; // return unsuccessful code +#endif +} + +static inline void end_transaction() { +#if __TBB_TSX_INTRINSICS_PRESENT + _xend(); +#endif +} + +static inline void abort_transaction() { +#if __TBB_TSX_INTRINSICS_PRESENT + _xabort(speculation_xabort_not_free); +#endif +} + +#if TBB_USE_ASSERT +static inline unsigned char is_in_transaction() { +#if __TBB_TSX_INTRINSICS_PRESENT + return _xtest(); +#else + return 0; +#endif +} +#endif // TBB_USE_ASSERT -} // namespace internal +} // namespace r1 +} // namespace detail } // namespace tbb #endif /* _TBB_tbb_misc_H */ diff --git a/src/tbb/src/tbb/tbb_misc_ex.cpp b/src/tbb/src/tbb/misc_ex.cpp similarity index 71% rename from src/tbb/src/tbb/tbb_misc_ex.cpp rename to src/tbb/src/tbb/misc_ex.cpp index 264abe6ae..13b7b04fb 100644 --- a/src/tbb/src/tbb/tbb_misc_ex.cpp +++ b/src/tbb/src/tbb/misc_ex.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2023 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,16 +12,12 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ // Source file for miscellaneous entities that are infrequently referenced by // an executing program, and implementation of which requires dynamic linking. -#include "tbb_misc.h" +#include "misc.h" #if !defined(__TBB_HardwareConcurrency) @@ -30,66 +26,72 @@ #include <limits.h> #if _WIN32||_WIN64 -#include "tbb/machine/windows_api.h" +#include <windows.h> #if __TBB_WIN8UI_SUPPORT #include <thread> #endif #else #include <unistd.h> +#if __unix__ #if __linux__ #include <sys/sysinfo.h> -#include <string.h> +#endif +#include <cstring> #include <sched.h> -#include <errno.h> +#include <cerrno> #elif __sun #include <sys/sysinfo.h> #elif __FreeBSD__ -#include <errno.h> -#include <string.h> +#include <cerrno> +#include <cstring> #include <sys/param.h> // Required by <sys/cpuset.h> #include <sys/cpuset.h> #endif #endif namespace tbb { -namespace internal { +namespace detail { +namespace r1 { #if __TBB_USE_OS_AFFINITY_SYSCALL -#if __linux__ +#if __unix__ // Handlers for interoperation with libiomp static int (*libiomp_try_restoring_original_mask)(); // Table for mapping to libiomp entry points static const dynamic_link_descriptor iompLinkTable[] = { - { "kmp_set_thread_affinity_mask_initial", (pointer_to_handler*)(void*)(&libiomp_try_restoring_original_mask) } + DLD_NOWEAK( kmp_set_thread_affinity_mask_initial, libiomp_try_restoring_original_mask ) }; #endif -static void set_thread_affinity_mask( size_t maskSize, const basic_mask_t* threadMask ) { -#if __linux__ - if( sched_setaffinity( 0, maskSize, threadMask ) ) -#else /* FreeBSD */ +static void set_thread_affinity_mask( std::size_t maskSize, const basic_mask_t* threadMask ) { +#if __FreeBSD__ || __NetBSD__ || __OpenBSD__ if( cpuset_setaffinity( CPU_LEVEL_WHICH, CPU_WHICH_TID, -1, maskSize, threadMask ) ) +#else /* __unix__ */ + if( sched_setaffinity( 0, maskSize, threadMask ) ) #endif + // Here and below the error severity is lowered from critical level + // because it may happen during TBB library unload because of not + // waiting for workers to complete (current RML policy, to be fixed). + // handle_perror( errno, "setaffinity syscall" ); runtime_warning( "setaffinity syscall failed" ); } -static void get_thread_affinity_mask( size_t maskSize, basic_mask_t* threadMask ) { -#if __linux__ - if( sched_getaffinity( 0, maskSize, threadMask ) ) -#else /* FreeBSD */ +static void get_thread_affinity_mask( std::size_t maskSize, basic_mask_t* threadMask ) { +#if __FreeBSD__ || __NetBSD__ || __OpenBSD__ if( cpuset_getaffinity( CPU_LEVEL_WHICH, CPU_WHICH_TID, -1, maskSize, threadMask ) ) +#else /* __unix__ */ + if( sched_getaffinity( 0, maskSize, threadMask ) ) #endif - runtime_warning( "getaffinity syscall failed" ); + runtime_warning( "getaffinity syscall failed" ); } static basic_mask_t* process_mask; static int num_masks; void destroy_process_mask() { - if( process_mask ) { - delete [] process_mask; - } + delete [] process_mask; + process_mask = nullptr; } #define curMaskSize sizeof(basic_mask_t) * num_masks @@ -102,9 +104,9 @@ affinity_helper::~affinity_helper() { } } void affinity_helper::protect_affinity_mask( bool restore_process_mask ) { - if( threadMask == NULL && num_masks ) { // TODO: assert num_masks validity? + if( threadMask == nullptr && num_masks ) { // TODO: assert num_masks validity? threadMask = new basic_mask_t [num_masks]; - memset( threadMask, 0, curMaskSize ); + std::memset( threadMask, 0, curMaskSize ); get_thread_affinity_mask( curMaskSize, threadMask ); if( restore_process_mask ) { __TBB_ASSERT( process_mask, "A process mask is requested but not yet stored" ); @@ -118,15 +120,13 @@ void affinity_helper::protect_affinity_mask( bool restore_process_mask ) { } } void affinity_helper::dismiss() { - if( threadMask ) { - delete [] threadMask; - threadMask = NULL; - } + delete [] threadMask; + threadMask = nullptr; is_changed = 0; } #undef curMaskSize -static atomic<do_once_state> hardware_concurrency_info; +static std::atomic<do_once_state> hardware_concurrency_info; static int theNumProcs; @@ -134,44 +134,31 @@ static void initialize_hardware_concurrency_info () { int err; int availableProcs = 0; int numMasks = 1; -#if __linux__ -#if __TBB_MAIN_THREAD_AFFINITY_BROKEN - int maxProcs = INT_MAX; // To check the entire mask. - int pid = 0; // Get the mask of the calling thread. -#else int maxProcs = sysconf(_SC_NPROCESSORS_ONLN); - int pid = getpid(); -#endif -#else /* FreeBSD >= 7.1 */ - int maxProcs = sysconf(_SC_NPROCESSORS_ONLN); -#endif basic_mask_t* processMask; - const size_t BasicMaskSize = sizeof(basic_mask_t); + const std::size_t BasicMaskSize = sizeof(basic_mask_t); for (;;) { const int curMaskSize = BasicMaskSize * numMasks; processMask = new basic_mask_t[numMasks]; - memset( processMask, 0, curMaskSize ); -#if __linux__ - err = sched_getaffinity( pid, curMaskSize, processMask ); - if ( !err || errno != EINVAL || curMaskSize * CHAR_BIT >= 256 * 1024 ) - break; -#else /* FreeBSD >= 7.1 */ + std::memset( processMask, 0, curMaskSize ); +#if __FreeBSD__ || __NetBSD__ || __OpenBSD__ // CPU_LEVEL_WHICH - anonymous (current) mask, CPU_LEVEL_CPUSET - assigned mask -#if __TBB_MAIN_THREAD_AFFINITY_BROKEN - err = cpuset_getaffinity( CPU_LEVEL_WHICH, CPU_WHICH_TID, -1, curMaskSize, processMask ); -#else err = cpuset_getaffinity( CPU_LEVEL_WHICH, CPU_WHICH_PID, -1, curMaskSize, processMask ); -#endif if ( !err || errno != ERANGE || curMaskSize * CHAR_BIT >= 16 * 1024 ) break; -#endif /* FreeBSD >= 7.1 */ +#else /* __unix__ */ + int pid = getpid(); + err = sched_getaffinity( pid, curMaskSize, processMask ); + if ( !err || errno != EINVAL || curMaskSize * CHAR_BIT >= 256 * 1024 ) + break; +#endif delete[] processMask; numMasks <<= 1; } if ( !err ) { // We have found the mask size and captured the process affinity mask into processMask. num_masks = numMasks; // do here because it's needed for affinity_helper to work -#if __linux__ +#if __unix__ // For better coexistence with libiomp which might have changed the mask already, // check for its presence and ask it to restore the mask. dynamic_link_handle libhandle; @@ -182,7 +169,7 @@ static void initialize_hardware_concurrency_info () { if ( libiomp_try_restoring_original_mask()==0 ) { // Now we have the right mask to capture, restored by libiomp. const int curMaskSize = BasicMaskSize * numMasks; - memset( processMask, 0, curMaskSize ); + std::memset( processMask, 0, curMaskSize ); get_thread_affinity_mask( curMaskSize, processMask ); } else affhelp.dismiss(); // thread mask has not changed @@ -191,7 +178,7 @@ static void initialize_hardware_concurrency_info () { } #endif for ( int m = 0; availableProcs < maxProcs && m < numMasks; ++m ) { - for ( size_t i = 0; (availableProcs < maxProcs) && (i < BasicMaskSize * CHAR_BIT); ++i ) { + for ( std::size_t i = 0; (availableProcs < maxProcs) && (i < BasicMaskSize * CHAR_BIT); ++i ) { if ( CPU_ISSET( i, processMask + m ) ) ++availableProcs; } @@ -204,7 +191,7 @@ static void initialize_hardware_concurrency_info () { delete[] processMask; } theNumProcs = availableProcs > 0 ? availableProcs : 1; // Fail safety strap - __TBB_ASSERT( theNumProcs <= sysconf(_SC_NPROCESSORS_ONLN), NULL ); + __TBB_ASSERT( theNumProcs <= sysconf(_SC_NPROCESSORS_ONLN), nullptr); } int AvailableHwConcurrency() { @@ -219,7 +206,7 @@ int AvailableHwConcurrency() { // Format of "present" file is: ([<int>-<int>|<int>],)+ int AvailableHwConcurrency() { FILE *fp = fopen("/sys/devices/system/cpu/present", "r"); - if (fp == NULL) return 1; + if (fp == nullptr) return 1; int num_args, lower, upper, num_cpus=0; while ((num_args = fscanf(fp, "%u-%u", &lower, &upper)) != EOF) { switch(num_args) { @@ -228,6 +215,7 @@ int AvailableHwConcurrency() { } fscanf(fp, ","); } + fclose(fp); return (num_cpus > 0) ? num_cpus : 1; } @@ -240,7 +228,7 @@ int AvailableHwConcurrency() { #elif _WIN32||_WIN64 -static atomic<do_once_state> hardware_concurrency_info; +static std::atomic<do_once_state> hardware_concurrency_info; static const WORD TBB_ALL_PROCESSOR_GROUPS = 0xffff; @@ -256,13 +244,13 @@ struct ProcessorGroupInfo { //! Total number of processor groups in the system static int NumGroups; - //! Index of the group with a slot reserved for the first master thread + //! Index of the group with a slot reserved for the first external thread /** In the context of multiple processor groups support current implementation - defines "the first master thread" as the first thread to invoke + defines "the first external thread" as the first thread to invoke AvailableHwConcurrency(). TODO: Implement a dynamic scheme remapping workers depending on the pending - master threads affinity. **/ + external threads affinity. **/ static int HoleIndex; }; @@ -270,15 +258,16 @@ int ProcessorGroupInfo::NumGroups = 1; int ProcessorGroupInfo::HoleIndex = 0; ProcessorGroupInfo theProcessorGroups[MaxProcessorGroups]; - +int calculate_numa[MaxProcessorGroups]; //Array needed for FindProcessorGroupIndex to calculate Processor Group when number of threads > number of cores to distribute threads evenly between processor groups +int numaSum; struct TBB_GROUP_AFFINITY { DWORD_PTR Mask; WORD Group; WORD Reserved[3]; }; -static DWORD (WINAPI *TBB_GetActiveProcessorCount)( WORD groupIndex ) = NULL; -static WORD (WINAPI *TBB_GetActiveProcessorGroupCount)() = NULL; +static DWORD (WINAPI *TBB_GetActiveProcessorCount)( WORD groupIndex ) = nullptr; +static WORD (WINAPI *TBB_GetActiveProcessorGroupCount)() = nullptr; static BOOL (WINAPI *TBB_SetThreadGroupAffinity)( HANDLE hThread, const TBB_GROUP_AFFINITY* newAff, TBB_GROUP_AFFINITY *prevAff ); static BOOL (WINAPI *TBB_GetThreadGroupAffinity)( HANDLE hThread, TBB_GROUP_AFFINITY* ); @@ -291,6 +280,7 @@ static const dynamic_link_descriptor ProcessorGroupsApiLinkTable[] = { }; static void initialize_hardware_concurrency_info () { + suppress_unused_warning(TBB_ALL_PROCESSOR_GROUPS); #if __TBB_WIN8UI_SUPPORT // For these applications processor groups info is unavailable // Setting up a number of processors for one processor group @@ -303,18 +293,18 @@ static void initialize_hardware_concurrency_info () { DWORD_PTR pam, sam, m = 1; GetProcessAffinityMask( GetCurrentProcess(), &pam, &sam ); int nproc = 0; - for ( size_t i = 0; i < sizeof(DWORD_PTR) * CHAR_BIT; ++i, m <<= 1 ) { + for ( std::size_t i = 0; i < sizeof(DWORD_PTR) * CHAR_BIT; ++i, m <<= 1 ) { if ( pam & m ) ++nproc; } - __TBB_ASSERT( nproc <= (int)si.dwNumberOfProcessors, NULL ); + __TBB_ASSERT( nproc <= (int)si.dwNumberOfProcessors, nullptr); // By default setting up a number of processors for one processor group theProcessorGroups[0].numProcs = theProcessorGroups[0].numProcsRunningTotal = nproc; // Setting up processor groups in case the process does not restrict affinity mask and more than one processor group is present if ( nproc == (int)si.dwNumberOfProcessors && TBB_GetActiveProcessorCount ) { // The process does not have restricting affinity mask and multiple processor groups are possible ProcessorGroupInfo::NumGroups = (int)TBB_GetActiveProcessorGroupCount(); - __TBB_ASSERT( ProcessorGroupInfo::NumGroups <= MaxProcessorGroups, NULL ); + __TBB_ASSERT( ProcessorGroupInfo::NumGroups <= MaxProcessorGroups, nullptr); // Fail safety bootstrap. Release versions will limit available concurrency // level, while debug ones would assert. if ( ProcessorGroupInfo::NumGroups > MaxProcessorGroups ) @@ -324,15 +314,27 @@ static void initialize_hardware_concurrency_info () { if ( TBB_GetThreadGroupAffinity( GetCurrentThread(), &ga ) ) ProcessorGroupInfo::HoleIndex = ga.Group; int nprocs = 0; + int min_procs = INT_MAX; for ( WORD i = 0; i < ProcessorGroupInfo::NumGroups; ++i ) { ProcessorGroupInfo &pgi = theProcessorGroups[i]; pgi.numProcs = (int)TBB_GetActiveProcessorCount(i); - __TBB_ASSERT( pgi.numProcs <= (int)sizeof(DWORD_PTR) * CHAR_BIT, NULL ); + if (pgi.numProcs < min_procs) min_procs = pgi.numProcs; //Finding the minimum number of processors in the Processor Groups + calculate_numa[i] = pgi.numProcs; + __TBB_ASSERT( pgi.numProcs <= (int)sizeof(DWORD_PTR) * CHAR_BIT, nullptr); pgi.mask = pgi.numProcs == sizeof(DWORD_PTR) * CHAR_BIT ? ~(DWORD_PTR)0 : (DWORD_PTR(1) << pgi.numProcs) - 1; pgi.numProcsRunningTotal = nprocs += pgi.numProcs; } - __TBB_ASSERT( nprocs == (int)TBB_GetActiveProcessorCount( TBB_ALL_PROCESSOR_GROUPS ), NULL ); + __TBB_ASSERT( nprocs == (int)TBB_GetActiveProcessorCount( TBB_ALL_PROCESSOR_GROUPS ), nullptr); + + calculate_numa[0] = (calculate_numa[0] / min_procs)-1; + for (WORD i = 1; i < ProcessorGroupInfo::NumGroups; ++i) { + calculate_numa[i] = calculate_numa[i-1] + (calculate_numa[i] / min_procs); + } + + numaSum = calculate_numa[ProcessorGroupInfo::NumGroups - 1]; + } + } #endif /* __TBB_WIN8UI_SUPPORT */ @@ -343,50 +345,41 @@ static void initialize_hardware_concurrency_info () { } int NumberOfProcessorGroups() { - __TBB_ASSERT( hardware_concurrency_info == initialization_complete, "NumberOfProcessorGroups is used before AvailableHwConcurrency" ); + __TBB_ASSERT( hardware_concurrency_info == do_once_state::initialized, "NumberOfProcessorGroups is used before AvailableHwConcurrency" ); return ProcessorGroupInfo::NumGroups; } -// Offset for the slot reserved for the first master thread -#define HoleAdjusted(procIdx, grpIdx) (procIdx + (holeIdx <= grpIdx)) - int FindProcessorGroupIndex ( int procIdx ) { - // In case of oversubscription spread extra workers in a round robin manner - int holeIdx; - const int numProcs = theProcessorGroups[ProcessorGroupInfo::NumGroups - 1].numProcsRunningTotal; - if ( procIdx >= numProcs - 1 ) { - holeIdx = INT_MAX; - procIdx = (procIdx - numProcs + 1) % numProcs; + int current_grp_idx = ProcessorGroupInfo::HoleIndex; + if (procIdx >= theProcessorGroups[current_grp_idx].numProcs && procIdx < theProcessorGroups[ProcessorGroupInfo::NumGroups - 1].numProcsRunningTotal) { + procIdx = procIdx - theProcessorGroups[current_grp_idx].numProcs; + do { + current_grp_idx = (current_grp_idx + 1) % (ProcessorGroupInfo::NumGroups); + procIdx = procIdx - theProcessorGroups[current_grp_idx].numProcs; + + } while (procIdx >= 0); } - else - holeIdx = ProcessorGroupInfo::HoleIndex; - __TBB_ASSERT( hardware_concurrency_info == initialization_complete, "FindProcessorGroupIndex is used before AvailableHwConcurrency" ); - // Approximate the likely group index assuming all groups are of the same size - int i = procIdx / theProcessorGroups[0].numProcs; - // Make sure the approximation is a valid group index - if (i >= ProcessorGroupInfo::NumGroups) i = ProcessorGroupInfo::NumGroups-1; - // Now adjust the approximation up or down - if ( theProcessorGroups[i].numProcsRunningTotal > HoleAdjusted(procIdx, i) ) { - while ( theProcessorGroups[i].numProcsRunningTotal - theProcessorGroups[i].numProcs > HoleAdjusted(procIdx, i) ) { - __TBB_ASSERT( i > 0, NULL ); - --i; + else if (procIdx >= theProcessorGroups[ProcessorGroupInfo::NumGroups - 1].numProcsRunningTotal) { + int temp_grp_index = 0; + procIdx = procIdx - theProcessorGroups[ProcessorGroupInfo::NumGroups - 1].numProcsRunningTotal; + procIdx = procIdx % (numaSum+1); //ProcIdx to stay between 0 and numaSum + + while (procIdx - calculate_numa[temp_grp_index] > 0) { + temp_grp_index = (temp_grp_index + 1) % ProcessorGroupInfo::NumGroups; } + current_grp_idx = temp_grp_index; } - else { - do { - ++i; - } while ( theProcessorGroups[i].numProcsRunningTotal <= HoleAdjusted(procIdx, i) ); - } - __TBB_ASSERT( i < ProcessorGroupInfo::NumGroups, NULL ); - return i; + __TBB_ASSERT(current_grp_idx < ProcessorGroupInfo::NumGroups, nullptr); + + return current_grp_idx; } void MoveThreadIntoProcessorGroup( void* hThread, int groupIndex ) { - __TBB_ASSERT( hardware_concurrency_info == initialization_complete, "MoveThreadIntoProcessorGroup is used before AvailableHwConcurrency" ); + __TBB_ASSERT( hardware_concurrency_info == do_once_state::initialized, "MoveThreadIntoProcessorGroup is used before AvailableHwConcurrency" ); if ( !TBB_SetThreadGroupAffinity ) return; TBB_GROUP_AFFINITY ga = { theProcessorGroups[groupIndex].mask, (WORD)groupIndex, {0,0,0} }; - TBB_SetThreadGroupAffinity( hThread, &ga, NULL ); + TBB_SetThreadGroupAffinity( hThread, &ga, nullptr); } int AvailableHwConcurrency() { @@ -399,7 +392,8 @@ int AvailableHwConcurrency() { #error AvailableHwConcurrency is not implemented for this OS #endif -} // namespace internal +} // namespace r1 +} // namespace detail } // namespace tbb #endif /* !__TBB_HardwareConcurrency */ diff --git a/src/tbb/src/tbb/mutex.cpp b/src/tbb/src/tbb/mutex.cpp deleted file mode 100644 index fd02d2605..000000000 --- a/src/tbb/src/tbb/mutex.cpp +++ /dev/null @@ -1,152 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#if _WIN32||_WIN64 -#include <errno.h> // EDEADLK -#endif -#include "tbb/mutex.h" -#include "itt_notify.h" -#if __TBB_TSX_AVAILABLE -#include "governor.h" // for speculation_enabled() -#endif - -namespace tbb { - void mutex::scoped_lock::internal_acquire( mutex& m ) { - -#if _WIN32||_WIN64 - switch( m.state ) { - case INITIALIZED: - case HELD: - EnterCriticalSection( &m.impl ); - // If a thread comes here, and another thread holds the lock, it will block - // in EnterCriticalSection. When it returns from EnterCriticalSection, - // m.state must be set to INITIALIZED. If the same thread tries to acquire a lock it - // already holds, the lock is in HELD state, thus will cause throwing the exception. - if (m.state==HELD) - tbb::internal::handle_perror(EDEADLK,"mutex::scoped_lock: deadlock caused by attempt to reacquire held mutex"); - m.state = HELD; - break; - case DESTROYED: - __TBB_ASSERT(false,"mutex::scoped_lock: mutex already destroyed"); - break; - default: - __TBB_ASSERT(false,"mutex::scoped_lock: illegal mutex state"); - break; - } -#else - int error_code = pthread_mutex_lock(&m.impl); - if( error_code ) - tbb::internal::handle_perror(error_code,"mutex::scoped_lock: pthread_mutex_lock failed"); -#endif /* _WIN32||_WIN64 */ - my_mutex = &m; - } - -void mutex::scoped_lock::internal_release() { - __TBB_ASSERT( my_mutex, "mutex::scoped_lock: not holding a mutex" ); -#if _WIN32||_WIN64 - switch( my_mutex->state ) { - case INITIALIZED: - __TBB_ASSERT(false,"mutex::scoped_lock: try to release the lock without acquisition"); - break; - case HELD: - my_mutex->state = INITIALIZED; - LeaveCriticalSection(&my_mutex->impl); - break; - case DESTROYED: - __TBB_ASSERT(false,"mutex::scoped_lock: mutex already destroyed"); - break; - default: - __TBB_ASSERT(false,"mutex::scoped_lock: illegal mutex state"); - break; - } -#else - int error_code = pthread_mutex_unlock(&my_mutex->impl); - __TBB_ASSERT_EX(!error_code, "mutex::scoped_lock: pthread_mutex_unlock failed"); -#endif /* _WIN32||_WIN64 */ - my_mutex = NULL; -} - -bool mutex::scoped_lock::internal_try_acquire( mutex& m ) { -#if _WIN32||_WIN64 - switch( m.state ) { - case INITIALIZED: - case HELD: - break; - case DESTROYED: - __TBB_ASSERT(false,"mutex::scoped_lock: mutex already destroyed"); - break; - default: - __TBB_ASSERT(false,"mutex::scoped_lock: illegal mutex state"); - break; - } -#endif /* _WIN32||_WIN64 */ - - bool result; -#if _WIN32||_WIN64 - result = TryEnterCriticalSection(&m.impl)!=0; - if( result ) { - __TBB_ASSERT(m.state!=HELD, "mutex::scoped_lock: deadlock caused by attempt to reacquire held mutex"); - m.state = HELD; - } -#else - result = pthread_mutex_trylock(&m.impl)==0; -#endif /* _WIN32||_WIN64 */ - if( result ) - my_mutex = &m; - return result; -} - -void mutex::internal_construct() { -#if _WIN32||_WIN64 - InitializeCriticalSectionEx(&impl, 4000, 0); - state = INITIALIZED; -#else - int error_code = pthread_mutex_init(&impl,NULL); - if( error_code ) - tbb::internal::handle_perror(error_code,"mutex: pthread_mutex_init failed"); -#endif /* _WIN32||_WIN64*/ - ITT_SYNC_CREATE(&impl, _T("tbb::mutex"), _T("")); -} - -void mutex::internal_destroy() { -#if _WIN32||_WIN64 - switch( state ) { - case INITIALIZED: - DeleteCriticalSection(&impl); - break; - case DESTROYED: - __TBB_ASSERT(false,"mutex: already destroyed"); - break; - default: - __TBB_ASSERT(false,"mutex: illegal state for destruction"); - break; - } - state = DESTROYED; -#else - int error_code = pthread_mutex_destroy(&impl); -#if __TBB_TSX_AVAILABLE - // For processors with speculative execution, skip the error code check due to glibc bug #16657 - if( tbb::internal::governor::speculation_enabled() ) return; -#endif - __TBB_ASSERT_EX(!error_code,"mutex: pthread_mutex_destroy failed"); -#endif /* _WIN32||_WIN64 */ -} - -} // namespace tbb diff --git a/src/tbb/src/tbb/observer_proxy.cpp b/src/tbb/src/tbb/observer_proxy.cpp index 2a9b2bf47..012c9e4a2 100644 --- a/src/tbb/src/tbb/observer_proxy.cpp +++ b/src/tbb/src/tbb/observer_proxy.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2022 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,205 +12,163 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ -#include "tbb/tbb_config.h" - -#if __TBB_SCHEDULER_OBSERVER +#include "oneapi/tbb/detail/_config.h" +#include "oneapi/tbb/detail/_utils.h" #include "observer_proxy.h" -#include "tbb_main.h" -#include "governor.h" -#include "scheduler.h" #include "arena.h" +#include "main.h" +#include "thread_data.h" -namespace tbb { -namespace internal { +#include <atomic> -padded<observer_list> the_global_observer_list; +namespace tbb { +namespace detail { +namespace r1 { #if TBB_USE_ASSERT -static atomic<int> observer_proxy_count; - -struct check_observer_proxy_count { - ~check_observer_proxy_count() { - if( observer_proxy_count!=0 ) { - runtime_warning( "Leaked %ld observer_proxy objects\n", long(observer_proxy_count) ); - } - } -}; - -static check_observer_proxy_count the_check_observer_proxy_count; +extern std::atomic<int> the_observer_proxy_count; #endif /* TBB_USE_ASSERT */ -#if __TBB_ARENA_OBSERVER || __TBB_SLEEP_PERMISSION -interface6::task_scheduler_observer* observer_proxy::get_v6_observer() { - if(my_version != 6) return NULL; - return static_cast<interface6::task_scheduler_observer*>(my_observer); -} -#endif - -#if __TBB_ARENA_OBSERVER -bool observer_proxy::is_global() { - return !get_v6_observer() || get_v6_observer()->my_context_tag == interface6::task_scheduler_observer::global_tag; -} -#endif /* __TBB_ARENA_OBSERVER */ - -observer_proxy::observer_proxy( task_scheduler_observer_v3& tso ) - : my_list(NULL), my_next(NULL), my_prev(NULL), my_observer(&tso) +observer_proxy::observer_proxy( d1::task_scheduler_observer& tso ) + : my_ref_count(1), my_list(nullptr), my_next(nullptr), my_prev(nullptr), my_observer(&tso) { #if TBB_USE_ASSERT - ++observer_proxy_count; + ++the_observer_proxy_count; #endif /* TBB_USE_ASSERT */ - // 1 for observer - my_ref_count = 1; - my_version = -#if __TBB_ARENA_OBSERVER - load<relaxed>(my_observer->my_busy_count) - == interface6::task_scheduler_observer::v6_trait ? 6 : -#endif - 0; - __TBB_ASSERT( my_version >= 6 || !load<relaxed>(my_observer->my_busy_count), NULL ); } -#if TBB_USE_ASSERT -observer_proxy::~observer_proxy () { +observer_proxy::~observer_proxy() { __TBB_ASSERT( !my_ref_count, "Attempt to destroy proxy still in use" ); poison_value(my_ref_count); poison_pointer(my_prev); poison_pointer(my_next); - --observer_proxy_count; -} +#if TBB_USE_ASSERT + --the_observer_proxy_count; #endif /* TBB_USE_ASSERT */ - -template<memory_semantics M, class T, class V> -T atomic_fetch_and_store ( T* addr, const V& val ) { - return (T)atomic_traits<sizeof(T), M>::fetch_and_store( addr, (T)val ); } -void observer_list::clear () { - __TBB_ASSERT( this != &the_global_observer_list, "Method clear() cannot be used on the list of global observers" ); - // Though the method will work fine for the empty list, we require the caller - // to check for the list emptiness before invoking it to avoid extra overhead. - __TBB_ASSERT( !empty(), NULL ); +void observer_list::clear() { { scoped_lock lock(mutex(), /*is_writer=*/true); - observer_proxy *next = my_head; + observer_proxy *next = my_head.load(std::memory_order_relaxed); while ( observer_proxy *p = next ) { - __TBB_ASSERT( p->my_version >= 6, NULL ); next = p->my_next; // Both proxy p and observer p->my_observer (if non-null) are guaranteed // to be alive while the list is locked. - task_scheduler_observer_v3 *obs = p->my_observer; + d1::task_scheduler_observer *obs = p->my_observer; // Make sure that possible concurrent observer destruction does not // conflict with the proxy list cleanup. - if ( !obs || !(p = (observer_proxy*)__TBB_FetchAndStoreW(&obs->my_proxy, 0)) ) + if (!obs || !(p = obs->my_proxy.exchange(nullptr))) { continue; + } // accessing 'obs' after detaching of obs->my_proxy leads to the race with observer destruction - __TBB_ASSERT( !next || p == next->my_prev, NULL ); - __TBB_ASSERT( is_alive(p->my_ref_count), "Observer's proxy died prematurely" ); - __TBB_ASSERT( p->my_ref_count == 1, "Reference for observer is missing" ); -#if TBB_USE_ASSERT - p->my_observer = NULL; - p->my_ref_count = 0; -#endif /* TBB_USE_ASSERT */ + __TBB_ASSERT(!next || p == next->my_prev, nullptr); + __TBB_ASSERT(is_alive(p->my_ref_count), "Observer's proxy died prematurely"); + __TBB_ASSERT(p->my_ref_count.load(std::memory_order_relaxed) == 1, "Reference for observer is missing"); + poison_pointer(p->my_observer); remove(p); + --p->my_ref_count; delete p; } } - while( my_head ) - __TBB_Yield(); + + // If observe(false) is called concurrently with the destruction of the arena, + // need to wait until all proxies are removed. + for (atomic_backoff backoff; ; backoff.pause()) { + scoped_lock lock(mutex(), /*is_writer=*/false); + if (my_head.load(std::memory_order_relaxed) == nullptr) { + break; + } + } + + __TBB_ASSERT(my_head.load(std::memory_order_relaxed) == nullptr && my_tail.load(std::memory_order_relaxed) == nullptr, nullptr); } -void observer_list::insert ( observer_proxy* p ) { +void observer_list::insert( observer_proxy* p ) { scoped_lock lock(mutex(), /*is_writer=*/true); - if ( my_head ) { - p->my_prev = my_tail; - my_tail->my_next = p; + if (my_head.load(std::memory_order_relaxed)) { + p->my_prev = my_tail.load(std::memory_order_relaxed); + my_tail.load(std::memory_order_relaxed)->my_next = p; + } else { + my_head.store(p, std::memory_order_relaxed); } - else - my_head = p; - my_tail = p; + my_tail.store(p, std::memory_order_relaxed); } -void observer_list::remove ( observer_proxy* p ) { - __TBB_ASSERT( my_head, "Attempt to remove an item from an empty list" ); - __TBB_ASSERT( !my_tail->my_next, "Last item's my_next must be NULL" ); - if( p == my_tail ) { - __TBB_ASSERT( !p->my_next, NULL ); - my_tail = p->my_prev; - } - else { - __TBB_ASSERT( p->my_next, NULL ); +void observer_list::remove(observer_proxy* p) { + __TBB_ASSERT(my_head.load(std::memory_order_relaxed), "Attempt to remove an item from an empty list"); + __TBB_ASSERT(!my_tail.load(std::memory_order_relaxed)->my_next, "Last item's my_next must be nullptr"); + if (p == my_tail.load(std::memory_order_relaxed)) { + __TBB_ASSERT(!p->my_next, nullptr); + my_tail.store(p->my_prev, std::memory_order_relaxed); + } else { + __TBB_ASSERT(p->my_next, nullptr); p->my_next->my_prev = p->my_prev; } - if ( p == my_head ) { - __TBB_ASSERT( !p->my_prev, NULL ); - my_head = p->my_next; - } - else { - __TBB_ASSERT( p->my_prev, NULL ); + if (p == my_head.load(std::memory_order_relaxed)) { + __TBB_ASSERT(!p->my_prev, nullptr); + my_head.store(p->my_next, std::memory_order_relaxed); + } else { + __TBB_ASSERT(p->my_prev, nullptr); p->my_prev->my_next = p->my_next; } - __TBB_ASSERT( (my_head && my_tail) || (!my_head && !my_tail), NULL ); + __TBB_ASSERT((my_head.load(std::memory_order_relaxed) && my_tail.load(std::memory_order_relaxed)) || + (!my_head.load(std::memory_order_relaxed) && !my_tail.load(std::memory_order_relaxed)), nullptr); } -void observer_list::remove_ref( observer_proxy* p ) { - int r = p->my_ref_count; - __TBB_ASSERT( is_alive(r), NULL ); - while(r>1) { - __TBB_ASSERT( r!=0, NULL ); - int r_old = p->my_ref_count.compare_and_swap(r-1,r); - if( r_old==r ) { - // Successfully decremented count. +void observer_list::remove_ref(observer_proxy* p) { + std::uintptr_t r = p->my_ref_count.load(std::memory_order_acquire); + __TBB_ASSERT(is_alive(r), nullptr); + while (r > 1) { + if (p->my_ref_count.compare_exchange_strong(r, r - 1)) { return; } - r = r_old; } - __TBB_ASSERT( r==1, NULL ); + __TBB_ASSERT(r == 1, nullptr); // Reference count might go to zero { // Use lock to avoid resurrection by a thread concurrently walking the list observer_list::scoped_lock lock(mutex(), /*is_writer=*/true); r = --p->my_ref_count; - if( !r ) + if (!r) { remove(p); + } } - __TBB_ASSERT( r || !p->my_ref_count, NULL ); - if( !r ) + __TBB_ASSERT(r || !p->my_ref_count, nullptr); + if (!r) { delete p; + } } -void observer_list::do_notify_entry_observers( observer_proxy*& last, bool worker ) { +void observer_list::do_notify_entry_observers(observer_proxy*& last, bool worker) { // Pointer p marches though the list from last (exclusively) to the end. - observer_proxy *p = last, *prev = p; - for(;;) { - task_scheduler_observer_v3* tso=NULL; + observer_proxy* p = last, * prev = p; + for (;;) { + d1::task_scheduler_observer* tso = nullptr; // Hold lock on list only long enough to advance to the next proxy in the list. { scoped_lock lock(mutex(), /*is_writer=*/false); do { - if( p ) { + if (p) { // We were already processing the list. - if( observer_proxy* q = p->my_next ) { - if( p == prev ) - remove_ref_fast(prev); // sets prev to NULL if successful + if (observer_proxy* q = p->my_next) { + if (p == prev) { + remove_ref_fast(prev); // sets prev to nullptr if successful + } p = q; - } - else { + } else { // Reached the end of the list. - if( p == prev ) { + if (p == prev) { // Keep the reference as we store the 'last' pointer in scheduler - __TBB_ASSERT(p->my_ref_count >= 1 + (p->my_observer?1:0), NULL); + __TBB_ASSERT(int(p->my_ref_count.load(std::memory_order_relaxed)) >= 1 + (p->my_observer ? 1 : 0), nullptr); } else { // The last few proxies were empty - __TBB_ASSERT(p->my_ref_count, NULL); + __TBB_ASSERT(int(p->my_ref_count.load(std::memory_order_relaxed)), nullptr); ++p->my_ref_count; - if( prev ) { + if (prev) { lock.release(); remove_ref(prev); } @@ -220,207 +178,153 @@ void observer_list::do_notify_entry_observers( observer_proxy*& last, bool worke } } else { // Starting pass through the list - p = my_head; - if( !p ) + p = my_head.load(std::memory_order_relaxed); + if (!p) { return; + } } tso = p->my_observer; - } while( !tso ); + } while (!tso); ++p->my_ref_count; ++tso->my_busy_count; } - __TBB_ASSERT( !prev || p!=prev, NULL ); + __TBB_ASSERT(!prev || p != prev, nullptr); // Release the proxy pinned before p - if( prev ) + if (prev) { remove_ref(prev); + } // Do not hold any locks on the list while calling user's code. // Do not intercept any exceptions that may escape the callback so that // they are either handled by the TBB scheduler or passed to the debugger. tso->on_scheduler_entry(worker); - __TBB_ASSERT(p->my_ref_count, NULL); + __TBB_ASSERT(p->my_ref_count.load(std::memory_order_relaxed), nullptr); intptr_t bc = --tso->my_busy_count; - __TBB_ASSERT_EX( bc>=0, "my_busy_count underflowed" ); + __TBB_ASSERT_EX(bc >= 0, "my_busy_count underflowed"); prev = p; } } -void observer_list::do_notify_exit_observers( observer_proxy* last, bool worker ) { +void observer_list::do_notify_exit_observers(observer_proxy* last, bool worker) { // Pointer p marches though the list from the beginning to last (inclusively). - observer_proxy *p = NULL, *prev = NULL; - for(;;) { - task_scheduler_observer_v3* tso=NULL; + observer_proxy* p = nullptr, * prev = nullptr; + for (;;) { + d1::task_scheduler_observer* tso = nullptr; // Hold lock on list only long enough to advance to the next proxy in the list. { scoped_lock lock(mutex(), /*is_writer=*/false); do { - if( p ) { + if (p) { // We were already processing the list. - if( p != last ) { - __TBB_ASSERT( p->my_next, "List items before 'last' must have valid my_next pointer" ); - if( p == prev ) - remove_ref_fast(prev); // sets prev to NULL if successful + if (p != last) { + __TBB_ASSERT(p->my_next, "List items before 'last' must have valid my_next pointer"); + if (p == prev) + remove_ref_fast(prev); // sets prev to nullptr if successful p = p->my_next; } else { // remove the reference from the last item remove_ref_fast(p); - if( p ) { + if (p) { lock.release(); + if (p != prev && prev) { + remove_ref(prev); + } remove_ref(p); } return; } } else { // Starting pass through the list - p = my_head; - __TBB_ASSERT( p, "Nonzero 'last' must guarantee that the global list is non-empty" ); + p = my_head.load(std::memory_order_relaxed); + __TBB_ASSERT(p, "Nonzero 'last' must guarantee that the global list is non-empty"); } tso = p->my_observer; - } while( !tso ); + } while (!tso); // The item is already refcounted - if ( p != last ) // the last is already referenced since entry notification + if (p != last) // the last is already referenced since entry notification ++p->my_ref_count; ++tso->my_busy_count; } - __TBB_ASSERT( !prev || p!=prev, NULL ); - if( prev ) + __TBB_ASSERT(!prev || p != prev, nullptr); + if (prev) remove_ref(prev); // Do not hold any locks on the list while calling user's code. // Do not intercept any exceptions that may escape the callback so that // they are either handled by the TBB scheduler or passed to the debugger. tso->on_scheduler_exit(worker); - __TBB_ASSERT(p->my_ref_count || p == last, NULL); - intptr_t bc = --tso->my_busy_count; - __TBB_ASSERT_EX( bc>=0, "my_busy_count underflowed" ); - prev = p; - } -} - -#if __TBB_SLEEP_PERMISSION -bool observer_list::ask_permission_to_leave() { - __TBB_ASSERT( this == &the_global_observer_list, "This method cannot be used on lists of arena observers" ); - if( !my_head ) return true; - // Pointer p marches though the list - observer_proxy *p = NULL, *prev = NULL; - bool result = true; - while( result ) { - task_scheduler_observer* tso = NULL; - // Hold lock on list only long enough to advance to the next proxy in the list. - { - scoped_lock lock(mutex(), /*is_writer=*/false); - do { - if( p ) { - // We were already processing the list. - observer_proxy* q = p->my_next; - // read next, remove the previous reference - if( p == prev ) - remove_ref_fast(prev); // sets prev to NULL if successful - if( q ) p = q; - else { - // Reached the end of the list. - if( prev ) { - lock.release(); - remove_ref(prev); - } - return result; - } - } else { - // Starting pass through the list - p = my_head; - if( !p ) - return result; - } - tso = p->get_v6_observer(); - } while( !tso ); - ++p->my_ref_count; - ++tso->my_busy_count; - } - __TBB_ASSERT( !prev || p!=prev, NULL ); - // Release the proxy pinned before p - if( prev ) - remove_ref(prev); - // Do not hold any locks on the list while calling user's code. - // Do not intercept any exceptions that may escape the callback so that - // they are either handled by the TBB scheduler or passed to the debugger. - result = tso->may_sleep(); - __TBB_ASSERT(p->my_ref_count, NULL); + __TBB_ASSERT(p->my_ref_count || p == last, nullptr); intptr_t bc = --tso->my_busy_count; - __TBB_ASSERT_EX( bc>=0, "my_busy_count underflowed" ); + __TBB_ASSERT_EX(bc >= 0, "my_busy_count underflowed"); prev = p; } - if( prev ) - remove_ref(prev); - return result; } -#endif//__TBB_SLEEP_PERMISSION -void task_scheduler_observer_v3::observe( bool enable ) { +void __TBB_EXPORTED_FUNC observe(d1::task_scheduler_observer &tso, bool enable) { if( enable ) { - if( !my_proxy ) { - my_proxy = new observer_proxy( *this ); - my_busy_count = 0; // proxy stores versioning information, clear it -#if __TBB_ARENA_OBSERVER - if ( !my_proxy->is_global() ) { - // Local observer activation - generic_scheduler* s = governor::local_scheduler_if_initialized(); - __TBB_ASSERT( my_proxy->get_v6_observer(), NULL ); - intptr_t tag = my_proxy->get_v6_observer()->my_context_tag; - if( tag != interface6::task_scheduler_observer::implicit_tag ) { // explicit arena - task_arena *a = reinterpret_cast<task_arena*>(tag); - a->initialize(); - my_proxy->my_list = &a->my_arena->my_observers; - } else { - if( !s ) - s = governor::init_scheduler( task_scheduler_init::automatic, 0, true ); - __TBB_ASSERT( __TBB_InitOnce::initialization_done(), NULL ); - __TBB_ASSERT( s && s->my_arena, NULL ); - my_proxy->my_list = &s->my_arena->my_observers; + if( !tso.my_proxy.load(std::memory_order_relaxed) ) { + observer_proxy* p = new observer_proxy(tso); + tso.my_proxy.store(p, std::memory_order_relaxed); + tso.my_busy_count.store(0, std::memory_order_relaxed); + + thread_data* td = governor::get_thread_data_if_initialized(); + if (p->my_observer->my_task_arena == nullptr) { + if (!(td && td->my_arena)) { + td = governor::get_thread_data(); } - my_proxy->my_list->insert(my_proxy); - // Notify newly activated observer and other pending ones if it belongs to current arena - if(s && &s->my_arena->my_observers == my_proxy->my_list ) - my_proxy->my_list->notify_entry_observers( s->my_last_local_observer, s->is_worker() ); - } else -#endif /* __TBB_ARENA_OBSERVER */ - { - // Obsolete. Global observer activation - if( !__TBB_InitOnce::initialization_done() ) - DoOneTimeInitializations(); - my_proxy->my_list = &the_global_observer_list; - my_proxy->my_list->insert(my_proxy); - if( generic_scheduler* s = governor::local_scheduler_if_initialized() ) { - // Notify newly created observer of its own thread. - // Any other pending observers are notified too. - the_global_observer_list.notify_entry_observers( s->my_last_global_observer, s->is_worker() ); + __TBB_ASSERT(__TBB_InitOnce::initialization_done(), nullptr); + __TBB_ASSERT(td && td->my_arena, nullptr); + p->my_list = &td->my_arena->my_observers; + } else { + d1::task_arena* ta = p->my_observer->my_task_arena; + arena* a = ta->my_arena.load(std::memory_order_acquire); + if (a == nullptr) { // Avoid recursion during arena initialization + ta->initialize(); + a = ta->my_arena.load(std::memory_order_relaxed); } + __TBB_ASSERT(a != nullptr, nullptr); + p->my_list = &a->my_observers; + } + p->my_list->insert(p); + // Notify newly activated observer and other pending ones if it belongs to current arena + if (td && td->my_arena && &td->my_arena->my_observers == p->my_list) { + p->my_list->notify_entry_observers(td->my_last_observer, td->my_is_worker); } } } else { // Make sure that possible concurrent proxy list cleanup does not conflict // with the observer destruction here. - if ( observer_proxy* proxy = (observer_proxy*)__TBB_FetchAndStoreW(&my_proxy, 0) ) { + if ( observer_proxy* proxy = tso.my_proxy.exchange(nullptr) ) { // List destruction should not touch this proxy after we've won the above interlocked exchange. - __TBB_ASSERT( proxy->my_observer == this, NULL ); - __TBB_ASSERT( is_alive(proxy->my_ref_count), "Observer's proxy died prematurely" ); - __TBB_ASSERT( proxy->my_ref_count >= 1, "reference for observer missing" ); + __TBB_ASSERT( proxy->my_observer == &tso, nullptr); + __TBB_ASSERT( is_alive(proxy->my_ref_count.load(std::memory_order_relaxed)), "Observer's proxy died prematurely" ); + __TBB_ASSERT( proxy->my_ref_count.load(std::memory_order_relaxed) >= 1, "reference for observer missing" ); observer_list &list = *proxy->my_list; { // Ensure that none of the list walkers relies on observer pointer validity observer_list::scoped_lock lock(list.mutex(), /*is_writer=*/true); - proxy->my_observer = NULL; + proxy->my_observer = nullptr; // Proxy may still be held by other threads (to track the last notified observer) if( !--proxy->my_ref_count ) {// nobody can increase it under exclusive lock list.remove(proxy); - __TBB_ASSERT( !proxy->my_ref_count, NULL ); + __TBB_ASSERT( !proxy->my_ref_count, nullptr); delete proxy; } } - while( my_busy_count ) // other threads are still accessing the callback - __TBB_Yield(); + spin_wait_until_eq(tso.my_busy_count, 0); // other threads are still accessing the callback } } } -} // namespace internal +} // namespace r1 +} // namespace detail } // namespace tbb -#endif /* __TBB_SCHEDULER_OBSERVER */ +namespace tbb { +namespace internal { + +void __TBB_EXPORTED_FUNC task_scheduler_observer_v3::observe( bool enable ) { + auto* tso = (tbb::detail::d1::task_scheduler_observer*) (this); + tbb::detail::r1::observe(*tso, enable); +} + +} // namespace internal +} // namespace tbb \ No newline at end of file diff --git a/src/tbb/src/tbb/observer_proxy.h b/src/tbb/src/tbb/observer_proxy.h index 7f2c4e39c..0ca2839d1 100644 --- a/src/tbb/src/tbb/observer_proxy.h +++ b/src/tbb/src/tbb/observer_proxy.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2022 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,24 +12,20 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ -#ifndef _TBB_observer_proxy_H -#define _TBB_observer_proxy_H +#ifndef __TBB_observer_proxy_H +#define __TBB_observer_proxy_H -#if __TBB_SCHEDULER_OBSERVER +#include "oneapi/tbb/detail/_config.h" +#include "oneapi/tbb/detail/_aligned_space.h" -#include "scheduler_common.h" // to include task.h -#include "tbb/task_scheduler_observer.h" -#include "tbb/spin_rw_mutex.h" -#include "tbb/aligned_space.h" +#include "oneapi/tbb/task_scheduler_observer.h" +#include "oneapi/tbb/spin_rw_mutex.h" namespace tbb { -namespace internal { +namespace detail { +namespace r1 { class observer_list { friend class arena; @@ -39,10 +35,10 @@ class observer_list { typedef aligned_space<spin_rw_mutex> my_mutex_type; //! Pointer to the head of this list. - observer_proxy* my_head; + std::atomic<observer_proxy*> my_head{nullptr}; //! Pointer to the tail of this list. - observer_proxy* my_tail; + std::atomic<observer_proxy*> my_tail{nullptr}; //! Mutex protecting this list. my_mutex_type my_mutex; @@ -51,7 +47,7 @@ class observer_list { arena* my_arena; //! Decrement refcount of the proxy p if there are other outstanding references. - /** In case of success sets p to NULL. Must be invoked from under the list lock. **/ + /** In case of success sets p to nullptr. Must be invoked from under the list lock. **/ inline static void remove_ref_fast( observer_proxy*& p ); //! Implements notify_entry_observers functionality. @@ -61,7 +57,7 @@ class observer_list { void do_notify_exit_observers( observer_proxy* last, bool worker ); public: - observer_list () : my_head(NULL), my_tail(NULL) {} + observer_list () = default; //! Removes and destroys all observer proxies from the list. /** Cannot be used concurrently with other methods. **/ @@ -83,18 +79,13 @@ class observer_list { //! Accessor to the reader-writer mutex associated with the list. spin_rw_mutex& mutex () { return my_mutex.begin()[0]; } - bool empty () const { return my_head == NULL; } - //! Call entry notifications on observers added after last was notified. /** Updates last to become the last notified observer proxy (in the global list) - or leaves it to be NULL. The proxy has its refcount incremented. **/ + or leaves it to be nullptr. The proxy has its refcount incremented. **/ inline void notify_entry_observers( observer_proxy*& last, bool worker ); //! Call exit notifications on last and observers added before it. inline void notify_exit_observers( observer_proxy*& last, bool worker ); - - //! Call may_sleep callbacks to ask for permission for a worker thread to leave market - bool ask_permission_to_leave(); }; // class observer_list //! Wrapper for an observer object @@ -102,75 +93,60 @@ class observer_list { object into a proxy so that a list item remained valid even after the corresponding proxy object is destroyed by the user code. **/ class observer_proxy { - friend class task_scheduler_observer_v3; + friend class d1::task_scheduler_observer; friend class observer_list; + friend void observe(d1::task_scheduler_observer&, bool); //! Reference count used for garbage collection. /** 1 for reference from my task_scheduler_observer. 1 for each task dispatcher's last observer pointer. No accounting for neighbors in the shared list. */ - atomic<int> my_ref_count; + std::atomic<std::uintptr_t> my_ref_count; //! Reference to the list this observer belongs to. observer_list* my_list; //! Pointer to next observer in the list specified by my_head. - /** NULL for the last item in the list. **/ + /** nullptr for the last item in the list. **/ observer_proxy* my_next; //! Pointer to the previous observer in the list specified by my_head. /** For the head of the list points to the last item. **/ observer_proxy* my_prev; //! Associated observer - task_scheduler_observer_v3* my_observer; - //! Version - char my_version; - -#if __TBB_ARENA_OBSERVER || __TBB_SLEEP_PERMISSION - interface6::task_scheduler_observer* get_v6_observer(); -#endif -#if __TBB_ARENA_OBSERVER - bool is_global(); //TODO: move them back inline when un-CPF'ing -#endif + d1::task_scheduler_observer* my_observer; //! Constructs proxy for the given observer and adds it to the specified list. - observer_proxy( task_scheduler_observer_v3& ); + observer_proxy( d1::task_scheduler_observer& ); -#if TBB_USE_ASSERT ~observer_proxy(); -#endif /* TBB_USE_ASSERT */ - - //! Shut up the warning - observer_proxy& operator = ( const observer_proxy& ); }; // class observer_proxy -inline void observer_list::remove_ref_fast( observer_proxy*& p ) { +void observer_list::remove_ref_fast( observer_proxy*& p ) { if( p->my_observer ) { // Can decrement refcount quickly, as it cannot drop to zero while under the lock. - int r = --p->my_ref_count; - __TBB_ASSERT_EX( r, NULL ); - p = NULL; + std::uintptr_t r = --p->my_ref_count; + __TBB_ASSERT_EX( r, nullptr); + p = nullptr; } else { // Use slow form of refcount decrementing, after the lock is released. } } -inline void observer_list::notify_entry_observers( observer_proxy*& last, bool worker ) { - if ( last == my_tail ) +void observer_list::notify_entry_observers(observer_proxy*& last, bool worker) { + if (last == my_tail.load(std::memory_order_relaxed)) return; - do_notify_entry_observers( last, worker ); + do_notify_entry_observers(last, worker); } -inline void observer_list::notify_exit_observers( observer_proxy*& last, bool worker ) { - if ( !last ) +void observer_list::notify_exit_observers( observer_proxy*& last, bool worker ) { + if (last == nullptr) { return; - __TBB_ASSERT(is_alive((uintptr_t)last), NULL); + } + __TBB_ASSERT(!is_poisoned(last), nullptr); do_notify_exit_observers( last, worker ); - __TBB_ASSERT(last, NULL); - poison_value(last); + __TBB_ASSERT(last != nullptr, nullptr); + poison_pointer(last); } -extern padded<observer_list> the_global_observer_list; - -} // namespace internal +} // namespace r1 +} // namespace detail } // namespace tbb -#endif /* __TBB_SCHEDULER_OBSERVER */ - -#endif /* _TBB_observer_proxy_H */ +#endif /* __TBB_observer_proxy_H */ diff --git a/src/tbb/src/tbb/parallel_pipeline.cpp b/src/tbb/src/tbb/parallel_pipeline.cpp new file mode 100644 index 000000000..bb8587b92 --- /dev/null +++ b/src/tbb/src/tbb/parallel_pipeline.cpp @@ -0,0 +1,471 @@ +/* + Copyright (c) 2005-2022 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "oneapi/tbb/parallel_pipeline.h" +#include "oneapi/tbb/spin_mutex.h" +#include "oneapi/tbb/tbb_allocator.h" +#include "oneapi/tbb/cache_aligned_allocator.h" +#include "itt_notify.h" +#include "tls.h" +#include "oneapi/tbb/detail/_exception.h" +#include "oneapi/tbb/detail/_small_object_pool.h" + +namespace tbb { +namespace detail { +namespace r1 { + +void handle_perror(int error_code, const char* aux_info); + +using Token = unsigned long; + +//! A processing pipeline that applies filters to items. +/** @ingroup algorithms */ +class pipeline { + friend void parallel_pipeline(d1::task_group_context&, std::size_t, const d1::filter_node&); +public: + + //! Construct empty pipeline. + pipeline(d1::task_group_context& cxt, std::size_t max_token) : + my_context(cxt), + first_filter(nullptr), + last_filter(nullptr), + input_tokens(Token(max_token)), + end_of_input(false), + wait_ctx(0) { + __TBB_ASSERT( max_token>0, "pipeline::run must have at least one token" ); + } + + ~pipeline(); + + //! Add filter to end of pipeline. + void add_filter( d1::base_filter& ); + + //! Traverse tree of fitler-node in-order and add filter for each leaf + void fill_pipeline(const d1::filter_node& root) { + if( root.left && root.right ) { + fill_pipeline(*root.left); + fill_pipeline(*root.right); + } + else { + __TBB_ASSERT(!root.left && !root.right, "tree should be full"); + add_filter(*root.create_filter()); + } + } + +private: + friend class stage_task; + friend class base_filter; + friend void set_end_of_input(d1::base_filter& bf); + + task_group_context& my_context; + + //! Pointer to first filter in the pipeline. + d1::base_filter* first_filter; + + //! Pointer to last filter in the pipeline. + d1::base_filter* last_filter; + + //! Number of idle tokens waiting for input stage. + std::atomic<Token> input_tokens; + + //! False until flow_control::stop() is called. + std::atomic<bool> end_of_input; + + d1::wait_context wait_ctx; +}; + +//! This structure is used to store task information in an input buffer +struct task_info { + void* my_object = nullptr; + //! Invalid unless a task went through an ordered stage. + Token my_token = 0; + //! False until my_token is set. + bool my_token_ready = false; + //! True if my_object is valid. + bool is_valid = false; + //! Set to initial state (no object, no token) + void reset() { + my_object = nullptr; + my_token = 0; + my_token_ready = false; + is_valid = false; + } +}; + +//! A buffer of input items for a filter. +/** Each item is a task_info, inserted into a position in the buffer corresponding to a Token. */ +class input_buffer { + friend class base_filter; + friend class stage_task; + friend class pipeline; + friend void set_end_of_input(d1::base_filter& bf); + + using size_type = Token; + + //! Array of deferred tasks that cannot yet start executing. + task_info* array; + + //! Size of array + /** Always 0 or a power of 2 */ + size_type array_size; + + //! Lowest token that can start executing. + /** All prior Token have already been seen. */ + Token low_token; + + //! Serializes updates. + spin_mutex array_mutex; + + //! Resize "array". + /** Caller is responsible to acquiring a lock on "array_mutex". */ + void grow( size_type minimum_size ); + + //! Initial size for "array" + /** Must be a power of 2 */ + static const size_type initial_buffer_size = 4; + + //! Used for out of order buffer, and for assigning my_token if is_ordered and my_token not already assigned + Token high_token; + + //! True for ordered filter, false otherwise. + const bool is_ordered; + + //! for parallel filters that accepts nullptrs, thread-local flag for reaching end_of_input + using end_of_input_tls_t = basic_tls<input_buffer*>; + end_of_input_tls_t end_of_input_tls; + bool end_of_input_tls_allocated; // no way to test pthread creation of TLS + +public: + input_buffer(const input_buffer&) = delete; + input_buffer& operator=(const input_buffer&) = delete; + + //! Construct empty buffer. + input_buffer( bool ordered) : + array(nullptr), + array_size(0), + low_token(0), + high_token(0), + is_ordered(ordered), + end_of_input_tls(), + end_of_input_tls_allocated(false) { + grow(initial_buffer_size); + __TBB_ASSERT( array, nullptr ); + } + + //! Destroy the buffer. + ~input_buffer() { + __TBB_ASSERT( array, nullptr ); + cache_aligned_allocator<task_info>().deallocate(array,array_size); + poison_pointer( array ); + if( end_of_input_tls_allocated ) { + destroy_my_tls(); + } + } + + //! Define order when the first filter is serial_in_order. + Token get_ordered_token(){ + return high_token++; + } + + //! Put a token into the buffer. + /** If task information was placed into buffer, returns true; + otherwise returns false, informing the caller to create and spawn a task. + */ + bool try_put_token( task_info& info ) { + info.is_valid = true; + spin_mutex::scoped_lock lock( array_mutex ); + Token token; + if( is_ordered ) { + if( !info.my_token_ready ) { + info.my_token = high_token++; + info.my_token_ready = true; + } + token = info.my_token; + } else + token = high_token++; + __TBB_ASSERT( (long)(token-low_token)>=0, nullptr ); + if( token!=low_token ) { + // Trying to put token that is beyond low_token. + // Need to wait until low_token catches up before dispatching. + if( token-low_token>=array_size ) + grow( token-low_token+1 ); + ITT_NOTIFY( sync_releasing, this ); + array[token&(array_size-1)] = info; + return true; + } + return false; + } + + //! Note that processing of a token is finished. + /** Fires up processing of the next token, if processing was deferred. */ + // Uses template to avoid explicit dependency on stage_task. + template<typename StageTask> + void try_to_spawn_task_for_next_token(StageTask& spawner, d1::execution_data& ed) { + task_info wakee; + { + spin_mutex::scoped_lock lock( array_mutex ); + // Wake the next task + task_info& item = array[++low_token & (array_size-1)]; + ITT_NOTIFY( sync_acquired, this ); + wakee = item; + item.is_valid = false; + } + if( wakee.is_valid ) + spawner.spawn_stage_task(wakee, ed); + } + + // end_of_input signal for parallel_pipeline, parallel input filters with 0 tokens allowed. + void create_my_tls() { + int status = end_of_input_tls.create(); + if(status) + handle_perror(status, "TLS not allocated for filter"); + end_of_input_tls_allocated = true; + } + void destroy_my_tls() { + int status = end_of_input_tls.destroy(); + if(status) + handle_perror(status, "Failed to destroy filter TLS"); + } + bool my_tls_end_of_input() { + return end_of_input_tls.get() != nullptr; + } + void set_my_tls_end_of_input() { + end_of_input_tls.set(this); + } +}; + +void input_buffer::grow( size_type minimum_size ) { + size_type old_size = array_size; + size_type new_size = old_size ? 2*old_size : initial_buffer_size; + while( new_size<minimum_size ) + new_size*=2; + task_info* new_array = cache_aligned_allocator<task_info>().allocate(new_size); + task_info* old_array = array; + for( size_type i=0; i<new_size; ++i ) + new_array[i].is_valid = false; + Token t=low_token; + for( size_type i=0; i<old_size; ++i, ++t ) + new_array[t&(new_size-1)] = old_array[t&(old_size-1)]; + array = new_array; + array_size = new_size; + if( old_array ) + cache_aligned_allocator<task_info>().deallocate(old_array,old_size); +} + +class stage_task : public d1::task, public task_info { +private: + friend class pipeline; + pipeline& my_pipeline; + d1::base_filter* my_filter; + d1::small_object_allocator m_allocator; + //! True if this task has not yet read the input. + bool my_at_start; + + //! True if this can be executed again. + bool execute_filter(d1::execution_data& ed); + + //! Spawn task if token is available. + void try_spawn_stage_task(d1::execution_data& ed) { + ITT_NOTIFY( sync_releasing, &my_pipeline.input_tokens ); + if( (my_pipeline.input_tokens.fetch_sub(1, std::memory_order_release)) > 1 ) { + d1::small_object_allocator alloc{}; + r1::spawn( *alloc.new_object<stage_task>(ed, my_pipeline, alloc ), my_pipeline.my_context ); + } + } + +public: + + //! Construct stage_task for first stage in a pipeline. + /** Such a stage has not read any input yet. */ + stage_task(pipeline& pipeline, d1::small_object_allocator& alloc ) : + my_pipeline(pipeline), + my_filter(pipeline.first_filter), + m_allocator(alloc), + my_at_start(true) + { + task_info::reset(); + my_pipeline.wait_ctx.reserve(); + } + //! Construct stage_task for a subsequent stage in a pipeline. + stage_task(pipeline& pipeline, d1::base_filter* filter, const task_info& info, d1::small_object_allocator& alloc) : + task_info(info), + my_pipeline(pipeline), + my_filter(filter), + m_allocator(alloc), + my_at_start(false) + { + my_pipeline.wait_ctx.reserve(); + } + //! Roughly equivalent to the constructor of input stage task + void reset() { + task_info::reset(); + my_filter = my_pipeline.first_filter; + my_at_start = true; + } + void finalize(d1::execution_data& ed) { + m_allocator.delete_object(this, ed); + } + //! The virtual task execution method + task* execute(d1::execution_data& ed) override { + if(!execute_filter(ed)) { + finalize(ed); + return nullptr; + } + return this; + } + task* cancel(d1::execution_data& ed) override { + finalize(ed); + return nullptr; + } + + ~stage_task() override { + if ( my_filter && my_object ) { + my_filter->finalize(my_object); + my_object = nullptr; + } + my_pipeline.wait_ctx.release(); + } + //! Creates and spawns stage_task from task_info + void spawn_stage_task(const task_info& info, d1::execution_data& ed) { + d1::small_object_allocator alloc{}; + stage_task* clone = alloc.new_object<stage_task>(ed, my_pipeline, my_filter, info, alloc); + r1::spawn(*clone, my_pipeline.my_context); + } +}; + +bool stage_task::execute_filter(d1::execution_data& ed) { + __TBB_ASSERT( !my_at_start || !my_object, "invalid state of task" ); + if( my_at_start ) { + if( my_filter->is_serial() ) { + my_object = (*my_filter)(my_object); + if( my_object || ( my_filter->object_may_be_null() && !my_pipeline.end_of_input.load(std::memory_order_relaxed)) ) { + if( my_filter->is_ordered() ) { + my_token = my_filter->my_input_buffer->get_ordered_token(); + my_token_ready = true; + } + if( !my_filter->next_filter_in_pipeline ) { // we're only filter in pipeline + reset(); + return true; + } else { + try_spawn_stage_task(ed); + } + } else { + my_pipeline.end_of_input.store(true, std::memory_order_relaxed); + return false; + } + } else /*not is_serial*/ { + if ( my_pipeline.end_of_input.load(std::memory_order_relaxed) ) { + return false; + } + + try_spawn_stage_task(ed); + + my_object = (*my_filter)(my_object); + if( !my_object && (!my_filter->object_may_be_null() || my_filter->my_input_buffer->my_tls_end_of_input()) ){ + my_pipeline.end_of_input.store(true, std::memory_order_relaxed); + return false; + } + } + my_at_start = false; + } else { + my_object = (*my_filter)(my_object); + if( my_filter->is_serial() ) + my_filter->my_input_buffer->try_to_spawn_task_for_next_token(*this, ed); + } + my_filter = my_filter->next_filter_in_pipeline; + if( my_filter ) { + // There is another filter to execute. + if( my_filter->is_serial() ) { + // The next filter must execute tokens when they are available (in order for serial_in_order) + if( my_filter->my_input_buffer->try_put_token(*this) ){ + my_filter = nullptr; // To prevent deleting my_object twice if exception occurs + return false; + } + } + } else { + // Reached end of the pipe. + std::size_t ntokens_avail = my_pipeline.input_tokens.fetch_add(1, std::memory_order_acquire); + + if( ntokens_avail>0 // Only recycle if there is one available token + || my_pipeline.end_of_input.load(std::memory_order_relaxed) ) { + return false; // No need to recycle for new input + } + ITT_NOTIFY( sync_acquired, &my_pipeline.input_tokens ); + // Recycle as an input stage task. + reset(); + } + return true; +} + +pipeline::~pipeline() { + while( first_filter ) { + d1::base_filter* f = first_filter; + if( input_buffer* b = f->my_input_buffer ) { + b->~input_buffer(); + deallocate_memory(b); + } + first_filter = f->next_filter_in_pipeline; + f->~base_filter(); + deallocate_memory(f); + } +} + +void pipeline::add_filter( d1::base_filter& new_fitler ) { + __TBB_ASSERT( new_fitler.next_filter_in_pipeline==d1::base_filter::not_in_pipeline(), "filter already part of pipeline?" ); + new_fitler.my_pipeline = this; + if ( first_filter == nullptr ) + first_filter = &new_fitler; + else + last_filter->next_filter_in_pipeline = &new_fitler; + new_fitler.next_filter_in_pipeline = nullptr; + last_filter = &new_fitler; + if( new_fitler.is_serial() ) { + new_fitler.my_input_buffer = new (allocate_memory(sizeof(input_buffer))) input_buffer( new_fitler.is_ordered() ); + } else { + if( first_filter == &new_fitler && new_fitler.object_may_be_null() ) { + //TODO: buffer only needed to hold TLS; could improve + new_fitler.my_input_buffer = new (allocate_memory(sizeof(input_buffer))) input_buffer( /*is_ordered*/false ); + new_fitler.my_input_buffer->create_my_tls(); + } + } +} + +void __TBB_EXPORTED_FUNC parallel_pipeline(d1::task_group_context& cxt, std::size_t max_token, const d1::filter_node& fn) { + pipeline pipe(cxt, max_token); + + pipe.fill_pipeline(fn); + + d1::small_object_allocator alloc{}; + stage_task& st = *alloc.new_object<stage_task>(pipe, alloc); + + // Start execution of tasks + r1::execute_and_wait(st, cxt, pipe.wait_ctx, cxt); +} + +void __TBB_EXPORTED_FUNC set_end_of_input(d1::base_filter& bf) { + __TBB_ASSERT(bf.my_input_buffer, nullptr); + __TBB_ASSERT(bf.object_may_be_null(), nullptr); + if(bf.is_serial() ) { + bf.my_pipeline->end_of_input.store(true, std::memory_order_relaxed); + } else { + __TBB_ASSERT(bf.my_input_buffer->end_of_input_tls_allocated, nullptr); + bf.my_input_buffer->set_my_tls_end_of_input(); + } +} + +} // namespace r1 +} // namespace detail +} // namespace tbb diff --git a/src/tbb/src/tbb/permit_manager.h b/src/tbb/src/tbb/permit_manager.h new file mode 100644 index 000000000..0a6a737c4 --- /dev/null +++ b/src/tbb/src/tbb/permit_manager.h @@ -0,0 +1,61 @@ +/* + Copyright (c) 2022-2023 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef _TBB_permit_manager_H +#define _TBB_permit_manager_H + +#include "oneapi/tbb/info.h" +#include "oneapi/tbb/detail/_utils.h" +#include "thread_request_serializer.h" + +namespace tbb { +namespace detail { +namespace r1 { + +class arena; +class pm_client; + +class permit_manager : no_copy { +public: + virtual ~permit_manager() {} + virtual pm_client* create_client(arena& a) = 0; + virtual void register_client(pm_client* client, d1::constraints& constraints) = 0; + virtual void unregister_and_destroy_client(pm_client& c) = 0; + + virtual void set_active_num_workers(int soft_limit) = 0; + virtual void adjust_demand(pm_client&, int mandatory_delta, int workers_delta) = 0; + + void set_thread_request_observer(thread_request_observer& tr_observer) { + __TBB_ASSERT(!my_thread_request_observer, "set_thread_request_observer was called already?"); + my_thread_request_observer = &tr_observer; + } +protected: + void notify_thread_request(int delta) { + __TBB_ASSERT(my_thread_request_observer, "set_thread_request_observer was not called?"); + if (delta) { + my_thread_request_observer->update(delta); + } + } +private: + thread_request_observer* my_thread_request_observer{nullptr}; +}; + + +} // namespace r1 +} // namespace detail +} // namespace tbb + +#endif // _TBB_permit_manager_H diff --git a/src/tbb/src/tbb/pipeline.cpp b/src/tbb/src/tbb/pipeline.cpp deleted file mode 100644 index 96a0aeeae..000000000 --- a/src/tbb/src/tbb/pipeline.cpp +++ /dev/null @@ -1,781 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#include "tbb/pipeline.h" -#include "tbb/spin_mutex.h" -#include "tbb/cache_aligned_allocator.h" -#include "itt_notify.h" -#include "semaphore.h" -#include "tls.h" // for parallel filters that do not use NULL as end_of_input - - -namespace tbb { - -namespace internal { - -//! This structure is used to store task information in a input buffer -struct task_info { - void* my_object; - //! Invalid unless a task went through an ordered stage. - Token my_token; - //! False until my_token is set. - bool my_token_ready; - //! True if my_object is valid. - bool is_valid; - //! Set to initial state (no object, no token) - void reset() { - my_object = NULL; - my_token = 0; - my_token_ready = false; - is_valid = false; - } -}; -//! A buffer of input items for a filter. -/** Each item is a task_info, inserted into a position in the buffer corresponding to a Token. */ -class input_buffer : no_copy { - friend class tbb::internal::pipeline_root_task; - friend class tbb::filter; - friend class tbb::thread_bound_filter; - friend class tbb::internal::stage_task; - friend class tbb::pipeline; - - typedef Token size_type; - - //! Array of deferred tasks that cannot yet start executing. - task_info* array; - - //! for thread-bound filter, semaphore for waiting, NULL otherwise. - semaphore* my_sem; - - //! Size of array - /** Always 0 or a power of 2 */ - size_type array_size; - - //! Lowest token that can start executing. - /** All prior Token have already been seen. */ - Token low_token; - - //! Serializes updates. - spin_mutex array_mutex; - - //! Resize "array". - /** Caller is responsible to acquiring a lock on "array_mutex". */ - void grow( size_type minimum_size ); - - //! Initial size for "array" - /** Must be a power of 2 */ - static const size_type initial_buffer_size = 4; - - //! Used for out of order buffer, and for assigning my_token if is_ordered and my_token not already assigned - Token high_token; - - //! True for ordered filter, false otherwise. - bool is_ordered; - - //! True for thread-bound filter, false otherwise. - bool is_bound; - - //! for parallel filters that accepts NULLs, thread-local flag for reaching end_of_input - typedef basic_tls<intptr_t> end_of_input_tls_t; - end_of_input_tls_t end_of_input_tls; - bool end_of_input_tls_allocated; // no way to test pthread creation of TLS - - void create_sema(size_t initial_tokens) { __TBB_ASSERT(!my_sem,NULL); my_sem = new internal::semaphore(initial_tokens); } - void free_sema() { __TBB_ASSERT(my_sem,NULL); delete my_sem; } - void sema_P() { __TBB_ASSERT(my_sem,NULL); my_sem->P(); } - void sema_V() { __TBB_ASSERT(my_sem,NULL); my_sem->V(); } - -public: - //! Construct empty buffer. - input_buffer( bool is_ordered_, bool is_bound_ ) : - array(NULL), my_sem(NULL), array_size(0), - low_token(0), high_token(0), - is_ordered(is_ordered_), is_bound(is_bound_), - end_of_input_tls_allocated(false) { - grow(initial_buffer_size); - __TBB_ASSERT( array, NULL ); - if(is_bound) create_sema(0); - } - - //! Destroy the buffer. - ~input_buffer() { - __TBB_ASSERT( array, NULL ); - cache_aligned_allocator<task_info>().deallocate(array,array_size); - poison_pointer( array ); - if(my_sem) { - free_sema(); - } - if(end_of_input_tls_allocated) { - destroy_my_tls(); - } - } - - //! Put a token into the buffer. - /** If task information was placed into buffer, returns true; - otherwise returns false, informing the caller to create and spawn a task. - If input buffer owned by thread-bound filter and the item at - low_token was not valid, issue a V() - If the input_buffer is owned by a successor to a thread-bound filter, - the force_put parameter should be true to ensure the token is inserted - in the buffer. - */ - bool put_token( task_info& info_, bool force_put = false ) { - { - info_.is_valid = true; - spin_mutex::scoped_lock lock( array_mutex ); - Token token; - bool was_empty = !array[low_token&(array_size-1)].is_valid; - if( is_ordered ) { - if( !info_.my_token_ready ) { - info_.my_token = high_token++; - info_.my_token_ready = true; - } - token = info_.my_token; - } else - token = high_token++; - __TBB_ASSERT( (tokendiff_t)(token-low_token)>=0, NULL ); - if( token!=low_token || is_bound || force_put ) { - // Trying to put token that is beyond low_token. - // Need to wait until low_token catches up before dispatching. - if( token-low_token>=array_size ) - grow( token-low_token+1 ); - ITT_NOTIFY( sync_releasing, this ); - array[token&(array_size-1)] = info_; - if(was_empty && is_bound) { - sema_V(); - } - return true; - } - } - return false; - } - - //! Note that processing of a token is finished. - /** Fires up processing of the next token, if processing was deferred. */ - // Using template to avoid explicit dependency on stage_task - // this is only called for serial filters, and is the reason for the - // advance parameter in return_item (we're incrementing low_token here.) - // Non-TBF serial stages don't advance the token at the start because the presence - // of the current token in the buffer keeps another stage from being spawned. - template<typename StageTask> - void note_done( Token token, StageTask& spawner ) { - task_info wakee; - wakee.reset(); - { - spin_mutex::scoped_lock lock( array_mutex ); - if( !is_ordered || token==low_token ) { - // Wake the next task - task_info& item = array[++low_token & (array_size-1)]; - ITT_NOTIFY( sync_acquired, this ); - wakee = item; - item.is_valid = false; - } - } - if( wakee.is_valid ) - spawner.spawn_stage_task(wakee); - } - -#if __TBB_TASK_GROUP_CONTEXT - //! The method destroys all data in filters to prevent memory leaks - void clear( filter* my_filter ) { - long t=low_token; - for( size_type i=0; i<array_size; ++i, ++t ){ - task_info& temp = array[t&(array_size-1)]; - if (temp.is_valid ) { - my_filter->finalize(temp.my_object); - temp.is_valid = false; - } - } - } -#endif - - //! return an item, invalidate the queued item, but only advance if advance - // advance == true for parallel filters. If the filter is serial, leave the - // item in the buffer to keep another stage from being spawned. - bool return_item(task_info& info, bool advance) { - spin_mutex::scoped_lock lock( array_mutex ); - task_info& item = array[low_token&(array_size-1)]; - ITT_NOTIFY( sync_acquired, this ); - if( item.is_valid ) { - info = item; - item.is_valid = false; - if (advance) low_token++; - return true; - } - return false; - } - - //! true if the current low_token is valid. - bool has_item() { spin_mutex::scoped_lock lock(array_mutex); return array[low_token&(array_size -1)].is_valid; } - - // end_of_input signal for parallel_pipeline, parallel input filters with 0 tokens allowed. - void create_my_tls() { int status = end_of_input_tls.create(); if(status) handle_perror(status, "TLS not allocated for filter"); end_of_input_tls_allocated = true; } - void destroy_my_tls() { int status = end_of_input_tls.destroy(); if(status) handle_perror(status, "Failed to destroy filter TLS"); } - bool my_tls_end_of_input() { return end_of_input_tls.get() != 0; } - void set_my_tls_end_of_input() { end_of_input_tls.set(1); } -}; - -void input_buffer::grow( size_type minimum_size ) { - size_type old_size = array_size; - size_type new_size = old_size ? 2*old_size : initial_buffer_size; - while( new_size<minimum_size ) - new_size*=2; - task_info* new_array = cache_aligned_allocator<task_info>().allocate(new_size); - task_info* old_array = array; - for( size_type i=0; i<new_size; ++i ) - new_array[i].is_valid = false; - long t=low_token; - for( size_type i=0; i<old_size; ++i, ++t ) - new_array[t&(new_size-1)] = old_array[t&(old_size-1)]; - array = new_array; - array_size = new_size; - if( old_array ) - cache_aligned_allocator<task_info>().deallocate(old_array,old_size); -} - -class stage_task: public task, public task_info { -private: - friend class tbb::pipeline; - pipeline& my_pipeline; - filter* my_filter; - //! True if this task has not yet read the input. - bool my_at_start; - -public: - //! Construct stage_task for first stage in a pipeline. - /** Such a stage has not read any input yet. */ - stage_task( pipeline& pipeline ) : - my_pipeline(pipeline), - my_filter(pipeline.filter_list), - my_at_start(true) - { - task_info::reset(); - } - //! Construct stage_task for a subsequent stage in a pipeline. - stage_task( pipeline& pipeline, filter* filter_, const task_info& info ) : - task_info(info), - my_pipeline(pipeline), - my_filter(filter_), - my_at_start(false) - {} - //! Roughly equivalent to the constructor of input stage task - void reset() { - task_info::reset(); - my_filter = my_pipeline.filter_list; - my_at_start = true; - } - //! The virtual task execution method - task* execute() __TBB_override; -#if __TBB_TASK_GROUP_CONTEXT - ~stage_task() - { - if (my_filter && my_object && (my_filter->my_filter_mode & filter::version_mask) >= __TBB_PIPELINE_VERSION(4)) { - __TBB_ASSERT(is_cancelled(), "Trying to finalize the task that wasn't cancelled"); - my_filter->finalize(my_object); - my_object = NULL; - } - } -#endif // __TBB_TASK_GROUP_CONTEXT - //! Creates and spawns stage_task from task_info - void spawn_stage_task(const task_info& info) - { - stage_task* clone = new (allocate_additional_child_of(*parent())) - stage_task( my_pipeline, my_filter, info ); - spawn(*clone); - } -}; - -task* stage_task::execute() { - __TBB_ASSERT( !my_at_start || !my_object, NULL ); - __TBB_ASSERT( !my_filter->is_bound(), NULL ); - if( my_at_start ) { - if( my_filter->is_serial() ) { - my_object = (*my_filter)(my_object); - if( my_object || ( my_filter->object_may_be_null() && !my_pipeline.end_of_input) ) - { - if( my_filter->is_ordered() ) { - my_token = my_pipeline.token_counter++; // ideally, with relaxed semantics - my_token_ready = true; - } else if( (my_filter->my_filter_mode & my_filter->version_mask) >= __TBB_PIPELINE_VERSION(5) ) { - if( my_pipeline.has_thread_bound_filters ) - my_pipeline.token_counter++; // ideally, with relaxed semantics - } - if( !my_filter->next_filter_in_pipeline ) { // we're only filter in pipeline - reset(); - goto process_another_stage; - } else { - ITT_NOTIFY( sync_releasing, &my_pipeline.input_tokens ); - if( --my_pipeline.input_tokens>0 ) - spawn( *new( allocate_additional_child_of(*parent()) ) stage_task( my_pipeline ) ); - } - } else { - my_pipeline.end_of_input = true; - return NULL; - } - } else /*not is_serial*/ { - if( my_pipeline.end_of_input ) - return NULL; - if( (my_filter->my_filter_mode & my_filter->version_mask) >= __TBB_PIPELINE_VERSION(5) ) { - if( my_pipeline.has_thread_bound_filters ) - my_pipeline.token_counter++; - } - ITT_NOTIFY( sync_releasing, &my_pipeline.input_tokens ); - if( --my_pipeline.input_tokens>0 ) - spawn( *new( allocate_additional_child_of(*parent()) ) stage_task( my_pipeline ) ); - my_object = (*my_filter)(my_object); - if( !my_object && (!my_filter->object_may_be_null() || my_filter->my_input_buffer->my_tls_end_of_input()) ) - { - my_pipeline.end_of_input = true; - if( (my_filter->my_filter_mode & my_filter->version_mask) >= __TBB_PIPELINE_VERSION(5) ) { - if( my_pipeline.has_thread_bound_filters ) - my_pipeline.token_counter--; // fix token_counter - } - return NULL; - } - } - my_at_start = false; - } else { - my_object = (*my_filter)(my_object); - if( my_filter->is_serial() ) - my_filter->my_input_buffer->note_done(my_token, *this); - } - my_filter = my_filter->next_filter_in_pipeline; - if( my_filter ) { - // There is another filter to execute. - if( my_filter->is_serial() ) { - // The next filter must execute tokens in order - if( my_filter->my_input_buffer->put_token(*this) ){ - // Can't proceed with the same item - if( my_filter->is_bound() ) { - // Find the next non-thread-bound filter - do { - my_filter = my_filter->next_filter_in_pipeline; - } while( my_filter && my_filter->is_bound() ); - // Check if there is an item ready to process - if( my_filter && my_filter->my_input_buffer->return_item(*this, !my_filter->is_serial())) - goto process_another_stage; - } - my_filter = NULL; // To prevent deleting my_object twice if exception occurs - return NULL; - } - } - } else { - // Reached end of the pipe. - size_t ntokens_avail = ++my_pipeline.input_tokens; - if(my_pipeline.filter_list->is_bound() ) { - if(ntokens_avail == 1) { - my_pipeline.filter_list->my_input_buffer->sema_V(); - } - return NULL; - } - if( ntokens_avail>1 // Only recycle if there is one available token - || my_pipeline.end_of_input ) { - return NULL; // No need to recycle for new input - } - ITT_NOTIFY( sync_acquired, &my_pipeline.input_tokens ); - // Recycle as an input stage task. - reset(); - } -process_another_stage: - /* A semi-hackish way to reexecute the same task object immediately without spawning. - recycle_as_continuation marks the task for future execution, - and then 'this' pointer is returned to bypass spawning. */ - recycle_as_continuation(); - return this; -} - -class pipeline_root_task: public task { - pipeline& my_pipeline; - bool do_segment_scanning; - - task* execute() __TBB_override { - if( !my_pipeline.end_of_input ) - if( !my_pipeline.filter_list->is_bound() ) - if( my_pipeline.input_tokens > 0 ) { - recycle_as_continuation(); - set_ref_count(1); - return new( allocate_child() ) stage_task( my_pipeline ); - } - if( do_segment_scanning ) { - filter* current_filter = my_pipeline.filter_list->next_segment; - /* first non-thread-bound filter that follows thread-bound one - and may have valid items to process */ - filter* first_suitable_filter = current_filter; - while( current_filter ) { - __TBB_ASSERT( !current_filter->is_bound(), "filter is thread-bound?" ); - __TBB_ASSERT( current_filter->prev_filter_in_pipeline->is_bound(), "previous filter is not thread-bound?" ); - if( !my_pipeline.end_of_input || current_filter->has_more_work()) - { - task_info info; - info.reset(); - if( current_filter->my_input_buffer->return_item(info, !current_filter->is_serial()) ) { - set_ref_count(1); - recycle_as_continuation(); - return new( allocate_child() ) stage_task( my_pipeline, current_filter, info); - } - current_filter = current_filter->next_segment; - if( !current_filter ) { - if( !my_pipeline.end_of_input ) { - recycle_as_continuation(); - return this; - } - current_filter = first_suitable_filter; - __TBB_Yield(); - } - } else { - /* The preceding pipeline segment is empty. - Fast-forward to the next post-TBF segment. */ - first_suitable_filter = first_suitable_filter->next_segment; - current_filter = first_suitable_filter; - } - } /* while( current_filter ) */ - return NULL; - } else { - if( !my_pipeline.end_of_input ) { - recycle_as_continuation(); - return this; - } - return NULL; - } - } -public: - pipeline_root_task( pipeline& pipeline ): my_pipeline(pipeline), do_segment_scanning(false) - { - __TBB_ASSERT( my_pipeline.filter_list, NULL ); - filter* first = my_pipeline.filter_list; - if( (first->my_filter_mode & first->version_mask) >= __TBB_PIPELINE_VERSION(5) ) { - // Scanning the pipeline for segments - filter* head_of_previous_segment = first; - for( filter* subfilter=first->next_filter_in_pipeline; - subfilter!=NULL; - subfilter=subfilter->next_filter_in_pipeline ) - { - if( subfilter->prev_filter_in_pipeline->is_bound() && !subfilter->is_bound() ) { - do_segment_scanning = true; - head_of_previous_segment->next_segment = subfilter; - head_of_previous_segment = subfilter; - } - } - } - } -}; - -#if _MSC_VER && !defined(__INTEL_COMPILER) - // Workaround for overzealous compiler warnings - // Suppress compiler warning about constant conditional expression - #pragma warning (disable: 4127) -#endif - -// The class destroys end_counter and clears all input buffers if pipeline was cancelled. -class pipeline_cleaner: internal::no_copy { - pipeline& my_pipeline; -public: - pipeline_cleaner(pipeline& _pipeline) : - my_pipeline(_pipeline) - {} - ~pipeline_cleaner(){ -#if __TBB_TASK_GROUP_CONTEXT - if (my_pipeline.end_counter->is_cancelled()) // Pipeline was cancelled - my_pipeline.clear_filters(); -#endif - my_pipeline.end_counter = NULL; - } -}; - -} // namespace internal - -void pipeline::inject_token( task& ) { - __TBB_ASSERT(false,"illegal call to inject_token"); -} - -#if __TBB_TASK_GROUP_CONTEXT -void pipeline::clear_filters() { - for( filter* f = filter_list; f; f = f->next_filter_in_pipeline ) { - if ((f->my_filter_mode & filter::version_mask) >= __TBB_PIPELINE_VERSION(4)) - if( internal::input_buffer* b = f->my_input_buffer ) - b->clear(f); - } -} -#endif - -pipeline::pipeline() : - filter_list(NULL), - filter_end(NULL), - end_counter(NULL), - end_of_input(false), - has_thread_bound_filters(false) -{ - token_counter = 0; - input_tokens = 0; -} - -pipeline::~pipeline() { - clear(); -} - -void pipeline::clear() { - filter* next; - for( filter* f = filter_list; f; f=next ) { - if( internal::input_buffer* b = f->my_input_buffer ) { - delete b; - f->my_input_buffer = NULL; - } - next=f->next_filter_in_pipeline; - f->next_filter_in_pipeline = filter::not_in_pipeline(); - if ( (f->my_filter_mode & filter::version_mask) >= __TBB_PIPELINE_VERSION(3) ) { - f->prev_filter_in_pipeline = filter::not_in_pipeline(); - f->my_pipeline = NULL; - } - if ( (f->my_filter_mode & filter::version_mask) >= __TBB_PIPELINE_VERSION(5) ) - f->next_segment = NULL; - } - filter_list = filter_end = NULL; -} - -void pipeline::add_filter( filter& filter_ ) { -#if TBB_USE_ASSERT - if ( (filter_.my_filter_mode & filter::version_mask) >= __TBB_PIPELINE_VERSION(3) ) - __TBB_ASSERT( filter_.prev_filter_in_pipeline==filter::not_in_pipeline(), "filter already part of pipeline?" ); - __TBB_ASSERT( filter_.next_filter_in_pipeline==filter::not_in_pipeline(), "filter already part of pipeline?" ); - __TBB_ASSERT( !end_counter, "invocation of add_filter on running pipeline" ); -#endif - if ( (filter_.my_filter_mode & filter::version_mask) >= __TBB_PIPELINE_VERSION(3) ) { - filter_.my_pipeline = this; - filter_.prev_filter_in_pipeline = filter_end; - if ( filter_list == NULL) - filter_list = &filter_; - else - filter_end->next_filter_in_pipeline = &filter_; - filter_.next_filter_in_pipeline = NULL; - filter_end = &filter_; - } - else - { - if( !filter_end ) - filter_end = reinterpret_cast<filter*>(&filter_list); - - *reinterpret_cast<filter**>(filter_end) = &filter_; - filter_end = reinterpret_cast<filter*>(&filter_.next_filter_in_pipeline); - *reinterpret_cast<filter**>(filter_end) = NULL; - } - if( (filter_.my_filter_mode & filter_.version_mask) >= __TBB_PIPELINE_VERSION(5) ) { - if( filter_.is_serial() ) { - if( filter_.is_bound() ) - has_thread_bound_filters = true; - filter_.my_input_buffer = new internal::input_buffer( filter_.is_ordered(), filter_.is_bound() ); - } - else { - if(filter_.prev_filter_in_pipeline) { - if(filter_.prev_filter_in_pipeline->is_bound()) { - // successors to bound filters must have an input_buffer - filter_.my_input_buffer = new internal::input_buffer( /*is_ordered*/false, false ); - } - } - else { // input filter - if(filter_.object_may_be_null() ) { - //TODO: buffer only needed to hold TLS; could improve - filter_.my_input_buffer = new internal::input_buffer( /*is_ordered*/false, false ); - filter_.my_input_buffer->create_my_tls(); - } - } - } - } else { - if( filter_.is_serial() ) { - filter_.my_input_buffer = new internal::input_buffer( filter_.is_ordered(), false ); - } - } - -} - -void pipeline::remove_filter( filter& filter_ ) { - __TBB_ASSERT( filter_.prev_filter_in_pipeline!=filter::not_in_pipeline(), "filter not part of pipeline" ); - __TBB_ASSERT( filter_.next_filter_in_pipeline!=filter::not_in_pipeline(), "filter not part of pipeline" ); - __TBB_ASSERT( !end_counter, "invocation of remove_filter on running pipeline" ); - if (&filter_ == filter_list) - filter_list = filter_.next_filter_in_pipeline; - else { - __TBB_ASSERT( filter_.prev_filter_in_pipeline, "filter list broken?" ); - filter_.prev_filter_in_pipeline->next_filter_in_pipeline = filter_.next_filter_in_pipeline; - } - if (&filter_ == filter_end) - filter_end = filter_.prev_filter_in_pipeline; - else { - __TBB_ASSERT( filter_.next_filter_in_pipeline, "filter list broken?" ); - filter_.next_filter_in_pipeline->prev_filter_in_pipeline = filter_.prev_filter_in_pipeline; - } - if( internal::input_buffer* b = filter_.my_input_buffer ) { - delete b; - filter_.my_input_buffer = NULL; - } - filter_.next_filter_in_pipeline = filter_.prev_filter_in_pipeline = filter::not_in_pipeline(); - if ( (filter_.my_filter_mode & filter::version_mask) >= __TBB_PIPELINE_VERSION(5) ) - filter_.next_segment = NULL; - filter_.my_pipeline = NULL; -} - -void pipeline::run( size_t max_number_of_live_tokens -#if __TBB_TASK_GROUP_CONTEXT - , tbb::task_group_context& context -#endif - ) { - __TBB_ASSERT( max_number_of_live_tokens>0, "pipeline::run must have at least one token" ); - __TBB_ASSERT( !end_counter, "pipeline already running?" ); - if( filter_list ) { - internal::pipeline_cleaner my_pipeline_cleaner(*this); - end_of_input = false; - input_tokens = internal::Token(max_number_of_live_tokens); - if(has_thread_bound_filters) { - // release input filter if thread-bound - if(filter_list->is_bound()) { - filter_list->my_input_buffer->sema_V(); - } - } -#if __TBB_TASK_GROUP_CONTEXT - end_counter = new( task::allocate_root(context) ) internal::pipeline_root_task( *this ); -#else - end_counter = new( task::allocate_root() ) internal::pipeline_root_task( *this ); -#endif - // Start execution of tasks - task::spawn_root_and_wait( *end_counter ); - - if(has_thread_bound_filters) { - for(filter* f = filter_list->next_filter_in_pipeline; f; f=f->next_filter_in_pipeline) { - if(f->is_bound()) { - f->my_input_buffer->sema_V(); // wake to end - } - } - } - } -} - -#if __TBB_TASK_GROUP_CONTEXT -void pipeline::run( size_t max_number_of_live_tokens ) { - if( filter_list ) { - // Construct task group context with the exception propagation mode expected - // by the pipeline caller. - uintptr_t ctx_traits = filter_list->my_filter_mode & filter::exact_exception_propagation ? - task_group_context::default_traits : - task_group_context::default_traits & ~task_group_context::exact_exception; - task_group_context context(task_group_context::bound, ctx_traits); - run(max_number_of_live_tokens, context); - } -} -#endif // __TBB_TASK_GROUP_CONTEXT - -bool filter::has_more_work() { - __TBB_ASSERT(my_pipeline, NULL); - __TBB_ASSERT(my_input_buffer, "has_more_work() called for filter with no input buffer"); - return (internal::tokendiff_t)(my_pipeline->token_counter - my_input_buffer->low_token) != 0; -} - -filter::~filter() { - if ( (my_filter_mode & version_mask) >= __TBB_PIPELINE_VERSION(3) ) { - if ( next_filter_in_pipeline != filter::not_in_pipeline() ) - my_pipeline->remove_filter(*this); - else - __TBB_ASSERT( prev_filter_in_pipeline == filter::not_in_pipeline(), "probably filter list is broken" ); - } else { - __TBB_ASSERT( next_filter_in_pipeline==filter::not_in_pipeline(), "cannot destroy filter that is part of pipeline" ); - } -} - -void -filter::set_end_of_input() { - __TBB_ASSERT(my_input_buffer, NULL); - __TBB_ASSERT(object_may_be_null(), NULL); - if(is_serial()) { - my_pipeline->end_of_input = true; - } - else { - __TBB_ASSERT(my_input_buffer->end_of_input_tls_allocated, NULL); - my_input_buffer->set_my_tls_end_of_input(); - } -} - -thread_bound_filter::result_type thread_bound_filter::process_item() { - return internal_process_item(true); -} - -thread_bound_filter::result_type thread_bound_filter::try_process_item() { - return internal_process_item(false); -} - -thread_bound_filter::result_type thread_bound_filter::internal_process_item(bool is_blocking) { - __TBB_ASSERT(my_pipeline != NULL,"It's not supposed that process_item is called for a filter that is not in a pipeline."); - internal::task_info info; - info.reset(); - - if( my_pipeline->end_of_input && !has_more_work() ) - return end_of_stream; - - if( !prev_filter_in_pipeline ) { - if( my_pipeline->end_of_input ) - return end_of_stream; - while( my_pipeline->input_tokens == 0 ) { - if( !is_blocking ) - return item_not_available; - my_input_buffer->sema_P(); - } - info.my_object = (*this)(info.my_object); - if( info.my_object ) { - __TBB_ASSERT(my_pipeline->input_tokens > 0, "Token failed in thread-bound filter"); - my_pipeline->input_tokens--; - if( is_ordered() ) { - info.my_token = my_pipeline->token_counter; - info.my_token_ready = true; - } - my_pipeline->token_counter++; // ideally, with relaxed semantics - } else { - my_pipeline->end_of_input = true; - return end_of_stream; - } - } else { /* this is not an input filter */ - while( !my_input_buffer->has_item() ) { - if( !is_blocking ) { - return item_not_available; - } - my_input_buffer->sema_P(); - if( my_pipeline->end_of_input && !has_more_work() ) { - return end_of_stream; - } - } - if( !my_input_buffer->return_item(info, /*advance*/true) ) { - __TBB_ASSERT(false,"return_item failed"); - } - info.my_object = (*this)(info.my_object); - } - if( next_filter_in_pipeline ) { - if ( !next_filter_in_pipeline->my_input_buffer->put_token(info,/*force_put=*/true) ) { - __TBB_ASSERT(false, "Couldn't put token after thread-bound buffer"); - } - } else { - size_t ntokens_avail = ++(my_pipeline->input_tokens); - if( my_pipeline->filter_list->is_bound() ) { - if( ntokens_avail == 1 ) { - my_pipeline->filter_list->my_input_buffer->sema_V(); - } - } - } - - return success; -} - -} // tbb - diff --git a/src/tbb/src/tbb/pm_client.h b/src/tbb/src/tbb/pm_client.h new file mode 100644 index 000000000..d08af8249 --- /dev/null +++ b/src/tbb/src/tbb/pm_client.h @@ -0,0 +1,76 @@ +/* + Copyright (c) 2022-2023 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef _TBB_pm_client_H +#define _TBB_pm_client_H + +#include "arena.h" + +namespace tbb { +namespace detail { +namespace r1 { + +class pm_client { +public: + pm_client(arena& a) : my_arena(a) {} + virtual ~pm_client() {} + + unsigned priority_level() { + return my_arena.priority_level(); + } + + void set_top_priority(bool b) { + my_arena.set_top_priority(b); + } + + int min_workers() const { + return my_min_workers; + } + + int max_workers() const { + return my_max_workers; + } + + int update_request(int mandatory_delta, int workers_delta) { + auto min_max_workers = my_arena.update_request(mandatory_delta, workers_delta); + int delta = min_max_workers.second - my_max_workers; + set_workers(min_max_workers.first, min_max_workers.second); + return delta; + } + + virtual void register_thread() = 0; + + virtual void unregister_thread() = 0; + + +protected: + void set_workers(int mn_w, int mx_w) { + __TBB_ASSERT(mn_w >= 0, nullptr); + __TBB_ASSERT(mx_w >= 0, nullptr); + my_min_workers = mn_w; + my_max_workers = mx_w; + } + + arena& my_arena; + int my_min_workers{0}; + int my_max_workers{0}; +}; + +} // namespace r1 +} // namespace detail +} // namespace tbb + +#endif // _TBB_pm_client_H diff --git a/src/tbb/src/tbb/private_server.cpp b/src/tbb/src/tbb/private_server.cpp index d9be168c2..6b04a85c9 100644 --- a/src/tbb/src/tbb/private_server.cpp +++ b/src/tbb/src/tbb/private_server.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2022 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,26 +12,27 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ +#include "oneapi/tbb/cache_aligned_allocator.h" +#include "oneapi/tbb/mutex.h" + #include "rml_tbb.h" -#include "../server/thread_monitor.h" -#include "tbb/atomic.h" -#include "tbb/cache_aligned_allocator.h" +#include "rml_thread_monitor.h" + #include "scheduler_common.h" #include "governor.h" -#include "tbb_misc.h" +#include "misc.h" + +#include <atomic> -using rml::internal::thread_monitor; namespace tbb { -namespace internal { +namespace detail { +namespace r1 { namespace rml { +using rml::internal::thread_monitor; typedef thread_monitor::handle_type thread_handle; class private_server; @@ -55,7 +56,7 @@ class private_worker: no_copy { //! Associated thread has ended normal life sequence and promises to never touch *this again. st_quit }; - atomic<state_t> my_state; + std::atomic<state_t> my_state; //! Associated server private_server& my_server; @@ -64,7 +65,7 @@ class private_worker: no_copy { tbb_client& my_client; //! index used for avoiding the 64K aliasing problem - const size_t my_index; + const std::size_t my_index; //! Monitor for sleeping when there is no work to do. /** The invariant that holds for sleeping workers is: @@ -80,7 +81,7 @@ class private_worker: no_copy { friend class private_server; //! Actions executed by the associated thread - void run(); + void run() noexcept; //! Wake up associated thread (or launch a thread if there is none) void wake_or_launch(); @@ -93,30 +94,27 @@ class private_worker: no_copy { static void release_handle(thread_handle my_handle, bool join); protected: - private_worker( private_server& server, tbb_client& client, const size_t i ) : - my_server(server), my_client(client), my_index(i), - my_thread_monitor(), my_handle(), my_next() - { - my_state = st_init; - } + private_worker( private_server& server, tbb_client& client, const std::size_t i ) : + my_state(st_init), my_server(server), my_client(client), my_index(i), + my_handle(), my_next() + {} }; -static const size_t cache_line_size = tbb::internal::NFS_MaxLineSize; - +static const std::size_t cache_line_size = tbb::detail::max_nfs_size; #if _MSC_VER && !defined(__INTEL_COMPILER) // Suppress overzealous compiler warnings about uninstantiable class - #pragma warning(push) - #pragma warning(disable:4510 4610) + // #pragma warning(push) + // #pragma warning(disable:4510 4610) #endif class padded_private_worker: public private_worker { char pad[cache_line_size - sizeof(private_worker)%cache_line_size]; public: - padded_private_worker( private_server& server, tbb_client& client, const size_t i ) + padded_private_worker( private_server& server, tbb_client& client, const std::size_t i ) : private_worker(server,client,i) { suppress_unused_warning(pad); } }; #if _MSC_VER && !defined(__INTEL_COMPILER) - #pragma warning(pop) + // #pragma warning(pop) #endif class private_server: public tbb_server, no_copy { @@ -127,29 +125,29 @@ class private_server: public tbb_server, no_copy { const tbb_client::size_type my_n_thread; //! Stack size for each thread. */ - const size_t my_stack_size; + const std::size_t my_stack_size; //! Number of jobs that could use their associated thread minus number of active threads. /** If negative, indicates oversubscription. If positive, indicates that more threads should run. Can be lowered asynchronously, but must be raised only while holding my_asleep_list_mutex, because raising it impacts the invariant for sleeping threads. */ - atomic<int> my_slack; + std::atomic<int> my_slack; //! Counter used to determine when to delete this. - atomic<int> my_ref_count; + std::atomic<int> my_ref_count; padded_private_worker* my_thread_array; //! List of workers that are asleep or committed to sleeping until notified by another thread. - tbb::atomic<private_worker*> my_asleep_list_root; + std::atomic<private_worker*> my_asleep_list_root; //! Protects my_asleep_list_root - typedef scheduler_mutex_type asleep_list_mutex_type; + typedef mutex asleep_list_mutex_type; asleep_list_mutex_type my_asleep_list_mutex; #if TBB_USE_ASSERT - atomic<int> my_net_slack_requests; + std::atomic<int> my_net_slack_requests; #endif /* TBB_USE_ASSERT */ //! Wake up to two sleeping workers, if there are any sleeping. @@ -157,7 +155,7 @@ class private_server: public tbb_server, no_copy { which in turn each wake up two threads, etc. */ void propagate_chain_reaction() { // First test of a double-check idiom. Second test is inside wake_some(0). - if( my_asleep_list_root ) + if( my_asleep_list_root.load(std::memory_order_relaxed) ) wake_some(0); } @@ -167,7 +165,7 @@ class private_server: public tbb_server, no_copy { //! Equivalent of adding additional_slack to my_slack and waking up to 2 threads if my_slack permits. void wake_some( int additional_slack ); - virtual ~private_server(); + ~private_server() override; void remove_server_ref() { if( --my_ref_count==0 ) { @@ -181,27 +179,27 @@ class private_server: public tbb_server, no_copy { public: private_server( tbb_client& client ); - version_type version() const __TBB_override { + version_type version() const override { return 0; } - void request_close_connection( bool /*exiting*/ ) __TBB_override { - for( size_t i=0; i<my_n_thread; ++i ) + void request_close_connection( bool /*exiting*/ ) override { + for( std::size_t i=0; i<my_n_thread; ++i ) my_thread_array[i].start_shutdown(); remove_server_ref(); } - void yield() __TBB_override {__TBB_Yield();} + void yield() override { d0::yield(); } - void independent_thread_number_changed( int ) __TBB_override {__TBB_ASSERT(false,NULL);} + void independent_thread_number_changed( int ) override {__TBB_ASSERT(false, nullptr);} - unsigned default_concurrency() const __TBB_override { return governor::default_num_threads() - 1; } + unsigned default_concurrency() const override { return governor::default_num_threads() - 1; } - void adjust_job_count_estimate( int delta ) __TBB_override; + void adjust_job_count_estimate( int delta ) override; -#if _WIN32||_WIN64 - void register_master ( ::rml::server::execution_resource_t& ) __TBB_override {} - void unregister_master ( ::rml::server::execution_resource_t ) __TBB_override {} +#if _WIN32 || _WIN64 + void register_external_thread ( ::rml::server::execution_resource_t& ) override {} + void unregister_external_thread ( ::rml::server::execution_resource_t ) override {} #endif /* _WIN32||_WIN64 */ }; @@ -210,8 +208,8 @@ class private_server: public tbb_server, no_copy { //------------------------------------------------------------------------ #if _MSC_VER && !defined(__INTEL_COMPILER) // Suppress overzealous compiler warnings about an initialized variable 'sink_for_alloca' not referenced - #pragma warning(push) - #pragma warning(disable:4189) + // #pragma warning(push) + // #pragma warning(disable:4189) #endif #if __MINGW32__ && __GNUC__==4 &&__GNUC_MINOR__>=2 && !__MINGW64__ // ensure that stack is properly aligned for TBB threads @@ -221,10 +219,11 @@ __RML_DECL_THREAD_ROUTINE private_worker::thread_routine( void* arg ) { private_worker* self = static_cast<private_worker*>(arg); AVOID_64K_ALIASING( self->my_index ); self->run(); + // return 0 instead of nullptr due to the difference in the type __RML_DECL_THREAD_ROUTINE on various OSs return 0; } #if _MSC_VER && !defined(__INTEL_COMPILER) - #pragma warning(pop) + // #pragma warning(pop) #endif void private_worker::release_handle(thread_handle handle, bool join) { @@ -235,13 +234,17 @@ void private_worker::release_handle(thread_handle handle, bool join) { } void private_worker::start_shutdown() { - state_t s; + __TBB_ASSERT(my_state.load(std::memory_order_relaxed) != st_quit, "The quit state is expected to be set only once"); - do { - s = my_state; - __TBB_ASSERT( s!=st_quit, NULL ); - } while( my_state.compare_and_swap( st_quit, s )!=s ); - if( s==st_normal || s==st_starting ) { + // `acq` to acquire my_handle + // `rel` to release market state + state_t prev_state = my_state.exchange(st_quit, std::memory_order_acq_rel); + + if (prev_state == st_init) { + // Perform action that otherwise would be performed by associated thread when it quits. + my_server.remove_server_ref(); + } else { + __TBB_ASSERT(prev_state == st_normal || prev_state == st_starting, nullptr); // May have invalidated invariant for sleeping, so wake up the thread. // Note that the notify() here occurs without maintaining invariants for my_slack. // It does not matter, because my_state==st_quit overrides checking of my_slack. @@ -249,15 +252,12 @@ void private_worker::start_shutdown() { // Do not need release handle in st_init state, // because in this case the thread wasn't started yet. // For st_starting release is done at launch site. - if (s==st_normal) + if (prev_state == st_normal) release_handle(my_handle, governor::does_client_join_workers(my_client)); - } else if( s==st_init ) { - // Perform action that otherwise would be performed by associated thread when it quits. - my_server.remove_server_ref(); } } -void private_worker::run() { +void private_worker::run() noexcept { my_server.propagate_chain_reaction(); // Transiting to st_normal here would require setting my_handle, @@ -265,21 +265,14 @@ void private_worker::run() { // complications in handle management on Windows. ::rml::job& j = *my_client.create_one_job(); - while( my_state!=st_quit ) { - if( my_server.my_slack>=0 ) { + // memory_order_seq_cst to be strictly ordered after thread_monitor::wait on the next iteration + while( my_state.load(std::memory_order_seq_cst)!=st_quit ) { + if( my_server.my_slack.load(std::memory_order_acquire)>=0 ) { my_client.process(j); - } else { - thread_monitor::cookie c; - // Prepare to wait - my_thread_monitor.prepare_wait(c); - // Check/set the invariant for sleeping - if( my_state!=st_quit && my_server.try_insert_in_asleep_list(*this) ) { - my_thread_monitor.commit_wait(c); - my_server.propagate_chain_reaction(); - } else { - // Invariant broken - my_thread_monitor.cancel_wait(); - } + } else if( my_server.try_insert_in_asleep_list(*this) ) { + my_thread_monitor.wait(); + __TBB_ASSERT(my_state.load(std::memory_order_relaxed) == st_quit || !my_next, "Thread monitor missed a spurious wakeup?" ); + my_server.propagate_chain_reaction(); } } my_client.cleanup(j); @@ -289,29 +282,43 @@ void private_worker::run() { } inline void private_worker::wake_or_launch() { - if( my_state==st_init && my_state.compare_and_swap( st_starting, st_init )==st_init ) { - // after this point, remove_server_ref() must be done by created thread -#if USE_WINTHREAD - my_handle = thread_monitor::launch( thread_routine, this, my_server.my_stack_size, &this->my_index ); -#elif USE_PTHREAD - { - affinity_helper fpa; - fpa.protect_affinity_mask( /*restore_process_mask=*/true ); - my_handle = thread_monitor::launch( thread_routine, this, my_server.my_stack_size ); - // Implicit destruction of fpa resets original affinity mask. - } -#endif /* USE_PTHREAD */ - state_t s = my_state.compare_and_swap( st_normal, st_starting ); - if (st_starting != s) { - // Do shutdown during startup. my_handle can't be released - // by start_shutdown, because my_handle value might be not set yet - // at time of transition from st_starting to st_quit. - __TBB_ASSERT( s==st_quit, NULL ); - release_handle(my_handle, governor::does_client_join_workers(my_client)); + state_t state = my_state.load(std::memory_order_relaxed); + + switch (state) { + case st_starting: + __TBB_fallthrough; + case st_normal: + __TBB_ASSERT(!my_next, "Should not wake a thread while it's still in asleep list"); + my_thread_monitor.notify(); + break; + case st_init: + if (my_state.compare_exchange_strong(state, st_starting)) { + // after this point, remove_server_ref() must be done by created thread +#if __TBB_USE_WINAPI + // Win thread_monitor::launch is designed on the assumption that the workers thread id go from 1 to Hard limit set by TBB market::global_market + const std::size_t worker_idx = my_server.my_n_thread - this->my_index; + my_handle = thread_monitor::launch(thread_routine, this, my_server.my_stack_size, &worker_idx); +#elif __TBB_USE_POSIX + { + affinity_helper fpa; + fpa.protect_affinity_mask( /*restore_process_mask=*/true); + my_handle = thread_monitor::launch(thread_routine, this, my_server.my_stack_size); + // Implicit destruction of fpa resets original affinity mask. + } +#endif /* __TBB_USE_POSIX */ + state = st_starting; + if (!my_state.compare_exchange_strong(state, st_normal)) { + // Do shutdown during startup. my_handle can't be released + // by start_shutdown, because my_handle value might be not set yet + // at time of transition from st_starting to st_quit. + __TBB_ASSERT(state == st_quit, nullptr); + release_handle(my_handle, governor::does_client_join_workers(my_client)); + } } + break; + default: + __TBB_ASSERT(state == st_quit, nullptr); } - else - my_thread_monitor.notify(); } //------------------------------------------------------------------------ @@ -321,28 +328,28 @@ private_server::private_server( tbb_client& client ) : my_client(client), my_n_thread(client.max_job_count()), my_stack_size(client.min_stack_size()), - my_thread_array(NULL) -{ - my_ref_count = my_n_thread+1; - my_slack = 0; + my_slack(0), + my_ref_count(my_n_thread+1), + my_thread_array(nullptr), + my_asleep_list_root(nullptr) #if TBB_USE_ASSERT - my_net_slack_requests = 0; + , my_net_slack_requests(0) #endif /* TBB_USE_ASSERT */ - my_asleep_list_root = NULL; +{ my_thread_array = tbb::cache_aligned_allocator<padded_private_worker>().allocate( my_n_thread ); - for( size_t i=0; i<my_n_thread; ++i ) { + for( std::size_t i=0; i<my_n_thread; ++i ) { private_worker* t = new( &my_thread_array[i] ) padded_private_worker( *this, client, i ); - t->my_next = my_asleep_list_root; - my_asleep_list_root = t; + t->my_next = my_asleep_list_root.load(std::memory_order_relaxed); + my_asleep_list_root.store(t, std::memory_order_relaxed); } } private_server::~private_server() { - __TBB_ASSERT( my_net_slack_requests==0, NULL ); - for( size_t i=my_n_thread; i--; ) + __TBB_ASSERT( my_net_slack_requests==0, nullptr); + for( std::size_t i=my_n_thread; i--; ) my_thread_array[i].~padded_private_worker(); tbb::cache_aligned_allocator<padded_private_worker>().deallocate( my_thread_array, my_n_thread ); - tbb::internal::poison_pointer( my_thread_array ); + tbb::detail::poison_pointer( my_thread_array ); } inline bool private_server::try_insert_in_asleep_list( private_worker& t ) { @@ -351,47 +358,59 @@ inline bool private_server::try_insert_in_asleep_list( private_worker& t ) { return false; // Contribute to slack under lock so that if another takes that unit of slack, // it sees us sleeping on the list and wakes us up. - int k = ++my_slack; - if( k<=0 ) { - t.my_next = my_asleep_list_root; - my_asleep_list_root = &t; - return true; - } else { - --my_slack; - return false; + auto expected = my_slack.load(std::memory_order_relaxed); + while (expected < 0) { + if (my_slack.compare_exchange_strong(expected, expected + 1)) { + t.my_next = my_asleep_list_root.load(std::memory_order_relaxed); + my_asleep_list_root.store(&t, std::memory_order_relaxed); + return true; + } } + + return false; } void private_server::wake_some( int additional_slack ) { - __TBB_ASSERT( additional_slack>=0, NULL ); + __TBB_ASSERT( additional_slack>=0, nullptr ); private_worker* wakee[2]; private_worker**w = wakee; - { + + if (additional_slack) { + // Contribute our unused slack to my_slack. + my_slack += additional_slack; + } + + int allotted_slack = 0; + while (allotted_slack < 2) { + // Chain reaction; Try to claim unit of slack + int old = my_slack.load(std::memory_order_relaxed); + do { + if (old <= 0) goto done; + } while (!my_slack.compare_exchange_strong(old, old - 1)); + ++allotted_slack; + } +done: + + if (allotted_slack) { asleep_list_mutex_type::scoped_lock lock(my_asleep_list_mutex); - while( my_asleep_list_root && w<wakee+2 ) { - if( additional_slack>0 ) { - if (additional_slack+my_slack<=0) // additional demand does not exceed surplus supply - break; - --additional_slack; - } else { - // Chain reaction; Try to claim unit of slack - int old; - do { - old = my_slack; - if( old<=0 ) goto done; - } while( my_slack.compare_and_swap(old-1,old)!=old ); - } + auto root = my_asleep_list_root.load(std::memory_order_relaxed); + while( root && w<wakee+2 && allotted_slack) { + --allotted_slack; // Pop sleeping worker to combine with claimed unit of slack - my_asleep_list_root = (*w++ = my_asleep_list_root)->my_next; + *w++ = root; + root = root->my_next; } - if( additional_slack ) { + my_asleep_list_root.store(root, std::memory_order_relaxed); + if(allotted_slack) { // Contribute our unused slack to my_slack. - my_slack += additional_slack; + my_slack += allotted_slack; } } -done: - while( w>wakee ) - (*--w)->wake_or_launch(); + while( w>wakee ) { + private_worker* ww = *--w; + ww->my_next = nullptr; + ww->wake_or_launch(); + } } void private_server::adjust_job_count_estimate( int delta ) { @@ -411,6 +430,7 @@ tbb_server* make_private_server( tbb_client& client ) { } } // namespace rml -} // namespace internal - +} // namespace r1 +} // namespace detail } // namespace tbb + diff --git a/src/tbb/src/tbb/profiling.cpp b/src/tbb/src/tbb/profiling.cpp new file mode 100644 index 000000000..3cf4da3ce --- /dev/null +++ b/src/tbb/src/tbb/profiling.cpp @@ -0,0 +1,265 @@ +/* + Copyright (c) 2005-2022 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "oneapi/tbb/detail/_config.h" +#include "oneapi/tbb/detail/_template_helpers.h" + +#include "main.h" +#include "itt_notify.h" + +#include "oneapi/tbb/profiling.h" + +#include <string.h> + +namespace tbb { +namespace detail { +namespace r1 { + +#if __TBB_USE_ITT_NOTIFY +bool ITT_Present; +static std::atomic<bool> ITT_InitializationDone; + +static __itt_domain *tbb_domains[d1::ITT_NUM_DOMAINS] = {}; + +struct resource_string { + const char *str; + __itt_string_handle *itt_str_handle; +}; + +// +// populate resource strings +// +#define TBB_STRING_RESOURCE( index_name, str ) { str, nullptr }, +static resource_string strings_for_itt[] = { + #include "oneapi/tbb/detail/_string_resource.h" + { "num_resource_strings", nullptr } +}; +#undef TBB_STRING_RESOURCE + +static __itt_string_handle* ITT_get_string_handle(std::uintptr_t idx) { + __TBB_ASSERT(idx < NUM_STRINGS, "string handle out of valid range"); + return idx < NUM_STRINGS ? strings_for_itt[idx].itt_str_handle : nullptr; +} + +static void ITT_init_domains() { + tbb_domains[d1::ITT_DOMAIN_MAIN] = __itt_domain_create( _T("tbb") ); + tbb_domains[d1::ITT_DOMAIN_MAIN]->flags = 1; + tbb_domains[d1::ITT_DOMAIN_FLOW] = __itt_domain_create( _T("tbb.flow") ); + tbb_domains[d1::ITT_DOMAIN_FLOW]->flags = 1; + tbb_domains[d1::ITT_DOMAIN_ALGO] = __itt_domain_create( _T("tbb.algorithm") ); + tbb_domains[d1::ITT_DOMAIN_ALGO]->flags = 1; +} + +static void ITT_init_strings() { + for ( std::uintptr_t i = 0; i < NUM_STRINGS; ++i ) { +#if _WIN32||_WIN64 + strings_for_itt[i].itt_str_handle = __itt_string_handle_createA( strings_for_itt[i].str ); +#else + strings_for_itt[i].itt_str_handle = __itt_string_handle_create( strings_for_itt[i].str ); +#endif + } +} + +static void ITT_init() { + ITT_init_domains(); + ITT_init_strings(); +} + +/** Thread-unsafe lazy one-time initialization of tools interop. + Used by both dummy handlers and general TBB one-time initialization routine. **/ +void ITT_DoUnsafeOneTimeInitialization () { + // Double check ITT_InitializationDone is necessary because the first check + // in ITT_DoOneTimeInitialization is not guarded with the __TBB_InitOnce lock. + if ( !ITT_InitializationDone ) { + ITT_Present = (__TBB_load_ittnotify()!=0); + if (ITT_Present) ITT_init(); + ITT_InitializationDone = true; + } +} + +/** Thread-safe lazy one-time initialization of tools interop. + Used by dummy handlers only. **/ +extern "C" +void ITT_DoOneTimeInitialization() { + if ( !ITT_InitializationDone ) { + __TBB_InitOnce::lock(); + ITT_DoUnsafeOneTimeInitialization(); + __TBB_InitOnce::unlock(); + } +} + +void create_itt_sync(void* ptr, const tchar* objtype, const tchar* objname) { + ITT_SYNC_CREATE(ptr, objtype, objname); +} + +void call_itt_notify(int t, void *ptr) { + switch (t) { + case 0: ITT_NOTIFY(sync_prepare, ptr); break; + case 1: ITT_NOTIFY(sync_cancel, ptr); break; + case 2: ITT_NOTIFY(sync_acquired, ptr); break; + case 3: ITT_NOTIFY(sync_releasing, ptr); break; + case 4: ITT_NOTIFY(sync_destroy, ptr); break; + } +} + +void itt_set_sync_name(void* obj, const tchar* name) { + __itt_sync_rename(obj, name); +} + +const __itt_id itt_null_id = { 0, 0, 0 }; + +static inline __itt_domain* get_itt_domain(d1::itt_domain_enum idx) { + if (tbb_domains[idx] == nullptr) { + ITT_DoOneTimeInitialization(); + } + return tbb_domains[idx]; +} + +static inline void itt_id_make(__itt_id* id, void* addr, unsigned long long extra) { + *id = __itt_id_make(addr, extra); +} + +static inline void itt_id_create(const __itt_domain* domain, __itt_id id) { + __itt_id_create(domain, id); +} + +void itt_make_task_group(d1::itt_domain_enum domain, void* group, unsigned long long group_extra, + void* parent, unsigned long long parent_extra, string_resource_index name_index) { + if (__itt_domain* d = get_itt_domain(domain)) { + __itt_id group_id = itt_null_id; + __itt_id parent_id = itt_null_id; + itt_id_make(&group_id, group, group_extra); + itt_id_create(d, group_id); + if (parent) { + itt_id_make(&parent_id, parent, parent_extra); + } + __itt_string_handle* n = ITT_get_string_handle(name_index); + __itt_task_group(d, group_id, parent_id, n); + } +} + +void __TBB_EXPORTED_FUNC itt_metadata_str_add(d1::itt_domain_enum domain, void *addr, unsigned long long addr_extra, + string_resource_index key, const char *value ) { + if ( __itt_domain *d = get_itt_domain( domain ) ) { + __itt_id id = itt_null_id; + itt_id_make( &id, addr, addr_extra ); + __itt_string_handle *k = ITT_get_string_handle(key); + size_t value_length = strlen( value ); +#if _WIN32||_WIN64 + __itt_metadata_str_addA(d, id, k, value, value_length); +#else + __itt_metadata_str_add(d, id, k, value, value_length); +#endif + } +} + +void __TBB_EXPORTED_FUNC itt_metadata_ptr_add(d1::itt_domain_enum domain, void *addr, unsigned long long addr_extra, + string_resource_index key, void *value ) { + if ( __itt_domain *d = get_itt_domain( domain ) ) { + __itt_id id = itt_null_id; + itt_id_make( &id, addr, addr_extra ); + __itt_string_handle *k = ITT_get_string_handle(key); +#if __TBB_x86_32 + __itt_metadata_add(d, id, k, __itt_metadata_u32, 1, value); +#else + __itt_metadata_add(d, id, k, __itt_metadata_u64, 1, value); +#endif + } +} + +void __TBB_EXPORTED_FUNC itt_relation_add(d1::itt_domain_enum domain, void *addr0, unsigned long long addr0_extra, + itt_relation relation, void *addr1, unsigned long long addr1_extra ) { + if ( __itt_domain *d = get_itt_domain( domain ) ) { + __itt_id id0 = itt_null_id; + __itt_id id1 = itt_null_id; + itt_id_make( &id0, addr0, addr0_extra ); + itt_id_make( &id1, addr1, addr1_extra ); + __itt_relation_add( d, id0, (__itt_relation)relation, id1 ); + } +} + +void __TBB_EXPORTED_FUNC itt_task_begin(d1::itt_domain_enum domain, void* task, unsigned long long task_extra, + void* parent, unsigned long long parent_extra, string_resource_index name_index) { + if (__itt_domain* d = get_itt_domain(domain)) { + __itt_id task_id = itt_null_id; + __itt_id parent_id = itt_null_id; + if (task) { + itt_id_make(&task_id, task, task_extra); + } + if (parent) { + itt_id_make(&parent_id, parent, parent_extra); + } + __itt_string_handle* n = ITT_get_string_handle(name_index); + __itt_task_begin(d, task_id, parent_id, n); + } +} + +void __TBB_EXPORTED_FUNC itt_task_end(d1::itt_domain_enum domain) { + if (__itt_domain* d = get_itt_domain(domain)) { + __itt_task_end(d); + } +} + +void __TBB_EXPORTED_FUNC itt_region_begin(d1::itt_domain_enum domain, void *region, unsigned long long region_extra, + void *parent, unsigned long long parent_extra, string_resource_index /* name_index */ ) { + if ( __itt_domain *d = get_itt_domain( domain ) ) { + __itt_id region_id = itt_null_id; + __itt_id parent_id = itt_null_id; + itt_id_make( ®ion_id, region, region_extra ); + if ( parent ) { + itt_id_make( &parent_id, parent, parent_extra ); + } + __itt_region_begin( d, region_id, parent_id, nullptr ); + } +} + +void __TBB_EXPORTED_FUNC itt_region_end(d1::itt_domain_enum domain, void *region, unsigned long long region_extra ) { + if ( __itt_domain *d = get_itt_domain( domain ) ) { + __itt_id region_id = itt_null_id; + itt_id_make( ®ion_id, region, region_extra ); + __itt_region_end( d, region_id ); + } +} + +#else +void create_itt_sync(void* /*ptr*/, const tchar* /*objtype*/, const tchar* /*objname*/) {} +void call_itt_notify(int /*t*/, void* /*ptr*/) {} +void itt_set_sync_name(void* /*obj*/, const tchar* /*name*/) {} +void itt_make_task_group(d1::itt_domain_enum /*domain*/, void* /*group*/, unsigned long long /*group_extra*/, + void* /*parent*/, unsigned long long /*parent_extra*/, string_resource_index /*name_index*/) {} +void itt_metadata_str_add(d1::itt_domain_enum /*domain*/, void* /*addr*/, unsigned long long /*addr_extra*/, + string_resource_index /*key*/, const char* /*value*/ ) { } +void itt_metadata_ptr_add(d1::itt_domain_enum /*domain*/, void * /*addr*/, unsigned long long /*addr_extra*/, + string_resource_index /*key*/, void * /*value*/ ) {} +void itt_relation_add(d1::itt_domain_enum /*domain*/, void* /*addr0*/, unsigned long long /*addr0_extra*/, + itt_relation /*relation*/, void* /*addr1*/, unsigned long long /*addr1_extra*/ ) { } +void itt_task_begin(d1::itt_domain_enum /*domain*/, void* /*task*/, unsigned long long /*task_extra*/, + void* /*parent*/, unsigned long long /*parent_extra*/, string_resource_index /*name_index*/ ) { } +void itt_task_end(d1::itt_domain_enum /*domain*/ ) { } +void itt_region_begin(d1::itt_domain_enum /*domain*/, void* /*region*/, unsigned long long /*region_extra*/, + void* /*parent*/, unsigned long long /*parent_extra*/, string_resource_index /*name_index*/ ) { } +void itt_region_end(d1::itt_domain_enum /*domain*/, void* /*region*/, unsigned long long /*region_extra*/ ) { } +#endif /* __TBB_USE_ITT_NOTIFY */ + +const tchar + *SyncType_Scheduler = _T("%Constant") + ; +const tchar + *SyncObj_ContextsList = _T("TBB Scheduler") + ; +} // namespace r1 +} // namespace detail +} // namespace tbb diff --git a/src/tbb/src/tbb/queuing_mutex.cpp b/src/tbb/src/tbb/queuing_mutex.cpp deleted file mode 100644 index 0071f8b9f..000000000 --- a/src/tbb/src/tbb/queuing_mutex.cpp +++ /dev/null @@ -1,108 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#include "tbb/queuing_mutex.h" -#include "tbb/tbb_machine.h" -#include "tbb/tbb_stddef.h" -#include "tbb_misc.h" -#include "itt_notify.h" - -namespace tbb { - -using namespace internal; - -//! A method to acquire queuing_mutex lock -void queuing_mutex::scoped_lock::acquire( queuing_mutex& m ) -{ - __TBB_ASSERT( !this->mutex, "scoped_lock is already holding a mutex"); - - // Must set all fields before the fetch_and_store, because once the - // fetch_and_store executes, *this becomes accessible to other threads. - mutex = &m; - next = NULL; - going = 0; - - // The fetch_and_store must have release semantics, because we are - // "sending" the fields initialized above to other processors. - scoped_lock* pred = m.q_tail.fetch_and_store<tbb::release>(this); - if( pred ) { - ITT_NOTIFY(sync_prepare, mutex); -#if TBB_USE_ASSERT - __TBB_control_consistency_helper(); // on "m.q_tail" - __TBB_ASSERT( !pred->next, "the predecessor has another successor!"); -#endif - pred->next = this; - spin_wait_while_eq( going, 0ul ); - } - ITT_NOTIFY(sync_acquired, mutex); - - // Force acquire so that user's critical section receives correct values - // from processor that was previously in the user's critical section. - __TBB_load_with_acquire(going); -} - -//! A method to acquire queuing_mutex if it is free -bool queuing_mutex::scoped_lock::try_acquire( queuing_mutex& m ) -{ - __TBB_ASSERT( !this->mutex, "scoped_lock is already holding a mutex"); - - // Must set all fields before the fetch_and_store, because once the - // fetch_and_store executes, *this becomes accessible to other threads. - next = NULL; - going = 0; - - // The CAS must have release semantics, because we are - // "sending" the fields initialized above to other processors. - if( m.q_tail.compare_and_swap<tbb::release>(this, NULL) ) - return false; - - // Force acquire so that user's critical section receives correct values - // from processor that was previously in the user's critical section. - __TBB_load_with_acquire(going); - mutex = &m; - ITT_NOTIFY(sync_acquired, mutex); - return true; -} - -//! A method to release queuing_mutex lock -void queuing_mutex::scoped_lock::release( ) -{ - __TBB_ASSERT(this->mutex!=NULL, "no lock acquired"); - - ITT_NOTIFY(sync_releasing, mutex); - if( !next ) { - if( this == mutex->q_tail.compare_and_swap<tbb::release>(NULL, this) ) { - // this was the only item in the queue, and the queue is now empty. - goto done; - } - // Someone in the queue - spin_wait_while_eq( next, (scoped_lock*)0 ); - } - __TBB_ASSERT(next,NULL); - __TBB_store_with_release(next->going, 1); -done: - initialize(); -} - -void queuing_mutex::internal_construct() { - ITT_SYNC_CREATE(this, _T("tbb::queuing_mutex"), _T("")); -} - -} // namespace tbb diff --git a/src/tbb/src/tbb/queuing_rw_mutex.cpp b/src/tbb/src/tbb/queuing_rw_mutex.cpp index f19a86bdb..b4590b5b7 100644 --- a/src/tbb/src/tbb/queuing_rw_mutex.cpp +++ b/src/tbb/src/tbb/queuing_rw_mutex.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2022 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,481 +12,606 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ /** Before making any changes in the implementation, please emulate algorithmic changes with SPIN tool using <TBB directory>/tools/spin_models/ReaderWriterMutex.pml. There could be some code looking as "can be restructured" but its structure does matter! */ -#include "tbb/queuing_rw_mutex.h" -#include "tbb/tbb_machine.h" -#include "tbb/tbb_stddef.h" -#include "tbb/tbb_machine.h" +#include "oneapi/tbb/queuing_rw_mutex.h" +#include "oneapi/tbb/detail/_assert.h" +#include "oneapi/tbb/detail/_utils.h" #include "itt_notify.h" - namespace tbb { - -using namespace internal; - -//! Flag bits in a state_t that specify information about a locking request. -enum state_t_flags { - STATE_NONE = 0, - STATE_WRITER = 1<<0, - STATE_READER = 1<<1, - STATE_READER_UNBLOCKNEXT = 1<<2, - STATE_ACTIVEREADER = 1<<3, - STATE_UPGRADE_REQUESTED = 1<<4, - STATE_UPGRADE_WAITING = 1<<5, - STATE_UPGRADE_LOSER = 1<<6, - STATE_COMBINED_WAITINGREADER = STATE_READER | STATE_READER_UNBLOCKNEXT, - STATE_COMBINED_READER = STATE_COMBINED_WAITINGREADER | STATE_ACTIVEREADER, - STATE_COMBINED_UPGRADING = STATE_UPGRADE_WAITING | STATE_UPGRADE_LOSER -}; - -const unsigned char RELEASED = 0; -const unsigned char ACQUIRED = 1; - -inline bool queuing_rw_mutex::scoped_lock::try_acquire_internal_lock() -{ - return as_atomic(my_internal_lock).compare_and_swap<tbb::acquire>(ACQUIRED,RELEASED) == RELEASED; -} - -inline void queuing_rw_mutex::scoped_lock::acquire_internal_lock() -{ - // Usually, we would use the test-test-and-set idiom here, with exponential backoff. - // But so far, experiments indicate there is no value in doing so here. - while( !try_acquire_internal_lock() ) { - __TBB_Pause(1); - } -} - -inline void queuing_rw_mutex::scoped_lock::release_internal_lock() -{ - __TBB_store_with_release(my_internal_lock,RELEASED); -} - -inline void queuing_rw_mutex::scoped_lock::wait_for_release_of_internal_lock() -{ - spin_wait_until_eq(my_internal_lock, RELEASED); -} - -inline void queuing_rw_mutex::scoped_lock::unblock_or_wait_on_internal_lock( uintptr_t flag ) { - if( flag ) - wait_for_release_of_internal_lock(); - else - release_internal_lock(); -} +namespace detail { +namespace r1 { #if defined(_MSC_VER) && !defined(__INTEL_COMPILER) // Workaround for overzealous compiler warnings - #pragma warning (push) - #pragma warning (disable: 4311 4312) + // #pragma warning (push) + // #pragma warning (disable: 4311 4312) #endif //! A view of a T* with additional functionality for twiddling low-order bits. template<typename T> -class tricky_atomic_pointer: no_copy { +class tricky_atomic_pointer { public: - typedef typename atomic_selector<sizeof(T*)>::word word; + using word = uintptr_t; + + static T* fetch_add( std::atomic<word>& location, word addend, std::memory_order memory_order ) { + return reinterpret_cast<T*>(location.fetch_add(addend, memory_order)); + } + + static T* exchange( std::atomic<word>& location, T* value, std::memory_order memory_order ) { + return reinterpret_cast<T*>(location.exchange(reinterpret_cast<word>(value), memory_order)); + } + + static T* compare_exchange_strong( std::atomic<word>& obj, const T* expected, const T* desired, std::memory_order memory_order ) { + word expd = reinterpret_cast<word>(expected); + obj.compare_exchange_strong(expd, reinterpret_cast<word>(desired), memory_order); + return reinterpret_cast<T*>(expd); + } - template<memory_semantics M> - static T* fetch_and_add( T* volatile * location, word addend ) { - return reinterpret_cast<T*>( atomic_traits<sizeof(T*),M>::fetch_and_add(location, addend) ); + static void store( std::atomic<word>& location, const T* value, std::memory_order memory_order ) { + location.store(reinterpret_cast<word>(value), memory_order); } - template<memory_semantics M> - static T* fetch_and_store( T* volatile * location, T* value ) { - return reinterpret_cast<T*>( atomic_traits<sizeof(T*),M>::fetch_and_store(location, reinterpret_cast<word>(value)) ); + + static T* load( std::atomic<word>& location, std::memory_order memory_order ) { + return reinterpret_cast<T*>(location.load(memory_order)); } - template<memory_semantics M> - static T* compare_and_swap( T* volatile * location, T* value, T* comparand ) { - return reinterpret_cast<T*>( - atomic_traits<sizeof(T*),M>::compare_and_swap(location, reinterpret_cast<word>(value), - reinterpret_cast<word>(comparand)) - ); + + static void spin_wait_while_eq(const std::atomic<word>& location, const T* value) { + tbb::detail::d0::spin_wait_while_eq(location, reinterpret_cast<word>(value) ); } T* & ref; tricky_atomic_pointer( T*& original ) : ref(original) {}; - tricky_atomic_pointer( T* volatile & original ) : ref(original) {}; - T* operator&( word operand2 ) const { + tricky_atomic_pointer(const tricky_atomic_pointer&) = delete; + tricky_atomic_pointer& operator=(const tricky_atomic_pointer&) = delete; + T* operator&( const word operand2 ) const { return reinterpret_cast<T*>( reinterpret_cast<word>(ref) & operand2 ); } - T* operator|( word operand2 ) const { + T* operator|( const word operand2 ) const { return reinterpret_cast<T*>( reinterpret_cast<word>(ref) | operand2 ); } }; -typedef tricky_atomic_pointer<queuing_rw_mutex::scoped_lock> tricky_pointer; +using tricky_pointer = tricky_atomic_pointer<queuing_rw_mutex::scoped_lock>; #if defined(_MSC_VER) && !defined(__INTEL_COMPILER) // Workaround for overzealous compiler warnings - #pragma warning (pop) + // #pragma warning (pop) #endif -//! Mask for low order bit of a pointer. -static const tricky_pointer::word FLAG = 0x1; +//! Flag bits in a state_t that specify information about a locking request. +enum state_t_flags : unsigned char { + STATE_NONE = 0, + STATE_WRITER = 1<<0, + STATE_READER = 1<<1, + STATE_READER_UNBLOCKNEXT = 1<<2, + STATE_ACTIVEREADER = 1<<3, + STATE_UPGRADE_REQUESTED = 1<<4, + STATE_UPGRADE_WAITING = 1<<5, + STATE_UPGRADE_LOSER = 1<<6, + STATE_COMBINED_WAITINGREADER = STATE_READER | STATE_READER_UNBLOCKNEXT, + STATE_COMBINED_READER = STATE_COMBINED_WAITINGREADER | STATE_ACTIVEREADER, + STATE_COMBINED_UPGRADING = STATE_UPGRADE_WAITING | STATE_UPGRADE_LOSER +}; -inline -uintptr_t get_flag( queuing_rw_mutex::scoped_lock* ptr ) { - return uintptr_t(ptr) & FLAG; -} +static const unsigned char RELEASED = 0; +static const unsigned char ACQUIRED = 1; -//------------------------------------------------------------------------ -// Methods of queuing_rw_mutex::scoped_lock -//------------------------------------------------------------------------ - -//! A method to acquire queuing_rw_mutex lock -void queuing_rw_mutex::scoped_lock::acquire( queuing_rw_mutex& m, bool write ) -{ - __TBB_ASSERT( !my_mutex, "scoped_lock is already holding a mutex"); - - // Must set all fields before the fetch_and_store, because once the - // fetch_and_store executes, *this becomes accessible to other threads. - my_mutex = &m; - __TBB_store_relaxed(my_prev , (scoped_lock*)0); - __TBB_store_relaxed(my_next , (scoped_lock*)0); - __TBB_store_relaxed(my_going, 0); - my_state = state_t(write ? STATE_WRITER : STATE_READER); - my_internal_lock = RELEASED; - - queuing_rw_mutex::scoped_lock* pred = m.q_tail.fetch_and_store<tbb::release>(this); - - if( write ) { // Acquiring for write - - if( pred ) { - ITT_NOTIFY(sync_prepare, my_mutex); - pred = tricky_pointer(pred) & ~FLAG; - __TBB_ASSERT( !( uintptr_t(pred) & FLAG ), "use of corrupted pointer!" ); -#if TBB_USE_ASSERT - __TBB_control_consistency_helper(); // on "m.q_tail" - __TBB_ASSERT( !__TBB_load_relaxed(pred->my_next), "the predecessor has another successor!"); -#endif - __TBB_store_with_release(pred->my_next,this); - spin_wait_until_eq(my_going, 1); - } +struct queuing_rw_mutex_impl { + //! Try to acquire the internal lock + /** Returns true if lock was successfully acquired. */ + static bool try_acquire_internal_lock(d1::queuing_rw_mutex::scoped_lock& s) + { + auto expected = RELEASED; + return s.my_internal_lock.compare_exchange_strong(expected, ACQUIRED); + } - } else { // Acquiring for read -#if DO_ITT_NOTIFY - bool sync_prepare_done = false; -#endif - if( pred ) { - unsigned short pred_state; - __TBB_ASSERT( !__TBB_load_relaxed(my_prev), "the predecessor is already set" ); - if( uintptr_t(pred) & FLAG ) { - /* this is only possible if pred is an upgrading reader and it signals us to wait */ - pred_state = STATE_UPGRADE_WAITING; - pred = tricky_pointer(pred) & ~FLAG; - } else { - // Load pred->my_state now, because once pred->my_next becomes - // non-NULL, we must assume that *pred might be destroyed. - pred_state = pred->my_state.compare_and_swap<tbb::acquire>(STATE_READER_UNBLOCKNEXT, STATE_READER); - } - __TBB_store_relaxed(my_prev, pred); - __TBB_ASSERT( !( uintptr_t(pred) & FLAG ), "use of corrupted pointer!" ); -#if TBB_USE_ASSERT - __TBB_control_consistency_helper(); // on "m.q_tail" - __TBB_ASSERT( !__TBB_load_relaxed(pred->my_next), "the predecessor has another successor!"); -#endif - __TBB_store_with_release(pred->my_next,this); - if( pred_state != STATE_ACTIVEREADER ) { -#if DO_ITT_NOTIFY - sync_prepare_done = true; - ITT_NOTIFY(sync_prepare, my_mutex); -#endif - spin_wait_until_eq(my_going, 1); - } + //! Acquire the internal lock + static void acquire_internal_lock(d1::queuing_rw_mutex::scoped_lock& s) + { + // Usually, we would use the test-test-and-set idiom here, with exponential backoff. + // But so far, experiments indicate there is no value in doing so here. + while( !try_acquire_internal_lock(s) ) { + machine_pause(1); } + } - // The protected state must have been acquired here before it can be further released to any other reader(s): - unsigned short old_state = my_state.compare_and_swap<tbb::acquire>(STATE_ACTIVEREADER, STATE_READER); - if( old_state!=STATE_READER ) { -#if DO_ITT_NOTIFY - if( !sync_prepare_done ) - ITT_NOTIFY(sync_prepare, my_mutex); -#endif - // Failed to become active reader -> need to unblock the next waiting reader first - __TBB_ASSERT( my_state==STATE_READER_UNBLOCKNEXT, "unexpected state" ); - spin_wait_while_eq(my_next, (scoped_lock*)NULL); - /* my_state should be changed before unblocking the next otherwise it might finish - and another thread can get our old state and left blocked */ - my_state = STATE_ACTIVEREADER; - __TBB_store_with_release(my_next->my_going,1); - } + //! Release the internal lock + static void release_internal_lock(d1::queuing_rw_mutex::scoped_lock& s) + { + s.my_internal_lock.store(RELEASED, std::memory_order_release); } - ITT_NOTIFY(sync_acquired, my_mutex); + //! Wait for internal lock to be released + static void wait_for_release_of_internal_lock(d1::queuing_rw_mutex::scoped_lock& s) + { + spin_wait_until_eq(s.my_internal_lock, RELEASED); + } - // Force acquire so that user's critical section receives correct values - // from processor that was previously in the user's critical section. - __TBB_load_with_acquire(my_going); -} + //! A helper function + static void unblock_or_wait_on_internal_lock(d1::queuing_rw_mutex::scoped_lock& s, uintptr_t flag ) { + if( flag ) { + wait_for_release_of_internal_lock(s); + } + else { + release_internal_lock(s); + } + } -//! A method to acquire queuing_rw_mutex if it is free -bool queuing_rw_mutex::scoped_lock::try_acquire( queuing_rw_mutex& m, bool write ) -{ - __TBB_ASSERT( !my_mutex, "scoped_lock is already holding a mutex"); - - if( load<relaxed>(m.q_tail) ) - return false; // Someone already took the lock - - // Must set all fields before the fetch_and_store, because once the - // fetch_and_store executes, *this becomes accessible to other threads. - __TBB_store_relaxed(my_prev, (scoped_lock*)0); - __TBB_store_relaxed(my_next, (scoped_lock*)0); - __TBB_store_relaxed(my_going, 0); // TODO: remove dead assignment? - my_state = state_t(write ? STATE_WRITER : STATE_ACTIVEREADER); - my_internal_lock = RELEASED; - - // The CAS must have release semantics, because we are - // "sending" the fields initialized above to other processors. - if( m.q_tail.compare_and_swap<tbb::release>(this, NULL) ) - return false; // Someone already took the lock - // Force acquire so that user's critical section receives correct values - // from processor that was previously in the user's critical section. - __TBB_load_with_acquire(my_going); - my_mutex = &m; - ITT_NOTIFY(sync_acquired, my_mutex); - return true; -} + //! Mask for low order bit of a pointer. + static const tricky_pointer::word FLAG = 0x1; -//! A method to release queuing_rw_mutex lock -void queuing_rw_mutex::scoped_lock::release( ) -{ - __TBB_ASSERT(my_mutex!=NULL, "no lock acquired"); + static uintptr_t get_flag( d1::queuing_rw_mutex::scoped_lock* ptr ) { + return reinterpret_cast<uintptr_t>(ptr) & FLAG; + } - ITT_NOTIFY(sync_releasing, my_mutex); + //------------------------------------------------------------------------ + // Methods of queuing_rw_mutex::scoped_lock + //------------------------------------------------------------------------ + + //! A method to acquire queuing_rw_mutex lock + static void acquire(d1::queuing_rw_mutex& m, d1::queuing_rw_mutex::scoped_lock& s, bool write) + { + __TBB_ASSERT( !s.my_mutex, "scoped_lock is already holding a mutex"); + + // Must set all fields before the exchange, because once the + // exchange executes, *this becomes accessible to other threads. + s.my_mutex = &m; + s.my_prev.store(0U, std::memory_order_relaxed); + s.my_next.store(0U, std::memory_order_relaxed); + s.my_going.store(0U, std::memory_order_relaxed); + s.my_state.store(d1::queuing_rw_mutex::scoped_lock::state_t(write ? STATE_WRITER : STATE_READER), std::memory_order_relaxed); + s.my_internal_lock.store(RELEASED, std::memory_order_relaxed); + + + // The CAS must have release semantics, because we are + // "sending" the fields initialized above to other actors. + // We need acquire semantics, because we are acquiring the predecessor (or mutex if no predecessor) + queuing_rw_mutex::scoped_lock* predecessor = m.q_tail.exchange(&s, std::memory_order_acq_rel); + + if( write ) { // Acquiring for write + + if( predecessor ) { + ITT_NOTIFY(sync_prepare, s.my_mutex); + predecessor = tricky_pointer(predecessor) & ~FLAG; + __TBB_ASSERT( !predecessor->my_next, "the predecessor has another successor!"); + tricky_pointer::store(predecessor->my_next, &s, std::memory_order_release); + // We are acquiring the mutex + spin_wait_until_eq(s.my_going, 1U, std::memory_order_acquire); + } - if( my_state == STATE_WRITER ) { // Acquired for write + } else { // Acquiring for read + #if __TBB_USE_ITT_NOTIFY + bool sync_prepare_done = false; + #endif + if( predecessor ) { + unsigned char pred_state{}; + __TBB_ASSERT( !s.my_prev.load(std::memory_order_relaxed), "the predecessor is already set" ); + if( tricky_pointer(predecessor) & FLAG ) { + /* this is only possible if predecessor is an upgrading reader and it signals us to wait */ + pred_state = STATE_UPGRADE_WAITING; + predecessor = tricky_pointer(predecessor) & ~FLAG; + } else { + // Load predecessor->my_state now, because once predecessor->my_next becomes + // non-null, we must assume that *predecessor might be destroyed. + pred_state = predecessor->my_state.load(std::memory_order_relaxed); + if (pred_state == STATE_READER) { + // Notify the previous reader to unblock us. + predecessor->my_state.compare_exchange_strong(pred_state, STATE_READER_UNBLOCKNEXT, std::memory_order_relaxed); + } + if (pred_state == STATE_ACTIVEREADER) { // either we initially read it or CAS failed + // Active reader means that the predecessor already acquired the mutex and cannot notify us. + // Therefore, we need to acquire the mutex ourselves by re-reading predecessor state. + (void)predecessor->my_state.load(std::memory_order_acquire); + } + } + tricky_pointer::store(s.my_prev, predecessor, std::memory_order_relaxed); + __TBB_ASSERT( !( tricky_pointer(predecessor) & FLAG ), "use of corrupted pointer!" ); + __TBB_ASSERT( !predecessor->my_next.load(std::memory_order_relaxed), "the predecessor has another successor!"); + tricky_pointer::store(predecessor->my_next, &s, std::memory_order_release); + if( pred_state != STATE_ACTIVEREADER ) { + #if __TBB_USE_ITT_NOTIFY + sync_prepare_done = true; + ITT_NOTIFY(sync_prepare, s.my_mutex); + #endif + // We are acquiring the mutex + spin_wait_until_eq(s.my_going, 1U, std::memory_order_acquire); + } + } - // The logic below is the same as "writerUnlock", but elides - // "return" from the middle of the routine. - // In the statement below, acquire semantics of reading my_next is required - // so that following operations with fields of my_next are safe. - scoped_lock* n = __TBB_load_with_acquire(my_next); - if( !n ) { - if( this == my_mutex->q_tail.compare_and_swap<tbb::release>(NULL, this) ) { - // this was the only item in the queue, and the queue is now empty. - goto done; + // The protected state must have been acquired here before it can be further released to any other reader(s): + unsigned char old_state = STATE_READER; + // When this reader is signaled by previous actor it acquires the mutex. + // We need to build happens-before relation with all other coming readers that will read our ACTIVEREADER + // without blocking on my_going. Therefore, we need to publish ACTIVEREADER with release semantics. + // On fail it is relaxed, because we will build happens-before on my_going. + s.my_state.compare_exchange_strong(old_state, STATE_ACTIVEREADER, std::memory_order_release, std::memory_order_relaxed); + if( old_state!=STATE_READER ) { +#if __TBB_USE_ITT_NOTIFY + if( !sync_prepare_done ) + ITT_NOTIFY(sync_prepare, s.my_mutex); +#endif + // Failed to become active reader -> need to unblock the next waiting reader first + __TBB_ASSERT( s.my_state.load(std::memory_order_relaxed)==STATE_READER_UNBLOCKNEXT, "unexpected state" ); + spin_wait_while_eq(s.my_next, 0U, std::memory_order_acquire); + /* my_state should be changed before unblocking the next otherwise it might finish + and another thread can get our old state and left blocked */ + s.my_state.store(STATE_ACTIVEREADER, std::memory_order_relaxed); + tricky_pointer::load(s.my_next, std::memory_order_relaxed)->my_going.store(1U, std::memory_order_release); } - spin_wait_while_eq( my_next, (scoped_lock*)NULL ); - n = __TBB_load_with_acquire(my_next); - } - __TBB_store_relaxed(n->my_going, 2); // protect next queue node from being destroyed too early - if( n->my_state==STATE_UPGRADE_WAITING ) { - // the next waiting for upgrade means this writer was upgraded before. - acquire_internal_lock(); - queuing_rw_mutex::scoped_lock* tmp = tricky_pointer::fetch_and_store<tbb::release>(&(n->my_prev), NULL); - n->my_state = STATE_UPGRADE_LOSER; - __TBB_store_with_release(n->my_going,1); - unblock_or_wait_on_internal_lock(get_flag(tmp)); - } else { - __TBB_ASSERT( my_state & (STATE_COMBINED_WAITINGREADER | STATE_WRITER), "unexpected state" ); - __TBB_ASSERT( !( uintptr_t(__TBB_load_relaxed(n->my_prev)) & FLAG ), "use of corrupted pointer!" ); - __TBB_store_relaxed(n->my_prev, (scoped_lock*)0); - __TBB_store_with_release(n->my_going,1); + __TBB_ASSERT(s.my_state.load(std::memory_order_relaxed) == STATE_ACTIVEREADER, "unlocked reader is active reader"); } - } else { // Acquired for read + ITT_NOTIFY(sync_acquired, s.my_mutex); + } - queuing_rw_mutex::scoped_lock *tmp = NULL; -retry: - // Addition to the original paper: Mark my_prev as in use - queuing_rw_mutex::scoped_lock *pred = tricky_pointer::fetch_and_add<tbb::acquire>(&my_prev, FLAG); + //! A method to acquire queuing_rw_mutex if it is free + static bool try_acquire(d1::queuing_rw_mutex& m, d1::queuing_rw_mutex::scoped_lock& s, bool write) + { + __TBB_ASSERT( !s.my_mutex, "scoped_lock is already holding a mutex"); + + if( m.q_tail.load(std::memory_order_relaxed) ) + return false; // Someone already took the lock + + // Must set all fields before the exchange, because once the + // exchange executes, *this becomes accessible to other threads. + s.my_prev.store(0U, std::memory_order_relaxed); + s.my_next.store(0U, std::memory_order_relaxed); + s.my_going.store(0U, std::memory_order_relaxed); // TODO: remove dead assignment? + s.my_state.store(d1::queuing_rw_mutex::scoped_lock::state_t(write ? STATE_WRITER : STATE_ACTIVEREADER), std::memory_order_relaxed); + s.my_internal_lock.store(RELEASED, std::memory_order_relaxed); + + // The CAS must have release semantics, because we are + // "sending" the fields initialized above to other actors. + // We need acquire semantics, because we are acquiring the mutex + d1::queuing_rw_mutex::scoped_lock* expected = nullptr; + if (!m.q_tail.compare_exchange_strong(expected, &s, std::memory_order_acq_rel)) + return false; // Someone already took the lock + s.my_mutex = &m; + ITT_NOTIFY(sync_acquired, s.my_mutex); + return true; + } - if( pred ) { - if( !(pred->try_acquire_internal_lock()) ) - { - // Failed to acquire the lock on pred. The predecessor either unlinks or upgrades. - // In the second case, it could or could not know my "in use" flag - need to check - tmp = tricky_pointer::compare_and_swap<tbb::release>(&my_prev, pred, tricky_pointer(pred) | FLAG ); - if( !(uintptr_t(tmp) & FLAG) ) { - // Wait for the predecessor to change my_prev (e.g. during unlink) - spin_wait_while_eq( my_prev, tricky_pointer(pred)|FLAG ); - // Now owner of pred is waiting for _us_ to release its lock - pred->release_internal_lock(); + //! A method to release queuing_rw_mutex lock + static void release(d1::queuing_rw_mutex::scoped_lock& s) { + __TBB_ASSERT(s.my_mutex!=nullptr, "no lock acquired"); + + ITT_NOTIFY(sync_releasing, s.my_mutex); + + if( s.my_state.load(std::memory_order_relaxed) == STATE_WRITER ) { // Acquired for write + + // The logic below is the same as "writerUnlock", but elides + // "return" from the middle of the routine. + // In the statement below, acquire semantics of reading my_next is required + // so that following operations with fields of my_next are safe. + d1::queuing_rw_mutex::scoped_lock* next = tricky_pointer::load(s.my_next, std::memory_order_acquire); + if( !next ) { + d1::queuing_rw_mutex::scoped_lock* expected = &s; + // Release mutex on success otherwise wait for successor publication + if( s.my_mutex->q_tail.compare_exchange_strong(expected, nullptr, + std::memory_order_release, std::memory_order_relaxed) ) + { + // this was the only item in the queue, and the queue is now empty. + goto done; } - // else the "in use" flag is back -> the predecessor didn't get it and will release itself; nothing to do - - tmp = NULL; - goto retry; + spin_wait_while_eq(s.my_next, 0U, std::memory_order_relaxed); + next = tricky_pointer::load(s.my_next, std::memory_order_acquire); + } + next->my_going.store(2U, std::memory_order_relaxed); // protect next queue node from being destroyed too early + // If the next is STATE_UPGRADE_WAITING, it is expected to acquire all other released readers via release + // sequence in next->my_state. In that case, we need to preserve release sequence in next->my_state + // contributed by other reader. So, there are two approaches not to break the release sequence: + // 1. Use read-modify-write (exchange) operation to store with release the UPGRADE_LOSER state; + // 2. Acquire the release sequence and store the sequence and UPGRADE_LOSER state. + // The second approach seems better on x86 because it does not involve interlocked operations. + // Therefore, we read next->my_state with acquire while it is not required for else branch to get the + // release sequence. + if( next->my_state.load(std::memory_order_acquire)==STATE_UPGRADE_WAITING ) { + // the next waiting for upgrade means this writer was upgraded before. + acquire_internal_lock(s); + // Responsibility transition, the one who reads uncorrupted my_prev will do release. + // Guarantee that above store of 2 into next->my_going happens-before resetting of next->my_prev + d1::queuing_rw_mutex::scoped_lock* tmp = tricky_pointer::exchange(next->my_prev, nullptr, std::memory_order_release); + // Pass the release sequence that we acquired with the above load of next->my_state. + next->my_state.store(STATE_UPGRADE_LOSER, std::memory_order_release); + // We are releasing the mutex + next->my_going.store(1U, std::memory_order_release); + unblock_or_wait_on_internal_lock(s, get_flag(tmp)); + } else { + // next->state cannot be STATE_UPGRADE_REQUESTED + __TBB_ASSERT( next->my_state.load(std::memory_order_relaxed) & (STATE_COMBINED_WAITINGREADER | STATE_WRITER), "unexpected state" ); + __TBB_ASSERT( !( next->my_prev.load(std::memory_order_relaxed) & FLAG ), "use of corrupted pointer!" ); + // Guarantee that above store of 2 into next->my_going happens-before resetting of next->my_prev + tricky_pointer::store(next->my_prev, nullptr, std::memory_order_release); + // We are releasing the mutex + next->my_going.store(1U, std::memory_order_release); } - __TBB_ASSERT(pred && pred->my_internal_lock==ACQUIRED, "predecessor's lock is not acquired"); - __TBB_store_relaxed(my_prev, pred); - acquire_internal_lock(); - __TBB_store_with_release(pred->my_next,static_cast<scoped_lock *>(NULL)); + } else { // Acquired for read + // The basic idea it to build happens-before relation with left and right readers via prev and next. In addition, + // the first reader should acquire the left (prev) signal and propagate to right (next). To simplify, we always + // build happens-before relation between left and right (left is happened before right). + queuing_rw_mutex::scoped_lock *tmp = nullptr; + retry: + // Addition to the original paper: Mark my_prev as in use + queuing_rw_mutex::scoped_lock *predecessor = tricky_pointer::fetch_add(s.my_prev, FLAG, std::memory_order_acquire); + + if( predecessor ) { + if( !(try_acquire_internal_lock(*predecessor)) ) + { + // Failed to acquire the lock on predecessor. The predecessor either unlinks or upgrades. + // In the second case, it could or could not know my "in use" flag - need to check + // Responsibility transition, the one who reads uncorrupted my_prev will do release. + tmp = tricky_pointer::compare_exchange_strong(s.my_prev, tricky_pointer(predecessor) | FLAG, predecessor, std::memory_order_acquire); + if( !(tricky_pointer(tmp) & FLAG) ) { + __TBB_ASSERT(tricky_pointer::load(s.my_prev, std::memory_order_relaxed) != (tricky_pointer(predecessor) | FLAG), nullptr); + // Now owner of predecessor is waiting for _us_ to release its lock + release_internal_lock(*predecessor); + } + // else the "in use" flag is back -> the predecessor didn't get it and will release itself; nothing to do + + tmp = nullptr; + goto retry; + } + __TBB_ASSERT(predecessor && predecessor->my_internal_lock.load(std::memory_order_relaxed)==ACQUIRED, "predecessor's lock is not acquired"); + tricky_pointer::store(s.my_prev, predecessor, std::memory_order_relaxed); + acquire_internal_lock(s); - if( !__TBB_load_relaxed(my_next) && this != my_mutex->q_tail.compare_and_swap<tbb::release>(pred, this) ) { - spin_wait_while_eq( my_next, (void*)NULL ); - } - __TBB_ASSERT( !get_flag(__TBB_load_relaxed(my_next)), "use of corrupted pointer" ); - - // ensure acquire semantics of reading 'my_next' - if( scoped_lock *const l_next = __TBB_load_with_acquire(my_next) ) { // I->next != nil, TODO: rename to n after clearing up and adapting the n in the comment two lines below - // Equivalent to I->next->prev = I->prev but protected against (prev[n]&FLAG)!=0 - tmp = tricky_pointer::fetch_and_store<tbb::release>(&(l_next->my_prev), pred); - // I->prev->next = I->next; - __TBB_ASSERT(__TBB_load_relaxed(my_prev)==pred, NULL); - __TBB_store_with_release(pred->my_next, my_next); - } - // Safe to release in the order opposite to acquiring which makes the code simpler - pred->release_internal_lock(); - - } else { // No predecessor when we looked - acquire_internal_lock(); // "exclusiveLock(&I->EL)" - scoped_lock* n = __TBB_load_with_acquire(my_next); - if( !n ) { - if( this != my_mutex->q_tail.compare_and_swap<tbb::release>(NULL, this) ) { - spin_wait_while_eq( my_next, (scoped_lock*)NULL ); - n = __TBB_load_relaxed(my_next); - } else { - goto unlock_self; + tricky_pointer::store(predecessor->my_next, nullptr, std::memory_order_release); + + d1::queuing_rw_mutex::scoped_lock* expected = &s; + if( !tricky_pointer::load(s.my_next, std::memory_order_acquire) && !s.my_mutex->q_tail.compare_exchange_strong(expected, predecessor, std::memory_order_release) ) { + spin_wait_while_eq( s.my_next, 0U, std::memory_order_acquire ); } + __TBB_ASSERT( !(s.my_next.load(std::memory_order_relaxed) & FLAG), "use of corrupted pointer" ); + + // my_next is acquired either with load or spin_wait. + if(d1::queuing_rw_mutex::scoped_lock *const l_next = tricky_pointer::load(s.my_next, std::memory_order_relaxed) ) { // I->next != nil, TODO: rename to next after clearing up and adapting the n in the comment two lines below + // Equivalent to I->next->prev = I->prev but protected against (prev[n]&FLAG)!=0 + tmp = tricky_pointer::exchange(l_next->my_prev, predecessor, std::memory_order_release); + // I->prev->next = I->next; + __TBB_ASSERT(tricky_pointer::load(s.my_prev, std::memory_order_relaxed)==predecessor, nullptr); + predecessor->my_next.store(s.my_next.load(std::memory_order_relaxed), std::memory_order_release); + } + // Safe to release in the order opposite to acquiring which makes the code simpler + release_internal_lock(*predecessor); + + } else { // No predecessor when we looked + acquire_internal_lock(s); // "exclusiveLock(&I->EL)" + d1::queuing_rw_mutex::scoped_lock* next = tricky_pointer::load(s.my_next, std::memory_order_acquire); + if( !next ) { + d1::queuing_rw_mutex::scoped_lock* expected = &s; + // Release mutex on success otherwise wait for successor publication + if( !s.my_mutex->q_tail.compare_exchange_strong(expected, nullptr, + std::memory_order_release, std::memory_order_relaxed) ) + { + spin_wait_while_eq( s.my_next, 0U, std::memory_order_relaxed ); + next = tricky_pointer::load(s.my_next, std::memory_order_acquire); + } else { + goto unlock_self; + } + } + next->my_going.store(2U, std::memory_order_relaxed); + // Responsibility transition, the one who reads uncorrupted my_prev will do release. + tmp = tricky_pointer::exchange(next->my_prev, nullptr, std::memory_order_release); + next->my_going.store(1U, std::memory_order_release); } - __TBB_store_relaxed(n->my_going, 2); // protect next queue node from being destroyed too early - tmp = tricky_pointer::fetch_and_store<tbb::release>(&(n->my_prev), NULL); - __TBB_store_with_release(n->my_going,1); + unlock_self: + unblock_or_wait_on_internal_lock(s, get_flag(tmp)); } -unlock_self: - unblock_or_wait_on_internal_lock(get_flag(tmp)); - } -done: - spin_wait_while_eq( my_going, 2 ); + done: + // Lifetime synchronization, no need to build happens-before relation + spin_wait_while_eq( s.my_going, 2U, std::memory_order_relaxed ); - initialize(); -} + s.initialize(); + } -bool queuing_rw_mutex::scoped_lock::downgrade_to_reader() -{ - __TBB_ASSERT( my_state==STATE_WRITER, "no sense to downgrade a reader" ); - - ITT_NOTIFY(sync_releasing, my_mutex); - my_state = STATE_READER; - if( ! __TBB_load_relaxed(my_next) ) { - // the following load of q_tail must not be reordered with setting STATE_READER above - if( this==my_mutex->q_tail.load<full_fence>() ) { - unsigned short old_state = my_state.compare_and_swap<tbb::release>(STATE_ACTIVEREADER, STATE_READER); - if( old_state==STATE_READER ) - return true; // Downgrade completed + static bool downgrade_to_reader(d1::queuing_rw_mutex::scoped_lock& s) { + if ( s.my_state.load(std::memory_order_relaxed) == STATE_ACTIVEREADER ) return true; // Already a reader + + ITT_NOTIFY(sync_releasing, s.my_mutex); + d1::queuing_rw_mutex::scoped_lock* next = tricky_pointer::load(s.my_next, std::memory_order_acquire); + if( !next ) { + s.my_state.store(STATE_READER, std::memory_order_seq_cst); + // the following load of q_tail must not be reordered with setting STATE_READER above + if( &s == s.my_mutex->q_tail.load(std::memory_order_seq_cst) ) { + unsigned char old_state = STATE_READER; + // When this reader is signaled by previous actor it acquires the mutex. + // We need to build happens-before relation with all other coming readers that will read our ACTIVEREADER + // without blocking on my_going. Therefore, we need to publish ACTIVEREADER with release semantics. + // On fail it is relaxed, because we will build happens-before on my_going. + s.my_state.compare_exchange_strong(old_state, STATE_ACTIVEREADER, std::memory_order_release, std::memory_order_relaxed); + if( old_state==STATE_READER ) + return true; // Downgrade completed + } + /* wait for the next to register */ + spin_wait_while_eq(s.my_next, 0U, std::memory_order_relaxed); + next = tricky_pointer::load(s.my_next, std::memory_order_acquire); } - /* wait for the next to register */ - spin_wait_while_eq( my_next, (void*)NULL ); + + __TBB_ASSERT( next, "still no successor at this point!" ); + if( next->my_state.load(std::memory_order_relaxed) & STATE_COMBINED_WAITINGREADER ) + next->my_going.store(1U, std::memory_order_release); + // If the next is STATE_UPGRADE_WAITING, it is expected to acquire all other released readers via release + // sequence in next->my_state. In that case, we need to preserve release sequence in next->my_state + // contributed by other reader. So, there are two approaches not to break the release sequence: + // 1. Use read-modify-write (exchange) operation to store with release the UPGRADE_LOSER state; + // 2. Acquire the release sequence and store the sequence and UPGRADE_LOSER state. + // The second approach seems better on x86 because it does not involve interlocked operations. + // Therefore, we read next->my_state with acquire while it is not required for else branch to get the + // release sequence. + else if( next->my_state.load(std::memory_order_acquire)==STATE_UPGRADE_WAITING ) + // the next waiting for upgrade means this writer was upgraded before. + // To safe release sequence on next->my_state read it with acquire + next->my_state.store(STATE_UPGRADE_LOSER, std::memory_order_release); + s.my_state.store(STATE_ACTIVEREADER, std::memory_order_release); + return true; } - scoped_lock *const n = __TBB_load_with_acquire(my_next); - __TBB_ASSERT( n, "still no successor at this point!" ); - if( n->my_state & STATE_COMBINED_WAITINGREADER ) - __TBB_store_with_release(n->my_going,1); - else if( n->my_state==STATE_UPGRADE_WAITING ) - // the next waiting for upgrade means this writer was upgraded before. - n->my_state = STATE_UPGRADE_LOSER; - my_state = STATE_ACTIVEREADER; - return true; -} -bool queuing_rw_mutex::scoped_lock::upgrade_to_writer() -{ - __TBB_ASSERT( my_state==STATE_ACTIVEREADER, "only active reader can be upgraded" ); - - queuing_rw_mutex::scoped_lock * tmp; - queuing_rw_mutex::scoped_lock * me = this; - - ITT_NOTIFY(sync_releasing, my_mutex); - my_state = STATE_UPGRADE_REQUESTED; -requested: - __TBB_ASSERT( !(uintptr_t(__TBB_load_relaxed(my_next)) & FLAG), "use of corrupted pointer!" ); - acquire_internal_lock(); - if( this != my_mutex->q_tail.compare_and_swap<tbb::release>(tricky_pointer(me)|FLAG, this) ) { - spin_wait_while_eq( my_next, (void*)NULL ); - queuing_rw_mutex::scoped_lock * n; - n = tricky_pointer::fetch_and_add<tbb::acquire>(&my_next, FLAG); - unsigned short n_state = n->my_state; - /* the next reader can be blocked by our state. the best thing to do is to unblock it */ - if( n_state & STATE_COMBINED_WAITINGREADER ) - __TBB_store_with_release(n->my_going,1); - tmp = tricky_pointer::fetch_and_store<tbb::release>(&(n->my_prev), this); - unblock_or_wait_on_internal_lock(get_flag(tmp)); - if( n_state & (STATE_COMBINED_READER | STATE_UPGRADE_REQUESTED) ) { - // save n|FLAG for simplicity of following comparisons - tmp = tricky_pointer(n)|FLAG; - for( atomic_backoff b; __TBB_load_relaxed(my_next)==tmp; b.pause() ) { - if( my_state & STATE_COMBINED_UPGRADING ) { - if( __TBB_load_with_acquire(my_next)==tmp ) - __TBB_store_relaxed(my_next, n); - goto waiting; + static bool upgrade_to_writer(d1::queuing_rw_mutex::scoped_lock& s) { + if (s.my_state.load(std::memory_order_relaxed) == STATE_WRITER) { + // Already a writer + return true; + } + + __TBB_ASSERT(s.my_state.load(std::memory_order_relaxed) == STATE_ACTIVEREADER, "only active reader can be updated"); + + queuing_rw_mutex::scoped_lock* tmp{}; + queuing_rw_mutex::scoped_lock* me = &s; + + ITT_NOTIFY(sync_releasing, s.my_mutex); + // Publish ourselves into my_state that other UPGRADE_WAITING actors can acquire our state. + s.my_state.store(STATE_UPGRADE_REQUESTED, std::memory_order_release); + requested: + __TBB_ASSERT( !(s.my_next.load(std::memory_order_relaxed) & FLAG), "use of corrupted pointer!" ); + acquire_internal_lock(s); + d1::queuing_rw_mutex::scoped_lock* expected = &s; + if( !s.my_mutex->q_tail.compare_exchange_strong(expected, tricky_pointer(me)|FLAG, std::memory_order_acq_rel) ) { + spin_wait_while_eq( s.my_next, 0U, std::memory_order_relaxed ); + queuing_rw_mutex::scoped_lock * next; + next = tricky_pointer::fetch_add(s.my_next, FLAG, std::memory_order_acquire); + // While we were READER the next READER might reach STATE_UPGRADE_WAITING state. + // Therefore, it did not build happens before relation with us and we need to acquire the + // next->my_state to build the happens before relation ourselves + unsigned short n_state = next->my_state.load(std::memory_order_acquire); + /* the next reader can be blocked by our state. the best thing to do is to unblock it */ + if( n_state & STATE_COMBINED_WAITINGREADER ) + next->my_going.store(1U, std::memory_order_release); + // Responsibility transition, the one who reads uncorrupted my_prev will do release. + tmp = tricky_pointer::exchange(next->my_prev, &s, std::memory_order_release); + unblock_or_wait_on_internal_lock(s, get_flag(tmp)); + if( n_state & (STATE_COMBINED_READER | STATE_UPGRADE_REQUESTED) ) { + // save next|FLAG for simplicity of following comparisons + tmp = tricky_pointer(next)|FLAG; + for( atomic_backoff b; tricky_pointer::load(s.my_next, std::memory_order_relaxed)==tmp; b.pause() ) { + if( s.my_state.load(std::memory_order_acquire) & STATE_COMBINED_UPGRADING ) { + if( tricky_pointer::load(s.my_next, std::memory_order_acquire)==tmp ) + tricky_pointer::store(s.my_next, next, std::memory_order_relaxed); + goto waiting; + } } + __TBB_ASSERT(tricky_pointer::load(s.my_next, std::memory_order_relaxed) != (tricky_pointer(next)|FLAG), nullptr); + goto requested; + } else { + __TBB_ASSERT( n_state & (STATE_WRITER | STATE_UPGRADE_WAITING), "unexpected state"); + __TBB_ASSERT( (tricky_pointer(next)|FLAG) == tricky_pointer::load(s.my_next, std::memory_order_relaxed), nullptr); + tricky_pointer::store(s.my_next, next, std::memory_order_relaxed); } - __TBB_ASSERT(__TBB_load_relaxed(my_next) != (tricky_pointer(n)|FLAG), NULL); - goto requested; } else { - __TBB_ASSERT( n_state & (STATE_WRITER | STATE_UPGRADE_WAITING), "unexpected state"); - __TBB_ASSERT( (tricky_pointer(n)|FLAG) == __TBB_load_relaxed(my_next), NULL); - __TBB_store_relaxed(my_next, n); + /* We are in the tail; whoever comes next is blocked by q_tail&FLAG */ + release_internal_lock(s); + } // if( this != my_mutex->q_tail... ) + { + unsigned char old_state = STATE_UPGRADE_REQUESTED; + // If we reach STATE_UPGRADE_WAITING state we do not build happens-before relation with READER on + // left. We delegate this responsibility to READER on left when it try upgrading. Therefore, we are releasing + // on success. + // Otherwise, on fail, we already acquired the next->my_state. + s.my_state.compare_exchange_strong(old_state, STATE_UPGRADE_WAITING, std::memory_order_release, std::memory_order_relaxed); } - } else { - /* We are in the tail; whoever comes next is blocked by q_tail&FLAG */ - release_internal_lock(); - } // if( this != my_mutex->q_tail... ) - my_state.compare_and_swap<tbb::acquire>(STATE_UPGRADE_WAITING, STATE_UPGRADE_REQUESTED); - -waiting: - __TBB_ASSERT( !( intptr_t(__TBB_load_relaxed(my_next)) & FLAG ), "use of corrupted pointer!" ); - __TBB_ASSERT( my_state & STATE_COMBINED_UPGRADING, "wrong state at upgrade waiting_retry" ); - __TBB_ASSERT( me==this, NULL ); - ITT_NOTIFY(sync_prepare, my_mutex); - /* if no one was blocked by the "corrupted" q_tail, turn it back */ - my_mutex->q_tail.compare_and_swap<tbb::release>( this, tricky_pointer(me)|FLAG ); - queuing_rw_mutex::scoped_lock * pred; - pred = tricky_pointer::fetch_and_add<tbb::acquire>(&my_prev, FLAG); - if( pred ) { - bool success = pred->try_acquire_internal_lock(); - pred->my_state.compare_and_swap<tbb::release>(STATE_UPGRADE_WAITING, STATE_UPGRADE_REQUESTED); - if( !success ) { - tmp = tricky_pointer::compare_and_swap<tbb::release>(&my_prev, pred, tricky_pointer(pred)|FLAG ); - if( uintptr_t(tmp) & FLAG ) { - spin_wait_while_eq(my_prev, pred); - pred = __TBB_load_relaxed(my_prev); + waiting: + __TBB_ASSERT( !( s.my_next.load(std::memory_order_relaxed) & FLAG ), "use of corrupted pointer!" ); + __TBB_ASSERT( s.my_state & STATE_COMBINED_UPGRADING, "wrong state at upgrade waiting_retry" ); + __TBB_ASSERT( me==&s, nullptr ); + ITT_NOTIFY(sync_prepare, s.my_mutex); + /* if no one was blocked by the "corrupted" q_tail, turn it back */ + expected = tricky_pointer(me)|FLAG; + s.my_mutex->q_tail.compare_exchange_strong(expected, &s, std::memory_order_release); + queuing_rw_mutex::scoped_lock * predecessor; + // Mark my_prev as 'in use' to prevent predecessor from releasing + predecessor = tricky_pointer::fetch_add(s.my_prev, FLAG, std::memory_order_acquire); + if( predecessor ) { + bool success = try_acquire_internal_lock(*predecessor); + { + // While the predecessor pointer (my_prev) is in use (FLAG is set), we can safely update the node`s state. + // Corrupted pointer transitions responsibility to release the predecessor`s node on us. + unsigned char old_state = STATE_UPGRADE_REQUESTED; + // Try to build happens before with the upgrading READER on left. If fail, the predecessor state is not + // important for us because it will acquire our state. + predecessor->my_state.compare_exchange_strong(old_state, STATE_UPGRADE_WAITING, std::memory_order_release, + std::memory_order_relaxed); + } + if( !success ) { + // Responsibility transition, the one who reads uncorrupted my_prev will do release. + tmp = tricky_pointer::compare_exchange_strong(s.my_prev, tricky_pointer(predecessor)|FLAG, predecessor, std::memory_order_acquire); + if( tricky_pointer(tmp) & FLAG ) { + tricky_pointer::spin_wait_while_eq(s.my_prev, predecessor); + predecessor = tricky_pointer::load(s.my_prev, std::memory_order_relaxed); + } else { + // TODO: spin_wait condition seems never reachable + tricky_pointer::spin_wait_while_eq(s.my_prev, tricky_pointer(predecessor)|FLAG); + release_internal_lock(*predecessor); + } } else { - spin_wait_while_eq( my_prev, tricky_pointer(pred)|FLAG ); - pred->release_internal_lock(); + tricky_pointer::store(s.my_prev, predecessor, std::memory_order_relaxed); + release_internal_lock(*predecessor); + tricky_pointer::spin_wait_while_eq(s.my_prev, predecessor); + predecessor = tricky_pointer::load(s.my_prev, std::memory_order_relaxed); } + if( predecessor ) + goto waiting; } else { - __TBB_store_relaxed(my_prev, pred); - pred->release_internal_lock(); - spin_wait_while_eq(my_prev, pred); - pred = __TBB_load_relaxed(my_prev); + tricky_pointer::store(s.my_prev, nullptr, std::memory_order_relaxed); } - if( pred ) - goto waiting; - } else { - // restore the corrupted my_prev field for possible further use (e.g. if downgrade back to reader) - __TBB_store_relaxed(my_prev, pred); + __TBB_ASSERT( !predecessor && !s.my_prev, nullptr ); + + // additional lifetime issue prevention checks + // wait for the successor to finish working with my fields + wait_for_release_of_internal_lock(s); + // now wait for the predecessor to finish working with my fields + spin_wait_while_eq( s.my_going, 2U ); + + bool result = ( s.my_state != STATE_UPGRADE_LOSER ); + s.my_state.store(STATE_WRITER, std::memory_order_relaxed); + s.my_going.store(1U, std::memory_order_relaxed); + + ITT_NOTIFY(sync_acquired, s.my_mutex); + return result; + } + + static bool is_writer(const d1::queuing_rw_mutex::scoped_lock& m) { + return m.my_state.load(std::memory_order_relaxed) == STATE_WRITER; } - __TBB_ASSERT( !pred && !__TBB_load_relaxed(my_prev), NULL ); - // additional lifetime issue prevention checks - // wait for the successor to finish working with my fields - wait_for_release_of_internal_lock(); - // now wait for the predecessor to finish working with my fields - spin_wait_while_eq( my_going, 2 ); + static void construct(d1::queuing_rw_mutex& m) { + suppress_unused_warning(m); + ITT_SYNC_CREATE(&m, _T("tbb::queuing_rw_mutex"), _T("")); + } +}; - // Acquire critical section indirectly from previous owner or directly from predecessor (TODO: not clear). - __TBB_control_consistency_helper(); // on either "my_mutex->q_tail" or "my_going" (TODO: not clear) +void __TBB_EXPORTED_FUNC acquire(d1::queuing_rw_mutex& m, d1::queuing_rw_mutex::scoped_lock& s, bool write) { + queuing_rw_mutex_impl::acquire(m, s, write); +} - bool result = ( my_state != STATE_UPGRADE_LOSER ); - my_state = STATE_WRITER; - __TBB_store_relaxed(my_going, 1); +bool __TBB_EXPORTED_FUNC try_acquire(d1::queuing_rw_mutex& m, d1::queuing_rw_mutex::scoped_lock& s, bool write) { + return queuing_rw_mutex_impl::try_acquire(m, s, write); +} + +void __TBB_EXPORTED_FUNC release(d1::queuing_rw_mutex::scoped_lock& s) { + queuing_rw_mutex_impl::release(s); +} + +bool __TBB_EXPORTED_FUNC upgrade_to_writer(d1::queuing_rw_mutex::scoped_lock& s) { + return queuing_rw_mutex_impl::upgrade_to_writer(s); +} + +bool __TBB_EXPORTED_FUNC is_writer(const d1::queuing_rw_mutex::scoped_lock& s) { + return queuing_rw_mutex_impl::is_writer(s); +} - ITT_NOTIFY(sync_acquired, my_mutex); - return result; +bool __TBB_EXPORTED_FUNC downgrade_to_reader(d1::queuing_rw_mutex::scoped_lock& s) { + return queuing_rw_mutex_impl::downgrade_to_reader(s); } -void queuing_rw_mutex::internal_construct() { - ITT_SYNC_CREATE(this, _T("tbb::queuing_rw_mutex"), _T("")); +void __TBB_EXPORTED_FUNC construct(d1::queuing_rw_mutex& m) { + queuing_rw_mutex_impl::construct(m); } +} // namespace r1 +} // namespace detail } // namespace tbb diff --git a/src/tbb/src/tbb/reader_writer_lock.cpp b/src/tbb/src/tbb/reader_writer_lock.cpp deleted file mode 100644 index 770ab25d9..000000000 --- a/src/tbb/src/tbb/reader_writer_lock.cpp +++ /dev/null @@ -1,347 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#include "tbb/reader_writer_lock.h" -#include "tbb/tbb_machine.h" -#include "tbb/tbb_exception.h" -#include "itt_notify.h" - -#if defined(_MSC_VER) && defined(_Wp64) - // Workaround for overzealous compiler warnings in /Wp64 mode - #pragma warning (disable: 4244) -#endif - -namespace tbb { -namespace interface5 { - -const uintptr_t WFLAG1 = 0x1; // writer interested or active -const uintptr_t WFLAG2 = 0x2; // writers interested, no entering readers -const uintptr_t RFLAG = 0x4; // reader interested but not active -const uintptr_t RC_INCR = 0x8; // to adjust reader count - - -// Perform an atomic bitwise-OR on the operand, and return its previous value. -inline uintptr_t fetch_and_or(atomic<uintptr_t>& operand, uintptr_t value) { - for (tbb::internal::atomic_backoff b;;b.pause()) { - uintptr_t old = operand; - uintptr_t result = operand.compare_and_swap(old|value, old); - if (result==old) return result; - } -} - -// Perform an atomic bitwise-AND on the operand, and return its previous value. -inline uintptr_t fetch_and_and(atomic<uintptr_t>& operand, uintptr_t value) { - for (tbb::internal::atomic_backoff b;;b.pause()) { - uintptr_t old = operand; - uintptr_t result = operand.compare_and_swap(old&value, old); - if (result==old) return result; - } -} - -//! Spin WHILE the value at the location is greater than or equal to a given value -/** T and U should be comparable types. */ -template<typename T, typename U> -void spin_wait_while_geq( const volatile T& location, U value ) { - tbb::internal::atomic_backoff backoff; - while( location>=value ) backoff.pause(); -} - -//! Spin UNTIL (location & value) is true. -/** T and U should be comparable types. */ -template<typename T, typename U> -void spin_wait_until_and( const volatile T& location, U value ) { - tbb::internal::atomic_backoff backoff; - while( !(location & value) ) backoff.pause(); -} - - -void reader_writer_lock::internal_construct() { - reader_head = NULL; - writer_head = NULL; - writer_tail = NULL; - rdr_count_and_flags = 0; - my_current_writer = tbb_thread::id(); -#if TBB_USE_THREADING_TOOLS - ITT_SYNC_CREATE(this, _T("tbb::reader_writer_lock"), _T("")); -#endif /* TBB_USE_THREADING_TOOLS */ -} - -void reader_writer_lock::internal_destroy() { - __TBB_ASSERT(rdr_count_and_flags==0, "reader_writer_lock destroyed with pending readers/writers."); - __TBB_ASSERT(reader_head==NULL, "reader_writer_lock destroyed with pending readers."); - __TBB_ASSERT(writer_tail==NULL, "reader_writer_lock destroyed with pending writers."); - __TBB_ASSERT(writer_head==NULL, "reader_writer_lock destroyed with pending/active writers."); -} - -// Acquires the reader_writer_lock for write. If the lock is currently held in write -// mode by another context, the writer will block by spinning on a local variable. -// Throws exception improper_lock if the context tries to acquire a -// reader_writer_lock that it already has write ownership of. -void reader_writer_lock::lock() { - if (is_current_writer()) { // recursive lock attempt - // we don't support recursive writer locks; throw exception - tbb::internal::throw_exception(tbb::internal::eid_improper_lock); - } - else { - scoped_lock *a_writer_lock = new scoped_lock(); - (void) start_write(a_writer_lock); - } -} - -// Tries to acquire the reader_writer_lock for write. This function does not block. -// Return Value: True or false, depending on whether the lock is acquired or not. -// If the lock is already held by this acquiring context, try_lock() returns false. -bool reader_writer_lock::try_lock() { - if (is_current_writer()) { // recursive lock attempt - return false; - } - else { - scoped_lock *a_writer_lock = new scoped_lock(); - a_writer_lock->status = waiting_nonblocking; - return start_write(a_writer_lock); - } -} - -bool reader_writer_lock::start_write(scoped_lock *I) { - tbb_thread::id id = this_tbb_thread::get_id(); - scoped_lock *pred = NULL; - if (I->status == waiting_nonblocking) { - if ((pred = writer_tail.compare_and_swap(I, NULL)) != NULL) { - delete I; - return false; - } - } - else { - ITT_NOTIFY(sync_prepare, this); - pred = writer_tail.fetch_and_store(I); - } - if (pred) - pred->next = I; - else { - set_next_writer(I); - if (I->status == waiting_nonblocking) { - if (I->next) { // potentially more writers - set_next_writer(I->next); - } - else { // no more writers - writer_head.fetch_and_store(NULL); - if (I != writer_tail.compare_and_swap(NULL, I)) { // an incoming writer is in the process of being added - spin_wait_while_eq(I->next, (scoped_lock *)NULL); // wait for new writer to be added - __TBB_ASSERT(I->next, "There should be a node following the last writer."); - set_next_writer(I->next); - } - } - delete I; - return false; - } - } - spin_wait_while_eq(I->status, waiting); - ITT_NOTIFY(sync_acquired, this); - my_current_writer = id; - return true; -} - -void reader_writer_lock::set_next_writer(scoped_lock *W) { - writer_head = W; - if (W->status == waiting_nonblocking) { - if (rdr_count_and_flags.compare_and_swap(WFLAG1+WFLAG2, 0) == 0) { - W->status = active; - } - } - else { - if (fetch_and_or(rdr_count_and_flags, WFLAG1) & RFLAG) { // reader present - spin_wait_until_and(rdr_count_and_flags, WFLAG2); // block until readers set WFLAG2 - } - else { // no reader in timing window - __TBB_AtomicOR(&rdr_count_and_flags, WFLAG2); - } - spin_wait_while_geq(rdr_count_and_flags, RC_INCR); // block until readers finish - W->status = active; - } -} - -// Acquires the reader_writer_lock for read. If the lock is currently held by a writer, -// this reader will block and wait until the writers are done. -// Throws exception improper_lock when the context tries to acquire a reader_writer_lock -// that it already has write ownership of. -void reader_writer_lock::lock_read() { - if (is_current_writer()) { // recursive lock attempt - // we don't support writer->reader downgrade; throw exception - tbb::internal::throw_exception(tbb::internal::eid_improper_lock); - } - else { - scoped_lock_read a_reader_lock; - start_read(&a_reader_lock); - } -} - -// Tries to acquire the reader_writer_lock for read. This function does not block. -// Return Value: True or false, depending on whether the lock is acquired or not. -bool reader_writer_lock::try_lock_read() { - if (is_current_writer()) { // recursive lock attempt - return false; - } - else { - if (rdr_count_and_flags.fetch_and_add(RC_INCR) & (WFLAG1+WFLAG2)) { // writers present - rdr_count_and_flags -= RC_INCR; - return false; - } - else { // no writers - ITT_NOTIFY(sync_acquired, this); - return true; - } - } -} - -void reader_writer_lock::start_read(scoped_lock_read *I) { - ITT_NOTIFY(sync_prepare, this); - I->next = reader_head.fetch_and_store(I); - if (!I->next) { // first arriving reader in my group; set RFLAG, test writer flags - // unblock and/or update statuses of non-blocking readers - if (!(fetch_and_or(rdr_count_and_flags, RFLAG) & (WFLAG1+WFLAG2))) { // no writers - unblock_readers(); - } - } - __TBB_ASSERT(I->status == waiting || I->status == active, "Lock requests should be waiting or active before blocking."); - spin_wait_while_eq(I->status, waiting); // block - if (I->next) { - __TBB_ASSERT(I->next->status == waiting, NULL); - rdr_count_and_flags += RC_INCR; - I->next->status = active; // wake successor - } - ITT_NOTIFY(sync_acquired, this); -} - -void reader_writer_lock::unblock_readers() { - // clear rdr interest flag, increment rdr count - __TBB_ASSERT(rdr_count_and_flags&RFLAG, NULL); - rdr_count_and_flags += RC_INCR-RFLAG; - __TBB_ASSERT(rdr_count_and_flags >= RC_INCR, NULL); - // indicate clear of window - if (rdr_count_and_flags & WFLAG1 && !(rdr_count_and_flags & WFLAG2)) { - __TBB_AtomicOR(&rdr_count_and_flags, WFLAG2); - } - // unblock waiting readers - scoped_lock_read *head = reader_head.fetch_and_store(NULL); - __TBB_ASSERT(head, NULL); - __TBB_ASSERT(head->status == waiting, NULL); - head->status = active; -} - -// Releases the reader_writer_lock -void reader_writer_lock::unlock() { - if( my_current_writer!=tbb_thread::id() ) { - // A writer owns the lock - __TBB_ASSERT(is_current_writer(), "caller of reader_writer_lock::unlock() does not own the lock."); - __TBB_ASSERT(writer_head, NULL); - __TBB_ASSERT(writer_head->status==active, NULL); - scoped_lock *a_writer_lock = writer_head; - end_write(a_writer_lock); - __TBB_ASSERT(a_writer_lock != writer_head, "Internal error: About to turn writer_head into dangling reference."); - delete a_writer_lock; - } else { - end_read(); - } -} - -void reader_writer_lock::end_write(scoped_lock *I) { - __TBB_ASSERT(I==writer_head, "Internal error: can't unlock a thread that is not holding the lock."); - my_current_writer = tbb_thread::id(); - ITT_NOTIFY(sync_releasing, this); - if (I->next) { // potentially more writers - writer_head = I->next; - writer_head->status = active; - } - else { // No more writers; clear writer flag, test reader interest flag - __TBB_ASSERT(writer_head, NULL); - if (fetch_and_and(rdr_count_and_flags, ~(WFLAG1+WFLAG2)) & RFLAG) { - unblock_readers(); - } - writer_head.fetch_and_store(NULL); - if (I != writer_tail.compare_and_swap(NULL, I)) { // an incoming writer is in the process of being added - spin_wait_while_eq(I->next, (scoped_lock *)NULL); // wait for new writer to be added - __TBB_ASSERT(I->next, "There should be a node following the last writer."); - set_next_writer(I->next); - } - } -} - -void reader_writer_lock::end_read() { - ITT_NOTIFY(sync_releasing, this); - __TBB_ASSERT(rdr_count_and_flags >= RC_INCR, "unlock() called but no readers hold the lock."); - rdr_count_and_flags -= RC_INCR; -} - -inline bool reader_writer_lock::is_current_writer() { - return my_current_writer==this_tbb_thread::get_id(); -} - -// Construct with a blocking attempt to acquire a write lock on the passed reader_writer_lock -void reader_writer_lock::scoped_lock::internal_construct (reader_writer_lock& lock) { - mutex = &lock; - next = NULL; - status = waiting; - if (mutex->is_current_writer()) { // recursive lock attempt - // we don't support recursive writer locks; throw exception - tbb::internal::throw_exception(tbb::internal::eid_improper_lock); - } - else { // this thread holds no locks - (void) mutex->start_write(this); - } -} - -inline reader_writer_lock::scoped_lock::scoped_lock() : mutex(NULL), next(NULL) { - status = waiting; -} - -// Construct with a blocking attempt to acquire a write lock on the passed reader_writer_lock -void reader_writer_lock::scoped_lock_read::internal_construct (reader_writer_lock& lock) { - mutex = &lock; - next = NULL; - status = waiting; - if (mutex->is_current_writer()) { // recursive lock attempt - // we don't support writer->reader downgrade; throw exception - tbb::internal::throw_exception(tbb::internal::eid_improper_lock); - } - else { // this thread holds no locks - mutex->start_read(this); - } -} - -inline reader_writer_lock::scoped_lock_read::scoped_lock_read() : mutex(NULL), next(NULL) { - status = waiting; -} - -void reader_writer_lock::scoped_lock::internal_destroy() { - if (mutex) { - __TBB_ASSERT(mutex->is_current_writer(), "~scoped_lock() destroyed by thread different than thread that holds lock."); - mutex->end_write(this); - } - status = invalid; -} - -void reader_writer_lock::scoped_lock_read::internal_destroy() { - if (mutex) - mutex->end_read(); - status = invalid; -} - -} // namespace interface5 -} // namespace tbb diff --git a/src/tbb/src/tbb/recursive_mutex.cpp b/src/tbb/src/tbb/recursive_mutex.cpp deleted file mode 100644 index 7d33202ce..000000000 --- a/src/tbb/src/tbb/recursive_mutex.cpp +++ /dev/null @@ -1,136 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#include "tbb/recursive_mutex.h" -#include "itt_notify.h" - -namespace tbb { - -void recursive_mutex::scoped_lock::internal_acquire( recursive_mutex& m ) { -#if _WIN32||_WIN64 - switch( m.state ) { - case INITIALIZED: - // since we cannot look into the internal of the CriticalSection object - // we won't know how many times the lock has been acquired, and thus - // we won't know when we may safely set the state back to INITIALIZED - // if we change the state to HELD as in mutex.cpp. thus, we won't change - // the state for recursive_mutex - EnterCriticalSection( &m.impl ); - break; - case DESTROYED: - __TBB_ASSERT(false,"recursive_mutex::scoped_lock: mutex already destroyed"); - break; - default: - __TBB_ASSERT(false,"recursive_mutex::scoped_lock: illegal mutex state"); - break; - } -#else - int error_code = pthread_mutex_lock(&m.impl); - if( error_code ) - tbb::internal::handle_perror(error_code,"recursive_mutex::scoped_lock: pthread_mutex_lock failed"); -#endif /* _WIN32||_WIN64 */ - my_mutex = &m; -} - -void recursive_mutex::scoped_lock::internal_release() { - __TBB_ASSERT( my_mutex, "recursive_mutex::scoped_lock: not holding a mutex" ); -#if _WIN32||_WIN64 - switch( my_mutex->state ) { - case INITIALIZED: - LeaveCriticalSection( &my_mutex->impl ); - break; - case DESTROYED: - __TBB_ASSERT(false,"recursive_mutex::scoped_lock: mutex already destroyed"); - break; - default: - __TBB_ASSERT(false,"recursive_mutex::scoped_lock: illegal mutex state"); - break; - } -#else - int error_code = pthread_mutex_unlock(&my_mutex->impl); - __TBB_ASSERT_EX(!error_code, "recursive_mutex::scoped_lock: pthread_mutex_unlock failed"); -#endif /* _WIN32||_WIN64 */ - my_mutex = NULL; -} - -bool recursive_mutex::scoped_lock::internal_try_acquire( recursive_mutex& m ) { -#if _WIN32||_WIN64 - switch( m.state ) { - case INITIALIZED: - break; - case DESTROYED: - __TBB_ASSERT(false,"recursive_mutex::scoped_lock: mutex already destroyed"); - break; - default: - __TBB_ASSERT(false,"recursive_mutex::scoped_lock: illegal mutex state"); - break; - } -#endif /* _WIN32||_WIN64 */ - bool result; -#if _WIN32||_WIN64 - result = TryEnterCriticalSection(&m.impl)!=0; -#else - result = pthread_mutex_trylock(&m.impl)==0; -#endif /* _WIN32||_WIN64 */ - if( result ) - my_mutex = &m; - return result; -} - -void recursive_mutex::internal_construct() { -#if _WIN32||_WIN64 - InitializeCriticalSectionEx(&impl, 4000, 0); - state = INITIALIZED; -#else - pthread_mutexattr_t mtx_attr; - int error_code = pthread_mutexattr_init( &mtx_attr ); - if( error_code ) - tbb::internal::handle_perror(error_code,"recursive_mutex: pthread_mutexattr_init failed"); - - pthread_mutexattr_settype( &mtx_attr, PTHREAD_MUTEX_RECURSIVE ); - error_code = pthread_mutex_init( &impl, &mtx_attr ); - if( error_code ) - tbb::internal::handle_perror(error_code,"recursive_mutex: pthread_mutex_init failed"); - pthread_mutexattr_destroy( &mtx_attr ); -#endif /* _WIN32||_WIN64*/ - ITT_SYNC_CREATE(&impl, _T("tbb::recursive_mutex"), _T("")); -} - -void recursive_mutex::internal_destroy() { -#if _WIN32||_WIN64 - switch( state ) { - case INITIALIZED: - DeleteCriticalSection(&impl); - break; - case DESTROYED: - __TBB_ASSERT(false,"recursive_mutex: already destroyed"); - break; - default: - __TBB_ASSERT(false,"recursive_mutex: illegal state for destruction"); - break; - } - state = DESTROYED; -#else - int error_code = pthread_mutex_destroy(&impl); - __TBB_ASSERT_EX(!error_code,"recursive_mutex: pthread_mutex_destroy failed"); -#endif /* _WIN32||_WIN64 */ -} - -} // namespace tbb diff --git a/src/tbb/src/rml/include/rml_base.h b/src/tbb/src/tbb/rml_base.h similarity index 84% rename from src/tbb/src/rml/include/rml_base.h rename to src/tbb/src/tbb/rml_base.h index 951026fa6..9e1705837 100644 --- a/src/tbb/src/rml/include/rml_base.h +++ b/src/tbb/src/tbb/rml_base.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ // Header guard and namespace names follow rml conventions. @@ -37,25 +33,20 @@ namespace rml { -//! Base class for denying assignment and copy constructor. -class no_copy { - void operator=( no_copy& ); - no_copy( no_copy& ); -public: - no_copy() {} -}; - class server; class versioned_object { public: //! A version number typedef unsigned version_type; - + + virtual ~versioned_object() {} + //! Get version of this object /** The version number is incremented when a incompatible change is introduced. The version number is invariant for the lifetime of the object. */ virtual version_type version() const RML_PURE(version_type) + }; //! Represents a client's job for an execution context. @@ -63,11 +54,6 @@ class versioned_object { Not derived from versioned_object because version is same as for client. */ class job { friend class server; - - //! Word for use by server - /** Typically the server uses it to speed up internal lookup. - Clients must not modify the word. */ - void* scratch_ptr; }; //! Information that client provides to server when asking for a server. @@ -80,7 +66,7 @@ class client: public versioned_object { //! Index of a job in a job pool typedef unsigned size_type; - //! Maximum number of threads that client can exploit profitably if nothing else is running on the machine. + //! Maximum number of threads that client can exploit profitably if nothing else is running on the machine. /** The returned value should remain invariant for the lifetime of the connection. [idempotent] */ virtual size_type max_job_count() const RML_PURE(size_type) @@ -95,23 +81,18 @@ class client: public versioned_object { after cleanup(job) has been called for each job. */ virtual void acknowledge_close_connection() RML_PURE(void) - enum policy_type {turnaround,throughput}; - - //! Inform server of desired policy. [idempotent] - virtual policy_type policy() const RML_PURE(policy_type) - - //! Inform client that server is done with *this. + //! Inform client that server is done with *this. /** Client should destroy the job. Not necessarily called by execution context represented by *this. Never called while any other thread is working on the job. */ virtual void cleanup( job& ) RML_PURE(void) - // In general, we should not add new virtual methods, because that would - // break derived classes. Think about reserving some vtable slots. + // In general, we should not add new virtual methods, because that would + // break derived classes. Think about reserving some vtable slots. }; // Information that server provides to client. -// Virtual functions are routines provided by the server for the client to call. +// Virtual functions are routines provided by the server for the client to call. class server: public versioned_object { public: //! Typedef for convenience of derived classes. @@ -123,32 +104,29 @@ class server: public versioned_object { //! Request that connection to server be closed. /** Causes each job associated with the client to have its cleanup method called, - possibly by a thread different than the thread that created the job. - This method can return before all cleanup methods return. - Actions that have to wait after all cleanup methods return should be part of - client::acknowledge_close_connection. + possibly by a thread different than the thread that created the job. + This method can return before all cleanup methods return. + Actions that have to wait after all cleanup methods return should be part of + client::acknowledge_close_connection. Pass true as exiting if request_close_connection() is called because exit() is called. In that case, it is the client's responsibility to make sure all threads are terminated. In all other cases, pass false. */ virtual void request_close_connection( bool exiting = false ) = 0; - //! Called by client thread when it reaches a point where it cannot make progress until other threads do. + //! Called by client thread when it reaches a point where it cannot make progress until other threads do. virtual void yield() = 0; //! Called by client to indicate a change in the number of non-RML threads that are running. - /** This is a performance hint to the RML to adjust how many threads it should let run + /** This is a performance hint to the RML to adjust how many threads it should let run concurrently. The delta is the change in the number of non-RML threads that are running. - For example, a value of 1 means the client has started running another thread, and a value + For example, a value of 1 means the client has started running another thread, and a value of -1 indicates that the client has blocked or terminated one of its threads. */ virtual void independent_thread_number_changed( int delta ) = 0; //! Default level of concurrency for which RML strives when there are no non-RML threads running. - /** Normally, the value is the hardware concurrency minus one. + /** Normally, the value is the hardware concurrency minus one. The "minus one" accounts for the thread created by main(). */ virtual unsigned default_concurrency() const = 0; - -protected: - static void*& scratch_ptr( job& j ) {return j.scratch_ptr;} }; class factory { @@ -161,9 +139,6 @@ class factory { st_incompatible }; - //! Scratch pointer for use by RML. - void* scratch_ptr; - protected: //! Pointer to routine that waits for server to indicate when client can close itself. status_type (*my_wait_to_close_routine)( factory& ); @@ -174,7 +149,7 @@ class factory { HMODULE library_handle; #else void* library_handle; -#endif /* _WIN32||_WIN64 */ +#endif /* _WIN32||_WIN64 */ //! Special marker to keep dll from being unloaded prematurely static const std::size_t c_dont_unload = 1; diff --git a/src/tbb/src/rml/client/rml_factory.h b/src/tbb/src/tbb/rml_tbb.cpp similarity index 64% rename from src/tbb/src/rml/client/rml_factory.h rename to src/tbb/src/tbb/rml_tbb.cpp index 0c4626a98..d1cd285c1 100644 --- a/src/tbb/src/rml/client/rml_factory.h +++ b/src/tbb/src/tbb/rml_tbb.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2023 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,13 +12,32 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. +*/ +#include "oneapi/tbb/detail/_assert.h" +#include "rml_tbb.h" +#include "dynamic_link.h" +namespace tbb { +namespace detail { +namespace r1 { +namespace rml { -*/ +#define MAKE_SERVER(x) DLD(__TBB_make_rml_server,x) +#define GET_INFO(x) DLD(__TBB_call_with_my_server_info,x) +#define SERVER tbb_server +#define CLIENT tbb_client +#define FACTORY tbb_factory -// No ifndef guard because this file is not a normal include file. +#if __TBB_WEAK_SYMBOLS_PRESENT + #pragma weak __TBB_make_rml_server + #pragma weak __TBB_call_with_my_server_info + extern "C" { + ::rml::factory::status_type __TBB_make_rml_server( rml::tbb_factory& f, rml::tbb_server*& server, rml::tbb_client& client ); + void __TBB_call_with_my_server_info( ::rml::server_info_callback_t cb, void* arg ); + } +#endif /* __TBB_WEAK_SYMBOLS_PRESENT */ #if TBB_USE_DEBUG #define DEBUG_SUFFIX "_debug" @@ -27,14 +46,14 @@ #endif /* TBB_USE_DEBUG */ // RML_SERVER_NAME is the name of the RML server library. -#if _WIN32||_WIN64 +#if _WIN32 || _WIN64 #define RML_SERVER_NAME "irml" DEBUG_SUFFIX ".dll" #elif __APPLE__ -#define RML_SERVER_NAME "libirml" DEBUG_SUFFIX ".dylib" -#elif __linux__ -#define RML_SERVER_NAME "libirml" DEBUG_SUFFIX ".so.1" -#elif __FreeBSD__ || __NetBSD__ || __sun || _AIX +#define RML_SERVER_NAME "libirml" DEBUG_SUFFIX ".1.dylib" +#elif __FreeBSD__ || __NetBSD__ || __OpenBSD__ || __sun || _AIX #define RML_SERVER_NAME "libirml" DEBUG_SUFFIX ".so" +#elif __unix__ +#define RML_SERVER_NAME "libirml" DEBUG_SUFFIX ".so.1" #else #error Unknown OS #endif @@ -52,7 +71,7 @@ const ::rml::versioned_object::version_type CLIENT_VERSION = 2; ::rml::factory::status_type FACTORY::open() { // Failure of following assertion indicates that factory is already open, or not zero-inited. - LIBRARY_ASSERT( !library_handle, NULL ); + __TBB_ASSERT_EX( !library_handle, nullptr); status_type (*open_factory_routine)( factory&, version_type&, version_type ); dynamic_link_descriptor server_link_table[4] = { DLD(__RML_open_factory,open_factory_routine), @@ -61,34 +80,33 @@ ::rml::factory::status_type FACTORY::open() { GET_INFO(my_call_with_server_info_routine), }; status_type result; - if( dynamic_link( RML_SERVER_NAME, server_link_table, 4, &library_handle ) ) { + if ( dynamic_link( RML_SERVER_NAME, server_link_table, 4, &library_handle ) ) { version_type server_version; result = (*open_factory_routine)( *this, server_version, CLIENT_VERSION ); // server_version can be checked here for incompatibility if necessary. } else { - library_handle = NULL; + library_handle = nullptr; result = st_not_found; } return result; } void FACTORY::close() { - if( library_handle ) + if ( library_handle ) (*my_wait_to_close_routine)(*this); - if( (size_t)library_handle>FACTORY::c_dont_unload ) { + if ( (size_t)library_handle>FACTORY::c_dont_unload ) { dynamic_unlink(library_handle); - library_handle = NULL; + library_handle = nullptr; } } ::rml::factory::status_type FACTORY::make_server( SERVER*& s, CLIENT& c) { // Failure of following assertion means that factory was not successfully opened. - LIBRARY_ASSERT( my_make_server_routine, NULL ); + __TBB_ASSERT_EX( my_make_server_routine, nullptr); return (*my_make_server_routine)(*this,s,c); } -void FACTORY::call_with_server_info( ::rml::server_info_callback_t cb, void* arg ) const { - // Failure of following assertion means that factory was not successfully opened. - LIBRARY_ASSERT( my_call_with_server_info_routine, NULL ); - (*my_call_with_server_info_routine)( cb, arg ); -} +} // namespace rml +} // namespace r1 +} // namespace detail +} // namespace tbb diff --git a/src/tbb/src/rml/include/rml_tbb.h b/src/tbb/src/tbb/rml_tbb.h similarity index 81% rename from src/tbb/src/rml/include/rml_tbb.h rename to src/tbb/src/tbb/rml_tbb.h index ce47f175b..61176f8d7 100644 --- a/src/tbb/src/rml/include/rml_tbb.h +++ b/src/tbb/src/tbb/rml_tbb.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ // Header guard and namespace names follow TBB conventions. @@ -23,31 +19,30 @@ #ifndef __TBB_rml_tbb_H #define __TBB_rml_tbb_H -#include "tbb/tbb_config.h" +#include "oneapi/tbb/version.h" #include "rml_base.h" namespace tbb { -namespace internal { +namespace detail { +namespace r1 { namespace rml { -class tbb_client; - //------------------------------------------------------------------------ // Classes instantiated by the server //------------------------------------------------------------------------ -//! Represents a set of tbb worker threads provided by the server. +//! Represents a set of oneTBB worker threads provided by the server. class tbb_server: public ::rml::server { public: //! Inform server of adjustments in the number of workers that the client can profitably use. virtual void adjust_job_count_estimate( int delta ) = 0; -#if _WIN32||_WIN64 - //! Inform server of a tbb master thread. - virtual void register_master( execution_resource_t& v ) = 0; +#if _WIN32 || _WIN64 + //! Inform server of a oneTBB external thread. + virtual void register_external_thread( execution_resource_t& v ) = 0; - //! Inform server that the tbb master thread is done with its work. - virtual void unregister_master( execution_resource_t v ) = 0; + //! Inform server that the oneTBB external thread is done with its work. + virtual void unregister_external_thread( execution_resource_t v ) = 0; #endif /* _WIN32||_WIN64 */ }; @@ -89,13 +84,11 @@ class tbb_factory: public ::rml::factory { //! Close factory void close(); - - //! Call the callback with the server build info - void call_with_server_info( ::rml::server_info_callback_t cb, void* arg ) const; }; } // namespace rml -} // namespace internal +} // namespace r1 +} // namespace detail } // namespace tbb #endif /*__TBB_rml_tbb_H */ diff --git a/src/tbb/src/rml/server/thread_monitor.h b/src/tbb/src/tbb/rml_thread_monitor.h similarity index 52% rename from src/tbb/src/rml/server/thread_monitor.h rename to src/tbb/src/tbb/rml_thread_monitor.h index e7ea79e45..57e9c30b0 100644 --- a/src/tbb/src/rml/server/thread_monitor.h +++ b/src/tbb/src/tbb/rml_thread_monitor.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2022 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ // All platform-specific threading support is encapsulated here. */ @@ -23,25 +19,28 @@ #ifndef __RML_thread_monitor_H #define __RML_thread_monitor_H -#if USE_WINTHREAD +#if __TBB_USE_WINAPI #include <windows.h> #include <process.h> #include <malloc.h> //_alloca -#include "tbb/tbb_misc.h" // support for processor groups -#if __TBB_WIN8UI_SUPPORT +#include "misc.h" // support for processor groups +#if __TBB_WIN8UI_SUPPORT && (_WIN32_WINNT < 0x0A00) #include <thread> #endif -#elif USE_PTHREAD +#elif __TBB_USE_POSIX #include <pthread.h> -#include <string.h> -#include <stdlib.h> +#include <cstring> +#include <cstdlib> +#include <time.h> #else #error Unsupported platform #endif -#include <stdio.h> -#include "tbb/itt_notify.h" -#include "tbb/atomic.h" -#include "tbb/semaphore.h" +#include <cstdio> + +#include "oneapi/tbb/detail/_template_helpers.h" + +#include "itt_notify.h" +#include "semaphore.h" // All platform-specific threading support is in this header. @@ -53,75 +52,63 @@ // See Knuth Vol 3. "Theorem S" for details on Fibonacci hashing. // The second statement is really does need "volatile", otherwise the compiler might remove the _alloca. #define AVOID_64K_ALIASING(idx) \ - size_t offset = (idx+1) * 40503U % (1U<<16); \ + std::size_t offset = (idx+1) * 40503U % (1U<<16); \ void* volatile sink_for_alloca = _alloca(offset); \ __TBB_ASSERT_EX(sink_for_alloca, "_alloca failed"); #else // Linux thread allocators avoid 64K aliasing. -#define AVOID_64K_ALIASING(idx) tbb::internal::suppress_unused_warning(idx) +#define AVOID_64K_ALIASING(idx) tbb::detail::suppress_unused_warning(idx) #endif /* _WIN32||_WIN64 */ -namespace rml { +namespace tbb { +namespace detail { +namespace r1 { +// Forward declaration: throws std::runtime_error with what() returning error_code description prefixed with aux_info +void handle_perror(int error_code, const char* aux_info); + +namespace rml { namespace internal { -#if DO_ITT_NOTIFY -static const ::tbb::tchar *SyncType_RML = _T("%Constant"); -static const ::tbb::tchar *SyncObj_ThreadMonitor = _T("RML Thr Monitor"); -#endif /* DO_ITT_NOTIFY */ +#if __TBB_USE_ITT_NOTIFY +static const ::tbb::detail::r1::tchar *SyncType_RML = _T("%Constant"); +static const ::tbb::detail::r1::tchar *SyncObj_ThreadMonitor = _T("RML Thr Monitor"); +#endif /* __TBB_USE_ITT_NOTIFY */ //! Monitor with limited two-phase commit form of wait. /** At most one thread should wait on an instance at a time. */ class thread_monitor { public: - class cookie { - friend class thread_monitor; - tbb::atomic<size_t> my_epoch; - }; - thread_monitor() : spurious(false), my_sema() { - my_cookie.my_epoch = 0; + thread_monitor() { ITT_SYNC_CREATE(&my_sema, SyncType_RML, SyncObj_ThreadMonitor); - in_wait = false; } ~thread_monitor() {} - //! If a thread is waiting or started a two-phase wait, notify it. + //! Notify waiting thread /** Can be called by any thread. */ void notify(); - //! Begin two-phase wait. - /** Should only be called by thread that owns the monitor. - The caller must either complete the wait or cancel it. */ - void prepare_wait( cookie& c ); - - //! Complete a two-phase wait and wait until notification occurs after the earlier prepare_wait. - void commit_wait( cookie& c ); + //! Wait for notification + void wait(); - //! Cancel a two-phase wait. - void cancel_wait(); - -#if USE_WINTHREAD +#if __TBB_USE_WINAPI typedef HANDLE handle_type; #define __RML_DECL_THREAD_ROUTINE unsigned WINAPI typedef unsigned (WINAPI *thread_routine_type)(void*); //! Launch a thread - static handle_type launch( thread_routine_type thread_routine, void* arg, size_t stack_size, const size_t* worker_index = NULL ); + static handle_type launch( thread_routine_type thread_routine, void* arg, std::size_t stack_size, const size_t* worker_index = nullptr ); -#elif USE_PTHREAD +#elif __TBB_USE_POSIX typedef pthread_t handle_type; #define __RML_DECL_THREAD_ROUTINE void* typedef void*(*thread_routine_type)(void*); //! Launch a thread - static handle_type launch( thread_routine_type thread_routine, void* arg, size_t stack_size ); -#endif /* USE_PTHREAD */ - - //! Yield control to OS - /** Affects the calling thread. **/ - static void yield(); + static handle_type launch( thread_routine_type thread_routine, void* arg, std::size_t stack_size ); +#endif /* __TBB_USE_POSIX */ //! Join thread static void join(handle_type handle); @@ -129,57 +116,55 @@ class thread_monitor { //! Detach thread static void detach_thread(handle_type handle); private: - cookie my_cookie; - tbb::atomic<bool> in_wait; - bool spurious; - tbb::internal::binary_semaphore my_sema; -#if USE_PTHREAD + // The protection from double notification of the binary semaphore + std::atomic<bool> my_notified{ false }; + binary_semaphore my_sema; +#if __TBB_USE_POSIX static void check( int error_code, const char* routine ); #endif }; -#if USE_WINTHREAD +#if __TBB_USE_WINAPI #ifndef STACK_SIZE_PARAM_IS_A_RESERVATION #define STACK_SIZE_PARAM_IS_A_RESERVATION 0x00010000 #endif -#if __TBB_WIN8UI_SUPPORT -inline thread_monitor::handle_type thread_monitor::launch( thread_routine_type thread_function, void* arg, size_t, const size_t*) { +// _beginthreadex API is not available in Windows 8 Store* applications, so use std::thread instead +#if __TBB_WIN8UI_SUPPORT && (_WIN32_WINNT < 0x0A00) +inline thread_monitor::handle_type thread_monitor::launch( thread_routine_type thread_function, void* arg, std::size_t, const std::size_t*) { //TODO: check that exception thrown from std::thread is not swallowed silently std::thread* thread_tmp=new std::thread(thread_function, arg); return thread_tmp->native_handle(); } -#else //__TBB_WIN8UI_SUPPORT -inline thread_monitor::handle_type thread_monitor::launch( thread_routine_type thread_routine, void* arg, size_t stack_size, const size_t* worker_index ) { +#else +inline thread_monitor::handle_type thread_monitor::launch( thread_routine_type thread_routine, void* arg, std::size_t stack_size, const std::size_t* worker_index ) { unsigned thread_id; - int number_of_processor_groups = ( worker_index ) ? tbb::internal::NumberOfProcessorGroups() : 0; + int number_of_processor_groups = ( worker_index ) ? NumberOfProcessorGroups() : 0; unsigned create_flags = ( number_of_processor_groups > 1 ) ? CREATE_SUSPENDED : 0; - HANDLE h = (HANDLE)_beginthreadex( NULL, unsigned(stack_size), thread_routine, arg, STACK_SIZE_PARAM_IS_A_RESERVATION | create_flags, &thread_id ); + HANDLE h = (HANDLE)_beginthreadex( nullptr, unsigned(stack_size), thread_routine, arg, STACK_SIZE_PARAM_IS_A_RESERVATION | create_flags, &thread_id ); if( !h ) { - fprintf(stderr,"thread_monitor::launch: _beginthreadex failed\n"); - exit(1); + handle_perror(0, "thread_monitor::launch: _beginthreadex failed\n"); } if ( number_of_processor_groups > 1 ) { - tbb::internal::MoveThreadIntoProcessorGroup( h, - tbb::internal::FindProcessorGroupIndex( static_cast<int>(*worker_index) ) ); + MoveThreadIntoProcessorGroup( h, FindProcessorGroupIndex( static_cast<int>(*worker_index) ) ); ResumeThread( h ); } return h; } -#endif //__TBB_WIN8UI_SUPPORT +#endif //__TBB_WIN8UI_SUPPORT && (_WIN32_WINNT < 0x0A00) void thread_monitor::join(handle_type handle) { #if TBB_USE_ASSERT DWORD res = #endif WaitForSingleObjectEx(handle, INFINITE, FALSE); - __TBB_ASSERT( res==WAIT_OBJECT_0, NULL ); + __TBB_ASSERT( res==WAIT_OBJECT_0, nullptr); #if TBB_USE_ASSERT BOOL val = #endif CloseHandle(handle); - __TBB_ASSERT( val, NULL ); + __TBB_ASSERT( val, nullptr); } void thread_monitor::detach_thread(handle_type handle) { @@ -187,84 +172,76 @@ void thread_monitor::detach_thread(handle_type handle) { BOOL val = #endif CloseHandle(handle); - __TBB_ASSERT( val, NULL ); + __TBB_ASSERT( val, nullptr); } -inline void thread_monitor::yield() { -// TODO: consider unification via __TBB_Yield or tbb::this_tbb_thread::yield -#if !__TBB_WIN8UI_SUPPORT - SwitchToThread(); -#else - std::this_thread::yield(); -#endif -} -#endif /* USE_WINTHREAD */ +#endif /* __TBB_USE_WINAPI */ -#if USE_PTHREAD -// TODO: can we throw exceptions instead of termination? +#if __TBB_USE_POSIX inline void thread_monitor::check( int error_code, const char* routine ) { if( error_code ) { - fprintf(stderr,"thread_monitor %s in %s\n", strerror(error_code), routine ); - exit(1); + handle_perror(error_code, routine); } } -inline thread_monitor::handle_type thread_monitor::launch( void* (*thread_routine)(void*), void* arg, size_t stack_size ) { +inline thread_monitor::handle_type thread_monitor::launch( void* (*thread_routine)(void*), void* arg, std::size_t stack_size ) { // FIXME - consider more graceful recovery than just exiting if a thread cannot be launched. // Note that there are some tricky situations to deal with, such that the thread is already // grabbed as part of an OpenMP team. pthread_attr_t s; - check(pthread_attr_init( &s ), "pthread_attr_init"); + check(pthread_attr_init( &s ), "pthread_attr_init has failed"); if( stack_size>0 ) - check(pthread_attr_setstacksize( &s, stack_size ), "pthread_attr_setstack_size" ); + check(pthread_attr_setstacksize( &s, stack_size ), "pthread_attr_setstack_size has failed" ); + + // pthread_create(2) can spuriously fail with EAGAIN. We retry + // max_num_tries times with progressively longer wait times. pthread_t handle; - check( pthread_create( &handle, &s, thread_routine, arg ), "pthread_create" ); - check( pthread_attr_destroy( &s ), "pthread_attr_destroy" ); + const int max_num_tries = 20; + int error = EAGAIN; + + for (int i = 0; i < max_num_tries && error == EAGAIN; i++) { + if (i != 0) { + // Wait i milliseconds + struct timespec ts = {0, i * 1000 * 1000}; + nanosleep(&ts, NULL); + } + error = pthread_create(&handle, &s, thread_routine, arg); + } + + if (error) + handle_perror(error, "pthread_create has failed"); + + check( pthread_attr_destroy( &s ), "pthread_attr_destroy has failed" ); return handle; } void thread_monitor::join(handle_type handle) { - check(pthread_join(handle, NULL), "pthread_join"); + check(pthread_join(handle, nullptr), "pthread_join has failed"); } void thread_monitor::detach_thread(handle_type handle) { - check(pthread_detach(handle), "pthread_detach"); + check(pthread_detach(handle), "pthread_detach has failed"); } - -inline void thread_monitor::yield() { - sched_yield(); -} -#endif /* USE_PTHREAD */ +#endif /* __TBB_USE_POSIX */ inline void thread_monitor::notify() { - my_cookie.my_epoch = my_cookie.my_epoch + 1; - bool do_signal = in_wait.fetch_and_store( false ); - if( do_signal ) + // Check that the semaphore is not notified twice + if (!my_notified.exchange(true, std::memory_order_release)) { my_sema.V(); -} - -inline void thread_monitor::prepare_wait( cookie& c ) { - if( spurious ) { - spurious = false; - // consumes a spurious posted signal. don't wait on my_sema. - my_sema.P(); } - c = my_cookie; - in_wait = true; - __TBB_full_memory_fence(); -} - -inline void thread_monitor::commit_wait( cookie& c ) { - bool do_it = ( c.my_epoch == my_cookie.my_epoch); - if( do_it ) my_sema.P(); - else cancel_wait(); } -inline void thread_monitor::cancel_wait() { - spurious = ! in_wait.fetch_and_store( false ); +inline void thread_monitor::wait() { + my_sema.P(); + // memory_order_seq_cst is required here to be ordered with + // further load checking shutdown state + my_notified.store(false, std::memory_order_seq_cst); } } // namespace internal } // namespace rml +} // namespace r1 +} // namespace detail +} // namespace tbb #endif /* __RML_thread_monitor_H */ diff --git a/src/tbb/src/tbb/rtm_mutex.cpp b/src/tbb/src/tbb/rtm_mutex.cpp new file mode 100644 index 000000000..f386735b9 --- /dev/null +++ b/src/tbb/src/tbb/rtm_mutex.cpp @@ -0,0 +1,121 @@ +/* + Copyright (c) 2005-2022 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "oneapi/tbb/detail/_assert.h" +#include "oneapi/tbb/detail/_rtm_mutex.h" +#include "itt_notify.h" +#include "governor.h" +#include "misc.h" + +#include <atomic> + +namespace tbb { +namespace detail { +namespace r1 { + + +struct rtm_mutex_impl { + // maximum number of times to retry + // TODO: experiment on retry values. + static constexpr int retry_threshold = 10; + using transaction_result_type = decltype(begin_transaction()); + + //! Release speculative mutex + static void release(d1::rtm_mutex::scoped_lock& s) { + switch(s.m_transaction_state) { + case d1::rtm_mutex::rtm_state::rtm_transacting: + __TBB_ASSERT(is_in_transaction(), "m_transaction_state && not speculating"); + end_transaction(); + s.m_mutex = nullptr; + break; + case d1::rtm_mutex::rtm_state::rtm_real: + s.m_mutex->unlock(); + s.m_mutex = nullptr; + break; + case d1::rtm_mutex::rtm_state::rtm_none: + __TBB_ASSERT(false, "mutex is not locked, but in release"); + break; + default: + __TBB_ASSERT(false, "invalid m_transaction_state"); + } + s.m_transaction_state = d1::rtm_mutex::rtm_state::rtm_none; + } + + //! Acquire lock on the given mutex. + static void acquire(d1::rtm_mutex& m, d1::rtm_mutex::scoped_lock& s, bool only_speculate) { + __TBB_ASSERT(s.m_transaction_state == d1::rtm_mutex::rtm_state::rtm_none, "scoped_lock already in transaction"); + if(governor::speculation_enabled()) { + int num_retries = 0; + transaction_result_type abort_code = 0; + do { + if(m.m_flag.load(std::memory_order_acquire)) { + if(only_speculate) return; + spin_wait_while_eq(m.m_flag, true); + } + // _xbegin returns -1 on success or the abort code, so capture it + if((abort_code = begin_transaction()) == transaction_result_type(speculation_successful_begin)) + { + // started speculation + if(m.m_flag.load(std::memory_order_relaxed)) { + abort_transaction(); + } + s.m_transaction_state = d1::rtm_mutex::rtm_state::rtm_transacting; + // Don not wrap the following assignment to a function, + // because it can abort the transaction in debug. Need mutex for release(). + s.m_mutex = &m; + return; // successfully started speculation + } + ++num_retries; + } while((abort_code & speculation_retry) != 0 && (num_retries < retry_threshold)); + } + + if(only_speculate) return; + s.m_mutex = &m; + s.m_mutex->lock(); + s.m_transaction_state = d1::rtm_mutex::rtm_state::rtm_real; + } + + //! Try to acquire lock on the given mutex. + static bool try_acquire(d1::rtm_mutex& m, d1::rtm_mutex::scoped_lock& s) { + acquire(m, s, /*only_speculate=*/true); + if (s.m_transaction_state == d1::rtm_mutex::rtm_state::rtm_transacting) { + return true; + } + __TBB_ASSERT(s.m_transaction_state == d1::rtm_mutex::rtm_state::rtm_none, nullptr); + // transacting acquire failed. try_lock the real mutex + if (m.try_lock()) { + s.m_mutex = &m; + s.m_transaction_state = d1::rtm_mutex::rtm_state::rtm_real; + return true; + } + return false; + } +}; + +void __TBB_EXPORTED_FUNC acquire(d1::rtm_mutex& m, d1::rtm_mutex::scoped_lock& s, bool only_speculate) { + rtm_mutex_impl::acquire(m, s, only_speculate); +} +bool __TBB_EXPORTED_FUNC try_acquire(d1::rtm_mutex& m, d1::rtm_mutex::scoped_lock& s) { + return rtm_mutex_impl::try_acquire(m, s); +} +void __TBB_EXPORTED_FUNC release(d1::rtm_mutex::scoped_lock& s) { + rtm_mutex_impl::release(s); +} + +} // namespace r1 +} // namespace detail +} // namespace tbb + diff --git a/src/tbb/src/tbb/rtm_rw_mutex.cpp b/src/tbb/src/tbb/rtm_rw_mutex.cpp new file mode 100644 index 000000000..fa87d0e39 --- /dev/null +++ b/src/tbb/src/tbb/rtm_rw_mutex.cpp @@ -0,0 +1,271 @@ +/* + Copyright (c) 2005-2022 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "oneapi/tbb/detail/_assert.h" +#include "oneapi/tbb/detail/_rtm_rw_mutex.h" +#include "itt_notify.h" +#include "governor.h" +#include "misc.h" + +#include <atomic> + +namespace tbb { +namespace detail { +namespace r1 { + +struct rtm_rw_mutex_impl { + // maximum number of times to retry + // TODO: experiment on retry values. + static constexpr int retry_threshold_read = 10; + static constexpr int retry_threshold_write = 10; + using transaction_result_type = decltype(begin_transaction()); + + //! Release speculative mutex + static void release(d1::rtm_rw_mutex::scoped_lock& s) { + switch(s.m_transaction_state) { + case d1::rtm_rw_mutex::rtm_type::rtm_transacting_writer: + case d1::rtm_rw_mutex::rtm_type::rtm_transacting_reader: + __TBB_ASSERT(is_in_transaction(), "m_transaction_state && not speculating"); + end_transaction(); + s.m_mutex = nullptr; + break; + case d1::rtm_rw_mutex::rtm_type::rtm_real_reader: + __TBB_ASSERT(!s.m_mutex->write_flag.load(std::memory_order_relaxed), "write_flag set but read lock acquired"); + s.m_mutex->unlock_shared(); + s.m_mutex = nullptr; + break; + case d1::rtm_rw_mutex::rtm_type::rtm_real_writer: + __TBB_ASSERT(s.m_mutex->write_flag.load(std::memory_order_relaxed), "write_flag unset but write lock acquired"); + s.m_mutex->write_flag.store(false, std::memory_order_relaxed); + s.m_mutex->unlock(); + s.m_mutex = nullptr; + break; + case d1::rtm_rw_mutex::rtm_type::rtm_not_in_mutex: + __TBB_ASSERT(false, "rtm_not_in_mutex, but in release"); + break; + default: + __TBB_ASSERT(false, "invalid m_transaction_state"); + } + s.m_transaction_state = d1::rtm_rw_mutex::rtm_type::rtm_not_in_mutex; + } + + //! Acquire write lock on the given mutex. + static void acquire_writer(d1::rtm_rw_mutex& m, d1::rtm_rw_mutex::scoped_lock& s, bool only_speculate) { + __TBB_ASSERT(s.m_transaction_state == d1::rtm_rw_mutex::rtm_type::rtm_not_in_mutex, "scoped_lock already in transaction"); + if(governor::speculation_enabled()) { + int num_retries = 0; + transaction_result_type abort_code = 0; + do { + if(m.m_state.load(std::memory_order_acquire)) { + if(only_speculate) return; + spin_wait_until_eq(m.m_state, d1::rtm_rw_mutex::state_type(0)); + } + // _xbegin returns -1 on success or the abort code, so capture it + if((abort_code = begin_transaction()) == transaction_result_type(speculation_successful_begin)) + { + // started speculation + if(m.m_state.load(std::memory_order_relaxed)) { // add spin_rw_mutex to read-set. + // reader or writer grabbed the lock, so abort. + abort_transaction(); + } + s.m_transaction_state = d1::rtm_rw_mutex::rtm_type::rtm_transacting_writer; + // Don not wrap the following assignment to a function, + // because it can abort the transaction in debug. Need mutex for release(). + s.m_mutex = &m; + return; // successfully started speculation + } + ++num_retries; + } while((abort_code & speculation_retry) != 0 && (num_retries < retry_threshold_write)); + } + + if(only_speculate) return; + s.m_mutex = &m; // should apply a real try_lock... + s.m_mutex->lock(); // kill transactional writers + __TBB_ASSERT(!m.write_flag.load(std::memory_order_relaxed), "After acquire for write, write_flag already true"); + m.write_flag.store(true, std::memory_order_relaxed); // kill transactional readers + s.m_transaction_state = d1::rtm_rw_mutex::rtm_type::rtm_real_writer; + } + + //! Acquire read lock on given mutex. + // only_speculate : true if we are doing a try_acquire. If true and we fail to speculate, don't + // really acquire the lock, return and do a try_acquire on the contained spin_rw_mutex. If + // the lock is already held by a writer, just return. + static void acquire_reader(d1::rtm_rw_mutex& m, d1::rtm_rw_mutex::scoped_lock& s, bool only_speculate) { + __TBB_ASSERT(s.m_transaction_state == d1::rtm_rw_mutex::rtm_type::rtm_not_in_mutex, "scoped_lock already in transaction"); + if(governor::speculation_enabled()) { + int num_retries = 0; + transaction_result_type abort_code = 0; + do { + // if in try_acquire, and lock is held as writer, don't attempt to speculate. + if(m.write_flag.load(std::memory_order_acquire)) { + if(only_speculate) return; + spin_wait_while_eq(m.write_flag, true); + } + // _xbegin returns -1 on success or the abort code, so capture it + if((abort_code = begin_transaction()) == transaction_result_type(speculation_successful_begin)) + { + // started speculation + if(m.write_flag.load(std::memory_order_relaxed)) { // add write_flag to read-set. + abort_transaction(); // writer grabbed the lock, so abort. + } + s.m_transaction_state = d1::rtm_rw_mutex::rtm_type::rtm_transacting_reader; + // Don not wrap the following assignment to a function, + // because it can abort the transaction in debug. Need mutex for release(). + s.m_mutex = &m; + return; // successfully started speculation + } + // fallback path + // retry only if there is any hope of getting into a transaction soon + // Retry in the following cases (from Section 8.3.5 of + // Intel(R) Architecture Instruction Set Extensions Programming Reference): + // 1. abort caused by XABORT instruction (bit 0 of EAX register is set) + // 2. the transaction may succeed on a retry (bit 1 of EAX register is set) + // 3. if another logical processor conflicted with a memory address + // that was part of the transaction that aborted (bit 2 of EAX register is set) + // That is, retry if (abort_code & 0x7) is non-zero + ++num_retries; + } while((abort_code & speculation_retry) != 0 && (num_retries < retry_threshold_read)); + } + + if(only_speculate) return; + s.m_mutex = &m; + s.m_mutex->lock_shared(); + s.m_transaction_state = d1::rtm_rw_mutex::rtm_type::rtm_real_reader; + } + + //! Upgrade reader to become a writer. + /** Returns whether the upgrade happened without releasing and re-acquiring the lock */ + static bool upgrade(d1::rtm_rw_mutex::scoped_lock& s) { + switch(s.m_transaction_state) { + case d1::rtm_rw_mutex::rtm_type::rtm_real_reader: { + s.m_transaction_state = d1::rtm_rw_mutex::rtm_type::rtm_real_writer; + bool no_release = s.m_mutex->upgrade(); + __TBB_ASSERT(!s.m_mutex->write_flag.load(std::memory_order_relaxed), "After upgrade, write_flag already true"); + s.m_mutex->write_flag.store(true, std::memory_order_relaxed); + return no_release; + } + case d1::rtm_rw_mutex::rtm_type::rtm_transacting_reader: { + d1::rtm_rw_mutex& m = *s.m_mutex; + if(m.m_state.load(std::memory_order_acquire)) { // add spin_rw_mutex to read-set. + // Real reader or writer holds the lock; so commit the read and re-acquire for write. + release(s); + acquire_writer(m, s, false); + return false; + } else + { + s.m_transaction_state = d1::rtm_rw_mutex::rtm_type::rtm_transacting_writer; + return true; + } + } + default: + __TBB_ASSERT(false, "Invalid state for upgrade"); + return false; + } + } + + //! Downgrade writer to a reader. + static bool downgrade(d1::rtm_rw_mutex::scoped_lock& s) { + switch (s.m_transaction_state) { + case d1::rtm_rw_mutex::rtm_type::rtm_real_writer: + s.m_transaction_state = d1::rtm_rw_mutex::rtm_type::rtm_real_reader; + __TBB_ASSERT(s.m_mutex->write_flag.load(std::memory_order_relaxed), "Before downgrade write_flag not true"); + s.m_mutex->write_flag.store(false, std::memory_order_relaxed); + s.m_mutex->downgrade(); + return true; + case d1::rtm_rw_mutex::rtm_type::rtm_transacting_writer: + s.m_transaction_state = d1::rtm_rw_mutex::rtm_type::rtm_transacting_reader; + return true; + default: + __TBB_ASSERT(false, "Invalid state for downgrade"); + return false; + } + } + + //! Try to acquire write lock on the given mutex. + // There may be reader(s) which acquired the spin_rw_mutex, as well as possibly + // transactional reader(s). If this is the case, the acquire will fail, and assigning + // write_flag will kill the transactors. So we only assign write_flag if we have successfully + // acquired the lock. + static bool try_acquire_writer(d1::rtm_rw_mutex& m, d1::rtm_rw_mutex::scoped_lock& s) { + acquire_writer(m, s, /*only_speculate=*/true); + if (s.m_transaction_state == d1::rtm_rw_mutex::rtm_type::rtm_transacting_writer) { + return true; + } + __TBB_ASSERT(s.m_transaction_state == d1::rtm_rw_mutex::rtm_type::rtm_not_in_mutex, nullptr); + // transacting write acquire failed. try_lock the real mutex + if (m.try_lock()) { + s.m_mutex = &m; + // only shoot down readers if we're not transacting ourselves + __TBB_ASSERT(!m.write_flag.load(std::memory_order_relaxed), "After try_acquire_writer, write_flag already true"); + m.write_flag.store(true, std::memory_order_relaxed); + s.m_transaction_state = d1::rtm_rw_mutex::rtm_type::rtm_real_writer; + return true; + } + return false; + } + + //! Try to acquire read lock on the given mutex. + static bool try_acquire_reader(d1::rtm_rw_mutex& m, d1::rtm_rw_mutex::scoped_lock& s) { + // speculatively acquire the lock. If this fails, do try_lock_shared on the spin_rw_mutex. + acquire_reader(m, s, /*only_speculate=*/true); + if (s.m_transaction_state == d1::rtm_rw_mutex::rtm_type::rtm_transacting_reader) { + return true; + } + __TBB_ASSERT(s.m_transaction_state == d1::rtm_rw_mutex::rtm_type::rtm_not_in_mutex, nullptr); + // transacting read acquire failed. try_lock_shared the real mutex + if (m.try_lock_shared()) { + s.m_mutex = &m; + s.m_transaction_state = d1::rtm_rw_mutex::rtm_type::rtm_real_reader; + return true; + } + return false; + } +}; + +void __TBB_EXPORTED_FUNC acquire_writer(d1::rtm_rw_mutex& m, d1::rtm_rw_mutex::scoped_lock& s, bool only_speculate) { + rtm_rw_mutex_impl::acquire_writer(m, s, only_speculate); +} +//! Internal acquire read lock. +// only_speculate == true if we're doing a try_lock, else false. +void __TBB_EXPORTED_FUNC acquire_reader(d1::rtm_rw_mutex& m, d1::rtm_rw_mutex::scoped_lock& s, bool only_speculate) { + rtm_rw_mutex_impl::acquire_reader(m, s, only_speculate); +} +//! Internal upgrade reader to become a writer. +bool __TBB_EXPORTED_FUNC upgrade(d1::rtm_rw_mutex::scoped_lock& s) { + return rtm_rw_mutex_impl::upgrade(s); +} +//! Internal downgrade writer to become a reader. +bool __TBB_EXPORTED_FUNC downgrade(d1::rtm_rw_mutex::scoped_lock& s) { + return rtm_rw_mutex_impl::downgrade(s); +} +//! Internal try_acquire write lock. +bool __TBB_EXPORTED_FUNC try_acquire_writer(d1::rtm_rw_mutex& m, d1::rtm_rw_mutex::scoped_lock& s) { + return rtm_rw_mutex_impl::try_acquire_writer(m, s); +} +//! Internal try_acquire read lock. +bool __TBB_EXPORTED_FUNC try_acquire_reader(d1::rtm_rw_mutex& m, d1::rtm_rw_mutex::scoped_lock& s) { + return rtm_rw_mutex_impl::try_acquire_reader(m, s); +} +//! Internal release lock. +void __TBB_EXPORTED_FUNC release(d1::rtm_rw_mutex::scoped_lock& s) { + rtm_rw_mutex_impl::release(s); +} + +} // namespace r1 +} // namespace detail +} // namespace tbb + + diff --git a/src/tbb/src/tbb/scheduler.cpp b/src/tbb/src/tbb/scheduler.cpp deleted file mode 100644 index adb01c808..000000000 --- a/src/tbb/src/tbb/scheduler.cpp +++ /dev/null @@ -1,1340 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#include "custom_scheduler.h" -#include "scheduler_utility.h" -#include "governor.h" -#include "market.h" -#include "arena.h" -#include "mailbox.h" -#include "observer_proxy.h" -#include "tbb/tbb_machine.h" -#include "tbb/atomic.h" - -namespace tbb { -namespace internal { - -//------------------------------------------------------------------------ -// Library initialization -//------------------------------------------------------------------------ - -/** Defined in tbb_main.cpp **/ -extern generic_scheduler* (*AllocateSchedulerPtr)( market& ); - -inline generic_scheduler* allocate_scheduler ( market& m ) { - return AllocateSchedulerPtr( m ); -} - -#if __TBB_TASK_GROUP_CONTEXT -context_state_propagation_mutex_type the_context_state_propagation_mutex; - -uintptr_t the_context_state_propagation_epoch = 0; - -//! Context to be associated with dummy tasks of worker threads schedulers. -/** It is never used for its direct purpose, and is introduced solely for the sake - of avoiding one extra conditional branch in the end of wait_for_all method. **/ -static task_group_context the_dummy_context(task_group_context::isolated); -#endif /* __TBB_TASK_GROUP_CONTEXT */ - -void Scheduler_OneTimeInitialization ( bool itt_present ) { - AllocateSchedulerPtr = itt_present ? &custom_scheduler<DefaultSchedulerTraits>::allocate_scheduler : - &custom_scheduler<IntelSchedulerTraits>::allocate_scheduler; -#if __TBB_TASK_GROUP_CONTEXT - // There must be no tasks belonging to this fake task group. Mark invalid for the assert - __TBB_ASSERT(!(task_group_context::low_unused_state_bit & (task_group_context::low_unused_state_bit-1)), NULL); - the_dummy_context.my_state = task_group_context::low_unused_state_bit; -#if __TBB_TASK_PRIORITY - // It should never prevent tasks from being passed to execution. - the_dummy_context.my_priority = num_priority_levels - 1; -#endif /* __TBB_TASK_PRIORITY */ -#endif /* __TBB_TASK_GROUP_CONTEXT */ -} - -//------------------------------------------------------------------------ -// scheduler interface -//------------------------------------------------------------------------ - -// A pure virtual destructor should still have a body -// so the one for tbb::internal::scheduler::~scheduler() is provided here -scheduler::~scheduler( ) {} - -//------------------------------------------------------------------------ -// generic_scheduler -//------------------------------------------------------------------------ - -#if _MSC_VER && !defined(__INTEL_COMPILER) - // Suppress overzealous compiler warning about using 'this' in base initializer list. - #pragma warning(push) - #pragma warning(disable:4355) -#endif - -generic_scheduler::generic_scheduler( market& m ) - : my_market(&m) - , my_random(this) - , my_ref_count(1) - , my_small_task_count(1) // Extra 1 is a guard reference -#if __TBB_SURVIVE_THREAD_SWITCH && TBB_USE_ASSERT - , my_cilk_state(cs_none) -#endif /* __TBB_SURVIVE_THREAD_SWITCH && TBB_USE_ASSERT */ -{ - __TBB_ASSERT( !my_arena_index, "constructor expects the memory being zero-initialized" ); - __TBB_ASSERT( governor::is_set(NULL), "scheduler is already initialized for this thread" ); - - my_innermost_running_task = my_dummy_task = &allocate_task( sizeof(task), __TBB_CONTEXT_ARG(NULL, &the_dummy_context) ); - my_properties.outermost = true; -#if __TBB_TASK_PRIORITY - my_ref_top_priority = &m.my_global_top_priority; - my_ref_reload_epoch = &m.my_global_reload_epoch; -#endif /* __TBB_TASK_PRIORITY */ -#if __TBB_TASK_GROUP_CONTEXT - // Sync up the local cancellation state with the global one. No need for fence here. - my_context_state_propagation_epoch = the_context_state_propagation_epoch; - my_context_list_head.my_prev = &my_context_list_head; - my_context_list_head.my_next = &my_context_list_head; - ITT_SYNC_CREATE(&my_context_list_mutex, SyncType_Scheduler, SyncObj_ContextsList); -#endif /* __TBB_TASK_GROUP_CONTEXT */ - ITT_SYNC_CREATE(&my_dummy_task->prefix().ref_count, SyncType_Scheduler, SyncObj_WorkerLifeCycleMgmt); - ITT_SYNC_CREATE(&my_return_list, SyncType_Scheduler, SyncObj_TaskReturnList); -} - -#if _MSC_VER && !defined(__INTEL_COMPILER) - #pragma warning(pop) -#endif // warning 4355 is back - -#if TBB_USE_ASSERT > 1 -void generic_scheduler::assert_task_pool_valid() const { - if ( !my_arena_slot ) - return; - acquire_task_pool(); - task** tp = my_arena_slot->task_pool_ptr; - if ( my_arena_slot->my_task_pool_size ) - __TBB_ASSERT( my_arena_slot->my_task_pool_size >= min_task_pool_size, NULL ); - const size_t H = __TBB_load_relaxed(my_arena_slot->head); // mirror - const size_t T = __TBB_load_relaxed(my_arena_slot->tail); // mirror - __TBB_ASSERT( H <= T, NULL ); - for ( size_t i = 0; i < H; ++i ) - __TBB_ASSERT( tp[i] == poisoned_ptr, "Task pool corrupted" ); - for ( size_t i = H; i < T; ++i ) { - if ( tp[i] ) { - assert_task_valid( tp[i] ); - __TBB_ASSERT( tp[i]->prefix().state == task::ready || - tp[i]->prefix().extra_state == es_task_proxy, "task in the deque has invalid state" ); - } - } - for ( size_t i = T; i < my_arena_slot->my_task_pool_size; ++i ) - __TBB_ASSERT( tp[i] == poisoned_ptr, "Task pool corrupted" ); - release_task_pool(); -} -#endif /* TBB_USE_ASSERT > 1 */ - -void generic_scheduler::init_stack_info () { - // Stacks are growing top-down. Highest address is called "stack base", - // and the lowest is "stack limit". - __TBB_ASSERT( !my_stealing_threshold, "Stealing threshold has already been calculated" ); - size_t stack_size = my_market->worker_stack_size(); -#if USE_WINTHREAD -#if defined(_MSC_VER)&&_MSC_VER<1400 && !_WIN64 - NT_TIB *pteb; - __asm mov eax, fs:[0x18] - __asm mov pteb, eax -#else - NT_TIB *pteb = (NT_TIB*)NtCurrentTeb(); -#endif - __TBB_ASSERT( &pteb < pteb->StackBase && &pteb > pteb->StackLimit, "invalid stack info in TEB" ); - __TBB_ASSERT( stack_size >0, "stack_size not initialized?" ); - // When a thread is created with the attribute STACK_SIZE_PARAM_IS_A_RESERVATION, stack limit - // in the TIB points to the committed part of the stack only. This renders the expression - // "(uintptr_t)pteb->StackBase / 2 + (uintptr_t)pteb->StackLimit / 2" virtually useless. - // Thus for worker threads we use the explicit stack size we used while creating them. - // And for master threads we rely on the following fact and assumption: - // - the default stack size of a master thread on Windows is 1M; - // - if it was explicitly set by the application it is at least as large as the size of a worker stack. - if ( is_worker() || stack_size < MByte ) - my_stealing_threshold = (uintptr_t)pteb->StackBase - stack_size / 2; - else - my_stealing_threshold = (uintptr_t)pteb->StackBase - MByte / 2; -#else /* USE_PTHREAD */ - // There is no portable way to get stack base address in Posix, so we use - // non-portable method (on all modern Linux) or the simplified approach - // based on the common sense assumptions. The most important assumption - // is that the main thread's stack size is not less than that of other threads. - // See also comment 3 at the end of this file - void *stack_base = &stack_size; -#if __linux__ && !__bg__ -#if __TBB_ipf - void *rsb_base = __TBB_get_bsp(); -#endif - size_t np_stack_size = 0; - void *stack_limit = NULL; - pthread_attr_t np_attr_stack; - if( 0 == pthread_getattr_np(pthread_self(), &np_attr_stack) ) { - if ( 0 == pthread_attr_getstack(&np_attr_stack, &stack_limit, &np_stack_size) ) { -#if __TBB_ipf - pthread_attr_t attr_stack; - if ( 0 == pthread_attr_init(&attr_stack) ) { - if ( 0 == pthread_attr_getstacksize(&attr_stack, &stack_size) ) { - if ( np_stack_size < stack_size ) { - // We are in a secondary thread. Use reliable data. - // IA-64 architecture stack is split into RSE backup and memory parts - rsb_base = stack_limit; - stack_size = np_stack_size/2; - // Limit of the memory part of the stack - stack_limit = (char*)stack_limit + stack_size; - } - // We are either in the main thread or this thread stack - // is bigger that that of the main one. As we cannot discern - // these cases we fall back to the default (heuristic) values. - } - pthread_attr_destroy(&attr_stack); - } - // IA-64 architecture stack is split into RSE backup and memory parts - my_rsb_stealing_threshold = (uintptr_t)((char*)rsb_base + stack_size/2); -#endif /* __TBB_ipf */ - // Size of the stack free part - stack_size = size_t((char*)stack_base - (char*)stack_limit); - } - pthread_attr_destroy(&np_attr_stack); - } -#endif /* __linux__ */ - __TBB_ASSERT( stack_size>0, "stack size must be positive" ); - my_stealing_threshold = (uintptr_t)((char*)stack_base - stack_size/2); -#endif /* USE_PTHREAD */ -} - -#if __TBB_TASK_GROUP_CONTEXT -/** The function uses synchronization scheme similar to the one in the destructor - of task_group_context augmented with interlocked state change of each context - object. The purpose of this algo is to prevent threads doing nonlocal context - destruction from accessing destroyed owner-scheduler instance still pointed to - by the context object. **/ -void generic_scheduler::cleanup_local_context_list () { - // Detach contexts remaining in the local list - bool wait_for_concurrent_destroyers_to_leave = false; - uintptr_t local_count_snapshot = my_context_state_propagation_epoch; - my_local_ctx_list_update.store<relaxed>(1); - { - // This is just a definition. Actual lock is acquired only in case of conflict. - spin_mutex::scoped_lock lock; - // Full fence prevents reordering of store to my_local_ctx_list_update with - // load from my_nonlocal_ctx_list_update. - atomic_fence(); - // Check for the conflict with concurrent destroyer or cancellation propagator - if ( my_nonlocal_ctx_list_update.load<relaxed>() || local_count_snapshot != the_context_state_propagation_epoch ) - lock.acquire(my_context_list_mutex); - // No acquire fence is necessary for loading my_context_list_head.my_next, - // as the list can be updated by this thread only. - context_list_node_t *node = my_context_list_head.my_next; - while ( node != &my_context_list_head ) { - task_group_context &ctx = __TBB_get_object_ref(task_group_context, my_node, node); - __TBB_ASSERT( __TBB_load_relaxed(ctx.my_kind) != task_group_context::binding_required, "Only a context bound to a root task can be detached" ); - node = node->my_next; - __TBB_ASSERT( is_alive(ctx.my_version_and_traits), "Walked into a destroyed context while detaching contexts from the local list" ); - // Synchronizes with ~task_group_context(). TODO: evaluate and perhaps relax - if ( internal::as_atomic(ctx.my_kind).fetch_and_store(task_group_context::detached) == task_group_context::dying ) - wait_for_concurrent_destroyers_to_leave = true; - } - } - my_local_ctx_list_update.store<release>(0); - // Wait until other threads referencing this scheduler object finish with it - if ( wait_for_concurrent_destroyers_to_leave ) - spin_wait_until_eq( my_nonlocal_ctx_list_update, 0u ); -} -#endif /* __TBB_TASK_GROUP_CONTEXT */ - -void generic_scheduler::free_scheduler() { - __TBB_ASSERT( !my_arena_slot, NULL ); -#if __TBB_TASK_GROUP_CONTEXT - cleanup_local_context_list(); -#endif /* __TBB_TASK_GROUP_CONTEXT */ - free_task<small_local_task>( *my_dummy_task ); - -#if __TBB_HOARD_NONLOCAL_TASKS - while( task* t = my_nonlocal_free_list ) { - task_prefix& p = t->prefix(); - my_nonlocal_free_list = p.next; - __TBB_ASSERT( p.origin && p.origin!=this, NULL ); - free_nonlocal_small_task(*t); - } -#endif - // k accounts for a guard reference and each task that we deallocate. - intptr_t k = 1; - for(;;) { - while( task* t = my_free_list ) { - my_free_list = t->prefix().next; - deallocate_task(*t); - ++k; - } - if( my_return_list==plugged_return_list() ) - break; - my_free_list = (task*)__TBB_FetchAndStoreW( &my_return_list, (intptr_t)plugged_return_list() ); - } -#if __TBB_COUNT_TASK_NODES - my_market->update_task_node_count( my_task_node_count ); -#endif /* __TBB_COUNT_TASK_NODES */ - // Update my_small_task_count last. Doing so sooner might cause another thread to free *this. - __TBB_ASSERT( my_small_task_count>=k, "my_small_task_count corrupted" ); - governor::sign_off(this); - if( __TBB_FetchAndAddW( &my_small_task_count, -k )==k ) - NFS_Free( this ); -} - -task& generic_scheduler::allocate_task( size_t number_of_bytes, - __TBB_CONTEXT_ARG(task* parent, task_group_context* context) ) { - GATHER_STATISTIC(++my_counters.active_tasks); - task *t; - if( number_of_bytes<=quick_task_size ) { -#if __TBB_HOARD_NONLOCAL_TASKS - if( (t = my_nonlocal_free_list) ) { - GATHER_STATISTIC(--my_counters.free_list_length); - __TBB_ASSERT( t->state()==task::freed, "free list of tasks is corrupted" ); - my_nonlocal_free_list = t->prefix().next; - } else -#endif - if( (t = my_free_list) ) { - GATHER_STATISTIC(--my_counters.free_list_length); - __TBB_ASSERT( t->state()==task::freed, "free list of tasks is corrupted" ); - my_free_list = t->prefix().next; - } else if( my_return_list ) { - // No fence required for read of my_return_list above, because __TBB_FetchAndStoreW has a fence. - t = (task*)__TBB_FetchAndStoreW( &my_return_list, 0 ); // with acquire - __TBB_ASSERT( t, "another thread emptied the my_return_list" ); - __TBB_ASSERT( t->prefix().origin==this, "task returned to wrong my_return_list" ); - ITT_NOTIFY( sync_acquired, &my_return_list ); - my_free_list = t->prefix().next; - } else { - t = (task*)((char*)NFS_Allocate( 1, task_prefix_reservation_size+quick_task_size, NULL ) + task_prefix_reservation_size ); -#if __TBB_COUNT_TASK_NODES - ++my_task_node_count; -#endif /* __TBB_COUNT_TASK_NODES */ - t->prefix().origin = this; - t->prefix().next = 0; - ++my_small_task_count; - } -#if __TBB_PREFETCHING - task *t_next = t->prefix().next; - if( !t_next ) { // the task was last in the list -#if __TBB_HOARD_NONLOCAL_TASKS - if( my_free_list ) - t_next = my_free_list; - else -#endif - if( my_return_list ) // enable prefetching, gives speedup - t_next = my_free_list = (task*)__TBB_FetchAndStoreW( &my_return_list, 0 ); - } - if( t_next ) { // gives speedup for both cache lines - __TBB_cl_prefetch(t_next); - __TBB_cl_prefetch(&t_next->prefix()); - } -#endif /* __TBB_PREFETCHING */ - } else { - GATHER_STATISTIC(++my_counters.big_tasks); - t = (task*)((char*)NFS_Allocate( 1, task_prefix_reservation_size+number_of_bytes, NULL ) + task_prefix_reservation_size ); -#if __TBB_COUNT_TASK_NODES - ++my_task_node_count; -#endif /* __TBB_COUNT_TASK_NODES */ - t->prefix().origin = NULL; - } - task_prefix& p = t->prefix(); -#if __TBB_TASK_GROUP_CONTEXT - p.context = context; -#endif /* __TBB_TASK_GROUP_CONTEXT */ - // Obsolete. But still in use, so has to be assigned correct value here. - p.owner = this; - p.ref_count = 0; - // Obsolete. Assign some not outrageously out-of-place value for a while. - p.depth = 0; - p.parent = parent; - // In TBB 2.1 and later, the constructor for task sets extra_state to indicate the version of the tbb/task.h header. - // In TBB 2.0 and earlier, the constructor leaves extra_state as zero. - p.extra_state = 0; - p.affinity = 0; - p.state = task::allocated; - __TBB_ISOLATION_EXPR( p.isolation = no_isolation ); - return *t; -} - -void generic_scheduler::free_nonlocal_small_task( task& t ) { - __TBB_ASSERT( t.state()==task::freed, NULL ); - generic_scheduler& s = *static_cast<generic_scheduler*>(t.prefix().origin); - __TBB_ASSERT( &s!=this, NULL ); - for(;;) { - task* old = s.my_return_list; - if( old==plugged_return_list() ) - break; - // Atomically insert t at head of s.my_return_list - t.prefix().next = old; - ITT_NOTIFY( sync_releasing, &s.my_return_list ); - if( as_atomic(s.my_return_list).compare_and_swap(&t, old )==old ) { -#if __TBB_PREFETCHING - __TBB_cl_evict(&t.prefix()); - __TBB_cl_evict(&t); -#endif - return; - } - } - deallocate_task(t); - if( __TBB_FetchAndDecrementWrelease( &s.my_small_task_count )==1 ) { - // We freed the last task allocated by scheduler s, so it's our responsibility - // to free the scheduler. - NFS_Free( &s ); - } -} - -inline size_t generic_scheduler::prepare_task_pool ( size_t num_tasks ) { - size_t T = __TBB_load_relaxed(my_arena_slot->tail); // mirror - if ( T + num_tasks <= my_arena_slot->my_task_pool_size ) - return T; - - size_t new_size = num_tasks; - - if ( !my_arena_slot->my_task_pool_size ) { - __TBB_ASSERT( !is_task_pool_published() && is_quiescent_local_task_pool_reset(), NULL ); - __TBB_ASSERT( !my_arena_slot->task_pool_ptr, NULL ); - if ( num_tasks < min_task_pool_size ) new_size = min_task_pool_size; - my_arena_slot->allocate_task_pool( new_size ); - return 0; - } - - acquire_task_pool(); - size_t H = __TBB_load_relaxed( my_arena_slot->head ); // mirror - task** task_pool = my_arena_slot->task_pool_ptr;; - __TBB_ASSERT( my_arena_slot->my_task_pool_size >= min_task_pool_size, NULL ); - // Count not skipped tasks. Consider using std::count_if. - for ( size_t i = H; i < T; ++i ) - if ( task_pool[i] ) ++new_size; - // If the free space at the beginning of the task pool is too short, we - // are likely facing a pathological single-producer-multiple-consumers - // scenario, and thus it's better to expand the task pool - bool allocate = new_size > my_arena_slot->my_task_pool_size - min_task_pool_size/4; - if ( allocate ) { - // Grow task pool. As this operation is rare, and its cost is asymptotically - // amortizable, we can tolerate new task pool allocation done under the lock. - if ( new_size < 2 * my_arena_slot->my_task_pool_size ) - new_size = 2 * my_arena_slot->my_task_pool_size; - my_arena_slot->allocate_task_pool( new_size ); // updates my_task_pool_size - } - // Filter out skipped tasks. Consider using std::copy_if. - size_t T1 = 0; - for ( size_t i = H; i < T; ++i ) - if ( task_pool[i] ) - my_arena_slot->task_pool_ptr[T1++] = task_pool[i]; - // Deallocate the previous task pool if a new one has been allocated. - if ( allocate ) - NFS_Free( task_pool ); - else - my_arena_slot->fill_with_canary_pattern( T1, my_arena_slot->tail ); - // Publish the new state. - commit_relocated_tasks( T1 ); - assert_task_pool_valid(); - return T1; -} - -/** ATTENTION: - This method is mostly the same as generic_scheduler::lock_task_pool(), with - a little different logic of slot state checks (slot is either locked or points - to our task pool). - Thus if either of them is changed, consider changing the counterpart as well. **/ -inline void generic_scheduler::acquire_task_pool() const { - if ( !is_task_pool_published() ) - return; // we are not in arena - nothing to lock - bool sync_prepare_done = false; - for( atomic_backoff b;;b.pause() ) { -#if TBB_USE_ASSERT - __TBB_ASSERT( my_arena_slot == my_arena->my_slots + my_arena_index, "invalid arena slot index" ); - // Local copy of the arena slot task pool pointer is necessary for the next - // assertion to work correctly to exclude asynchronous state transition effect. - task** tp = my_arena_slot->task_pool; - __TBB_ASSERT( tp == LockedTaskPool || tp == my_arena_slot->task_pool_ptr, "slot ownership corrupt?" ); -#endif - if( my_arena_slot->task_pool != LockedTaskPool && - as_atomic(my_arena_slot->task_pool).compare_and_swap(LockedTaskPool, my_arena_slot->task_pool_ptr ) == my_arena_slot->task_pool_ptr ) - { - // We acquired our own slot - ITT_NOTIFY(sync_acquired, my_arena_slot); - break; - } - else if( !sync_prepare_done ) { - // Start waiting - ITT_NOTIFY(sync_prepare, my_arena_slot); - sync_prepare_done = true; - } - // Someone else acquired a lock, so pause and do exponential backoff. - } - __TBB_ASSERT( my_arena_slot->task_pool == LockedTaskPool, "not really acquired task pool" ); -} // generic_scheduler::acquire_task_pool - -inline void generic_scheduler::release_task_pool() const { - if ( !is_task_pool_published() ) - return; // we are not in arena - nothing to unlock - __TBB_ASSERT( my_arena_slot, "we are not in arena" ); - __TBB_ASSERT( my_arena_slot->task_pool == LockedTaskPool, "arena slot is not locked" ); - ITT_NOTIFY(sync_releasing, my_arena_slot); - __TBB_store_with_release( my_arena_slot->task_pool, my_arena_slot->task_pool_ptr ); -} - -/** ATTENTION: - This method is mostly the same as generic_scheduler::acquire_task_pool(), - with a little different logic of slot state checks (slot can be empty, locked - or point to any task pool other than ours, and asynchronous transitions between - all these states are possible). - Thus if any of them is changed, consider changing the counterpart as well **/ -inline task** generic_scheduler::lock_task_pool( arena_slot* victim_arena_slot ) const { - task** victim_task_pool; - bool sync_prepare_done = false; - for( atomic_backoff backoff;; /*backoff pause embedded in the loop*/) { - victim_task_pool = victim_arena_slot->task_pool; - // NOTE: Do not use comparison of head and tail indices to check for - // the presence of work in the victim's task pool, as they may give - // incorrect indication because of task pool relocations and resizes. - if ( victim_task_pool == EmptyTaskPool ) { - // The victim thread emptied its task pool - nothing to lock - if( sync_prepare_done ) - ITT_NOTIFY(sync_cancel, victim_arena_slot); - break; - } - if( victim_task_pool != LockedTaskPool && - as_atomic(victim_arena_slot->task_pool).compare_and_swap(LockedTaskPool, victim_task_pool ) == victim_task_pool ) - { - // We've locked victim's task pool - ITT_NOTIFY(sync_acquired, victim_arena_slot); - break; - } - else if( !sync_prepare_done ) { - // Start waiting - ITT_NOTIFY(sync_prepare, victim_arena_slot); - sync_prepare_done = true; - } - GATHER_STATISTIC( ++my_counters.thieves_conflicts ); - // Someone else acquired a lock, so pause and do exponential backoff. -#if __TBB_STEALING_ABORT_ON_CONTENTION - if(!backoff.bounded_pause()) { - // the 16 was acquired empirically and a theory behind it supposes - // that number of threads becomes much bigger than number of - // tasks which can be spawned by one thread causing excessive contention. - // TODO: However even small arenas can benefit from the abort on contention - // if preemption of a thief is a problem - if(my_arena->my_limit >= 16) - return EmptyTaskPool; - __TBB_Yield(); - } -#else - backoff.pause(); -#endif - } - __TBB_ASSERT( victim_task_pool == EmptyTaskPool || - (victim_arena_slot->task_pool == LockedTaskPool && victim_task_pool != LockedTaskPool), - "not really locked victim's task pool?" ); - return victim_task_pool; -} // generic_scheduler::lock_task_pool - -inline void generic_scheduler::unlock_task_pool( arena_slot* victim_arena_slot, - task** victim_task_pool ) const { - __TBB_ASSERT( victim_arena_slot, "empty victim arena slot pointer" ); - __TBB_ASSERT( victim_arena_slot->task_pool == LockedTaskPool, "victim arena slot is not locked" ); - ITT_NOTIFY(sync_releasing, victim_arena_slot); - __TBB_store_with_release( victim_arena_slot->task_pool, victim_task_pool ); -} - - -inline task* generic_scheduler::prepare_for_spawning( task* t ) { - __TBB_ASSERT( t->state()==task::allocated, "attempt to spawn task that is not in 'allocated' state" ); - t->prefix().state = task::ready; -#if TBB_USE_ASSERT - if( task* parent = t->parent() ) { - internal::reference_count ref_count = parent->prefix().ref_count; - __TBB_ASSERT( ref_count>=0, "attempt to spawn task whose parent has a ref_count<0" ); - __TBB_ASSERT( ref_count!=0, "attempt to spawn task whose parent has a ref_count==0 (forgot to set_ref_count?)" ); - parent->prefix().extra_state |= es_ref_count_active; - } -#endif /* TBB_USE_ASSERT */ - affinity_id dst_thread = t->prefix().affinity; - __TBB_ASSERT( dst_thread == 0 || is_version_3_task(*t), - "backwards compatibility to TBB 2.0 tasks is broken" ); -#if __TBB_TASK_ISOLATION - isolation_tag isolation = my_innermost_running_task->prefix().isolation; - t->prefix().isolation = isolation; -#endif /* __TBB_TASK_ISOLATION */ - if( dst_thread != 0 && dst_thread != my_affinity_id ) { - task_proxy& proxy = (task_proxy&)allocate_task( sizeof(task_proxy), - __TBB_CONTEXT_ARG(NULL, NULL) ); - // Mark as a proxy - proxy.prefix().extra_state = es_task_proxy; - proxy.outbox = &my_arena->mailbox(dst_thread); - // Mark proxy as present in both locations (sender's task pool and destination mailbox) - proxy.task_and_tag = intptr_t(t) | task_proxy::location_mask; -#if __TBB_TASK_PRIORITY - poison_pointer( proxy.prefix().context ); -#endif /* __TBB_TASK_PRIORITY */ - __TBB_ISOLATION_EXPR( proxy.prefix().isolation = isolation ); - ITT_NOTIFY( sync_releasing, proxy.outbox ); - // Mail the proxy - after this point t may be destroyed by another thread at any moment. - proxy.outbox->push(&proxy); - return &proxy; - } - return t; -} - -/** Conceptually, this method should be a member of class scheduler. - But doing so would force us to publish class scheduler in the headers. */ -void generic_scheduler::local_spawn( task* first, task*& next ) { - __TBB_ASSERT( first, NULL ); - __TBB_ASSERT( governor::is_set(this), NULL ); -#if __TBB_TODO - // We need to consider capping the max task pool size and switching - // to in-place task execution whenever it is reached. -#endif - if ( &first->prefix().next == &next ) { - // Single task is being spawned -#if __TBB_TODO - // TODO: - // In the future we need to add overloaded spawn method for a single task, - // and a method accepting an array of task pointers (we may also want to - // change the implementation of the task_list class). But since such changes - // may affect the binary compatibility, we postpone them for a while. -#endif - size_t T = prepare_task_pool( 1 ); - my_arena_slot->task_pool_ptr[T] = prepare_for_spawning( first ); - commit_spawned_tasks( T + 1 ); - } - else { - // Task list is being spawned -#if __TBB_TODO - // TODO: add task_list::front() and implement&document the local execution ordering which is - // opposite to the current implementation. The idea is to remove hackish fast_reverse_vector - // and use push_back/push_front when accordingly LIFO and FIFO order of local execution is - // desired. It also requires refactoring of the reload_tasks method and my_offloaded_tasks list. - // Additional benefit may come from adding counter to the task_list so that it can reserve enough - // space in the task pool in advance and move all the tasks directly without any intermediate - // storages. But it requires dealing with backward compatibility issues and still supporting - // counter-less variant (though not necessarily fast implementation). -#endif - task *arr[min_task_pool_size]; - fast_reverse_vector<task*> tasks(arr, min_task_pool_size); - task *t_next = NULL; - for( task* t = first; ; t = t_next ) { - // If t is affinitized to another thread, it may already be executed - // and destroyed by the time prepare_for_spawning returns. - // So milk it while it is alive. - bool end = &t->prefix().next == &next; - t_next = t->prefix().next; - tasks.push_back( prepare_for_spawning(t) ); - if( end ) - break; - } - size_t num_tasks = tasks.size(); - size_t T = prepare_task_pool( num_tasks ); - tasks.copy_memory( my_arena_slot->task_pool_ptr + T ); - commit_spawned_tasks( T + num_tasks ); - } - if ( !is_task_pool_published() ) - publish_task_pool(); - my_arena->advertise_new_work<arena::work_spawned>(); - assert_task_pool_valid(); -} - -void generic_scheduler::local_spawn_root_and_wait( task* first, task*& next ) { - __TBB_ASSERT( governor::is_set(this), NULL ); - __TBB_ASSERT( first, NULL ); - auto_empty_task dummy( __TBB_CONTEXT_ARG(this, first->prefix().context) ); - internal::reference_count n = 0; - for( task* t=first; ; t=t->prefix().next ) { - ++n; - __TBB_ASSERT( !t->prefix().parent, "not a root task, or already running" ); - t->prefix().parent = &dummy; - if( &t->prefix().next==&next ) break; -#if __TBB_TASK_GROUP_CONTEXT - __TBB_ASSERT( t->prefix().context == t->prefix().next->prefix().context, - "all the root tasks in list must share the same context"); -#endif /* __TBB_TASK_GROUP_CONTEXT */ - } - dummy.prefix().ref_count = n+1; - if( n>1 ) - local_spawn( first->prefix().next, next ); - local_wait_for_all( dummy, first ); -} - -void tbb::internal::generic_scheduler::spawn( task& first, task*& next ) { - governor::local_scheduler()->local_spawn( &first, next ); -} - -void tbb::internal::generic_scheduler::spawn_root_and_wait( task& first, task*& next ) { - governor::local_scheduler()->local_spawn_root_and_wait( &first, next ); -} - -void tbb::internal::generic_scheduler::enqueue( task& t, void* prio ) { - generic_scheduler *s = governor::local_scheduler(); - // these redirections are due to bw-compatibility, consider reworking some day - __TBB_ASSERT( s->my_arena, "thread is not in any arena" ); - s->my_arena->enqueue_task(t, (intptr_t)prio, s->my_random ); -} - -#if __TBB_TASK_PRIORITY -class auto_indicator : no_copy { - volatile bool& my_indicator; -public: - auto_indicator ( volatile bool& indicator ) : my_indicator(indicator) { my_indicator = true ;} - ~auto_indicator () { my_indicator = false; } -}; - -task *generic_scheduler::get_task_and_activate_task_pool( size_t H0, __TBB_ISOLATION_ARG( size_t T0, isolation_tag isolation ) ) { - __TBB_ASSERT( is_local_task_pool_quiescent(), NULL ); - - // Go through the task pool to find an available task for execution. - task *t = NULL; -#if __TBB_TASK_ISOLATION - size_t T = T0; - bool tasks_omitted = false; - while ( !t && T>H0 ) { - t = get_task( --T, isolation, tasks_omitted ); - if ( !tasks_omitted ) { - poison_pointer( my_arena_slot->task_pool_ptr[T] ); - --T0; - } - } - // Make a hole if some tasks have been skipped. - if ( t && tasks_omitted ) { - my_arena_slot->task_pool_ptr[T] = NULL; - if ( T == H0 ) { - // The obtained task is on the head. So we can move the head instead of making a hole. - ++H0; - poison_pointer( my_arena_slot->task_pool_ptr[T] ); - } - } -#else - while ( !t && T0 ) { - t = get_task( --T0 ); - poison_pointer( my_arena_slot->task_pool_ptr[T0] ); - } -#endif /* __TBB_TASK_ISOLATION */ - - if ( H0 < T0 ) { - // There are some tasks in the task pool. Publish them. - __TBB_store_relaxed( my_arena_slot->head, H0 ); - __TBB_store_relaxed( my_arena_slot->tail, T0 ); - if ( is_task_pool_published() ) - release_task_pool(); - else - publish_task_pool(); - } else { - __TBB_store_relaxed( my_arena_slot->head, 0 ); - __TBB_store_relaxed( my_arena_slot->tail, 0 ); - if ( is_task_pool_published() ) - leave_task_pool(); - } - -#if __TBB_TASK_ISOLATION - // Now it is safe to call note_affinity because the task pool is restored. - if ( tasks_omitted && my_innermost_running_task == t ) { - assert_task_valid( t ); - t->note_affinity( my_affinity_id ); - } -#endif /* __TBB_TASK_ISOLATION */ - - assert_task_pool_valid(); - return t; -} - -task* generic_scheduler::winnow_task_pool( __TBB_ISOLATION_EXPR( isolation_tag isolation ) ) { - GATHER_STATISTIC( ++my_counters.prio_winnowings ); - __TBB_ASSERT( is_task_pool_published(), NULL ); - __TBB_ASSERT( my_offloaded_tasks, "At least one task is expected to be already offloaded" ); - // To eliminate possible sinking of the store to the indicator below the subsequent - // store to my_arena_slot->tail, the stores should have either been separated - // by full fence or both use release fences. And resetting indicator should have - // been done with release fence. But since this is just an optimization, and - // the corresponding checking sequence in arena::is_out_of_work() is not atomic - // anyway, fences aren't used, so that not to penalize warmer path. - auto_indicator indicator( my_pool_reshuffling_pending ); - - // Locking the task pool unconditionally produces simpler code, - // scalability of which should not suffer unless priority jitter takes place. - // TODO: consider the synchronization algorithm here is for the owner thread - // to avoid locking task pool most of the time. - acquire_task_pool(); - size_t T0 = __TBB_load_relaxed( my_arena_slot->tail ); - size_t H0 = __TBB_load_relaxed( my_arena_slot->head ); - size_t T1 = 0; - for ( size_t src = H0; src<T0; ++src ) { - if ( task *t = my_arena_slot->task_pool_ptr[src] ) { - // We cannot offload a proxy task (check the priority of it) because it can be already consumed. - if ( !is_proxy( *t ) ) { - intptr_t p = priority( *t ); - if ( p<*my_ref_top_priority ) { - offload_task( *t, p ); - continue; - } - } - my_arena_slot->task_pool_ptr[T1++] = t; - } - } - __TBB_ASSERT( T1<=T0, NULL ); - - // Choose max(T1, H0) because ranges [0, T1) and [H0, T0) can overlap. - my_arena_slot->fill_with_canary_pattern( max( T1, H0 ), T0 ); - return get_task_and_activate_task_pool( 0, __TBB_ISOLATION_ARG( T1, isolation ) ); -} - -task* generic_scheduler::reload_tasks ( task*& offloaded_tasks, task**& offloaded_task_list_link, __TBB_ISOLATION_ARG( intptr_t top_priority, isolation_tag isolation ) ) { - GATHER_STATISTIC( ++my_counters.prio_reloads ); -#if __TBB_TASK_ISOLATION - // In many cases, locking the task pool is no-op here because the task pool is in the empty - // state. However, isolation allows entering stealing loop with non-empty task pool. - // In principle, it is possible to process reloaded tasks without locking but it will - // complicate the logic of get_task_and_activate_task_pool (TODO: evaluate). - acquire_task_pool(); -#else - __TBB_ASSERT( !is_task_pool_published(), NULL ); -#endif - task *arr[min_task_pool_size]; - fast_reverse_vector<task*> tasks(arr, min_task_pool_size); - task **link = &offloaded_tasks; - while ( task *t = *link ) { - task** next_ptr = &t->prefix().next_offloaded; - __TBB_ASSERT( !is_proxy(*t), "The proxy tasks cannot be offloaded" ); - if ( priority(*t) >= top_priority ) { - tasks.push_back( t ); - // Note that owner is an alias of next_offloaded. Thus the following - // assignment overwrites *next_ptr - task* next = *next_ptr; - t->prefix().owner = this; - __TBB_ASSERT( t->prefix().state == task::ready, NULL ); - *link = next; - } - else { - link = next_ptr; - } - } - if ( link == &offloaded_tasks ) { - offloaded_tasks = NULL; -#if TBB_USE_ASSERT - offloaded_task_list_link = NULL; -#endif /* TBB_USE_ASSERT */ - } - else { - __TBB_ASSERT( link, NULL ); - // Mark end of list - *link = NULL; - offloaded_task_list_link = link; - } - __TBB_ASSERT( link, NULL ); - size_t num_tasks = tasks.size(); - if ( !num_tasks ) { - __TBB_ISOLATION_EXPR( release_task_pool() ); - return NULL; - } - - // Copy found tasks into the task pool. - GATHER_STATISTIC( ++my_counters.prio_tasks_reloaded ); - size_t T = prepare_task_pool( num_tasks ); - tasks.copy_memory( my_arena_slot->task_pool_ptr + T ); - - // Find a task available for execution. - task *t = get_task_and_activate_task_pool( __TBB_load_relaxed( my_arena_slot->head ), __TBB_ISOLATION_ARG( T + num_tasks, isolation ) ); - if ( t ) --num_tasks; - if ( num_tasks ) - my_arena->advertise_new_work<arena::work_spawned>(); - - return t; -} - -task* generic_scheduler::reload_tasks( __TBB_ISOLATION_EXPR( isolation_tag isolation ) ) { - uintptr_t reload_epoch = *my_ref_reload_epoch; - __TBB_ASSERT( my_offloaded_tasks, NULL ); - __TBB_ASSERT( my_local_reload_epoch <= reload_epoch - || my_local_reload_epoch - reload_epoch > uintptr_t(-1)/2, - "Reload epoch counter overflow?" ); - if ( my_local_reload_epoch == reload_epoch ) - return NULL; - __TBB_ASSERT( my_offloaded_tasks, NULL ); - intptr_t top_priority = effective_reference_priority(); - __TBB_ASSERT( (uintptr_t)top_priority < (uintptr_t)num_priority_levels, NULL ); - task *t = reload_tasks( my_offloaded_tasks, my_offloaded_task_list_tail_link, __TBB_ISOLATION_ARG( top_priority, isolation ) ); - if ( my_offloaded_tasks && (my_arena->my_bottom_priority >= top_priority || !my_arena->my_num_workers_requested) ) { - // Safeguard against deliberately relaxed synchronization while checking - // for the presence of work in arena (so that not to impact hot paths). - // Arena may be reset to empty state when offloaded low priority tasks - // are still present. This results in both bottom and top priority bounds - // becoming 'normal', which makes offloaded low priority tasks unreachable. - // Update arena's bottom priority to accommodate them. - // NOTE: If the number of priority levels is increased, we may want - // to calculate minimum of priorities in my_offloaded_tasks. - - // First indicate the presence of lower-priority tasks - my_market->update_arena_priority( *my_arena, priority(*my_offloaded_tasks) ); - // Then mark arena as full to unlock arena priority level adjustment - // by arena::is_out_of_work(), and ensure worker's presence - my_arena->advertise_new_work<arena::wakeup>(); - } - my_local_reload_epoch = reload_epoch; - return t; -} -#endif /* __TBB_TASK_PRIORITY */ - -#if __TBB_TASK_ISOLATION -inline task* generic_scheduler::get_task( size_t T, isolation_tag isolation, bool& tasks_omitted ) -#else -inline task* generic_scheduler::get_task( size_t T ) -#endif /* __TBB_TASK_ISOLATION */ -{ - __TBB_ASSERT( __TBB_load_relaxed( my_arena_slot->tail ) <= T - || is_local_task_pool_quiescent(), "Is it safe to get a task at position T?" ); - - task* result = my_arena_slot->task_pool_ptr[T]; - __TBB_ASSERT( !is_poisoned( result ), "The poisoned task is going to be processed" ); -#if __TBB_TASK_ISOLATION - if ( !result ) - return NULL; - - bool omit = isolation != no_isolation && isolation != result->prefix().isolation; - if ( !omit && !is_proxy( *result ) ) - return result; - else if ( omit ) { - tasks_omitted = true; - return NULL; - } -#else - poison_pointer( my_arena_slot->task_pool_ptr[T] ); - if ( !result || !is_proxy( *result ) ) - return result; -#endif /* __TBB_TASK_ISOLATION */ - - task_proxy& tp = static_cast<task_proxy&>(*result); - if ( task *t = tp.extract_task<task_proxy::pool_bit>() ) { - GATHER_STATISTIC( ++my_counters.proxies_executed ); - // Following assertion should be true because TBB 2.0 tasks never specify affinity, and hence are not proxied. - __TBB_ASSERT( is_version_3_task( *t ), "backwards compatibility with TBB 2.0 broken" ); - __TBB_ASSERT( my_innermost_running_task != t, NULL ); - my_innermost_running_task = t; // prepare for calling note_affinity() -#if __TBB_TASK_ISOLATION - // Task affinity has changed. Postpone calling note_affinity because the task pool is in invalid state. - if ( !tasks_omitted ) -#endif /* __TBB_TASK_ISOLATION */ - { - poison_pointer( my_arena_slot->task_pool_ptr[T] ); - t->note_affinity( my_affinity_id ); - } - return t; - } - - // Proxy was empty, so it's our responsibility to free it - free_task<small_task>( tp ); -#if __TBB_TASK_ISOLATION - if ( tasks_omitted ) - my_arena_slot->task_pool_ptr[T] = NULL; -#endif /* __TBB_TASK_ISOLATION */ - return NULL; -} - -inline task* generic_scheduler::get_task( __TBB_ISOLATION_EXPR( isolation_tag isolation ) ) { - __TBB_ASSERT( is_task_pool_published(), NULL ); - // The current task position in the task pool. - size_t T0 = __TBB_load_relaxed( my_arena_slot->tail ); - // The bounds of available tasks in the task pool. H0 is only used when the head bound is reached. - size_t H0 = (size_t)-1, T = T0; - task* result = NULL; - bool task_pool_empty = false; - __TBB_ISOLATION_EXPR( bool tasks_omitted = false ); - do { - __TBB_ASSERT( !result, NULL ); - __TBB_store_relaxed( my_arena_slot->tail, --T ); - atomic_fence(); - if ( (intptr_t)__TBB_load_relaxed( my_arena_slot->head ) > (intptr_t)T ) { - acquire_task_pool(); - H0 = __TBB_load_relaxed( my_arena_slot->head ); - if ( (intptr_t)H0 > (intptr_t)T ) { - // The thief has not backed off - nothing to grab. - __TBB_ASSERT( H0 == __TBB_load_relaxed( my_arena_slot->head ) - && T == __TBB_load_relaxed( my_arena_slot->tail ) - && H0 == T + 1, "victim/thief arbitration algorithm failure" ); - reset_task_pool_and_leave(); - // No tasks in the task pool. - task_pool_empty = true; - break; - } else if ( H0 == T ) { - // There is only one task in the task pool. - reset_task_pool_and_leave(); - task_pool_empty = true; - } else { - // Release task pool if there are still some tasks. - // After the release, the tail will be less than T, thus a thief - // will not attempt to get a task at position T. - release_task_pool(); - } - } - __TBB_control_consistency_helper(); // on my_arena_slot->head -#if __TBB_TASK_ISOLATION - result = get_task( T, isolation, tasks_omitted ); - if ( result ) { - poison_pointer( my_arena_slot->task_pool_ptr[T] ); - break; - } else if ( !tasks_omitted ) { - poison_pointer( my_arena_slot->task_pool_ptr[T] ); - __TBB_ASSERT( T0 == T+1, NULL ); - T0 = T; - } -#else - result = get_task( T ); -#endif /* __TBB_TASK_ISOLATION */ - } while ( !result && !task_pool_empty ); - -#if __TBB_TASK_ISOLATION - if ( tasks_omitted ) { - if ( task_pool_empty ) { - // All tasks have been checked. The task pool should be in reset state. - // We just restore the bounds for the available tasks. - // TODO: Does it have sense to move them to the beginning of the task pool? - __TBB_ASSERT( is_quiescent_local_task_pool_reset(), NULL ); - if ( result ) { - // If we have a task, it should be at H0 position. - __TBB_ASSERT( H0 == T, NULL ); - ++H0; - } - __TBB_ASSERT( H0 <= T0, NULL ); - if ( H0 < T0 ) { - // Restore the task pool if there are some tasks. - __TBB_store_relaxed( my_arena_slot->head, H0 ); - __TBB_store_relaxed( my_arena_slot->tail, T0 ); - // The release fence is used in publish_task_pool. - publish_task_pool(); - // Synchronize with snapshot as we published some tasks. - my_arena->advertise_new_work<arena::wakeup>(); - } - } else { - // A task has been obtained. We need to make a hole in position T. - __TBB_ASSERT( is_task_pool_published(), NULL ); - __TBB_ASSERT( result, NULL ); - my_arena_slot->task_pool_ptr[T] = NULL; - __TBB_store_with_release( my_arena_slot->tail, T0 ); - // Synchronize with snapshot as we published some tasks. - // TODO: consider some approach not to call wakeup for each time. E.g. check if the tail reached the head. - my_arena->advertise_new_work<arena::wakeup>(); - } - - // Now it is safe to call note_affinity because the task pool is restored. - if ( my_innermost_running_task == result ) { - assert_task_valid( result ); - result->note_affinity( my_affinity_id ); - } - } -#endif /* __TBB_TASK_ISOLATION */ - __TBB_ASSERT( (intptr_t)__TBB_load_relaxed( my_arena_slot->tail ) >= 0, NULL ); - __TBB_ASSERT( result || __TBB_ISOLATION_EXPR( tasks_omitted || ) is_quiescent_local_task_pool_reset(), NULL ); - return result; -} // generic_scheduler::get_task - -task* generic_scheduler::steal_task( __TBB_ISOLATION_ARG( arena_slot& victim_slot, isolation_tag isolation ) ) { - task** victim_pool = lock_task_pool( &victim_slot ); - if ( !victim_pool ) - return NULL; - task* result = NULL; - size_t H = __TBB_load_relaxed(victim_slot.head); // mirror - size_t H0 = H; - bool tasks_omitted = false; - do { - __TBB_store_relaxed( victim_slot.head, ++H ); - atomic_fence(); - if ( (intptr_t)H > (intptr_t)__TBB_load_relaxed( victim_slot.tail ) ) { - // Stealing attempt failed, deque contents has not been changed by us - GATHER_STATISTIC( ++my_counters.thief_backoffs ); - __TBB_store_relaxed( victim_slot.head, /*dead: H = */ H0 ); - __TBB_ASSERT( !result, NULL ); - goto unlock; - } - __TBB_control_consistency_helper(); // on victim_slot.tail - result = victim_pool[H-1]; - __TBB_ASSERT( !is_poisoned( result ), NULL ); - - if ( result ) { - __TBB_ISOLATION_EXPR( if ( isolation == no_isolation || isolation == result->prefix().isolation ) ) - { - if ( !is_proxy( *result ) ) - break; - task_proxy& tp = *static_cast<task_proxy*>(result); - // If mailed task is likely to be grabbed by its destination thread, skip it. - if ( !(task_proxy::is_shared( tp.task_and_tag ) && tp.outbox->recipient_is_idle()) ) - break; - GATHER_STATISTIC( ++my_counters.proxies_bypassed ); - } - // The task cannot be executed either due to isolation or proxy contraints. - result = NULL; - tasks_omitted = true; - } else if ( !tasks_omitted ) { - // Cleanup the task pool from holes until a task is skipped. - __TBB_ASSERT( H0 == H-1, NULL ); - poison_pointer( victim_pool[H0] ); - H0 = H; - } - } while ( !result ); - __TBB_ASSERT( result, NULL ); - - // emit "task was consumed" signal - ITT_NOTIFY( sync_acquired, (void*)((uintptr_t)&victim_slot+sizeof( uintptr_t )) ); - poison_pointer( victim_pool[H-1] ); - if ( tasks_omitted ) { - // Some proxies in the task pool have been omitted. Set the stolen task to NULL. - victim_pool[H-1] = NULL; - __TBB_store_relaxed( victim_slot.head, /*dead: H = */ H0 ); - } -unlock: - unlock_task_pool( &victim_slot, victim_pool ); -#if __TBB_PREFETCHING - __TBB_cl_evict(&victim_slot.head); - __TBB_cl_evict(&victim_slot.tail); -#endif - if ( tasks_omitted ) - // Synchronize with snapshot as the head and tail can be bumped which can falsely trigger EMPTY state - my_arena->advertise_new_work<arena::wakeup>(); - return result; -} - -task* generic_scheduler::get_mailbox_task( __TBB_ISOLATION_EXPR( isolation_tag isolation ) ) { - __TBB_ASSERT( my_affinity_id>0, "not in arena" ); - while ( task_proxy* const tp = my_inbox.pop( __TBB_ISOLATION_EXPR( isolation ) ) ) { - if ( task* result = tp->extract_task<task_proxy::mailbox_bit>() ) { - ITT_NOTIFY( sync_acquired, my_inbox.outbox() ); - result->prefix().extra_state |= es_task_is_stolen; - return result; - } - // We have exclusive access to the proxy, and can destroy it. - free_task<no_cache_small_task>(*tp); - } - return NULL; -} - -inline void generic_scheduler::publish_task_pool() { - __TBB_ASSERT ( my_arena, "no arena: initialization not completed?" ); - __TBB_ASSERT ( my_arena_index < my_arena->my_num_slots, "arena slot index is out-of-bound" ); - __TBB_ASSERT ( my_arena_slot == &my_arena->my_slots[my_arena_index], NULL); - __TBB_ASSERT ( my_arena_slot->task_pool == EmptyTaskPool, "someone else grabbed my arena slot?" ); - __TBB_ASSERT ( __TBB_load_relaxed(my_arena_slot->head) < __TBB_load_relaxed(my_arena_slot->tail), - "entering arena without tasks to share" ); - // Release signal on behalf of previously spawned tasks (when this thread was not in arena yet) - ITT_NOTIFY(sync_releasing, my_arena_slot); - __TBB_store_with_release( my_arena_slot->task_pool, my_arena_slot->task_pool_ptr ); -} - -inline void generic_scheduler::leave_task_pool() { - __TBB_ASSERT( is_task_pool_published(), "Not in arena" ); - // Do not reset my_arena_index. It will be used to (attempt to) re-acquire the slot next time - __TBB_ASSERT( &my_arena->my_slots[my_arena_index] == my_arena_slot, "arena slot and slot index mismatch" ); - __TBB_ASSERT ( my_arena_slot->task_pool == LockedTaskPool, "Task pool must be locked when leaving arena" ); - __TBB_ASSERT ( is_quiescent_local_task_pool_empty(), "Cannot leave arena when the task pool is not empty" ); - ITT_NOTIFY(sync_releasing, &my_arena->my_slots[my_arena_index]); - // No release fence is necessary here as this assignment precludes external - // accesses to the local task pool when becomes visible. Thus it is harmless - // if it gets hoisted above preceding local bookkeeping manipulations. - __TBB_store_relaxed( my_arena_slot->task_pool, EmptyTaskPool ); -} - -generic_scheduler* generic_scheduler::create_worker( market& m, size_t index ) { - generic_scheduler* s = allocate_scheduler( m ); - __TBB_ASSERT(index, "workers should have index > 0"); - s->my_arena_index = index; // index is not a real slot in arena yet - s->my_dummy_task->prefix().ref_count = 2; - s->my_properties.type = scheduler_properties::worker; - // Do not call init_stack_info before the scheduler is set as master or worker. - s->init_stack_info(); - governor::sign_on(s); - return s; -} - -// TODO: make it a member method -generic_scheduler* generic_scheduler::create_master( arena* a ) { - // add an internal market reference; the public reference is possibly added in create_arena - generic_scheduler* s = allocate_scheduler( market::global_market(/*is_public=*/false) ); - __TBB_ASSERT( !s->my_arena, NULL ); - __TBB_ASSERT( s->my_market, NULL ); - task& t = *s->my_dummy_task; - s->my_properties.type = scheduler_properties::master; - t.prefix().ref_count = 1; -#if __TBB_TASK_GROUP_CONTEXT - t.prefix().context = new ( NFS_Allocate(1, sizeof(task_group_context), NULL) ) - task_group_context( task_group_context::isolated, task_group_context::default_traits ); -#if __TBB_FP_CONTEXT - s->default_context()->capture_fp_settings(); -#endif - // Do not call init_stack_info before the scheduler is set as master or worker. - s->init_stack_info(); - context_state_propagation_mutex_type::scoped_lock lock(the_context_state_propagation_mutex); - s->my_market->my_masters.push_front( *s ); - lock.release(); -#endif /* __TBB_TASK_GROUP_CONTEXT */ - if( a ) { - // Master thread always occupies the first slot - s->attach_arena( a, /*index*/0, /*is_master*/true ); - s->my_arena_slot->my_scheduler = s; - a->my_default_ctx = s->default_context(); // also transfers implied ownership - } - __TBB_ASSERT( s->my_arena_index == 0, "Master thread must occupy the first slot in its arena" ); - governor::sign_on(s); - -#if _WIN32||_WIN64 - s->my_market->register_master( s->master_exec_resource ); -#endif /* _WIN32||_WIN64 */ - // Process any existing observers. -#if __TBB_ARENA_OBSERVER - __TBB_ASSERT( !a || a->my_observers.empty(), "Just created arena cannot have any observers associated with it" ); -#endif -#if __TBB_SCHEDULER_OBSERVER - the_global_observer_list.notify_entry_observers( s->my_last_global_observer, /*worker=*/false ); -#endif /* __TBB_SCHEDULER_OBSERVER */ - return s; -} - -void generic_scheduler::cleanup_worker( void* arg, bool worker ) { - generic_scheduler& s = *(generic_scheduler*)arg; - __TBB_ASSERT( !s.my_arena_slot, "cleaning up attached worker" ); -#if __TBB_SCHEDULER_OBSERVER - if ( worker ) // can be called by master for worker, do not notify master twice - the_global_observer_list.notify_exit_observers( s.my_last_global_observer, /*worker=*/true ); -#endif /* __TBB_SCHEDULER_OBSERVER */ - s.free_scheduler(); -} - -bool generic_scheduler::cleanup_master( bool blocking_terminate ) { - arena* const a = my_arena; - market * const m = my_market; - __TBB_ASSERT( my_market, NULL ); - if( a && is_task_pool_published() ) { - acquire_task_pool(); - if ( my_arena_slot->task_pool == EmptyTaskPool || - __TBB_load_relaxed(my_arena_slot->head) >= __TBB_load_relaxed(my_arena_slot->tail) ) - { - // Local task pool is empty - leave_task_pool(); - } - else { - // Master's local task pool may e.g. contain proxies of affinitized tasks. - release_task_pool(); - __TBB_ASSERT ( governor::is_set(this), "TLS slot is cleared before the task pool cleanup" ); - local_wait_for_all( *my_dummy_task, NULL ); - __TBB_ASSERT( !is_task_pool_published(), NULL ); - __TBB_ASSERT ( governor::is_set(this), "Other thread reused our TLS key during the task pool cleanup" ); - } - } -#if __TBB_ARENA_OBSERVER - if( a ) - a->my_observers.notify_exit_observers( my_last_local_observer, /*worker=*/false ); -#endif -#if __TBB_SCHEDULER_OBSERVER - the_global_observer_list.notify_exit_observers( my_last_global_observer, /*worker=*/false ); -#endif /* __TBB_SCHEDULER_OBSERVER */ -#if _WIN32||_WIN64 - m->unregister_master( master_exec_resource ); -#endif /* _WIN32||_WIN64 */ - if( a ) { - __TBB_ASSERT(a->my_slots+0 == my_arena_slot, NULL); -#if __TBB_STATISTICS - *my_arena_slot->my_counters += my_counters; -#endif /* __TBB_STATISTICS */ - __TBB_store_with_release(my_arena_slot->my_scheduler, (generic_scheduler*)NULL); - } -#if __TBB_TASK_GROUP_CONTEXT - else { // task_group_context ownership was not transferred to arena - default_context()->~task_group_context(); - NFS_Free(default_context()); - } - context_state_propagation_mutex_type::scoped_lock lock(the_context_state_propagation_mutex); - my_market->my_masters.remove( *this ); - lock.release(); -#endif /* __TBB_TASK_GROUP_CONTEXT */ - my_arena_slot = NULL; // detached from slot - free_scheduler(); // do not use scheduler state after this point - - if( a ) - a->on_thread_leaving<arena::ref_external>(); - // If there was an associated arena, it added a public market reference - return m->release( /*is_public*/ a != NULL, blocking_terminate ); -} - -} // namespace internal -} // namespace tbb - -/* - Comments: - -1. The premise of the cancellation support implementation is that cancellations are - not part of the hot path of the program execution. Therefore all changes in its - implementation in order to reduce the overhead of the cancellation control flow - should be done only in ways that do not increase overhead of the normal execution. - - In general contexts are used by all threads and their descendants are created in - different threads as well. In order to minimize impact of the cross-thread tree - maintenance (first of all because of the synchronization), the tree of contexts - is split into pieces, each of which is handled by the only thread. Such pieces - are represented as lists of contexts, members of which are contexts that were - bound to their parents in the given thread. - - The context tree maintenance and cancellation propagation algorithms is designed - in such a manner that cross-thread access to a context list will take place only - when cancellation signal is sent (by user or when an exception happens), and - synchronization is necessary only then. Thus the normal execution flow (without - exceptions and cancellation) remains free from any synchronization done on - behalf of exception handling and cancellation support. - -2. Consider parallel cancellations at the different levels of the context tree: - - Ctx1 <- Cancelled by Thread1 |- Thread2 started processing - | | - Ctx2 |- Thread1 started processing - | T1 |- Thread2 finishes and syncs up local counters - Ctx3 <- Cancelled by Thread2 | - | |- Ctx5 is bound to Ctx2 - Ctx4 | - T2 |- Thread1 reaches Ctx2 - - Thread-propagator of each cancellation increments global counter. However the thread - propagating the cancellation from the outermost context (Thread1) may be the last - to finish. Which means that the local counters may be synchronized earlier (by Thread2, - at Time1) than it propagated cancellation into Ctx2 (at time Time2). If a new context - (Ctx5) is created and bound to Ctx2 between Time1 and Time2, checking its parent only - (Ctx2) may result in cancellation request being lost. - - This issue is solved by doing the whole propagation under the lock. - - If we need more concurrency while processing parallel cancellations, we could try - the following modification of the propagation algorithm: - - advance global counter and remember it - for each thread: - scan thread's list of contexts - for each thread: - sync up its local counter only if the global counter has not been changed - - However this version of the algorithm requires more analysis and verification. - -3. There is no portable way to get stack base address in Posix, however the modern - Linux versions provide pthread_attr_np API that can be used to obtain thread's - stack size and base address. Unfortunately even this function does not provide - enough information for the main thread on IA-64 architecture (RSE spill area - and memory stack are allocated as two separate discontinuous chunks of memory), - and there is no portable way to discern the main and the secondary threads. - Thus for macOS* and IA-64 architecture for Linux* OS we use the TBB worker stack size for - all threads and use the current stack top as the stack base. This simplified - approach is based on the following assumptions: - 1) If the default stack size is insufficient for the user app needs, the - required amount will be explicitly specified by the user at the point of the - TBB scheduler initialization (as an argument to tbb::task_scheduler_init - constructor). - 2) When a master thread initializes the scheduler, it has enough space on its - stack. Here "enough" means "at least as much as worker threads have". - 3) If the user app strives to conserve the memory by cutting stack size, it - should do this for TBB workers too (as in the #1). -*/ diff --git a/src/tbb/src/tbb/scheduler.h b/src/tbb/src/tbb/scheduler.h deleted file mode 100644 index 8dda929fb..000000000 --- a/src/tbb/src/tbb/scheduler.h +++ /dev/null @@ -1,732 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef _TBB_scheduler_H -#define _TBB_scheduler_H - -#include "scheduler_common.h" -#include "tbb/spin_mutex.h" -#include "mailbox.h" -#include "tbb_misc.h" // for FastRandom -#include "itt_notify.h" -#include "../rml/include/rml_tbb.h" - -#include "intrusive_list.h" - -#if __TBB_SURVIVE_THREAD_SWITCH -#include "cilk-tbb-interop.h" -#endif /* __TBB_SURVIVE_THREAD_SWITCH */ - -namespace tbb { -namespace internal { - -template<typename SchedulerTraits> class custom_scheduler; - -//------------------------------------------------------------------------ -// generic_scheduler -//------------------------------------------------------------------------ - -#define EmptyTaskPool ((task**)0) -#define LockedTaskPool ((task**)~(intptr_t)0) - -//! Bit-field representing properties of a sheduler -struct scheduler_properties { - static const bool worker = false; - static const bool master = true; - //! Indicates that a scheduler acts as a master or a worker. - bool type : 1; - //! Indicates that a scheduler is on outermost level. - /** Note that the explicit execute method will set this property. **/ - bool outermost : 1; - //! Reserved bits - unsigned char : 6; -}; - -struct scheduler_state { - //! Index of the arena slot the scheduler occupies now, or occupied last time. - size_t my_arena_index; // TODO: make it unsigned and pair with my_affinity_id to fit into cache line - - //! Pointer to the slot in the arena we own at the moment. - arena_slot* my_arena_slot; - - //! The arena that I own (if master) or am servicing at the moment (if worker) - arena* my_arena; - - //! Innermost task whose task::execute() is running. A dummy task on the outermost level. - task* my_innermost_running_task; - - mail_inbox my_inbox; - - //! The mailbox id assigned to this scheduler. - /** The id is assigned upon first entry into the arena. - TODO: how are id's being garbage collected? - TODO: master thread may enter arena and leave and then reenter. - We want to give it the same affinity_id upon reentry, if practical. - TODO: investigate if it makes sense to merge this field into scheduler_properties. - */ - affinity_id my_affinity_id; - - scheduler_properties my_properties; - -#if __TBB_SCHEDULER_OBSERVER - //! Last observer in the global observers list processed by this scheduler - observer_proxy* my_last_global_observer; -#endif - -#if __TBB_ARENA_OBSERVER - //! Last observer in the local observers list processed by this scheduler - observer_proxy* my_last_local_observer; -#endif -#if __TBB_TASK_PRIORITY - //! Latest known highest priority of tasks in the market or arena. - /** Master threads currently tracks only tasks in their arenas, while workers - take into account global top priority (among all arenas in the market). **/ - volatile intptr_t *my_ref_top_priority; - - //! Pointer to market's (for workers) or current arena's (for the master) reload epoch counter. - volatile uintptr_t *my_ref_reload_epoch; -#endif /* __TBB_TASK_PRIORITY */ -}; - -//! Work stealing task scheduler. -/** None of the fields here are ever read or written by threads other than - the thread that creates the instance. - - Class generic_scheduler is an abstract base class that contains most of the scheduler, - except for tweaks specific to processors and tools (e.g. VTune). - The derived template class custom_scheduler<SchedulerTraits> fills in the tweaks. */ -class generic_scheduler: public scheduler - , public ::rml::job - , public intrusive_list_node - , public scheduler_state { -public: // almost every class in TBB uses generic_scheduler - - //! If sizeof(task) is <=quick_task_size, it is handled on a free list instead of malloc'd. - static const size_t quick_task_size = 256-task_prefix_reservation_size; - - static bool is_version_3_task( task& t ) { - return (t.prefix().extra_state & 0x0F)>=0x1; - } - - //! Position in the call stack specifying its maximal filling when stealing is still allowed - uintptr_t my_stealing_threshold; -#if __TBB_ipf - //! Position in the RSE backup area specifying its maximal filling when stealing is still allowed - uintptr_t my_rsb_stealing_threshold; -#endif - - static const size_t null_arena_index = ~size_t(0); - - inline bool is_task_pool_published () const; - - inline bool is_local_task_pool_quiescent () const; - - inline bool is_quiescent_local_task_pool_empty () const; - - inline bool is_quiescent_local_task_pool_reset () const; - - //! The market I am in - market* my_market; - - //! Random number generator used for picking a random victim from which to steal. - FastRandom my_random; - - //! Free list of small tasks that can be reused. - task* my_free_list; - -#if __TBB_HOARD_NONLOCAL_TASKS - //! Free list of small non-local tasks that should be returned or can be reused. - task* my_nonlocal_free_list; -#endif - //! Fake root task created by slave threads. - /** The task is used as the "parent" argument to method wait_for_all. */ - task* my_dummy_task; - - //! Reference count for scheduler - /** Number of task_scheduler_init objects that point to this scheduler */ - long my_ref_count; - - inline void attach_mailbox( affinity_id id ); - - /* A couple of bools can be located here because space is otherwise just padding after my_affinity_id. */ - - //! True if *this was created by automatic TBB initialization - bool my_auto_initialized; - -#if __TBB_COUNT_TASK_NODES - //! Net number of big task objects that have been allocated but not yet freed. - intptr_t my_task_node_count; -#endif /* __TBB_COUNT_TASK_NODES */ - - //! Sets up the data necessary for the stealing limiting heuristics - void init_stack_info (); - - //! Returns true if stealing is allowed - bool can_steal () { - int anchor; - // TODO IDEA: Add performance warning? -#if __TBB_ipf - return my_stealing_threshold < (uintptr_t)&anchor && (uintptr_t)__TBB_get_bsp() < my_rsb_stealing_threshold; -#else - return my_stealing_threshold < (uintptr_t)&anchor; -#endif - } - - //! Used by workers to enter the task pool - /** Does not lock the task pool in case if arena slot has been successfully grabbed. **/ - void publish_task_pool(); - - //! Leave the task pool - /** Leaving task pool automatically releases the task pool if it is locked. **/ - void leave_task_pool(); - - //! Resets head and tail indices to 0, and leaves task pool - /** The task pool must be locked by the owner (via acquire_task_pool).**/ - inline void reset_task_pool_and_leave (); - - //! Locks victim's task pool, and returns pointer to it. The pointer can be NULL. - /** Garbles victim_arena_slot->task_pool for the duration of the lock. **/ - task** lock_task_pool( arena_slot* victim_arena_slot ) const; - - //! Unlocks victim's task pool - /** Restores victim_arena_slot->task_pool munged by lock_task_pool. **/ - void unlock_task_pool( arena_slot* victim_arena_slot, task** victim_task_pool ) const; - - //! Locks the local task pool - /** Garbles my_arena_slot->task_pool for the duration of the lock. Requires - correctly set my_arena_slot->task_pool_ptr. **/ - void acquire_task_pool() const; - - //! Unlocks the local task pool - /** Restores my_arena_slot->task_pool munged by acquire_task_pool. Requires - correctly set my_arena_slot->task_pool_ptr. **/ - void release_task_pool() const; - - //! Checks if t is affinitized to another thread, and if so, bundles it as proxy. - /** Returns either t or proxy containing t. **/ - task* prepare_for_spawning( task* t ); - - //! Makes newly spawned tasks visible to thieves - inline void commit_spawned_tasks( size_t new_tail ); - - //! Makes relocated tasks visible to thieves and releases the local task pool. - /** Obviously, the task pool must be locked when calling this method. **/ - inline void commit_relocated_tasks( size_t new_tail ); - - //! Get a task from the local pool. - /** Called only by the pool owner. - Returns the pointer to the task or NULL if a suitable task is not found. - Resets the pool if it is empty. **/ - task* get_task( __TBB_ISOLATION_EXPR( isolation_tag isolation ) ); - - //! Get a task from the local pool at specified location T. - /** Returns the pointer to the task or NULL if the task cannot be executed, - e.g. proxy has been deallocated or isolation constraint is not met. - tasks_omitted tells if some tasks have been omitted. - Called only by the pool owner. The caller should guarantee that the - position T is not available for a thief. **/ -#if __TBB_TASK_ISOLATION - task* get_task( size_t T, isolation_tag isolation, bool& tasks_omitted ); -#else - task* get_task( size_t T ); -#endif /* __TBB_TASK_ISOLATION */ - //! Attempt to get a task from the mailbox. - /** Gets a task only if it has not been executed by its sender or a thief - that has stolen it from the sender's task pool. Otherwise returns NULL. - - This method is intended to be used only by the thread extracting the proxy - from its mailbox. (In contrast to local task pool, mailbox can be read only - by its owner). **/ - task* get_mailbox_task( __TBB_ISOLATION_EXPR( isolation_tag isolation ) ); - - //! True if t is a task_proxy - static bool is_proxy( const task& t ) { - return t.prefix().extra_state==es_task_proxy; - } - - //! Steal task from another scheduler's ready pool. - task* steal_task( __TBB_ISOLATION_ARG( arena_slot& victim_arena_slot, isolation_tag isolation ) ); - - /** Initial size of the task deque sufficient to serve without reallocation - 4 nested parallel_for calls with iteration space of 65535 grains each. **/ - static const size_t min_task_pool_size = 64; - - //! Makes sure that the task pool can accommodate at least n more elements - /** If necessary relocates existing task pointers or grows the ready task deque. - Returns (possible updated) tail index (not accounting for n). **/ - size_t prepare_task_pool( size_t n ); - - //! Initialize a scheduler for a master thread. - static generic_scheduler* create_master( arena* a ); - - //! Perform necessary cleanup when a master thread stops using TBB. - bool cleanup_master( bool blocking_terminate ); - - //! Initialize a scheduler for a worker thread. - static generic_scheduler* create_worker( market& m, size_t index ); - - //! Perform necessary cleanup when a worker thread finishes. - static void cleanup_worker( void* arg, bool worker ); - -protected: - template<typename SchedulerTraits> friend class custom_scheduler; - generic_scheduler( market & ); - -public: -#if TBB_USE_ASSERT > 1 - //! Check that internal data structures are in consistent state. - /** Raises __TBB_ASSERT failure if inconsistency is found. */ - void assert_task_pool_valid() const; -#else - void assert_task_pool_valid() const {} -#endif /* TBB_USE_ASSERT <= 1 */ - - void attach_arena( arena*, size_t index, bool is_master ); - void nested_arena_entry( arena*, size_t ); - void nested_arena_exit(); - void wait_until_empty(); - - void spawn( task& first, task*& next ) __TBB_override; - - void spawn_root_and_wait( task& first, task*& next ) __TBB_override; - - void enqueue( task&, void* reserved ) __TBB_override; - - void local_spawn( task* first, task*& next ); - void local_spawn_root_and_wait( task* first, task*& next ); - virtual void local_wait_for_all( task& parent, task* child ) = 0; - - //! Destroy and deallocate this scheduler object - void free_scheduler(); - - //! Allocate task object, either from the heap or a free list. - /** Returns uninitialized task object with initialized prefix. */ - task& allocate_task( size_t number_of_bytes, - __TBB_CONTEXT_ARG(task* parent, task_group_context* context) ); - - //! Put task on free list. - /** Does not call destructor. */ - template<free_task_hint h> - void free_task( task& t ); - - //! Return task object to the memory allocator. - inline void deallocate_task( task& t ); - - //! True if running on a worker thread, false otherwise. - inline bool is_worker() const; - - //! True if the scheduler is on the outermost dispatch level. - inline bool outermost_level() const; - - //! True if the scheduler is on the outermost dispatch level in a master thread. - /** Returns true when this scheduler instance is associated with an application - thread, and is not executing any TBB task. This includes being in a TBB - dispatch loop (one of wait_for_all methods) invoked directly from that thread. **/ - inline bool master_outermost_level () const; - - //! True if the scheduler is on the outermost dispatch level in a worker thread. - inline bool worker_outermost_level () const; - - //! Returns the concurrency limit of the current arena. - unsigned max_threads_in_arena(); - -#if __TBB_COUNT_TASK_NODES - intptr_t get_task_node_count( bool count_arena_workers = false ); -#endif /* __TBB_COUNT_TASK_NODES */ - - //! Special value used to mark my_return_list as not taking any more entries. - static task* plugged_return_list() {return (task*)(intptr_t)(-1);} - - //! Number of small tasks that have been allocated by this scheduler. - __TBB_atomic intptr_t my_small_task_count; - - //! List of small tasks that have been returned to this scheduler by other schedulers. - // TODO IDEA: see if putting my_return_list on separate cache line improves performance - task* my_return_list; - - //! Try getting a task from other threads (via mailbox, stealing, FIFO queue, orphans adoption). - /** Returns obtained task or NULL if all attempts fail. */ - virtual task* receive_or_steal_task( __TBB_ISOLATION_ARG( __TBB_atomic reference_count& completion_ref_count, isolation_tag isolation ) ) = 0; - - //! Free a small task t that that was allocated by a different scheduler - void free_nonlocal_small_task( task& t ); - -#if __TBB_TASK_GROUP_CONTEXT - //! Returns task group context used by this scheduler instance. - /** This context is associated with root tasks created by a master thread - without explicitly specified context object outside of any running task. - - Note that the default context of a worker thread is never accessed by - user code (directly or indirectly). **/ - inline task_group_context* default_context (); - - //! Padding isolating thread-local members from members that can be written to by other threads. - char _padding1[NFS_MaxLineSize - sizeof(context_list_node_t)]; - - //! Head of the thread specific list of task group contexts. - context_list_node_t my_context_list_head; - - //! Mutex protecting access to the list of task group contexts. - // TODO: check whether it can be deadly preempted and replace by spinning/sleeping mutex - spin_mutex my_context_list_mutex; - - //! Last state propagation epoch known to this thread - /** Together with the_context_state_propagation_epoch constitute synchronization protocol - that keeps hot path of task group context construction destruction mostly - lock-free. - When local epoch equals the global one, the state of task group contexts - registered with this thread is consistent with that of the task group trees - they belong to. **/ - uintptr_t my_context_state_propagation_epoch; - - //! Flag indicating that a context is being destructed by its owner thread - /** Together with my_nonlocal_ctx_list_update constitute synchronization protocol - that keeps hot path of context destruction (by the owner thread) mostly - lock-free. **/ - tbb::atomic<uintptr_t> my_local_ctx_list_update; - -#if __TBB_TASK_PRIORITY - //! Returns reference priority used to decide whether a task should be offloaded. - inline intptr_t effective_reference_priority () const; - - // TODO: move into slots and fix is_out_of_work - //! Task pool for offloading tasks with priorities lower than the current top priority. - task* my_offloaded_tasks; - - //! Points to the last offloaded task in the my_offloaded_tasks list. - task** my_offloaded_task_list_tail_link; - - //! Indicator of how recently the offload area was checked for the presence of top priority tasks. - uintptr_t my_local_reload_epoch; - - //! Indicates that the pool is likely non-empty even if appears so from outside - volatile bool my_pool_reshuffling_pending; - - //! Searches offload area for top priority tasks and reloads found ones into primary task pool. - /** Returns one of the found tasks or NULL. **/ - task* reload_tasks( __TBB_ISOLATION_EXPR( isolation_tag isolation ) ); - - task* reload_tasks( task*& offloaded_tasks, task**& offloaded_task_list_link, __TBB_ISOLATION_ARG( intptr_t top_priority, isolation_tag isolation ) ); - - //! Moves tasks with priority below the top one from primary task pool into offload area. - /** Returns the next execution candidate task or NULL. **/ - task* winnow_task_pool ( __TBB_ISOLATION_EXPR( isolation_tag isolation ) ); - - //! Get a task from locked or empty pool in range [H0, T0). Releases or unlocks the task pool. - /** Returns the found task or NULL. **/ - task *get_task_and_activate_task_pool( size_t H0 , __TBB_ISOLATION_ARG( size_t T0, isolation_tag isolation ) ); - - //! Unconditionally moves the task into offload area. - inline void offload_task ( task& t, intptr_t task_priority ); -#endif /* __TBB_TASK_PRIORITY */ - - //! Detaches abandoned contexts - /** These contexts must be destroyed by other threads. **/ - void cleanup_local_context_list (); - - //! Finds all contexts registered by this scheduler affected by the state change - //! and propagates the new state to them. - template <typename T> - void propagate_task_group_state ( T task_group_context::*mptr_state, task_group_context& src, T new_state ); - - // check consistency - static void assert_context_valid(const task_group_context *tgc) { - suppress_unused_warning(tgc); -#if TBB_USE_ASSERT - __TBB_ASSERT(tgc, NULL); - uintptr_t ctx = tgc->my_version_and_traits; - __TBB_ASSERT(is_alive(ctx), "referenced task_group_context was destroyed"); - static const char *msg = "task_group_context is invalid"; - __TBB_ASSERT(!(ctx&~(3|(7<<task_group_context::traits_offset))), msg); // the value fits known values of versions and traits - __TBB_ASSERT(tgc->my_kind < task_group_context::dying, msg); - __TBB_ASSERT(tgc->my_cancellation_requested == 0 || tgc->my_cancellation_requested == 1, msg); - __TBB_ASSERT(tgc->my_state < task_group_context::low_unused_state_bit, msg); - if(tgc->my_kind != task_group_context::isolated) { - __TBB_ASSERT(tgc->my_owner, msg); - __TBB_ASSERT(tgc->my_node.my_next && tgc->my_node.my_prev, msg); - } -#if __TBB_TASK_PRIORITY - assert_priority_valid(tgc->my_priority); -#endif - if(tgc->my_parent) -#if TBB_USE_ASSERT > 1 - assert_context_valid(tgc->my_parent); -#else - __TBB_ASSERT(is_alive(tgc->my_parent->my_version_and_traits), msg); -#endif -#endif - } -#endif /* __TBB_TASK_GROUP_CONTEXT */ - -#if _WIN32||_WIN64 -private: - //! Handle returned by RML when registering a master with RML - ::rml::server::execution_resource_t master_exec_resource; -public: -#endif /* _WIN32||_WIN64 */ - -#if __TBB_TASK_GROUP_CONTEXT - //! Flag indicating that a context is being destructed by non-owner thread. - /** See also my_local_ctx_list_update. **/ - tbb::atomic<uintptr_t> my_nonlocal_ctx_list_update; -#endif /* __TBB_TASK_GROUP_CONTEXT */ - -#if __TBB_SURVIVE_THREAD_SWITCH - __cilk_tbb_unwatch_thunk my_cilk_unwatch_thunk; -#if TBB_USE_ASSERT - //! State values used to check interface contract with cilkrts. - /** Names of cs_running...cs_freed derived from state machine diagram in cilk-tbb-interop.h */ - enum cilk_state_t { - cs_none=0xF000, // Start at nonzero value so that we can detect use of zeroed memory. - cs_running, - cs_limbo, - cs_freed - }; - cilk_state_t my_cilk_state; -#endif /* TBB_USE_ASSERT */ -#endif /* __TBB_SURVIVE_THREAD_SWITCH */ - -#if __TBB_STATISTICS - //! Set of counters to track internal statistics on per thread basis - /** Placed at the end of the class definition to minimize the disturbance of - the core logic memory operations. **/ - mutable statistics_counters my_counters; -#endif /* __TBB_STATISTICS */ - -}; // class generic_scheduler - - -} // namespace internal -} // namespace tbb - -#include "arena.h" -#include "governor.h" - -namespace tbb { -namespace internal { - -inline bool generic_scheduler::is_task_pool_published () const { - __TBB_ASSERT(my_arena_slot, 0); - return my_arena_slot->task_pool != EmptyTaskPool; -} - -inline bool generic_scheduler::is_local_task_pool_quiescent () const { - __TBB_ASSERT(my_arena_slot, 0); - task** tp = my_arena_slot->task_pool; - return tp == EmptyTaskPool || tp == LockedTaskPool; -} - -inline bool generic_scheduler::is_quiescent_local_task_pool_empty () const { - __TBB_ASSERT( is_local_task_pool_quiescent(), "Task pool is not quiescent" ); - return __TBB_load_relaxed(my_arena_slot->head) == __TBB_load_relaxed(my_arena_slot->tail); -} - -inline bool generic_scheduler::is_quiescent_local_task_pool_reset () const { - __TBB_ASSERT( is_local_task_pool_quiescent(), "Task pool is not quiescent" ); - return __TBB_load_relaxed(my_arena_slot->head) == 0 && __TBB_load_relaxed(my_arena_slot->tail) == 0; -} - -inline bool generic_scheduler::outermost_level () const { - return my_properties.outermost; -} - -inline bool generic_scheduler::master_outermost_level () const { - return !is_worker() && outermost_level(); -} - -inline bool generic_scheduler::worker_outermost_level () const { - return is_worker() && outermost_level(); -} - -#if __TBB_TASK_GROUP_CONTEXT -inline task_group_context* generic_scheduler::default_context () { - return my_dummy_task->prefix().context; -} -#endif /* __TBB_TASK_GROUP_CONTEXT */ - -inline void generic_scheduler::attach_mailbox( affinity_id id ) { - __TBB_ASSERT(id>0,NULL); - my_inbox.attach( my_arena->mailbox(id) ); - my_affinity_id = id; -} - -inline bool generic_scheduler::is_worker() const { - return my_properties.type == scheduler_properties::worker; -} - -inline unsigned generic_scheduler::max_threads_in_arena() { - __TBB_ASSERT(my_arena, NULL); - return my_arena->my_num_slots; -} - -//! Return task object to the memory allocator. -inline void generic_scheduler::deallocate_task( task& t ) { -#if TBB_USE_ASSERT - task_prefix& p = t.prefix(); - p.state = 0xFF; - p.extra_state = 0xFF; - poison_pointer(p.next); -#endif /* TBB_USE_ASSERT */ - NFS_Free((char*)&t-task_prefix_reservation_size); -#if __TBB_COUNT_TASK_NODES - --my_task_node_count; -#endif /* __TBB_COUNT_TASK_NODES */ -} - -#if __TBB_COUNT_TASK_NODES -inline intptr_t generic_scheduler::get_task_node_count( bool count_arena_workers ) { - return my_task_node_count + (count_arena_workers? my_arena->workers_task_node_count(): 0); -} -#endif /* __TBB_COUNT_TASK_NODES */ - -inline void generic_scheduler::reset_task_pool_and_leave () { - __TBB_ASSERT( my_arena_slot->task_pool == LockedTaskPool, "Task pool must be locked when resetting task pool" ); - __TBB_store_relaxed( my_arena_slot->tail, 0 ); - __TBB_store_relaxed( my_arena_slot->head, 0 ); - leave_task_pool(); -} - -//TODO: move to arena_slot -inline void generic_scheduler::commit_spawned_tasks( size_t new_tail ) { - __TBB_ASSERT ( new_tail <= my_arena_slot->my_task_pool_size, "task deque end was overwritten" ); - // emit "task was released" signal - ITT_NOTIFY(sync_releasing, (void*)((uintptr_t)my_arena_slot+sizeof(uintptr_t))); - // Release fence is necessary to make sure that previously stored task pointers - // are visible to thieves. - __TBB_store_with_release( my_arena_slot->tail, new_tail ); -} - -void generic_scheduler::commit_relocated_tasks ( size_t new_tail ) { - __TBB_ASSERT( is_local_task_pool_quiescent(), - "Task pool must be locked when calling commit_relocated_tasks()" ); - __TBB_store_relaxed( my_arena_slot->head, 0 ); - // Tail is updated last to minimize probability of a thread making arena - // snapshot being misguided into thinking that this task pool is empty. - __TBB_store_release( my_arena_slot->tail, new_tail ); - release_task_pool(); -} - -template<free_task_hint hint> -void generic_scheduler::free_task( task& t ) { -#if __TBB_HOARD_NONLOCAL_TASKS - static const int h = hint&(~local_task); -#else - static const free_task_hint h = hint; -#endif - GATHER_STATISTIC(--my_counters.active_tasks); - task_prefix& p = t.prefix(); - // Verify that optimization hints are correct. - __TBB_ASSERT( h!=small_local_task || p.origin==this, NULL ); - __TBB_ASSERT( !(h&small_task) || p.origin, NULL ); - __TBB_ASSERT( !(h&local_task) || (!p.origin || uintptr_t(p.origin) > uintptr_t(4096)), "local_task means allocated"); - poison_value(p.depth); - poison_value(p.ref_count); - poison_pointer(p.owner); - __TBB_ASSERT( 1L<<t.state() & (1L<<task::executing|1L<<task::allocated), NULL ); - p.state = task::freed; - if( h==small_local_task || p.origin==this ) { - GATHER_STATISTIC(++my_counters.free_list_length); - p.next = my_free_list; - my_free_list = &t; - } else if( !(h&local_task) && p.origin && uintptr_t(p.origin) < uintptr_t(4096) ) { - // a special value reserved for future use, do nothing since - // origin is not pointing to a scheduler instance - } else if( !(h&local_task) && p.origin ) { - GATHER_STATISTIC(++my_counters.free_list_length); -#if __TBB_HOARD_NONLOCAL_TASKS - if( !(h&no_cache) ) { - p.next = my_nonlocal_free_list; - my_nonlocal_free_list = &t; - } else -#endif - free_nonlocal_small_task(t); - } else { - GATHER_STATISTIC(--my_counters.big_tasks); - deallocate_task(t); - } -} - -#if __TBB_TASK_PRIORITY -inline intptr_t generic_scheduler::effective_reference_priority () const { - // Workers on the outermost dispatch level (i.e. with empty stack) use market's - // priority as a reference point (to speedup discovering process level priority - // changes). But when there are enough workers to service (even if only partially) - // a lower priority arena, they should use arena's priority as a reference, lest - // be trapped in a futile spinning (because market's priority would prohibit - // executing ANY tasks in this arena). - return !worker_outermost_level() || - (my_arena->my_num_workers_allotted < my_arena->num_workers_active() -#if __TBB_ENQUEUE_ENFORCED_CONCURRENCY - && my_arena->my_concurrency_mode!=arena_base::cm_enforced_global -#endif - ) ? *my_ref_top_priority : my_arena->my_top_priority; -} - -inline void generic_scheduler::offload_task ( task& t, intptr_t /*priority*/ ) { - GATHER_STATISTIC( ++my_counters.prio_tasks_offloaded ); - __TBB_ASSERT( !is_proxy(t), "The proxy task cannot be offloaded" ); - __TBB_ASSERT( my_offloaded_task_list_tail_link && !*my_offloaded_task_list_tail_link, NULL ); -#if TBB_USE_ASSERT - t.prefix().state = task::ready; -#endif /* TBB_USE_ASSERT */ - t.prefix().next_offloaded = my_offloaded_tasks; - my_offloaded_tasks = &t; -} -#endif /* __TBB_TASK_PRIORITY */ - -#if __TBB_FP_CONTEXT -class cpu_ctl_env_helper { - cpu_ctl_env guard_cpu_ctl_env; - cpu_ctl_env curr_cpu_ctl_env; -public: - cpu_ctl_env_helper() { - guard_cpu_ctl_env.get_env(); - curr_cpu_ctl_env = guard_cpu_ctl_env; - } - ~cpu_ctl_env_helper() { - if ( curr_cpu_ctl_env != guard_cpu_ctl_env ) - guard_cpu_ctl_env.set_env(); - } - void set_env( const task_group_context *ctx ) { - generic_scheduler::assert_context_valid(ctx); - const cpu_ctl_env &ctl = *punned_cast<cpu_ctl_env*>(&ctx->my_cpu_ctl_env); - if ( ctl != curr_cpu_ctl_env ) { - curr_cpu_ctl_env = ctl; - curr_cpu_ctl_env.set_env(); - } - } - void restore_default() { - if ( curr_cpu_ctl_env != guard_cpu_ctl_env ) { - guard_cpu_ctl_env.set_env(); - curr_cpu_ctl_env = guard_cpu_ctl_env; - } - } -}; -#else -struct cpu_ctl_env_helper { - void set_env( __TBB_CONTEXT_ARG1(task_group_context *) ) {} - void restore_default() {} -}; -#endif /* __TBB_FP_CONTEXT */ - -} // namespace internal -} // namespace tbb - -#endif /* _TBB_scheduler_H */ diff --git a/src/tbb/src/tbb/scheduler_common.h b/src/tbb/src/tbb/scheduler_common.h index 02a39d250..d689bf1d6 100644 --- a/src/tbb/src/tbb/scheduler_common.h +++ b/src/tbb/src/tbb/scheduler_common.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2024 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,131 +12,288 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef _TBB_scheduler_common_H #define _TBB_scheduler_common_H -#include "tbb/tbb_machine.h" -#include "tbb/cache_aligned_allocator.h" - -#include <string.h> // for memset, memcpy, memmove - -#include "tbb_statistics.h" - -#if TBB_USE_ASSERT > 1 -#include <stdio.h> -#endif /* TBB_USE_ASSERT > 1 */ - -/* Temporarily change "private" to "public" while including "tbb/task.h". - This hack allows us to avoid publishing internal types and methods - in the public header files just for sake of friend declarations. */ -#ifndef private - #define private public - #define undef_private -#endif - -#include "tbb/task.h" -#include "tbb/tbb_exception.h" - -#ifdef undef_private - #undef private -#endif +#include "oneapi/tbb/detail/_utils.h" +#include "oneapi/tbb/detail/_template_helpers.h" +#include "oneapi/tbb/detail/_task.h" +#include "oneapi/tbb/detail/_machine.h" +#include "oneapi/tbb/task_group.h" +#include "oneapi/tbb/cache_aligned_allocator.h" +#include "oneapi/tbb/tbb_allocator.h" +#include "itt_notify.h" +#include "co_context.h" +#include "misc.h" +#include "governor.h" #ifndef __TBB_SCHEDULER_MUTEX_TYPE #define __TBB_SCHEDULER_MUTEX_TYPE tbb::spin_mutex #endif // TODO: add conditional inclusion based on specified type -#include "tbb/spin_mutex.h" - -// This macro is an attempt to get rid of ugly ifdefs in the shared parts of the code. -// It drops the second argument depending on whether the controlling macro is defined. -// The first argument is just a convenience allowing to keep comma before the macro usage. -#if __TBB_TASK_GROUP_CONTEXT - #define __TBB_CONTEXT_ARG1(context) context - #define __TBB_CONTEXT_ARG(arg1, context) arg1, context -#else /* !__TBB_TASK_GROUP_CONTEXT */ - #define __TBB_CONTEXT_ARG1(context) - #define __TBB_CONTEXT_ARG(arg1, context) arg1 -#endif /* !__TBB_TASK_GROUP_CONTEXT */ - -#if __TBB_TASK_ISOLATION - #define __TBB_ISOLATION_EXPR(isolation) isolation - #define __TBB_ISOLATION_ARG(arg1, isolation) arg1, isolation -#else - #define __TBB_ISOLATION_EXPR(isolation) - #define __TBB_ISOLATION_ARG(arg1, isolation) arg1 -#endif /* __TBB_TASK_ISOLATION */ +#include "oneapi/tbb/spin_mutex.h" +#include "oneapi/tbb/mutex.h" +#if TBB_USE_ASSERT +#include <atomic> +#endif -#if DO_TBB_TRACE -#include <cstdio> -#define TBB_TRACE(x) ((void)std::printf x) -#else -#define TBB_TRACE(x) ((void)(0)) -#endif /* DO_TBB_TRACE */ +#include <cstdint> +#include <exception> +#include <memory> // unique_ptr +#include <unordered_map> -#if !__TBB_CPU_CTL_ENV_PRESENT -#include <fenv.h> -#endif +//! Mutex type for global locks in the scheduler +using scheduler_mutex_type = __TBB_SCHEDULER_MUTEX_TYPE; #if _MSC_VER && !defined(__INTEL_COMPILER) // Workaround for overzealous compiler warnings // These particular warnings are so ubiquitous that no attempt is made to narrow // the scope of the warnings. - #pragma warning (disable: 4100 4127 4312 4244 4267 4706) + // #pragma warning (disable: 4100 4127 4312 4244 4267 4706) #endif namespace tbb { -namespace interface7 { -namespace internal { -class task_arena_base; -class delegated_task; -class wait_task; -}} -namespace internal { -using namespace interface7::internal; +namespace detail { +namespace r1 { class arena; -template<typename SchedulerTraits> class custom_scheduler; -class generic_scheduler; -class governor; +class mail_inbox; class mail_outbox; class market; class observer_proxy; -class task_scheduler_observer_v3; -#if __TBB_TASK_PRIORITY -static const intptr_t num_priority_levels = 3; -static const intptr_t normalized_normal_priority = (num_priority_levels - 1) / 2; +enum task_stream_accessor_type { front_accessor = 0, back_nonnull_accessor }; +template<task_stream_accessor_type> class task_stream; -inline intptr_t normalize_priority ( priority_t p ) { - return intptr_t(p - priority_low) / priority_stride_v4; +using isolation_type = std::intptr_t; +constexpr isolation_type no_isolation = 0; + +struct cache_aligned_deleter { + template <typename T> + void operator() (T* ptr) const { + ptr->~T(); + cache_aligned_deallocate(ptr); + } +}; + +template <typename T> +using cache_aligned_unique_ptr = std::unique_ptr<T, cache_aligned_deleter>; + +template <typename T, typename ...Args> +cache_aligned_unique_ptr<T> make_cache_aligned_unique(Args&& ...args) { + return cache_aligned_unique_ptr<T>(new (cache_aligned_allocate(sizeof(T))) T(std::forward<Args>(args)...)); } -static const priority_t priority_from_normalized_rep[num_priority_levels] = { - priority_low, priority_normal, priority_high +//------------------------------------------------------------------------ +// Extended execute data +//------------------------------------------------------------------------ + +//! Execute data used on a task dispatcher side, reflects a current execution state +struct execution_data_ext : d1::execution_data { + task_dispatcher* task_disp{}; + isolation_type isolation{}; + d1::wait_context* wait_ctx{}; }; -inline void assert_priority_valid ( intptr_t p ) { - __TBB_ASSERT_EX( p >= 0 && p < num_priority_levels, NULL ); +//------------------------------------------------------------------------ +// Task accessor +//------------------------------------------------------------------------ + +//! Interpretation of reserved task fields inside a task dispatcher +struct task_accessor { + static constexpr std::uint64_t proxy_task_trait = 1; + static constexpr std::uint64_t resume_task_trait = 2; + static d1::task_group_context*& context(d1::task& t) { + task_group_context** tgc = reinterpret_cast<task_group_context**>(&t.m_reserved[0]); + return *tgc; + } + static isolation_type& isolation(d1::task& t) { + isolation_type* tag = reinterpret_cast<isolation_type*>(&t.m_reserved[2]); + return *tag; + } + static void set_proxy_trait(d1::task& t) { + // TODO: refactor proxy tasks not to work on uninitialized memory. + //__TBB_ASSERT((t.m_version_and_traits & proxy_task_trait) == 0, nullptr); + t.m_version_and_traits |= proxy_task_trait; + } + static bool is_proxy_task(d1::task& t) { + return (t.m_version_and_traits & proxy_task_trait) != 0; + } + static void set_resume_trait(d1::task& t) { + __TBB_ASSERT((t.m_version_and_traits & resume_task_trait) == 0, nullptr); + t.m_version_and_traits |= resume_task_trait; + } + static bool is_resume_task(d1::task& t) { + return (t.m_version_and_traits & resume_task_trait) != 0; + } +}; + +//------------------------------------------------------------------------ +//! Extended variant of the standard offsetof macro +/** The standard offsetof macro is not sufficient for TBB as it can be used for + POD-types only. The constant 0x1000 (not nullptr) is necessary to appease GCC. **/ +#define __TBB_offsetof(class_name, member_name) \ + ((ptrdiff_t)&(reinterpret_cast<class_name*>(0x1000)->member_name) - 0x1000) + +//! Returns address of the object containing a member with the given name and address +#define __TBB_get_object_ref(class_name, member_name, member_addr) \ + (*reinterpret_cast<class_name*>((char*)member_addr - __TBB_offsetof(class_name, member_name))) + +//! Helper class for tracking floating point context and task group context switches +/** Assuming presence of an itt collector, in addition to keeping track of floating + point context, this class emits itt events to indicate begin and end of task group + context execution **/ +template <bool report_tasks> +class context_guard_helper { + const d1::task_group_context* curr_ctx; + d1::cpu_ctl_env guard_cpu_ctl_env; + d1::cpu_ctl_env curr_cpu_ctl_env; +public: + context_guard_helper() : curr_ctx(nullptr) { + guard_cpu_ctl_env.get_env(); + curr_cpu_ctl_env = guard_cpu_ctl_env; + } + ~context_guard_helper() { + if (curr_cpu_ctl_env != guard_cpu_ctl_env) + guard_cpu_ctl_env.set_env(); + if (report_tasks && curr_ctx) + ITT_TASK_END; + } + // The function is called from bypass dispatch loop on the hot path. + // Consider performance issues when refactoring. + void set_ctx(const d1::task_group_context* ctx) { + if (!ctx) + return; + const d1::cpu_ctl_env* ctl = reinterpret_cast<const d1::cpu_ctl_env*>(&ctx->my_cpu_ctl_env); + // Compare the FPU settings directly because the context can be reused between parallel algorithms. + if (*ctl != curr_cpu_ctl_env) { + curr_cpu_ctl_env = *ctl; + curr_cpu_ctl_env.set_env(); + } + if (report_tasks && ctx != curr_ctx) { + // if task group context was active, report end of current execution frame. + if (curr_ctx) + ITT_TASK_END; + // reporting begin of new task group context execution frame. + // using address of task group context object to group tasks (parent). + // id of task execution frame is nullptr and reserved for future use. + ITT_TASK_BEGIN(ctx, ctx->my_name, nullptr); + curr_ctx = ctx; + } + } +#if _WIN64 + void restore_default() { + if (curr_cpu_ctl_env != guard_cpu_ctl_env) { + guard_cpu_ctl_env.set_env(); + curr_cpu_ctl_env = guard_cpu_ctl_env; + } + } +#endif // _WIN64 +}; + +#if (_WIN32 || _WIN64 || __unix__ || __APPLE__) && (__TBB_x86_32 || __TBB_x86_64) +#if _MSC_VER +#pragma intrinsic(__rdtsc) +#endif +inline std::uint64_t machine_time_stamp() { +#if __INTEL_COMPILER + return _rdtsc(); +#elif _MSC_VER + return __rdtsc(); +#else + std::uint32_t hi, lo; + __asm__ __volatile__("rdtsc" : "=d"(hi), "=a"(lo)); + return (std::uint64_t(hi) << 32) | lo; +#endif +} + +inline void prolonged_pause_impl() { + // Assumption based on practice: 1000-2000 ticks seems to be a suitable invariant for the + // majority of platforms. Currently, skip platforms that define __TBB_STEALING_PAUSE + // because these platforms require very careful tuning. + std::uint64_t prev = machine_time_stamp(); + const std::uint64_t finish = prev + 1000; + atomic_backoff backoff; + do { + backoff.bounded_pause(); + std::uint64_t curr = machine_time_stamp(); + if (curr <= prev) + // Possibly, the current logical thread is moved to another hardware thread or overflow is occurred. + break; + prev = curr; + } while (prev < finish); } +#else +inline void prolonged_pause_impl() { +#ifdef __TBB_ipf + static const long PauseTime = 1500; +#else + static const long PauseTime = 80; +#endif + // TODO IDEA: Update PauseTime adaptively? + machine_pause(PauseTime); +} +#endif -inline intptr_t& priority ( task& t ) { - return t.prefix().context->my_priority; +inline void prolonged_pause() { +#if __TBB_WAITPKG_INTRINSICS_PRESENT + if (governor::wait_package_enabled()) { + std::uint64_t time_stamp = machine_time_stamp(); + // _tpause function directs the processor to enter an implementation-dependent optimized state + // until the Time Stamp Counter reaches or exceeds the value specified in second parameter. + // Constant "1000" is ticks to wait for. + // TODO : Modify this parameter based on empirical study of benchmarks. + // First parameter 0 selects between a lower power (cleared) or faster wakeup (set) optimized state. + _tpause(0, time_stamp + 1000); + } + else +#endif + prolonged_pause_impl(); } -#else /* __TBB_TASK_PRIORITY */ -static const intptr_t num_priority_levels = 1; -#endif /* __TBB_TASK_PRIORITY */ -//! Mutex type for global locks in the scheduler -typedef __TBB_SCHEDULER_MUTEX_TYPE scheduler_mutex_type; +// TODO: investigate possibility to work with number of CPU cycles +// because for different configurations this number of pauses + yields +// will be calculated in different amount of CPU cycles +// for example use rdtsc for it +class stealing_loop_backoff { + const int my_pause_threshold; + const int my_yield_threshold; + int my_pause_count; + int my_yield_count; +public: + // my_yield_threshold = 100 is an experimental value. Ideally, once we start calling __TBB_Yield(), + // the time spent spinning before calling out_of_work() should be approximately + // the time it takes for a thread to be woken up. Doing so would guarantee that we do + // no worse than 2x the optimal spin time. Or perhaps a time-slice quantum is the right amount. + stealing_loop_backoff(int num_workers, int yields_multiplier) + : my_pause_threshold{ 2 * (num_workers + 1) } + , my_yield_threshold{100 * yields_multiplier} + , my_pause_count{} + , my_yield_count{} + {} + bool pause() { + prolonged_pause(); + if (my_pause_count++ >= my_pause_threshold) { + my_pause_count = my_pause_threshold; + d0::yield(); + if (my_yield_count++ >= my_yield_threshold) { + my_yield_count = my_yield_threshold; + return true; + } + } + return false; + } + void reset_wait() { + my_pause_count = my_yield_count = 0; + } +}; -#if __TBB_TASK_GROUP_CONTEXT +//------------------------------------------------------------------------ +// Exception support +//------------------------------------------------------------------------ //! Task group state change propagation global epoch /** Together with generic_scheduler::my_context_state_propagation_epoch forms cross-thread signaling mechanism that allows to avoid locking at the hot path @@ -147,298 +304,308 @@ typedef __TBB_SCHEDULER_MUTEX_TYPE scheduler_mutex_type; and thus registration/deregistration routines take slower branch that may block (at most one thread of the pool can be blocked at any moment). Otherwise the control path is lock-free and fast. **/ -extern uintptr_t the_context_state_propagation_epoch; +extern std::atomic<std::uintptr_t> the_context_state_propagation_epoch; //! Mutex guarding state change propagation across task groups forest. /** Also protects modification of related data structures. **/ typedef scheduler_mutex_type context_state_propagation_mutex_type; extern context_state_propagation_mutex_type the_context_state_propagation_mutex; -#endif /* __TBB_TASK_GROUP_CONTEXT */ - -//! Alignment for a task object -const size_t task_alignment = 32; - -//! Number of bytes reserved for a task prefix -/** If not exactly sizeof(task_prefix), the extra bytes *precede* the task_prefix. */ -const size_t task_prefix_reservation_size = ((sizeof(internal::task_prefix)-1)/task_alignment+1)*task_alignment; - -//! Definitions for bits in task_prefix::extra_state -enum task_extra_state { - //! Tag for v1 tasks (i.e. tasks in TBB 1.0 and 2.0) - es_version_1_task = 0, - //! Tag for v3 tasks (i.e. tasks in TBB 2.1-2.2) - es_version_3_task = 1, - //! Tag for enqueued tasks - es_task_enqueued = 0x10, - //! Tag for v3 task_proxy. - es_task_proxy = 0x20, - //! Set if ref_count might be changed by another thread. Used for debugging. - es_ref_count_active = 0x40, - //! Set if the task has been stolen - es_task_is_stolen = 0x80 -}; -inline void reset_extra_state ( task *t ) { - t->prefix().extra_state &= ~(es_task_is_stolen | es_task_enqueued); -} +class tbb_exception_ptr { + std::exception_ptr my_ptr; +public: + static tbb_exception_ptr* allocate() noexcept; -//! Optimization hint to free_task that enables it omit unnecessary tests and code. -enum free_task_hint { - //! No hint - no_hint=0, - //! Task is known to have been allocated by this scheduler - local_task=1, - //! Task is known to be a small task. - /** Task should be returned to the free list of *some* scheduler, possibly not this scheduler. */ - small_task=2, - //! Bitwise-OR of local_task and small_task. - /** Task should be returned to free list of this scheduler. */ - small_local_task=3, - //! Disable caching for a small task. - no_cache = 4, - //! Task is known to be a small task and must not be cached. - no_cache_small_task = no_cache | small_task -}; + //! Destroys this objects + /** Note that objects of this type can be created only by the allocate() method. **/ + void destroy() noexcept; + + //! Throws the contained exception . + void throw_self(); + +private: + tbb_exception_ptr(const std::exception_ptr& src) : my_ptr(src) {} +}; // class tbb_exception_ptr //------------------------------------------------------------------------ // Debugging support //------------------------------------------------------------------------ #if TBB_USE_ASSERT +static const std::uintptr_t venom = tbb::detail::select_size_t_constant<0xDEADBEEFU, 0xDDEEAADDDEADBEEFULL>::value; -static const uintptr_t venom = tbb::internal::select_size_t_constant<0xDEADBEEFU,0xDDEEAADDDEADBEEFULL>::value; +inline void poison_value(std::uintptr_t& val) { val = venom; } -template <typename T> -void poison_value ( T& val ) { val = * punned_cast<T*>(&venom); } +inline void poison_value(std::atomic<std::uintptr_t>& val) { val.store(venom, std::memory_order_relaxed); } /** Expected to be used in assertions only, thus no empty form is defined. **/ -inline bool is_alive( uintptr_t v ) { return v != venom; } +inline bool is_alive(std::uintptr_t v) { return v != venom; } /** Logically, this method should be a member of class task. But we do not want to publish it, so it is here instead. */ -inline void assert_task_valid( const task* task ) { - __TBB_ASSERT( task!=NULL, NULL ); - __TBB_ASSERT( !is_poisoned(&task), NULL ); - __TBB_ASSERT( (uintptr_t)task % task_alignment == 0, "misaligned task" ); -#if __TBB_RECYCLE_TO_ENQUEUE - __TBB_ASSERT( (unsigned)task->state()<=(unsigned)task::to_enqueue, "corrupt task (invalid state)" ); -#else - __TBB_ASSERT( (unsigned)task->state()<=(unsigned)task::recycle, "corrupt task (invalid state)" ); -#endif +inline void assert_task_valid(const d1::task* t) { + assert_pointer_valid(t); } - #else /* !TBB_USE_ASSERT */ /** In contrast to debug version poison_value() is a macro here because the variable used as its argument may be undefined in release builds. **/ #define poison_value(g) ((void)0) -inline void assert_task_valid( const task* ) {} +inline void assert_task_valid(const d1::task*) {} #endif /* !TBB_USE_ASSERT */ -//------------------------------------------------------------------------ -// Helpers -//------------------------------------------------------------------------ - -#if __TBB_TASK_GROUP_CONTEXT -inline bool ConcurrentWaitsEnabled ( task& t ) { - return (t.prefix().context->my_version_and_traits & task_group_context::concurrent_wait) != 0; -} - -inline bool CancellationInfoPresent ( task& t ) { - return t.prefix().context->my_cancellation_requested != 0; -} - -#if TBB_USE_CAPTURED_EXCEPTION - inline tbb_exception* TbbCurrentException( task_group_context*, tbb_exception* src) { return src->move(); } - inline tbb_exception* TbbCurrentException( task_group_context* c, captured_exception* src) { - if( c->my_version_and_traits & task_group_context::exact_exception ) - runtime_warning( "Exact exception propagation is requested by application but the linked library is built without support for it"); - return src; +struct suspend_point_type { +#if __TBB_RESUMABLE_TASKS + //! The arena related to this task_dispatcher + arena* m_arena{ nullptr }; + //! The random for the resume task + FastRandom m_random; + //! The flag is raised when the original owner should return to this task dispatcher. + std::atomic<bool> m_is_owner_recalled{ false }; + //! Inicates if the resume task should be placed to the critical task stream. + bool m_is_critical{ false }; + //! Associated coroutine + co_context m_co_context; + //! Supend point before resume + suspend_point_type* m_prev_suspend_point{nullptr}; + + // Possible state transitions: + // A -> S -> N -> A + // A -> N -> S -> N -> A + enum class stack_state { + active, // some thread is working with this stack + suspended, // no thread is working with this stack + notified // some thread tried to resume this stack + }; + + //! The flag required to protect suspend finish and resume call + std::atomic<stack_state> m_stack_state{stack_state::active}; + + void resume(suspend_point_type* sp) { + __TBB_ASSERT(m_stack_state.load(std::memory_order_relaxed) != stack_state::suspended, "The stack is expected to be active"); + + sp->m_prev_suspend_point = this; + + // Do not access sp after resume + m_co_context.resume(sp->m_co_context); + __TBB_ASSERT(m_stack_state.load(std::memory_order_relaxed) != stack_state::active, nullptr); + + finilize_resume(); } - #define TbbRethrowException(TbbCapturedException) (TbbCapturedException)->throw_self() -#else - // Using macro instead of an inline function here allows to avoid evaluation of the - // TbbCapturedException expression when exact propagation is enabled for the context. - #define TbbCurrentException(context, TbbCapturedException) \ - context->my_version_and_traits & task_group_context::exact_exception \ - ? tbb_exception_ptr::allocate() \ - : tbb_exception_ptr::allocate( *(TbbCapturedException) ); - #define TbbRethrowException(TbbCapturedException) \ - { \ - if( governor::rethrow_exception_broken() ) fix_broken_rethrow(); \ - (TbbCapturedException)->throw_self(); \ - } -#endif /* !TBB_USE_CAPTURED_EXCEPTION */ -#define TbbRegisterCurrentException(context, TbbCapturedException) \ - if ( context->cancel_group_execution() ) { \ - /* We are the first to signal cancellation, so store the exception that caused it. */ \ - context->my_exception = TbbCurrentException( context, TbbCapturedException ); \ + void finilize_resume() { + m_stack_state.store(stack_state::active, std::memory_order_relaxed); + // Set the suspended state for the stack that we left. If the state is already notified, it means that + // someone already tried to resume our previous stack but failed. So, we need to resume it. + // m_prev_suspend_point might be nullptr when destroying co_context based on threads + if (m_prev_suspend_point && m_prev_suspend_point->m_stack_state.exchange(stack_state::suspended) == stack_state::notified) { + r1::resume(m_prev_suspend_point); + } + m_prev_suspend_point = nullptr; } -#define TbbCatchAll(context) \ - catch ( tbb_exception& exc ) { \ - TbbRegisterCurrentException( context, &exc ); \ - } catch ( std::exception& exc ) { \ - TbbRegisterCurrentException( context, captured_exception::allocate(typeid(exc).name(), exc.what()) ); \ - } catch ( ... ) { \ - TbbRegisterCurrentException( context, captured_exception::allocate("...", "Unidentified exception") );\ + bool try_notify_resume() { + // Check that stack is already suspended. Return false if not yet. + return m_stack_state.exchange(stack_state::notified) == stack_state::suspended; } -#else /* !__TBB_TASK_GROUP_CONTEXT */ + void recall_owner() { + __TBB_ASSERT(m_stack_state.load(std::memory_order_relaxed) == stack_state::suspended, nullptr); + m_stack_state.store(stack_state::notified, std::memory_order_relaxed); + m_is_owner_recalled.store(true, std::memory_order_release); + } -inline bool ConcurrentWaitsEnabled ( task& t ) { return false; } + struct resume_task final : public d1::task { + task_dispatcher& m_target; + explicit resume_task(task_dispatcher& target) : m_target(target) { + task_accessor::set_resume_trait(*this); + } + d1::task* execute(d1::execution_data& ed) override; + d1::task* cancel(d1::execution_data&) override { + __TBB_ASSERT(false, "The resume task cannot be canceled"); + return nullptr; + } + } m_resume_task; -#endif /* __TBB_TASK_GROUP_CONTEXT */ + suspend_point_type(arena* a, std::size_t stack_size, task_dispatcher& target); +#endif /*__TBB_RESUMABLE_TASKS */ +}; -inline void prolonged_pause() { -#if defined(__TBB_time_stamp) && !__TBB_STEALING_PAUSE - // Assumption based on practice: 1000-2000 ticks seems to be a suitable invariant for the - // majority of platforms. Currently, skip platforms that define __TBB_STEALING_PAUSE - // because these platforms require very careful tuning. - machine_tsc_t prev = __TBB_time_stamp(); - const machine_tsc_t finish = prev + 1000; - atomic_backoff backoff; - do { - backoff.bounded_pause(); - machine_tsc_t curr = __TBB_time_stamp(); - if ( curr <= prev ) - // Possibly, the current logical thread is moved to another hardware thread or overflow is occurred. - break; - prev = curr; - } while ( prev < finish ); -#else -#ifdef __TBB_STEALING_PAUSE - static const long PauseTime = __TBB_STEALING_PAUSE; -#elif __TBB_ipf - static const long PauseTime = 1500; -#else - static const long PauseTime = 80; -#endif - // TODO IDEA: Update PauseTime adaptively? - __TBB_Pause(PauseTime); +#if _MSC_VER && !defined(__INTEL_COMPILER) +// structure was padded due to alignment specifier +// #pragma warning( push ) +// #pragma warning( disable: 4324 ) #endif -} -//------------------------------------------------------------------------ -// arena_slot -//------------------------------------------------------------------------ -struct arena_slot_line1 { - //TODO: make this tbb:atomic<>. - //! Scheduler of the thread attached to the slot - /** Marks the slot as busy, and is used to iterate through the schedulers belonging to this arena **/ - generic_scheduler* my_scheduler; - - // Synchronization of access to Task pool - /** Also is used to specify if the slot is empty or locked: - 0 - empty - -1 - locked **/ - task* *__TBB_atomic task_pool; - - //! Index of the first ready task in the deque. - /** Modified by thieves, and by the owner during compaction/reallocation **/ - __TBB_atomic size_t head; -}; +class alignas (max_nfs_size) task_dispatcher { +public: + // TODO: reconsider low level design to better organize dependencies and files. + friend class thread_data; + friend class arena_slot; + friend class nested_arena_context; + friend class delegated_task; + friend struct base_waiter; + + //! The list of possible post resume actions. + enum class post_resume_action { + invalid, + register_waiter, + cleanup, + notify, + none + }; + + //! The data of the current thread attached to this task_dispatcher + thread_data* m_thread_data{ nullptr }; + + //! The current execution data + execution_data_ext m_execute_data_ext; + + //! Properties + struct properties { + bool outermost{ true }; + bool fifo_tasks_allowed{ true }; + bool critical_task_allowed{ true }; + } m_properties; + + //! Position in the call stack when stealing is still allowed. + std::uintptr_t m_stealing_threshold{}; + + //! Suspend point (null if this task dispatcher has been never suspended) + suspend_point_type* m_suspend_point{ nullptr }; + + //! Used to improve scalability of d1::wait_context by using per thread reference_counter + std::unordered_map<d1::wait_tree_vertex_interface*, d1::reference_vertex*, + std::hash<d1::wait_tree_vertex_interface*>, std::equal_to<d1::wait_tree_vertex_interface*>, + tbb_allocator<std::pair<d1::wait_tree_vertex_interface* const, d1::reference_vertex*>> + > + m_reference_vertex_map; + + //! Attempt to get a task from the mailbox. + /** Gets a task only if it has not been executed by its sender or a thief + that has stolen it from the sender's task pool. Otherwise returns nullptr. + This method is intended to be used only by the thread extracting the proxy + from its mailbox. (In contrast to local task pool, mailbox can be read only + by its owner). **/ + d1::task* get_mailbox_task(mail_inbox& my_inbox, execution_data_ext& ed, isolation_type isolation); + + d1::task* get_critical_task(d1::task*, execution_data_ext&, isolation_type, bool); + + template <bool ITTPossible, typename Waiter> + d1::task* receive_or_steal_task(thread_data& tls, execution_data_ext& ed, Waiter& waiter, + isolation_type isolation, bool outermost, bool criticality_absence); + + template <bool ITTPossible, typename Waiter> + d1::task* local_wait_for_all(d1::task * t, Waiter& waiter); + + task_dispatcher(const task_dispatcher&) = delete; + + bool can_steal(); +public: + task_dispatcher(arena* a); -struct arena_slot_line2 { - //! Hint provided for operations with the container of starvation-resistant tasks. - /** Modified by the owner thread (during these operations). **/ - unsigned hint_for_pop; + ~task_dispatcher() { + if (m_suspend_point) { + m_suspend_point->~suspend_point_type(); + cache_aligned_deallocate(m_suspend_point); + } - //! Index of the element following the last ready task in the deque. - /** Modified by the owner thread. **/ - __TBB_atomic size_t tail; + for (auto& elem : m_reference_vertex_map) { + d1::reference_vertex*& node = elem.second; + node->~reference_vertex(); + cache_aligned_deallocate(node); + poison_pointer(node); + } - //! Capacity of the primary task pool (number of elements - pointers to task). - size_t my_task_pool_size; + poison_pointer(m_thread_data); + poison_pointer(m_suspend_point); + } - // Task pool of the scheduler that owns this slot - task* *__TBB_atomic task_pool_ptr; + template <typename Waiter> + d1::task* local_wait_for_all(d1::task* t, Waiter& waiter); -#if __TBB_STATISTICS - //! Set of counters to accumulate internal statistics related to this arena - statistics_counters *my_counters; -#endif /* __TBB_STATISTICS */ -}; + bool allow_fifo_task(bool new_state) { + bool old_state = m_properties.fifo_tasks_allowed; + m_properties.fifo_tasks_allowed = new_state; + return old_state; + } -struct arena_slot : padded<arena_slot_line1>, padded<arena_slot_line2> { -#if TBB_USE_ASSERT - void fill_with_canary_pattern ( size_t first, size_t last ) { - for ( size_t i = first; i < last; ++i ) - poison_pointer(task_pool_ptr[i]); + isolation_type set_isolation(isolation_type isolation) { + isolation_type prev = m_execute_data_ext.isolation; + m_execute_data_ext.isolation = isolation; + return prev; } -#else - void fill_with_canary_pattern ( size_t, size_t ) {} -#endif /* TBB_USE_ASSERT */ - - void allocate_task_pool( size_t n ) { - size_t byte_size = ((n * sizeof(task*) + NFS_MaxLineSize - 1) / NFS_MaxLineSize) * NFS_MaxLineSize; - my_task_pool_size = byte_size / sizeof(task*); - task_pool_ptr = (task**)NFS_Allocate( 1, byte_size, NULL ); - // No need to clear the fresh deque since valid items are designated by the head and tail members. - // But fill it with a canary pattern in the high vigilance debug mode. - fill_with_canary_pattern( 0, my_task_pool_size ); + + thread_data& get_thread_data() { + __TBB_ASSERT(m_thread_data, nullptr); + return *m_thread_data; } - //! Deallocate task pool that was allocated by means of allocate_task_pool. - void free_task_pool( ) { - // TODO: understand the assertion and modify - // __TBB_ASSERT( !task_pool /*TODO: == EmptyTaskPool*/, NULL); - if( task_pool_ptr ) { - __TBB_ASSERT( my_task_pool_size, NULL); - NFS_Free( task_pool_ptr ); - task_pool_ptr = NULL; - my_task_pool_size = 0; - } + static void execute_and_wait(d1::task* t, d1::wait_context& wait_ctx, d1::task_group_context& w_ctx); + + void set_stealing_threshold(std::uintptr_t stealing_threshold) { + bool assert_condition = (stealing_threshold == 0 && m_stealing_threshold != 0) || + (stealing_threshold != 0 && m_stealing_threshold == 0); + __TBB_ASSERT_EX( assert_condition, nullptr ); + m_stealing_threshold = stealing_threshold; } + + d1::task* get_inbox_or_critical_task(execution_data_ext&, mail_inbox&, isolation_type, bool); + d1::task* get_stream_or_critical_task(execution_data_ext&, arena&, task_stream<front_accessor>&, + unsigned& /*hint_for_stream*/, isolation_type, + bool /*critical_allowed*/); + d1::task* steal_or_get_critical(execution_data_ext&, arena&, unsigned /*arena_index*/, FastRandom&, + isolation_type, bool /*critical_allowed*/); + +#if __TBB_RESUMABLE_TASKS + /* [[noreturn]] */ void co_local_wait_for_all() noexcept; + void suspend(suspend_callback_type suspend_callback, void* user_callback); + void internal_suspend(); + void do_post_resume_action(); + + bool resume(task_dispatcher& target); + suspend_point_type* get_suspend_point(); + void init_suspend_point(arena* a, std::size_t stack_size); + friend void internal_resume(suspend_point_type*); + void recall_point(); +#endif /* __TBB_RESUMABLE_TASKS */ }; -#if !__TBB_CPU_CTL_ENV_PRESENT -class cpu_ctl_env { - fenv_t *my_fenv_ptr; -public: - cpu_ctl_env() : my_fenv_ptr(NULL) {} - ~cpu_ctl_env() { - if ( my_fenv_ptr ) - tbb::internal::NFS_Free( (void*)my_fenv_ptr ); - } - // It is possible not to copy memory but just to copy pointers but the following issues should be addressed: - // 1. The arena lifetime and the context lifetime are independent; - // 2. The user is allowed to recapture different FPU settings to context so 'current FPU settings' inside - // dispatch loop may become invalid. - // But do we really want to improve the fenv implementation? It seems to be better to replace the fenv implementation - // with a platform specific implementation. - cpu_ctl_env( const cpu_ctl_env &src ) : my_fenv_ptr(NULL) { - *this = src; - } - cpu_ctl_env& operator=( const cpu_ctl_env &src ) { - __TBB_ASSERT( src.my_fenv_ptr, NULL ); - if ( !my_fenv_ptr ) - my_fenv_ptr = (fenv_t*)tbb::internal::NFS_Allocate(1, sizeof(fenv_t), NULL); - *my_fenv_ptr = *src.my_fenv_ptr; - return *this; - } - bool operator!=( const cpu_ctl_env &ctl ) const { - __TBB_ASSERT( my_fenv_ptr, "cpu_ctl_env is not initialized." ); - __TBB_ASSERT( ctl.my_fenv_ptr, "cpu_ctl_env is not initialized." ); - return memcmp( (void*)my_fenv_ptr, (void*)ctl.my_fenv_ptr, sizeof(fenv_t) ); - } - void get_env () { - if ( !my_fenv_ptr ) - my_fenv_ptr = (fenv_t*)tbb::internal::NFS_Allocate(1, sizeof(fenv_t), NULL); - fegetenv( my_fenv_ptr ); - } - const cpu_ctl_env& set_env () const { - __TBB_ASSERT( my_fenv_ptr, "cpu_ctl_env is not initialized." ); - fesetenv( my_fenv_ptr ); - return *this; - } +#if _MSC_VER && !defined(__INTEL_COMPILER) +// #pragma warning( pop ) +#endif + +inline std::uintptr_t calculate_stealing_threshold(std::uintptr_t base, std::size_t stack_size) { + __TBB_ASSERT(stack_size != 0, "Stack size cannot be zero"); + __TBB_ASSERT(base > stack_size / 2, "Stack anchor calculation overflow"); + return base - stack_size / 2; +} + +struct task_group_context_impl { + static void destroy(d1::task_group_context&); + static void initialize(d1::task_group_context&); + static void register_with(d1::task_group_context&, thread_data*); + static void bind_to_impl(d1::task_group_context&, thread_data*); + static void bind_to(d1::task_group_context&, thread_data*); + static void propagate_task_group_state(d1::task_group_context&, std::atomic<uint32_t> d1::task_group_context::*, d1::task_group_context&, uint32_t); + static bool cancel_group_execution(d1::task_group_context&); + static bool is_group_execution_cancelled(const d1::task_group_context&); + static void reset(d1::task_group_context&); + static void capture_fp_settings(d1::task_group_context&); + static void copy_fp_settings(d1::task_group_context& ctx, const d1::task_group_context& src); }; -#endif /* !__TBB_CPU_CTL_ENV_PRESENT */ -} // namespace internal + +//! Forward declaration for scheduler entities +bool gcc_rethrow_exception_broken(); +void fix_broken_rethrow(); +//! Forward declaration: throws std::runtime_error with what() returning error_code description prefixed with aux_info +void handle_perror(int error_code, const char* aux_info); + +} // namespace r1 +} // namespace detail } // namespace tbb #endif /* _TBB_scheduler_common_H */ diff --git a/src/tbb/src/tbb/scheduler_utility.h b/src/tbb/src/tbb/scheduler_utility.h deleted file mode 100644 index 5c2285987..000000000 --- a/src/tbb/src/tbb/scheduler_utility.h +++ /dev/null @@ -1,133 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef _TBB_scheduler_utility_H -#define _TBB_scheduler_utility_H - -#include "scheduler.h" - -namespace tbb { -namespace internal { - -//------------------------------------------------------------------------ -// auto_empty_task -//------------------------------------------------------------------------ - -//! Smart holder for the empty task class with automatic destruction -class auto_empty_task { - task* my_task; - generic_scheduler* my_scheduler; -public: - auto_empty_task ( __TBB_CONTEXT_ARG(generic_scheduler *s, task_group_context* context) ) - : my_task( new(&s->allocate_task(sizeof(empty_task), __TBB_CONTEXT_ARG(NULL, context))) empty_task ) - , my_scheduler(s) - {} - // empty_task has trivial destructor, so there's no need to call it. - ~auto_empty_task () { my_scheduler->free_task<small_local_task>(*my_task); } - - operator task& () { return *my_task; } - task* operator & () { return my_task; } - task_prefix& prefix () { return my_task->prefix(); } -}; // class auto_empty_task - -//------------------------------------------------------------------------ -// fast_reverse_vector -//------------------------------------------------------------------------ - -//! Vector that grows without reallocations, and stores items in the reverse order. -/** Requires to initialize its first segment with a preallocated memory chunk - (usually it is static array or an array allocated on the stack). - The second template parameter specifies maximal number of segments. Each next - segment is twice as large as the previous one. **/ -template<typename T, size_t max_segments = 16> -class fast_reverse_vector -{ -public: - fast_reverse_vector ( T* initial_segment, size_t segment_size ) - : m_cur_segment(initial_segment) - , m_cur_segment_size(segment_size) - , m_pos(segment_size) - , m_num_segments(0) - , m_size(0) - { - __TBB_ASSERT ( initial_segment && segment_size, "Nonempty initial segment must be supplied"); - } - - ~fast_reverse_vector () - { - for ( size_t i = 1; i < m_num_segments; ++i ) - NFS_Free( m_segments[i] ); - } - - size_t size () const { return m_size + m_cur_segment_size - m_pos; } - - void push_back ( const T& val ) - { - if ( !m_pos ) { - if ( !m_num_segments ) m_segments[m_num_segments++] = m_cur_segment; - m_size += m_cur_segment_size; - m_cur_segment_size *= 2; - m_pos = m_cur_segment_size; - m_segments[m_num_segments++] = m_cur_segment = (T*)NFS_Allocate( m_cur_segment_size, sizeof(T), NULL ); - __TBB_ASSERT ( m_num_segments < max_segments, "Maximal capacity exceeded" ); - } - m_cur_segment[--m_pos] = val; - } - - //! Copies the contents of the vector into the dst array. - /** Can only be used when T is a POD type, as copying does not invoke copy constructors. **/ - void copy_memory ( T* dst ) const - { - size_t sz = m_cur_segment_size - m_pos; - memcpy( dst, m_cur_segment + m_pos, sz * sizeof(T) ); - dst += sz; - sz = m_cur_segment_size / 2; - for ( long i = (long)m_num_segments - 2; i >= 0; --i ) { - memcpy( dst, m_segments[i], sz * sizeof(T) ); - dst += sz; - sz /= 2; - } - } - -protected: - //! The current (not completely filled) segment - T *m_cur_segment; - - //! Capacity of m_cur_segment - size_t m_cur_segment_size; - - //! Insertion position in m_cur_segment - size_t m_pos; - - //! Array of segments (has fixed size specified by the second template parameter) - T *m_segments[max_segments]; - - //! Number of segments (the size of m_segments) - size_t m_num_segments; - - //! Number of items in the segments in m_segments - size_t m_size; - -}; // class fast_reverse_vector - -} // namespace internal -} // namespace tbb - -#endif /* _TBB_scheduler_utility_H */ diff --git a/src/tbb/src/tbb/semaphore.cpp b/src/tbb/src/tbb/semaphore.cpp index 4c6d8532d..211091d1f 100644 --- a/src/tbb/src/tbb/semaphore.cpp +++ b/src/tbb/src/tbb/semaphore.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2022 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include "semaphore.h" @@ -25,17 +21,18 @@ #endif namespace tbb { -namespace internal { +namespace detail { +namespace r1 { // TODO: For new win UI port, we can use SRWLock API without dynamic_link etc. #if __TBB_USE_SRWLOCK -static atomic<do_once_state> concmon_module_inited; +static std::atomic<do_once_state> concmon_module_inited; void WINAPI init_binsem_using_event( SRWLOCK* h_ ) { srwl_or_handle* shptr = (srwl_or_handle*) h_; - shptr->h = CreateEventEx( NULL, NULL, 0, EVENT_ALL_ACCESS|SEMAPHORE_ALL_ACCESS ); + shptr->h = CreateEventEx( nullptr, nullptr, 0, EVENT_ALL_ACCESS|SEMAPHORE_ALL_ACCESS ); } void WINAPI acquire_binsem_using_event( SRWLOCK* h_ ) @@ -63,11 +60,11 @@ static const dynamic_link_descriptor SRWLLinkTable[] = { inline void init_concmon_module() { - __TBB_ASSERT( (uintptr_t)__TBB_init_binsem==(uintptr_t)&init_binsem_using_event, NULL ); + __TBB_ASSERT( (uintptr_t)__TBB_init_binsem==(uintptr_t)&init_binsem_using_event, nullptr); if( dynamic_link( "Kernel32.dll", SRWLLinkTable, sizeof(SRWLLinkTable)/sizeof(dynamic_link_descriptor) ) ) { - __TBB_ASSERT( (uintptr_t)__TBB_init_binsem!=(uintptr_t)&init_binsem_using_event, NULL ); - __TBB_ASSERT( (uintptr_t)__TBB_acquire_binsem!=(uintptr_t)&acquire_binsem_using_event, NULL ); - __TBB_ASSERT( (uintptr_t)__TBB_release_binsem!=(uintptr_t)&release_binsem_using_event, NULL ); + __TBB_ASSERT( (uintptr_t)__TBB_init_binsem!=(uintptr_t)&init_binsem_using_event, nullptr); + __TBB_ASSERT( (uintptr_t)__TBB_acquire_binsem!=(uintptr_t)&acquire_binsem_using_event, nullptr); + __TBB_ASSERT( (uintptr_t)__TBB_release_binsem!=(uintptr_t)&release_binsem_using_event, nullptr); } } @@ -90,5 +87,6 @@ void binary_semaphore::V() { __TBB_release_binsem( &my_sem.lock ); } #endif /* __TBB_USE_SRWLOCK */ -} // namespace internal +} // namespace r1 +} // namespace detail } // namespace tbb diff --git a/src/tbb/src/tbb/semaphore.h b/src/tbb/src/tbb/semaphore.h index 43af24aa6..9d27f3ac9 100644 --- a/src/tbb/src/tbb/semaphore.h +++ b/src/tbb/src/tbb/semaphore.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2022 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,37 +12,116 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ -#ifndef __TBB_tbb_semaphore_H -#define __TBB_tbb_semaphore_H +#ifndef __TBB_semaphore_H +#define __TBB_semaphore_H -#include "tbb/tbb_stddef.h" +#include "oneapi/tbb/detail/_utils.h" #if _WIN32||_WIN64 -#include "tbb/machine/windows_api.h" - +#include <windows.h> #elif __APPLE__ -#include <mach/semaphore.h> -#include <mach/task.h> -#include <mach/mach_init.h> -#include <mach/error.h> - +#include <dispatch/dispatch.h> #else #include <semaphore.h> #ifdef TBB_USE_DEBUG -#include <errno.h> +#include <cerrno> #endif #endif /*_WIN32||_WIN64*/ +#include <atomic> + +#if __unix__ +#if defined(__has_include) +#define __TBB_has_include __has_include +#else +#define __TBB_has_include(x) 0 +#endif + +/* Futex definitions */ +#include <unistd.h> +#if defined(__linux__) || __TBB_has_include(<sys/syscall.h>) +#include <sys/syscall.h> +#endif + +#if defined(SYS_futex) + +/* This section is included for Linux and some other systems that may support futexes.*/ + +#define __TBB_USE_FUTEX 1 + +/* +If available, use typical headers where futex API is defined. While Linux and OpenBSD +are known to provide such headers, other systems might have them as well. +*/ +#if defined(__linux__) || __TBB_has_include(<linux/futex.h>) +#include <linux/futex.h> +#elif defined(__OpenBSD__) || __TBB_has_include(<sys/futex.h>) +#include <sys/futex.h> +#endif + +#include <climits> +#include <cerrno> + +/* +Some systems might not define the macros or use different names. In such case we expect +the actual parameter values to match Linux: 0 for wait, 1 for wake. +*/ +#if defined(FUTEX_WAIT_PRIVATE) +#define __TBB_FUTEX_WAIT FUTEX_WAIT_PRIVATE +#elif defined(FUTEX_WAIT) +#define __TBB_FUTEX_WAIT FUTEX_WAIT +#else +#define __TBB_FUTEX_WAIT 0 +#endif + +#if defined(FUTEX_WAKE_PRIVATE) +#define __TBB_FUTEX_WAKE FUTEX_WAKE_PRIVATE +#elif defined(FUTEX_WAKE) +#define __TBB_FUTEX_WAKE FUTEX_WAKE +#else +#define __TBB_FUTEX_WAKE 1 +#endif + +#endif // SYS_futex +#endif // __unix__ + namespace tbb { -namespace internal { +namespace detail { +namespace r1 { + +//////////////////////////////////////////////////////////////////////////////////////////////////// +// Futex implementation +//////////////////////////////////////////////////////////////////////////////////////////////////// + +#if __TBB_USE_FUTEX + +static inline int futex_wait( void *futex, int comparand ) { + int r = ::syscall(SYS_futex, futex, __TBB_FUTEX_WAIT, comparand, nullptr, nullptr, 0); +#if TBB_USE_ASSERT + int e = errno; + __TBB_ASSERT(r == 0 || r == EWOULDBLOCK || (r == -1 && (e == EAGAIN || e == EINTR)), "futex_wait failed."); +#endif /* TBB_USE_ASSERT */ + return r; +} + +static inline int futex_wakeup_one( void *futex ) { + int r = ::syscall(SYS_futex, futex, __TBB_FUTEX_WAKE, 1, nullptr, nullptr, 0); + __TBB_ASSERT(r == 0 || r == 1, "futex_wakeup_one: more than one thread woken up?"); + return r; +} +// Additional possible methods that are not required right now +// static inline int futex_wakeup_all( void *futex ) { +// int r = ::syscall( SYS_futex,futex,__TBB_FUTEX_WAKE,INT_MAX,nullptr,nullptr,0 ); +// __TBB_ASSERT( r>=0, "futex_wakeup_all: error in waking up threads" ); +// return r; +// } +#endif // __TBB_USE_FUTEX + +//////////////////////////////////////////////////////////////////////////////////////////////////// #if _WIN32||_WIN64 typedef LONG sem_count_t; //! Edsger Dijkstra's counting semaphore @@ -56,11 +135,11 @@ class semaphore : no_copy { //! wait/acquire void P() {WaitForSingleObjectEx( sem, INFINITE, FALSE );} //! post/release - void V() {ReleaseSemaphore( sem, 1, NULL );} + void V() {ReleaseSemaphore( sem, 1, nullptr);} private: HANDLE sem; void init_semaphore(size_t start_cnt_) { - sem = CreateSemaphoreEx( NULL, LONG(start_cnt_), max_semaphore_cnt, NULL, 0, SEMAPHORE_ALL_ACCESS ); + sem = CreateSemaphoreEx( nullptr, LONG(start_cnt_), max_semaphore_cnt, nullptr, 0, SEMAPHORE_ALL_ACCESS ); } }; #elif __APPLE__ @@ -68,28 +147,18 @@ class semaphore : no_copy { class semaphore : no_copy { public: //! ctor - semaphore(int start_cnt_ = 0) : sem(start_cnt_) { init_semaphore(start_cnt_); } + semaphore(int start_cnt_ = 0) { my_sem = dispatch_semaphore_create(start_cnt_); } //! dtor - ~semaphore() { - kern_return_t ret = semaphore_destroy( mach_task_self(), sem ); - __TBB_ASSERT_EX( ret==err_none, NULL ); - } + ~semaphore() { dispatch_release(my_sem); } //! wait/acquire void P() { - int ret; - do { - ret = semaphore_wait( sem ); - } while( ret==KERN_ABORTED ); - __TBB_ASSERT( ret==KERN_SUCCESS, "semaphore_wait() failed" ); + std::intptr_t ret = dispatch_semaphore_wait(my_sem, DISPATCH_TIME_FOREVER); + __TBB_ASSERT_EX(ret == 0, "dispatch_semaphore_wait() failed"); } //! post/release - void V() { semaphore_signal( sem ); } + void V() { dispatch_semaphore_signal(my_sem); } private: - semaphore_t sem; - void init_semaphore(int start_cnt_) { - kern_return_t ret = semaphore_create( mach_task_self(), &sem, SYNC_POLICY_FIFO, start_cnt_ ); - __TBB_ASSERT_EX( ret==err_none, "failed to create a semaphore" ); - } + dispatch_semaphore_t my_sem; }; #else /* Linux/Unix */ typedef uint32_t sem_count_t; @@ -102,12 +171,12 @@ class semaphore : no_copy { //! dtor ~semaphore() { int ret = sem_destroy( &sem ); - __TBB_ASSERT_EX( !ret, NULL ); + __TBB_ASSERT_EX( !ret, nullptr); } //! wait/acquire void P() { while( sem_wait( &sem )!=0 ) - __TBB_ASSERT( errno==EINTR, NULL ); + __TBB_ASSERT( errno==EINTR, nullptr); } //! post/release void V() { sem_post( &sem ); } @@ -115,7 +184,7 @@ class semaphore : no_copy { sem_t sem; void init_semaphore(int start_cnt_) { int ret = sem_init( &sem, /*shared among threads*/ 0, start_cnt_ ); - __TBB_ASSERT_EX( !ret, NULL ); + __TBB_ASSERT_EX( !ret, nullptr); } }; #endif /* _WIN32||_WIN64 */ @@ -128,7 +197,7 @@ class semaphore : no_copy { class binary_semaphore : no_copy { public: //! ctor - binary_semaphore() { my_sem = CreateEventEx( NULL, NULL, 0, EVENT_ALL_ACCESS ); } + binary_semaphore() { my_sem = CreateEventEx( nullptr, nullptr, 0, EVENT_ALL_ACCESS ); } //! dtor ~binary_semaphore() { CloseHandle( my_sem ); } //! wait/acquire @@ -162,35 +231,13 @@ class binary_semaphore : no_copy { #endif /* !__TBB_USE_SRWLOCK */ #elif __APPLE__ //! binary_semaphore for concurrent monitor -class binary_semaphore : no_copy { -public: - //! ctor - binary_semaphore() : my_sem(0) { - kern_return_t ret = semaphore_create( mach_task_self(), &my_sem, SYNC_POLICY_FIFO, 0 ); - __TBB_ASSERT_EX( ret==err_none, "failed to create a semaphore" ); - } - //! dtor - ~binary_semaphore() { - kern_return_t ret = semaphore_destroy( mach_task_self(), my_sem ); - __TBB_ASSERT_EX( ret==err_none, NULL ); - } - //! wait/acquire - void P() { - int ret; - do { - ret = semaphore_wait( my_sem ); - } while( ret==KERN_ABORTED ); - __TBB_ASSERT( ret==KERN_SUCCESS, "semaphore_wait() failed" ); - } - //! post/release - void V() { semaphore_signal( my_sem ); } -private: - semaphore_t my_sem; -}; +using binary_semaphore = semaphore; #else /* Linux/Unix */ #if __TBB_USE_FUTEX class binary_semaphore : no_copy { +// The implementation is equivalent to the "Mutex, Take 3" one +// in the paper "Futexes Are Tricky" by Ulrich Drepper public: //! ctor binary_semaphore() { my_sem = 1; } @@ -198,27 +245,24 @@ class binary_semaphore : no_copy { ~binary_semaphore() {} //! wait/acquire void P() { - int s; - if( (s = my_sem.compare_and_swap( 1, 0 ))!=0 ) { + int s = 0; + if( !my_sem.compare_exchange_strong( s, 1 ) ) { if( s!=2 ) - s = my_sem.fetch_and_store( 2 ); - while( s!=0 ) { + s = my_sem.exchange( 2 ); + while( s!=0 ) { // This loop deals with spurious wakeup futex_wait( &my_sem, 2 ); - s = my_sem.fetch_and_store( 2 ); + s = my_sem.exchange( 2 ); } } } //! post/release void V() { - __TBB_ASSERT( my_sem>=1, "multiple V()'s in a row?" ); - if( my_sem--!=1 ) { - //if old value was 2 - my_sem = 0; + __TBB_ASSERT( my_sem.load(std::memory_order_relaxed)>=1, "multiple V()'s in a row?" ); + if( my_sem.exchange( 0 )==2 ) futex_wakeup_one( &my_sem ); - } } private: - atomic<int> my_sem; + std::atomic<int> my_sem; // 0 - open; 1 - closed, no waits; 2 - closed, possible waits }; #else typedef uint32_t sem_count_t; @@ -228,17 +272,17 @@ class binary_semaphore : no_copy { //! ctor binary_semaphore() { int ret = sem_init( &my_sem, /*shared among threads*/ 0, 0 ); - __TBB_ASSERT_EX( !ret, NULL ); + __TBB_ASSERT_EX( !ret, nullptr); } //! dtor ~binary_semaphore() { int ret = sem_destroy( &my_sem ); - __TBB_ASSERT_EX( !ret, NULL ); + __TBB_ASSERT_EX( !ret, nullptr); } //! wait/acquire void P() { while( sem_wait( &my_sem )!=0 ) - __TBB_ASSERT( errno==EINTR, NULL ); + __TBB_ASSERT( errno==EINTR, nullptr); } //! post/release void V() { sem_post( &my_sem ); } @@ -248,7 +292,8 @@ class binary_semaphore : no_copy { #endif /* __TBB_USE_FUTEX */ #endif /* _WIN32||_WIN64 */ -} // namespace internal +} // namespace r1 +} // namespace detail } // namespace tbb -#endif /* __TBB_tbb_semaphore_H */ +#endif /* __TBB_semaphore_H */ diff --git a/src/tbb/src/tbb/small_object_pool.cpp b/src/tbb/src/tbb/small_object_pool.cpp new file mode 100644 index 000000000..28d11d011 --- /dev/null +++ b/src/tbb/src/tbb/small_object_pool.cpp @@ -0,0 +1,154 @@ +/* + Copyright (c) 2020-2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "oneapi/tbb/cache_aligned_allocator.h" +#include "oneapi/tbb/detail/_small_object_pool.h" +#include "oneapi/tbb/detail/_task.h" +#include "governor.h" +#include "thread_data.h" +#include "task_dispatcher.h" + +#include <cstddef> + +namespace tbb { +namespace detail { +namespace r1 { + +small_object_pool_impl::small_object* const small_object_pool_impl::dead_public_list = + reinterpret_cast<small_object_pool_impl::small_object*>(1); + +void* __TBB_EXPORTED_FUNC allocate(d1::small_object_pool*& allocator, std::size_t number_of_bytes, const d1::execution_data& ed) { + auto& tls = static_cast<const execution_data_ext&>(ed).task_disp->get_thread_data(); + auto pool = tls.my_small_object_pool; + return pool->allocate_impl(allocator, number_of_bytes); +} + +void* __TBB_EXPORTED_FUNC allocate(d1::small_object_pool*& allocator, std::size_t number_of_bytes) { + // TODO: optimize if the allocator contains a valid pool. + auto tls = governor::get_thread_data(); + auto pool = tls->my_small_object_pool; + return pool->allocate_impl(allocator, number_of_bytes); +} + +void* small_object_pool_impl::allocate_impl(d1::small_object_pool*& allocator, std::size_t number_of_bytes) +{ + small_object* obj{nullptr}; + + if (number_of_bytes <= small_object_size) { + if (m_private_list) { + obj = m_private_list; + m_private_list = m_private_list->next; + } else if (m_public_list.load(std::memory_order_relaxed)) { + // No fence required for read of my_public_list above, because std::atomic::exchange() has a fence. + obj = m_public_list.exchange(nullptr); + __TBB_ASSERT( obj, "another thread emptied the my_public_list" ); + m_private_list = obj->next; + } else { + obj = new (cache_aligned_allocate(small_object_size)) small_object{nullptr}; + ++m_private_counter; + } + } else { + obj = new (cache_aligned_allocate(number_of_bytes)) small_object{nullptr}; + } + allocator = this; + + // Return uninitialized memory for further construction on user side. + obj->~small_object(); + return obj; +} + +void __TBB_EXPORTED_FUNC deallocate(d1::small_object_pool& allocator, void* ptr, std::size_t number_of_bytes) { + auto pool = static_cast<small_object_pool_impl*>(&allocator); + auto tls = governor::get_thread_data(); + pool->deallocate_impl(ptr, number_of_bytes, *tls); +} + +void __TBB_EXPORTED_FUNC deallocate(d1::small_object_pool& allocator, void* ptr, std::size_t number_of_bytes, const d1::execution_data& ed) { + auto& tls = static_cast<const execution_data_ext&>(ed).task_disp->get_thread_data(); + auto pool = static_cast<small_object_pool_impl*>(&allocator); + pool->deallocate_impl(ptr, number_of_bytes, tls); +} + +void small_object_pool_impl::deallocate_impl(void* ptr, std::size_t number_of_bytes, thread_data& td) { + __TBB_ASSERT(ptr != nullptr, "pointer to deallocate should not be null"); + __TBB_ASSERT(number_of_bytes >= sizeof(small_object), "number of bytes should be at least sizeof(small_object)"); + + if (number_of_bytes <= small_object_size) { + auto obj = new (ptr) small_object{nullptr}; + if (td.my_small_object_pool == this) { + obj->next = m_private_list; + m_private_list = obj; + } else { + auto old_public_list = m_public_list.load(std::memory_order_relaxed); + + for (;;) { + if (old_public_list == dead_public_list) { + obj->~small_object(); + cache_aligned_deallocate(obj); + if (++m_public_counter == 0) + { + this->~small_object_pool_impl(); + cache_aligned_deallocate(this); + } + break; + } + obj->next = old_public_list; + if (m_public_list.compare_exchange_strong(old_public_list, obj)) { + break; + } + } + } + } else { + cache_aligned_deallocate(ptr); + } +} + +std::int64_t small_object_pool_impl::cleanup_list(small_object* list) +{ + std::int64_t removed_count{}; + + while (list) { + small_object* current = list; + list = list->next; + current->~small_object(); + cache_aligned_deallocate(current); + ++removed_count; + } + return removed_count; +} + +void small_object_pool_impl::destroy() +{ + // clean up private list and subtract the removed count from private counter + m_private_counter -= cleanup_list(m_private_list); + // Grab public list and place dead mark + small_object* public_list = m_public_list.exchange(dead_public_list); + // clean up public list and subtract from private (intentionally) counter + m_private_counter -= cleanup_list(public_list); + __TBB_ASSERT(m_private_counter >= 0, "Private counter may not be less than 0"); + // Equivalent to fetch_sub(m_private_counter) - m_private_counter. But we need to do it + // atomically with operator-= not to access m_private_counter after the subtraction. + auto new_value = m_public_counter -= m_private_counter; + // check if this method is responsible to clean up the resources + if (new_value == 0) { + this->~small_object_pool_impl(); + cache_aligned_deallocate(this); + } +} + +} // namespace r1 +} // namespace detail +} // namespace tbb diff --git a/src/tbb/src/tbb/small_object_pool_impl.h b/src/tbb/src/tbb/small_object_pool_impl.h new file mode 100644 index 000000000..a6b664bea --- /dev/null +++ b/src/tbb/src/tbb/small_object_pool_impl.h @@ -0,0 +1,59 @@ +/* + Copyright (c) 2020-2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_small_object_pool_impl_H +#define __TBB_small_object_pool_impl_H + +#include "oneapi/tbb/detail/_small_object_pool.h" +#include "oneapi/tbb/detail/_utils.h" + +#include <cstddef> +#include <cstdint> +#include <atomic> + + +namespace tbb { +namespace detail { +namespace r1 { + +class thread_data; + +class small_object_pool_impl : public d1::small_object_pool +{ + static constexpr std::size_t small_object_size = 256; + struct small_object { + small_object* next; + }; + static small_object* const dead_public_list; +public: + void* allocate_impl(small_object_pool*& allocator, std::size_t number_of_bytes); + void deallocate_impl(void* ptr, std::size_t number_of_bytes, thread_data& td); + void destroy(); +private: + static std::int64_t cleanup_list(small_object* list); + ~small_object_pool_impl() = default; +private: + alignas(max_nfs_size) small_object* m_private_list; + std::int64_t m_private_counter{}; + alignas(max_nfs_size) std::atomic<small_object*> m_public_list; + std::atomic<std::int64_t> m_public_counter{}; +}; + +} // namespace r1 +} // namespace detail +} // namespace tbb + +#endif /* __TBB_small_object_pool_impl_H */ diff --git a/src/tbb/src/tbb/spin_mutex.cpp b/src/tbb/src/tbb/spin_mutex.cpp deleted file mode 100644 index 6fab769a9..000000000 --- a/src/tbb/src/tbb/spin_mutex.cpp +++ /dev/null @@ -1,58 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#include "tbb/tbb_machine.h" -#include "tbb/spin_mutex.h" -#include "itt_notify.h" -#include "tbb_misc.h" - -namespace tbb { - -void spin_mutex::scoped_lock::internal_acquire( spin_mutex& m ) { - __TBB_ASSERT( !my_mutex, "already holding a lock on a spin_mutex" ); - ITT_NOTIFY(sync_prepare, &m); - __TBB_LockByte(m.flag); - my_mutex = &m; - ITT_NOTIFY(sync_acquired, &m); -} - -void spin_mutex::scoped_lock::internal_release() { - __TBB_ASSERT( my_mutex, "release on spin_mutex::scoped_lock that is not holding a lock" ); - - ITT_NOTIFY(sync_releasing, my_mutex); - __TBB_UnlockByte(my_mutex->flag); - my_mutex = NULL; -} - -bool spin_mutex::scoped_lock::internal_try_acquire( spin_mutex& m ) { - __TBB_ASSERT( !my_mutex, "already holding a lock on a spin_mutex" ); - bool result = bool( __TBB_TryLockByte(m.flag) ); - if( result ) { - my_mutex = &m; - ITT_NOTIFY(sync_acquired, &m); - } - return result; -} - -void spin_mutex::internal_construct() { - ITT_SYNC_CREATE(this, _T("tbb::spin_mutex"), _T("")); -} - -} // namespace tbb diff --git a/src/tbb/src/tbb/spin_rw_mutex.cpp b/src/tbb/src/tbb/spin_rw_mutex.cpp deleted file mode 100644 index 9b14cc1c9..000000000 --- a/src/tbb/src/tbb/spin_rw_mutex.cpp +++ /dev/null @@ -1,159 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#include "tbb/spin_rw_mutex.h" -#include "tbb/tbb_machine.h" -#include "tbb/atomic.h" -#include "itt_notify.h" - -#if defined(_MSC_VER) && defined(_Wp64) - // Workaround for overzealous compiler warnings in /Wp64 mode - #pragma warning (disable: 4244) -#endif - -namespace tbb { - -template<typename T> // a template can work with private spin_rw_mutex::state_t -static inline T CAS(volatile T &addr, T newv, T oldv) { - // ICC (9.1 and 10.1 tried) unable to do implicit conversion - // from "volatile T*" to "volatile void*", so explicit cast added. - return tbb::internal::as_atomic(addr).compare_and_swap( newv, oldv ); -} - -//! Acquire write lock on the given mutex. -bool spin_rw_mutex_v3::internal_acquire_writer() -{ - ITT_NOTIFY(sync_prepare, this); - for( internal::atomic_backoff backoff;;backoff.pause() ){ - state_t s = const_cast<volatile state_t&>(state); // ensure reloading - if( !(s & BUSY) ) { // no readers, no writers - if( CAS(state, WRITER, s)==s ) - break; // successfully stored writer flag - backoff.reset(); // we could be very close to complete op. - } else if( !(s & WRITER_PENDING) ) { // no pending writers - __TBB_AtomicOR(&state, WRITER_PENDING); - } - } - ITT_NOTIFY(sync_acquired, this); - return false; -} - -//! Release writer lock on the given mutex -void spin_rw_mutex_v3::internal_release_writer() -{ - ITT_NOTIFY(sync_releasing, this); - __TBB_AtomicAND( &state, READERS ); -} - -//! Acquire read lock on given mutex. -void spin_rw_mutex_v3::internal_acquire_reader() -{ - ITT_NOTIFY(sync_prepare, this); - for( internal::atomic_backoff b;;b.pause() ){ - state_t s = const_cast<volatile state_t&>(state); // ensure reloading - if( !(s & (WRITER|WRITER_PENDING)) ) { // no writer or write requests - state_t t = (state_t)__TBB_FetchAndAddW( &state, (intptr_t) ONE_READER ); - if( !( t&WRITER )) - break; // successfully stored increased number of readers - // writer got there first, undo the increment - __TBB_FetchAndAddW( &state, -(intptr_t)ONE_READER ); - } - } - - ITT_NOTIFY(sync_acquired, this); - __TBB_ASSERT( state & READERS, "invalid state of a read lock: no readers" ); -} - -//! Upgrade reader to become a writer. -/** Returns whether the upgrade happened without releasing and re-acquiring the lock */ -bool spin_rw_mutex_v3::internal_upgrade() -{ - state_t s = state; - __TBB_ASSERT( s & READERS, "invalid state before upgrade: no readers " ); - // check and set writer-pending flag - // required conditions: either no pending writers, or we are the only reader - // (with multiple readers and pending writer, another upgrade could have been requested) - while( (s & READERS)==ONE_READER || !(s & WRITER_PENDING) ) { - state_t old_s = s; - if( (s=CAS(state, s | WRITER | WRITER_PENDING, s))==old_s ) { - ITT_NOTIFY(sync_prepare, this); - internal::atomic_backoff backoff; - while( (state & READERS) != ONE_READER ) backoff.pause(); - __TBB_ASSERT((state&(WRITER_PENDING|WRITER))==(WRITER_PENDING|WRITER),"invalid state when upgrading to writer"); - // both new readers and writers are blocked at this time - __TBB_FetchAndAddW( &state, - (intptr_t)(ONE_READER+WRITER_PENDING)); - ITT_NOTIFY(sync_acquired, this); - return true; // successfully upgraded - } - } - // slow reacquire - internal_release_reader(); - return internal_acquire_writer(); // always returns false -} - -//! Downgrade writer to a reader -void spin_rw_mutex_v3::internal_downgrade() { - ITT_NOTIFY(sync_releasing, this); - __TBB_FetchAndAddW( &state, (intptr_t)(ONE_READER-WRITER)); - __TBB_ASSERT( state & READERS, "invalid state after downgrade: no readers" ); -} - -//! Release read lock on the given mutex -void spin_rw_mutex_v3::internal_release_reader() -{ - __TBB_ASSERT( state & READERS, "invalid state of a read lock: no readers" ); - ITT_NOTIFY(sync_releasing, this); // release reader - __TBB_FetchAndAddWrelease( &state,-(intptr_t)ONE_READER); -} - -//! Try to acquire write lock on the given mutex -bool spin_rw_mutex_v3::internal_try_acquire_writer() -{ - // for a writer: only possible to acquire if no active readers or writers - state_t s = state; - if( !(s & BUSY) ) // no readers, no writers; mask is 1..1101 - if( CAS(state, WRITER, s)==s ) { - ITT_NOTIFY(sync_acquired, this); - return true; // successfully stored writer flag - } - return false; -} - -//! Try to acquire read lock on the given mutex -bool spin_rw_mutex_v3::internal_try_acquire_reader() -{ - // for a reader: acquire if no active or waiting writers - state_t s = state; - if( !(s & (WRITER|WRITER_PENDING)) ) { // no writers - state_t t = (state_t)__TBB_FetchAndAddW( &state, (intptr_t) ONE_READER ); - if( !( t&WRITER )) { // got the lock - ITT_NOTIFY(sync_acquired, this); - return true; // successfully stored increased number of readers - } - // writer got there first, undo the increment - __TBB_FetchAndAddW( &state, -(intptr_t)ONE_READER ); - } - return false; -} - -void spin_rw_mutex_v3::internal_construct() { - ITT_SYNC_CREATE(this, _T("tbb::spin_rw_mutex"), _T("")); -} -} // namespace tbb diff --git a/src/tbb/src/tbb/task.cpp b/src/tbb/src/tbb/task.cpp index 12307a918..84b4278f0 100644 --- a/src/tbb/src/tbb/task.cpp +++ b/src/tbb/src/tbb/task.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2024 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,266 +12,246 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ // Do not include task.h directly. Use scheduler_common.h instead #include "scheduler_common.h" #include "governor.h" -#include "scheduler.h" +#include "arena.h" +#include "thread_data.h" +#include "task_dispatcher.h" +#include "waiters.h" #include "itt_notify.h" -#include "tbb/cache_aligned_allocator.h" -#include "tbb/partitioner.h" +#include "oneapi/tbb/detail/_task.h" +#include "oneapi/tbb/partitioner.h" +#include "oneapi/tbb/task.h" -#include <new> +#include <cstring> namespace tbb { - -using namespace std; - -namespace internal { +namespace detail { +namespace r1 { //------------------------------------------------------------------------ -// Methods of allocate_root_proxy +// resumable tasks //------------------------------------------------------------------------ -task& allocate_root_proxy::allocate( size_t size ) { - internal::generic_scheduler* v = governor::local_scheduler_weak(); - __TBB_ASSERT( v, "thread did not activate a task_scheduler_init object?" ); -#if __TBB_TASK_GROUP_CONTEXT - task_prefix& p = v->my_innermost_running_task->prefix(); - - ITT_STACK_CREATE(p.context->itt_caller); -#endif - // New root task becomes part of the currently running task's cancellation context - return v->allocate_task( size, __TBB_CONTEXT_ARG(NULL, p.context) ); -} +#if __TBB_RESUMABLE_TASKS -void allocate_root_proxy::free( task& task ) { - internal::generic_scheduler* v = governor::local_scheduler_weak(); - __TBB_ASSERT( v, "thread does not have initialized task_scheduler_init object?" ); -#if __TBB_TASK_GROUP_CONTEXT - // No need to do anything here as long as there is no context -> task connection -#endif /* __TBB_TASK_GROUP_CONTEXT */ - v->free_task<local_task>( task ); +void suspend(suspend_callback_type suspend_callback, void* user_callback) { + thread_data& td = *governor::get_thread_data(); + td.my_task_dispatcher->suspend(suspend_callback, user_callback); + // Do not access td after suspend. } -#if __TBB_TASK_GROUP_CONTEXT -//------------------------------------------------------------------------ -// Methods of allocate_root_with_context_proxy -//------------------------------------------------------------------------ -task& allocate_root_with_context_proxy::allocate( size_t size ) const { - internal::generic_scheduler* s = governor::local_scheduler_weak(); - __TBB_ASSERT( s, "Scheduler auto-initialization failed?" ); - __TBB_ASSERT( &my_context, "allocate_root(context) argument is a dereferenced NULL pointer" ); - task& t = s->allocate_task( size, NULL, &my_context ); - // Supported usage model prohibits concurrent initial binding. Thus we do not - // need interlocked operations or fences to manipulate with my_context.my_kind - if ( __TBB_load_relaxed(my_context.my_kind) == task_group_context::binding_required ) { - // If we are in the outermost task dispatch loop of a master thread, then - // there is nothing to bind this context to, and we skip the binding part - // treating the context as isolated. - if ( s->master_outermost_level() ) - __TBB_store_relaxed(my_context.my_kind, task_group_context::isolated); - else - my_context.bind_to( s ); +void resume(suspend_point_type* sp) { + assert_pointers_valid(sp, sp->m_arena); + task_dispatcher& task_disp = sp->m_resume_task.m_target; + + if (sp->try_notify_resume()) { + // TODO: remove this work-around + // Prolong the arena's lifetime while all coroutines are alive + // (otherwise the arena can be destroyed while some tasks are suspended). + arena& a = *sp->m_arena; + a.my_references += arena::ref_worker; + + if (task_disp.m_properties.critical_task_allowed) { + // The target is not in the process of executing critical task, so the resume task is not critical. + a.my_resume_task_stream.push(&sp->m_resume_task, random_lane_selector(sp->m_random)); + } else { + #if __TBB_PREVIEW_CRITICAL_TASKS + // The target is in the process of executing critical task, so the resume task is critical. + a.my_critical_task_stream.push(&sp->m_resume_task, random_lane_selector(sp->m_random)); + #endif + } + // Do not access target after that point. + a.advertise_new_work<arena::wakeup>(); + // Release our reference to my_arena. + a.on_thread_leaving(arena::ref_worker); } -#if __TBB_FP_CONTEXT - if ( __TBB_load_relaxed(my_context.my_kind) == task_group_context::isolated && - !(my_context.my_version_and_traits & task_group_context::fp_settings) ) - my_context.copy_fp_settings( *s->default_context() ); -#endif - ITT_STACK_CREATE(my_context.itt_caller); - return t; -} -void allocate_root_with_context_proxy::free( task& task ) const { - internal::generic_scheduler* v = governor::local_scheduler_weak(); - __TBB_ASSERT( v, "thread does not have initialized task_scheduler_init object?" ); - // No need to do anything here as long as unbinding is performed by context destructor only. - v->free_task<local_task>( task ); } -#endif /* __TBB_TASK_GROUP_CONTEXT */ -//------------------------------------------------------------------------ -// Methods of allocate_continuation_proxy -//------------------------------------------------------------------------ -task& allocate_continuation_proxy::allocate( size_t size ) const { - task* t = (task*)this; - assert_task_valid(t); - generic_scheduler* s = governor::local_scheduler_weak(); - task* parent = t->parent(); - t->prefix().parent = NULL; - return s->allocate_task( size, __TBB_CONTEXT_ARG(parent, t->prefix().context) ); +suspend_point_type* current_suspend_point() { + thread_data& td = *governor::get_thread_data(); + return td.my_task_dispatcher->get_suspend_point(); } -void allocate_continuation_proxy::free( task& mytask ) const { - // Restore the parent as it was before the corresponding allocate was called. - ((task*)this)->prefix().parent = mytask.parent(); - governor::local_scheduler_weak()->free_task<local_task>(mytask); +task_dispatcher& create_coroutine(thread_data& td) { + // We may have some task dispatchers cached + task_dispatcher* task_disp = td.my_arena->my_co_cache.pop(); + if (!task_disp) { + void* ptr = cache_aligned_allocate(sizeof(task_dispatcher)); + task_disp = new(ptr) task_dispatcher(td.my_arena); + task_disp->init_suspend_point(td.my_arena, td.my_arena->my_threading_control->worker_stack_size()); + } + // Prolong the arena's lifetime until all coroutines is alive + // (otherwise the arena can be destroyed while some tasks are suspended). + // TODO: consider behavior if there are more than 4K external references. + td.my_arena->my_references += arena::ref_external; + return *task_disp; } -//------------------------------------------------------------------------ -// Methods of allocate_child_proxy -//------------------------------------------------------------------------ -task& allocate_child_proxy::allocate( size_t size ) const { - task* t = (task*)this; - assert_task_valid(t); - generic_scheduler* s = governor::local_scheduler_weak(); - return s->allocate_task( size, __TBB_CONTEXT_ARG(t, t->prefix().context) ); -} +void task_dispatcher::internal_suspend() { + __TBB_ASSERT(m_thread_data != nullptr, nullptr); -void allocate_child_proxy::free( task& mytask ) const { - governor::local_scheduler_weak()->free_task<local_task>(mytask); -} + arena_slot* slot = m_thread_data->my_arena_slot; + __TBB_ASSERT(slot != nullptr, nullptr); -//------------------------------------------------------------------------ -// Methods of allocate_additional_child_of_proxy -//------------------------------------------------------------------------ -task& allocate_additional_child_of_proxy::allocate( size_t size ) const { - parent.increment_ref_count(); - generic_scheduler* s = governor::local_scheduler_weak(); - return s->allocate_task( size, __TBB_CONTEXT_ARG(&parent, parent.prefix().context) ); -} + task_dispatcher& default_task_disp = slot->default_task_dispatcher(); + // TODO: simplify the next line, e.g. is_task_dispatcher_recalled( task_dispatcher& ) + bool is_recalled = default_task_disp.get_suspend_point()->m_is_owner_recalled.load(std::memory_order_acquire); + task_dispatcher& target = is_recalled ? default_task_disp : create_coroutine(*m_thread_data); -void allocate_additional_child_of_proxy::free( task& task ) const { - // Undo the increment. We do not check the result of the fetch-and-decrement. - // We could consider be spawning the task if the fetch-and-decrement returns 1. - // But we do not know that was the programmer's intention. - // Furthermore, if it was the programmer's intention, the program has a fundamental - // race condition (that we warn about in Reference manual), because the - // reference count might have become zero before the corresponding call to - // allocate_additional_child_of_proxy::allocate. - parent.internal_decrement_ref_count(); - governor::local_scheduler_weak()->free_task<local_task>(task); + resume(target); + + if (m_properties.outermost) { + recall_point(); + } } -//------------------------------------------------------------------------ -// Support for auto_partitioner -//------------------------------------------------------------------------ -size_t get_initial_auto_partitioner_divisor() { - const size_t X_FACTOR = 4; - return X_FACTOR * governor::local_scheduler()->max_threads_in_arena(); +void task_dispatcher::suspend(suspend_callback_type suspend_callback, void* user_callback) { + __TBB_ASSERT(suspend_callback != nullptr, nullptr); + __TBB_ASSERT(user_callback != nullptr, nullptr); + suspend_callback(user_callback, get_suspend_point()); + + __TBB_ASSERT(m_thread_data != nullptr, nullptr); + __TBB_ASSERT(m_thread_data->my_post_resume_action == post_resume_action::none, nullptr); + __TBB_ASSERT(m_thread_data->my_post_resume_arg == nullptr, nullptr); + internal_suspend(); } -//------------------------------------------------------------------------ -// Methods of affinity_partitioner_base_v3 -//------------------------------------------------------------------------ -void affinity_partitioner_base_v3::resize( unsigned factor ) { - // Check factor to avoid asking for number of workers while there might be no arena. - size_t new_size = factor ? factor*governor::local_scheduler()->max_threads_in_arena() : 0; - if( new_size!=my_size ) { - if( my_array ) { - NFS_Free( my_array ); - // Following two assignments must be done here for sake of exception safety. - my_array = NULL; - my_size = 0; - } - if( new_size ) { - my_array = static_cast<affinity_id*>(NFS_Allocate(new_size,sizeof(affinity_id), NULL )); - memset( my_array, 0, sizeof(affinity_id)*new_size ); - my_size = new_size; +bool task_dispatcher::resume(task_dispatcher& target) { + // Do not create non-trivial objects on the stack of this function. They might never be destroyed + { + thread_data* td = m_thread_data; + __TBB_ASSERT(&target != this, "We cannot resume to ourself"); + __TBB_ASSERT(td != nullptr, "This task dispatcher must be attach to a thread data"); + __TBB_ASSERT(td->my_task_dispatcher == this, "Thread data must be attached to this task dispatcher"); + + // Change the task dispatcher + td->detach_task_dispatcher(); + td->attach_task_dispatcher(target); + } + __TBB_ASSERT(m_suspend_point != nullptr, "Suspend point must be created"); + __TBB_ASSERT(target.m_suspend_point != nullptr, "Suspend point must be created"); + // Swap to the target coroutine. + + m_suspend_point->resume(target.m_suspend_point); + // Pay attention that m_thread_data can be changed after resume + if (m_thread_data) { + thread_data* td = m_thread_data; + __TBB_ASSERT(td != nullptr, "This task dispatcher must be attach to a thread data"); + __TBB_ASSERT(td->my_task_dispatcher == this, "Thread data must be attached to this task dispatcher"); + do_post_resume_action(); + + // Remove the recall flag if the thread in its original task dispatcher + arena_slot* slot = td->my_arena_slot; + __TBB_ASSERT(slot != nullptr, nullptr); + if (this == slot->my_default_task_dispatcher) { + __TBB_ASSERT(m_suspend_point != nullptr, nullptr); + m_suspend_point->m_is_owner_recalled.store(false, std::memory_order_relaxed); } + return true; } + return false; } -} // namespace internal - -using namespace tbb::internal; +void task_dispatcher::do_post_resume_action() { + thread_data* td = m_thread_data; + switch (td->my_post_resume_action) { + case post_resume_action::register_waiter: + { + __TBB_ASSERT(td->my_post_resume_arg, "The post resume action must have an argument"); + static_cast<thread_control_monitor::resume_context*>(td->my_post_resume_arg)->notify(); + break; + } + case post_resume_action::cleanup: + { + __TBB_ASSERT(td->my_post_resume_arg, "The post resume action must have an argument"); + task_dispatcher* to_cleanup = static_cast<task_dispatcher*>(td->my_post_resume_arg); + // Release coroutine's reference to my_arena + td->my_arena->on_thread_leaving(arena::ref_external); + // Cache the coroutine for possible later re-usage + td->my_arena->my_co_cache.push(to_cleanup); + break; + } + case post_resume_action::notify: + { + __TBB_ASSERT(td->my_post_resume_arg, "The post resume action must have an argument"); + suspend_point_type* sp = static_cast<suspend_point_type*>(td->my_post_resume_arg); + sp->recall_owner(); + // Do not access sp because it can be destroyed after recall + + auto is_our_suspend_point = [sp] (market_context ctx) { + return std::uintptr_t(sp) == ctx.my_uniq_addr; + }; + td->my_arena->get_waiting_threads_monitor().notify(is_our_suspend_point); + break; + } + default: + __TBB_ASSERT(td->my_post_resume_action == post_resume_action::none, "Unknown post resume action"); + __TBB_ASSERT(td->my_post_resume_arg == nullptr, "The post resume argument should not be set"); + } + td->clear_post_resume_action(); +} -//------------------------------------------------------------------------ -// task -//------------------------------------------------------------------------ +#else -void task::internal_set_ref_count( int count ) { - __TBB_ASSERT( count>=0, "count must not be negative" ); - task_prefix &p = prefix(); - __TBB_ASSERT(p.ref_count==1 && p.state==allocated && self().parent()==this - || !(p.extra_state & es_ref_count_active), "ref_count race detected"); - ITT_NOTIFY(sync_releasing, &p.ref_count); - p.ref_count = count; +void suspend(suspend_callback_type, void*) { + __TBB_ASSERT_RELEASE(false, "Resumable tasks are unsupported on this platform"); } -internal::reference_count task::internal_decrement_ref_count() { - ITT_NOTIFY( sync_releasing, &prefix().ref_count ); - internal::reference_count k = __TBB_FetchAndDecrementWrelease( &prefix().ref_count ); - __TBB_ASSERT( k>=1, "task's reference count underflowed" ); - if( k==1 ) - ITT_NOTIFY( sync_acquired, &prefix().ref_count ); - return k-1; +void resume(suspend_point_type*) { + __TBB_ASSERT_RELEASE(false, "Resumable tasks are unsupported on this platform"); } -task& task::self() { - generic_scheduler *v = governor::local_scheduler_weak(); - v->assert_task_pool_valid(); - __TBB_ASSERT( v->my_innermost_running_task, NULL ); - return *v->my_innermost_running_task; +suspend_point_type* current_suspend_point() { + __TBB_ASSERT_RELEASE(false, "Resumable tasks are unsupported on this platform"); + return nullptr; } -bool task::is_owned_by_current_thread() const { - return true; -} +#endif /* __TBB_RESUMABLE_TASKS */ -void interface5::internal::task_base::destroy( task& victim ) { - // 1 may be a guard reference for wait_for_all, which was not reset because - // of concurrent_wait mode or because prepared root task was not actually used - // for spawning tasks (as in structured_task_group). - __TBB_ASSERT( (intptr_t)victim.prefix().ref_count <= 1, "Task being destroyed must not have children" ); - __TBB_ASSERT( victim.state()==task::allocated, "illegal state for victim task" ); - task* parent = victim.parent(); - victim.~task(); - if( parent ) { - __TBB_ASSERT( parent->state()!=task::freed && parent->state()!=task::ready, - "attempt to destroy child of running or corrupted parent?" ); - // 'reexecute' and 'executing' are also signs of a race condition, since most tasks - // set their ref_count upon entry but "es_ref_count_active" should detect this - parent->internal_decrement_ref_count(); - // Even if the last reference to *parent is removed, it should not be spawned (documented behavior). - } - governor::local_scheduler_weak()->free_task<no_cache>( victim ); -} +void notify_waiters(std::uintptr_t wait_ctx_addr) { + auto is_related_wait_ctx = [&] (market_context context) { + return wait_ctx_addr == context.my_uniq_addr; + }; -void task::spawn_and_wait_for_all( task_list& list ) { - generic_scheduler* s = governor::local_scheduler(); - task* t = list.first; - if( t ) { - if( &t->prefix().next!=list.next_ptr ) - s->local_spawn( t->prefix().next, *list.next_ptr ); - list.clear(); - } - s->local_wait_for_all( *this, t ); + governor::get_thread_data()->my_arena->get_waiting_threads_monitor().notify(is_related_wait_ctx); } -/** Defined out of line so that compiler does not replicate task's vtable. - It's pointless to define it inline anyway, because all call sites to it are virtual calls - that the compiler is unlikely to optimize. */ -void task::note_affinity( affinity_id ) { -} +d1::wait_tree_vertex_interface* get_thread_reference_vertex(d1::wait_tree_vertex_interface* top_wait_context) { + __TBB_ASSERT(top_wait_context, nullptr); + auto& dispatcher = *governor::get_thread_data()->my_task_dispatcher; + + d1::reference_vertex* ref_counter{nullptr}; + auto& reference_map = dispatcher.m_reference_vertex_map; + auto pos = reference_map.find(top_wait_context); + if (pos != reference_map.end()) { + ref_counter = pos->second; + } else { + constexpr std::size_t max_reference_vertex_map_size = 1000; + if (reference_map.size() > max_reference_vertex_map_size) { + // TODO: Research the possibility of using better approach for a clean-up + for (auto it = reference_map.begin(); it != reference_map.end();) { + if (it->second->get_num_child() == 0) { + it->second->~reference_vertex(); + cache_aligned_deallocate(it->second); + it = reference_map.erase(it); + } else { + ++it; + } + } + } -#if __TBB_TASK_GROUP_CONTEXT -void task::change_group ( task_group_context& ctx ) { - prefix().context = &ctx; - internal::generic_scheduler* s = governor::local_scheduler_weak(); - if ( __TBB_load_relaxed(ctx.my_kind) == task_group_context::binding_required ) { - // If we are in the outermost task dispatch loop of a master thread, then - // there is nothing to bind this context to, and we skip the binding part - // treating the context as isolated. - if ( s->master_outermost_level() ) - __TBB_store_relaxed(ctx.my_kind, task_group_context::isolated); - else - ctx.bind_to( s ); + reference_map[top_wait_context] = ref_counter = + new (cache_aligned_allocate(sizeof(d1::reference_vertex))) d1::reference_vertex(top_wait_context, 0); } -#if __TBB_FP_CONTEXT - if ( __TBB_load_relaxed(ctx.my_kind) == task_group_context::isolated && - !(ctx.my_version_and_traits & task_group_context::fp_settings) ) - ctx.copy_fp_settings( *s->default_context() ); -#endif - ITT_STACK_CREATE(ctx.itt_caller); + + return ref_counter; } -#endif /* __TBB_TASK_GROUP_CONTEXT */ +} // namespace r1 +} // namespace detail } // namespace tbb - diff --git a/src/tbb/src/tbb/task_dispatcher.cpp b/src/tbb/src/tbb/task_dispatcher.cpp new file mode 100644 index 000000000..5ea7d3f53 --- /dev/null +++ b/src/tbb/src/tbb/task_dispatcher.cpp @@ -0,0 +1,244 @@ +/* + Copyright (c) 2020-2022 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "task_dispatcher.h" +#include "waiters.h" + +namespace tbb { +namespace detail { +namespace r1 { + +static inline void spawn_and_notify(d1::task& t, arena_slot* slot, arena* a) { + slot->spawn(t); + a->advertise_new_work<arena::work_spawned>(); + // TODO: TBB_REVAMP_TODO slot->assert_task_pool_valid(); +} + +void __TBB_EXPORTED_FUNC spawn(d1::task& t, d1::task_group_context& ctx) { + thread_data* tls = governor::get_thread_data(); + task_group_context_impl::bind_to(ctx, tls); + arena* a = tls->my_arena; + arena_slot* slot = tls->my_arena_slot; + // Capture current context + task_accessor::context(t) = &ctx; + // Mark isolation + task_accessor::isolation(t) = tls->my_task_dispatcher->m_execute_data_ext.isolation; + spawn_and_notify(t, slot, a); +} + +void __TBB_EXPORTED_FUNC spawn(d1::task& t, d1::task_group_context& ctx, d1::slot_id id) { + thread_data* tls = governor::get_thread_data(); + task_group_context_impl::bind_to(ctx, tls); + arena* a = tls->my_arena; + arena_slot* slot = tls->my_arena_slot; + execution_data_ext& ed = tls->my_task_dispatcher->m_execute_data_ext; + + // Capture context + task_accessor::context(t) = &ctx; + // Mark isolation + task_accessor::isolation(t) = ed.isolation; + + if ( id != d1::no_slot && id != tls->my_arena_index && id < a->my_num_slots) { + // Allocate proxy task + d1::small_object_allocator alloc{}; + auto proxy = alloc.new_object<task_proxy>(static_cast<d1::execution_data&>(ed)); + // Mark as a proxy + task_accessor::set_proxy_trait(*proxy); + // Mark isolation for the proxy task + task_accessor::isolation(*proxy) = ed.isolation; + // Deallocation hint (tls) from the task allocator + proxy->allocator = alloc; + proxy->slot = id; + proxy->outbox = &a->mailbox(id); + // Mark proxy as present in both locations (sender's task pool and destination mailbox) + proxy->task_and_tag = intptr_t(&t) | task_proxy::location_mask; + // Mail the proxy - after this point t may be destroyed by another thread at any moment. + proxy->outbox->push(proxy); + // Spawn proxy to the local task pool + spawn_and_notify(*proxy, slot, a); + } else { + spawn_and_notify(t, slot, a); + } +} + +void __TBB_EXPORTED_FUNC submit(d1::task& t, d1::task_group_context& ctx, arena* a, std::uintptr_t as_critical) { + suppress_unused_warning(as_critical); + assert_pointer_valid(a); + thread_data& tls = *governor::get_thread_data(); + + // TODO revamp: for each use case investigate neccesity to make this call + task_group_context_impl::bind_to(ctx, &tls); + task_accessor::context(t) = &ctx; + // TODO revamp: consider respecting task isolation if this call is being made by external thread + task_accessor::isolation(t) = tls.my_task_dispatcher->m_execute_data_ext.isolation; + + // TODO: consider code refactoring when lane selection mechanism is unified. + + if ( tls.is_attached_to(a) ) { + arena_slot* slot = tls.my_arena_slot; +#if __TBB_PREVIEW_CRITICAL_TASKS + if( as_critical ) { + a->my_critical_task_stream.push( &t, subsequent_lane_selector(slot->critical_hint()) ); + } else +#endif + { + slot->spawn(t); + } + } else { + random_lane_selector lane_selector{tls.my_random}; +#if !__TBB_PREVIEW_CRITICAL_TASKS + suppress_unused_warning(as_critical); +#else + if ( as_critical ) { + a->my_critical_task_stream.push( &t, lane_selector ); + } else +#endif + { + // Avoid joining the arena the thread is not currently in. + a->my_fifo_task_stream.push( &t, lane_selector ); + } + } + // It is assumed that some thread will explicitly wait in the arena the task is submitted + // into. Therefore, no need to utilize mandatory concurrency here. + a->advertise_new_work<arena::work_spawned>(); +} + +void __TBB_EXPORTED_FUNC execute_and_wait(d1::task& t, d1::task_group_context& t_ctx, d1::wait_context& wait_ctx, d1::task_group_context& w_ctx) { + task_accessor::context(t) = &t_ctx; + task_dispatcher::execute_and_wait(&t, wait_ctx, w_ctx); +} + +void __TBB_EXPORTED_FUNC wait(d1::wait_context& wait_ctx, d1::task_group_context& w_ctx) { + // Enter the task dispatch loop without a task + task_dispatcher::execute_and_wait(nullptr, wait_ctx, w_ctx); +} + +d1::slot_id __TBB_EXPORTED_FUNC execution_slot(const d1::execution_data* ed) { + if (ed) { + const execution_data_ext* ed_ext = static_cast<const execution_data_ext*>(ed); + assert_pointers_valid(ed_ext->task_disp, ed_ext->task_disp->m_thread_data); + return ed_ext->task_disp->m_thread_data->my_arena_index; + } else { + thread_data* td = governor::get_thread_data_if_initialized(); + return td ? td->my_arena_index : d1::slot_id(-1); + } +} + +d1::task_group_context* __TBB_EXPORTED_FUNC current_context() { + thread_data* td = governor::get_thread_data(); + assert_pointers_valid(td, td->my_task_dispatcher); + + task_dispatcher* task_disp = td->my_task_dispatcher; + if (task_disp->m_properties.outermost) { + // No one task is executed, so no execute_data. + return nullptr; + } else { + return td->my_task_dispatcher->m_execute_data_ext.context; + } +} + +void task_dispatcher::execute_and_wait(d1::task* t, d1::wait_context& wait_ctx, d1::task_group_context& w_ctx) { + // Get an associated task dispatcher + thread_data* tls = governor::get_thread_data(); + __TBB_ASSERT(tls->my_task_dispatcher != nullptr, nullptr); + task_dispatcher& local_td = *tls->my_task_dispatcher; + + // TODO: factor out the binding to execute_and_wait_impl + if (t) { + task_group_context_impl::bind_to(*task_accessor::context(*t), tls); + // Propagate the isolation to the task executed without spawn. + task_accessor::isolation(*t) = tls->my_task_dispatcher->m_execute_data_ext.isolation; + } + + // Waiting on special object tied to a waiting thread. + external_waiter waiter{ *tls->my_arena, wait_ctx }; + t = local_td.local_wait_for_all(t, waiter); + __TBB_ASSERT_EX(t == nullptr, "External waiter must not leave dispatch loop with a task"); + + // The external thread couldn't exit the dispatch loop in an idle state + if (local_td.m_thread_data->my_inbox.is_idle_state(true)) { + local_td.m_thread_data->my_inbox.set_is_idle(false); + } + + auto exception = w_ctx.my_exception.load(std::memory_order_acquire); + if (exception) { + __TBB_ASSERT(w_ctx.is_group_execution_cancelled(), "The task group context with an exception should be canceled."); + exception->throw_self(); + } +} + +#if __TBB_RESUMABLE_TASKS + +#if _WIN32 +/* [[noreturn]] */ void __stdcall co_local_wait_for_all(void* addr) noexcept +#else +/* [[noreturn]] */ void co_local_wait_for_all(unsigned hi, unsigned lo) noexcept +#endif +{ +#if !_WIN32 + std::uintptr_t addr = lo; + __TBB_ASSERT(sizeof(addr) == 8 || hi == 0, nullptr); + addr += std::uintptr_t(std::uint64_t(hi) << 32); +#endif + task_dispatcher& task_disp = *reinterpret_cast<task_dispatcher*>(addr); + assert_pointers_valid(task_disp.m_thread_data, task_disp.m_thread_data->my_arena); + task_disp.set_stealing_threshold(task_disp.m_thread_data->my_arena->calculate_stealing_threshold()); + __TBB_ASSERT(task_disp.can_steal(), nullptr); + task_disp.co_local_wait_for_all(); + // This code is unreachable +} + +/* [[noreturn]] */ void task_dispatcher::co_local_wait_for_all() noexcept { + // Do not create non-trivial objects on the stack of this function. They will never be destroyed. + assert_pointer_valid(m_thread_data); + + m_suspend_point->finilize_resume(); + // Basically calls the user callback passed to the tbb::task::suspend function + do_post_resume_action(); + + // Endless loop here because coroutine could be reused + d1::task* resume_task{}; + do { + arena* a = m_thread_data->my_arena; + coroutine_waiter waiter(*a); + resume_task = local_wait_for_all(nullptr, waiter); + assert_task_valid(resume_task); + __TBB_ASSERT(this == m_thread_data->my_task_dispatcher, nullptr); + + m_thread_data->set_post_resume_action(post_resume_action::cleanup, this); + + } while (resume(static_cast<suspend_point_type::resume_task*>(resume_task)->m_target)); + // This code might be unreachable +} + +d1::suspend_point task_dispatcher::get_suspend_point() { + if (m_suspend_point == nullptr) { + assert_pointer_valid(m_thread_data); + // 0 means that we attach this task dispatcher to the current stack + init_suspend_point(m_thread_data->my_arena, 0); + } + assert_pointer_valid(m_suspend_point); + return m_suspend_point; +} +void task_dispatcher::init_suspend_point(arena* a, std::size_t stack_size) { + __TBB_ASSERT(m_suspend_point == nullptr, nullptr); + m_suspend_point = new(cache_aligned_allocate(sizeof(suspend_point_type))) + suspend_point_type(a, stack_size, *this); +} +#endif /* __TBB_RESUMABLE_TASKS */ +} // namespace r1 +} // namespace detail +} // namespace tbb diff --git a/src/tbb/src/tbb/task_dispatcher.h b/src/tbb/src/tbb/task_dispatcher.h new file mode 100644 index 000000000..c818934e5 --- /dev/null +++ b/src/tbb/src/tbb/task_dispatcher.h @@ -0,0 +1,479 @@ +/* + Copyright (c) 2020-2024 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef _TBB_task_dispatcher_H +#define _TBB_task_dispatcher_H + +#include "oneapi/tbb/detail/_utils.h" +#include "oneapi/tbb/detail/_task.h" +#include "oneapi/tbb/global_control.h" + +#include "scheduler_common.h" +#include "waiters.h" +#include "arena_slot.h" +#include "arena.h" +#include "thread_data.h" +#include "mailbox.h" +#include "itt_notify.h" +#include "concurrent_monitor.h" +#include "threading_control.h" + +#include <atomic> + +#if !__TBB_CPU_CTL_ENV_PRESENT +#include <fenv.h> // +#endif + +namespace tbb { +namespace detail { +namespace r1 { + +inline d1::task* get_self_recall_task(arena_slot& slot) { + suppress_unused_warning(slot); + d1::task* t = nullptr; +#if __TBB_RESUMABLE_TASKS + suspend_point_type* sp = slot.default_task_dispatcher().m_suspend_point; + if (sp && sp->m_is_owner_recalled.load(std::memory_order_acquire)) { + t = &sp->m_resume_task; + __TBB_ASSERT(sp->m_resume_task.m_target.m_thread_data == nullptr, nullptr); + } +#endif /* __TBB_RESUMABLE_TASKS */ + return t; +} + +// Defined in exception.cpp +/*[[noreturn]]*/void do_throw_noexcept(void (*throw_exception)()) noexcept; + +//------------------------------------------------------------------------ +// Suspend point +//------------------------------------------------------------------------ +#if __TBB_RESUMABLE_TASKS + +inline d1::task* suspend_point_type::resume_task::execute(d1::execution_data& ed) { + execution_data_ext& ed_ext = static_cast<execution_data_ext&>(ed); + + if (ed_ext.wait_ctx) { + thread_control_monitor::resume_context monitor_node{{std::uintptr_t(ed_ext.wait_ctx), nullptr}, ed_ext, m_target}; + // The wait_ctx is present only in external_waiter. In that case we leave the current stack + // in the abandoned state to resume when waiting completes. + thread_data* td = ed_ext.task_disp->m_thread_data; + td->set_post_resume_action(task_dispatcher::post_resume_action::register_waiter, &monitor_node); + + thread_control_monitor& wait_list = td->my_arena->get_waiting_threads_monitor(); + + if (wait_list.wait([&] { return !ed_ext.wait_ctx->continue_execution(); }, monitor_node)) { + return nullptr; + } + + td->clear_post_resume_action(); + r1::resume(ed_ext.task_disp->get_suspend_point()); + } else { + // If wait_ctx is null, it can be only a worker thread on outermost level because + // coroutine_waiter interrupts bypass loop before the resume_task execution. + ed_ext.task_disp->m_thread_data->set_post_resume_action(task_dispatcher::post_resume_action::notify, + ed_ext.task_disp->get_suspend_point()); + } + // Do not access this task because it might be destroyed + ed_ext.task_disp->resume(m_target); + return nullptr; +} + +inline suspend_point_type::suspend_point_type(arena* a, size_t stack_size, task_dispatcher& task_disp) + : m_arena(a) + , m_random(this) + , m_co_context(stack_size, &task_disp) + , m_resume_task(task_disp) +{ + assert_pointer_valid(m_arena); + assert_pointer_valid(m_arena->my_default_ctx); + task_accessor::context(m_resume_task) = m_arena->my_default_ctx; + task_accessor::isolation(m_resume_task) = no_isolation; + // Initialize the itt_caller for the context of the resume task. + // It will be bound to the stack of the first suspend call. + task_group_context_impl::bind_to(*task_accessor::context(m_resume_task), task_disp.m_thread_data); +} + +#endif /* __TBB_RESUMABLE_TASKS */ + +//------------------------------------------------------------------------ +// Task Dispatcher +//------------------------------------------------------------------------ +inline task_dispatcher::task_dispatcher(arena* a) { + m_execute_data_ext.context = a->my_default_ctx; + m_execute_data_ext.task_disp = this; +} + +inline bool task_dispatcher::can_steal() { + __TBB_ASSERT(m_stealing_threshold != 0, nullptr); + stack_anchor_type anchor{}; + return reinterpret_cast<std::uintptr_t>(&anchor) > m_stealing_threshold; +} + +inline d1::task* task_dispatcher::get_inbox_or_critical_task( + execution_data_ext& ed, mail_inbox& inbox, isolation_type isolation, bool critical_allowed) +{ + if (inbox.empty()) + return nullptr; + d1::task* result = get_critical_task(nullptr, ed, isolation, critical_allowed); + if (result) + return result; + // Check if there are tasks mailed to this thread via task-to-thread affinity mechanism. + result = get_mailbox_task(inbox, ed, isolation); + // There is a race with a thread adding a new task (possibly with suitable isolation) + // to our mailbox, so the below conditions might result in a false positive. + // Then set_is_idle(false) allows that task to be stolen; it's OK. + if (isolation != no_isolation && !result && !inbox.empty() && inbox.is_idle_state(true)) { + // We have proxy tasks in our mailbox but the isolation blocks their execution. + // So publish the proxy tasks in mailbox to be available for stealing from owner's task pool. + inbox.set_is_idle( false ); + } + return result; +} + +inline d1::task* task_dispatcher::get_stream_or_critical_task( + execution_data_ext& ed, arena& a, task_stream<front_accessor>& stream, unsigned& hint, + isolation_type isolation, bool critical_allowed) +{ + if (stream.empty()) + return nullptr; + d1::task* result = get_critical_task(nullptr, ed, isolation, critical_allowed); + if (result) + return result; + return a.get_stream_task(stream, hint); +} + +inline d1::task* task_dispatcher::steal_or_get_critical( + execution_data_ext& ed, arena& a, unsigned arena_index, FastRandom& random, + isolation_type isolation, bool critical_allowed) +{ + if (d1::task* t = a.steal_task(arena_index, random, ed, isolation)) { + ed.context = task_accessor::context(*t); + ed.isolation = task_accessor::isolation(*t); + return get_critical_task(t, ed, isolation, critical_allowed); + } + return nullptr; +} + +template <bool ITTPossible, typename Waiter> +d1::task* task_dispatcher::receive_or_steal_task( + thread_data& tls, execution_data_ext& ed, Waiter& waiter, isolation_type isolation, + bool fifo_allowed, bool critical_allowed) +{ + __TBB_ASSERT(governor::is_thread_data_set(&tls), nullptr); + // Task to return + d1::task* t = nullptr; + // Get tls data (again) + arena& a = *tls.my_arena; + arena_slot& slot = *tls.my_arena_slot; + unsigned arena_index = tls.my_arena_index; + mail_inbox& inbox = tls.my_inbox; + task_stream<front_accessor>& resume_stream = a.my_resume_task_stream; + unsigned& resume_hint = slot.hint_for_resume_stream; + task_stream<front_accessor>& fifo_stream = a.my_fifo_task_stream; + unsigned& fifo_hint = slot.hint_for_fifo_stream; + + waiter.reset_wait(); + // Thread is in idle state now + inbox.set_is_idle(true); + + bool stealing_is_allowed = can_steal(); + + // Stealing loop mailbox/enqueue/other_slots + for (;;) { + __TBB_ASSERT(t == nullptr, nullptr); + // Check if the resource manager requires our arena to relinquish some threads + // For the external thread restore idle state to true after dispatch loop + if (!waiter.continue_execution(slot, t)) { + __TBB_ASSERT(t == nullptr, nullptr); + break; + } + // Start searching + if (t != nullptr) { + // continue_execution returned a task + } + else if ((t = get_inbox_or_critical_task(ed, inbox, isolation, critical_allowed))) { + // Successfully got the task from mailbox or critical task + } + else if ((t = get_stream_or_critical_task(ed, a, resume_stream, resume_hint, isolation, critical_allowed))) { + // Successfully got the resume or critical task + } + else if (fifo_allowed && isolation == no_isolation + && (t = get_stream_or_critical_task(ed, a, fifo_stream, fifo_hint, isolation, critical_allowed))) { + // Checked if there are tasks in starvation-resistant stream. Only allowed at the outermost dispatch level without isolation. + } + else if (stealing_is_allowed + && (t = steal_or_get_critical(ed, a, arena_index, tls.my_random, isolation, critical_allowed))) { + // Stole a task from a random arena slot + } + else { + t = get_critical_task(t, ed, isolation, critical_allowed); + } + + if (t != nullptr) { + ed.context = task_accessor::context(*t); + ed.isolation = task_accessor::isolation(*t); + a.my_observers.notify_entry_observers(tls.my_last_observer, tls.my_is_worker); + break; // Stealing success, end of stealing attempt + } + // Nothing to do, pause a little. + waiter.pause(slot); + } // end of nonlocal task retrieval loop + + __TBB_ASSERT(is_alive(a.my_guard), nullptr); + if (inbox.is_idle_state(true)) { + inbox.set_is_idle(false); + } + return t; +} + +template <bool ITTPossible, typename Waiter> +d1::task* task_dispatcher::local_wait_for_all(d1::task* t, Waiter& waiter ) { + assert_pointer_valid(m_thread_data); + __TBB_ASSERT(m_thread_data->my_task_dispatcher == this, nullptr); + + // Guard an outer/default execution state + struct dispatch_loop_guard { + task_dispatcher& task_disp; + execution_data_ext old_execute_data_ext; + properties old_properties; + bool is_initially_registered; + + ~dispatch_loop_guard() { + task_disp.m_execute_data_ext = old_execute_data_ext; + task_disp.m_properties = old_properties; + + if (!is_initially_registered) { + task_disp.m_thread_data->my_arena->my_tc_client.get_pm_client()->unregister_thread(); + task_disp.m_thread_data->my_is_registered = false; + } + + __TBB_ASSERT(task_disp.m_thread_data && governor::is_thread_data_set(task_disp.m_thread_data), nullptr); + __TBB_ASSERT(task_disp.m_thread_data->my_task_dispatcher == &task_disp, nullptr); + } + } dl_guard{ *this, m_execute_data_ext, m_properties, m_thread_data->my_is_registered }; + + // The context guard to track fp setting and itt tasks. + context_guard_helper</*report_tasks=*/ITTPossible> context_guard; + + // Current isolation context + const isolation_type isolation = dl_guard.old_execute_data_ext.isolation; + + // Critical work inflection point. Once turned false current execution context has taken + // critical task on the previous stack frame and cannot take more until that critical path is + // finished. + bool critical_allowed = dl_guard.old_properties.critical_task_allowed; + + // Extended execution data that is used for dispatching. + // Base version is passed to the task::execute method. + execution_data_ext& ed = m_execute_data_ext; + ed.context = t ? task_accessor::context(*t) : nullptr; + ed.original_slot = m_thread_data->my_arena_index; + ed.affinity_slot = d1::no_slot; + ed.task_disp = this; + ed.wait_ctx = waiter.wait_ctx(); + + m_properties.outermost = false; + m_properties.fifo_tasks_allowed = false; + + if (!dl_guard.is_initially_registered) { + m_thread_data->my_arena->my_tc_client.get_pm_client()->register_thread(); + m_thread_data->my_is_registered = true; + } + + t = get_critical_task(t, ed, isolation, critical_allowed); + if (t && m_thread_data->my_inbox.is_idle_state(true)) { + // The thread has a work to do. Therefore, marking its inbox as not idle so that + // affinitized tasks can be stolen from it. + m_thread_data->my_inbox.set_is_idle(false); + } + + // Infinite exception loop + for (;;) { + try { + // Main execution loop + do { + // We assume that bypass tasks are from the same task group. + context_guard.set_ctx(ed.context); + // Inner level evaluates tasks coming from nesting loops and those returned + // by just executed tasks (bypassing spawn or enqueue calls). + while (t != nullptr) { + assert_task_valid(t); + assert_pointer_valid</*alignment = */alignof(void*)>(ed.context); + __TBB_ASSERT(ed.context->my_state == d1::task_group_context::state::bound || + ed.context->my_state == d1::task_group_context::state::isolated, nullptr); + __TBB_ASSERT(m_thread_data->my_inbox.is_idle_state(false), nullptr); + __TBB_ASSERT(task_accessor::is_resume_task(*t) || isolation == no_isolation || isolation == ed.isolation, nullptr); + // Check premature leave + if (Waiter::postpone_execution(*t)) { + __TBB_ASSERT(task_accessor::is_resume_task(*t) && dl_guard.old_properties.outermost, + "Currently, the bypass loop can be interrupted only for resume task on outermost level"); + return t; + } + // Copy itt_caller to a stack because the context might be destroyed after t->execute. + void* itt_caller = ed.context->my_itt_caller; + suppress_unused_warning(itt_caller); + + ITT_CALLEE_ENTER(ITTPossible, t, itt_caller); + + if (ed.context->is_group_execution_cancelled()) { + t = t->cancel(ed); + } else { + t = t->execute(ed); + } + + ITT_CALLEE_LEAVE(ITTPossible, itt_caller); + + // The task affinity in execution data is set for affinitized tasks. + // So drop it after the task execution. + ed.affinity_slot = d1::no_slot; + // Reset task owner id for bypassed task + ed.original_slot = m_thread_data->my_arena_index; + t = get_critical_task(t, ed, isolation, critical_allowed); + } + __TBB_ASSERT(m_thread_data && governor::is_thread_data_set(m_thread_data), nullptr); + __TBB_ASSERT(m_thread_data->my_task_dispatcher == this, nullptr); + // When refactoring, pay attention that m_thread_data can be changed after t->execute() + __TBB_ASSERT(m_thread_data->my_arena_slot != nullptr, nullptr); + arena_slot& slot = *m_thread_data->my_arena_slot; + if (!waiter.continue_execution(slot, t)) { + break; + } + // Retrieve the task from local task pool + if (t || (slot.is_task_pool_published() && (t = slot.get_task(ed, isolation)))) { + __TBB_ASSERT(ed.original_slot == m_thread_data->my_arena_index, nullptr); + ed.context = task_accessor::context(*t); + ed.isolation = task_accessor::isolation(*t); + continue; + } + // Retrieve the task from global sources + t = receive_or_steal_task<ITTPossible>( + *m_thread_data, ed, waiter, isolation, dl_guard.old_properties.fifo_tasks_allowed, + critical_allowed + ); + } while (t != nullptr); // main dispatch loop + break; // Exit exception loop; + } catch (...) { + if (global_control::active_value(global_control::terminate_on_exception) == 1) { + do_throw_noexcept([] { throw; }); + } + if (ed.context->cancel_group_execution()) { + /* We are the first to signal cancellation, so store the exception that caused it. */ + ed.context->my_exception.store(tbb_exception_ptr::allocate(), std::memory_order_release); + } + } + } // Infinite exception loop + __TBB_ASSERT(t == nullptr, nullptr); + + +#if __TBB_RESUMABLE_TASKS + if (dl_guard.old_properties.outermost) { + recall_point(); + } +#endif /* __TBB_RESUMABLE_TASKS */ + + return nullptr; +} + +#if __TBB_RESUMABLE_TASKS +inline void task_dispatcher::recall_point() { + if (this != &m_thread_data->my_arena_slot->default_task_dispatcher()) { + __TBB_ASSERT(m_suspend_point != nullptr, nullptr); + __TBB_ASSERT(m_suspend_point->m_is_owner_recalled.load(std::memory_order_relaxed) == false, nullptr); + + m_thread_data->set_post_resume_action(post_resume_action::notify, get_suspend_point()); + internal_suspend(); + + if (m_thread_data->my_inbox.is_idle_state(true)) { + m_thread_data->my_inbox.set_is_idle(false); + } + } +} +#endif /* __TBB_RESUMABLE_TASKS */ + +#if __TBB_PREVIEW_CRITICAL_TASKS +inline d1::task* task_dispatcher::get_critical_task(d1::task* t, execution_data_ext& ed, isolation_type isolation, bool critical_allowed) { + __TBB_ASSERT( critical_allowed || !m_properties.critical_task_allowed, nullptr ); + + if (!critical_allowed) { + // The stack is already in the process of critical path execution. Cannot take another + // critical work until finish with the current one. + __TBB_ASSERT(!m_properties.critical_task_allowed, nullptr); + return t; + } + + assert_pointers_valid(m_thread_data, m_thread_data->my_arena, m_thread_data->my_arena_slot); + thread_data& td = *m_thread_data; + arena& a = *td.my_arena; + arena_slot& slot = *td.my_arena_slot; + + d1::task* crit_t = a.get_critical_task(slot.hint_for_critical_stream, isolation); + if (crit_t != nullptr) { + assert_task_valid(crit_t); + if (t != nullptr) { + assert_pointer_valid</*alignment = */alignof(void*)>(ed.context); + r1::spawn(*t, *ed.context); + } + ed.context = task_accessor::context(*crit_t); + ed.isolation = task_accessor::isolation(*crit_t); + + // We cannot execute more than one critical task on the same stack. + // In other words, we prevent nested critical tasks. + m_properties.critical_task_allowed = false; + + // TODO: add a test that the observer is called when critical task is taken. + a.my_observers.notify_entry_observers(td.my_last_observer, td.my_is_worker); + t = crit_t; + } else { + // Was unable to find critical work in the queue. Allow inspecting the queue in nested + // invocations. Handles the case when critical task has been just completed. + m_properties.critical_task_allowed = true; + } + return t; +} +#else +inline d1::task* task_dispatcher::get_critical_task(d1::task* t, execution_data_ext&, isolation_type, bool /*critical_allowed*/) { + return t; +} +#endif + +inline d1::task* task_dispatcher::get_mailbox_task(mail_inbox& my_inbox, execution_data_ext& ed, isolation_type isolation) { + while (task_proxy* const tp = my_inbox.pop(isolation)) { + if (d1::task* result = tp->extract_task<task_proxy::mailbox_bit>()) { + ed.original_slot = (unsigned short)(-2); + ed.affinity_slot = ed.task_disp->m_thread_data->my_arena_index; + return result; + } + // We have exclusive access to the proxy, and can destroy it. + tp->allocator.delete_object(tp, ed); + } + return nullptr; +} + +template <typename Waiter> +d1::task* task_dispatcher::local_wait_for_all(d1::task* t, Waiter& waiter) { + if (governor::is_itt_present()) { + return local_wait_for_all</*ITTPossible = */ true>(t, waiter); + } else { + return local_wait_for_all</*ITTPossible = */ false>(t, waiter); + } +} + +} // namespace r1 +} // namespace detail +} // namespace tbb + +#endif // _TBB_task_dispatcher_H + diff --git a/src/tbb/src/tbb/task_group_context.cpp b/src/tbb/src/tbb/task_group_context.cpp index 4036a6668..c20b2790f 100644 --- a/src/tbb/src/tbb/task_group_context.cpp +++ b/src/tbb/src/tbb/task_group_context.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2023 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,267 +12,123 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ -#include "scheduler.h" - +#include "oneapi/tbb/detail/_config.h" +#include "oneapi/tbb/tbb_allocator.h" +#include "oneapi/tbb/task_group.h" +#include "governor.h" +#include "thread_data.h" +#include "scheduler_common.h" #include "itt_notify.h" +#include "task_dispatcher.h" -namespace tbb { +#include <type_traits> -#if __TBB_TASK_GROUP_CONTEXT - -using namespace internal; +namespace tbb { +namespace detail { +namespace r1 { //------------------------------------------------------------------------ -// captured_exception +// tbb_exception_ptr //------------------------------------------------------------------------ - -inline char* duplicate_string ( const char* src ) { - char* dst = NULL; - if ( src ) { - size_t len = strlen(src) + 1; - dst = (char*)allocate_via_handler_v3(len); - strncpy (dst, src, len); - } - return dst; +tbb_exception_ptr* tbb_exception_ptr::allocate() noexcept { + tbb_exception_ptr* eptr = (tbb_exception_ptr*)allocate_memory(sizeof(tbb_exception_ptr)); + return eptr ? new (eptr) tbb_exception_ptr(std::current_exception()) : nullptr; } -captured_exception::~captured_exception () throw() { - clear(); +void tbb_exception_ptr::destroy() noexcept { + this->~tbb_exception_ptr(); + deallocate_memory(this); } -void captured_exception::set ( const char* a_name, const char* info ) throw() { - my_exception_name = duplicate_string( a_name ); - my_exception_info = duplicate_string( info ); +void tbb_exception_ptr::throw_self() { + if (governor::rethrow_exception_broken()) fix_broken_rethrow(); + std::rethrow_exception(my_ptr); } -void captured_exception::clear () throw() { - deallocate_via_handler_v3 (const_cast<char*>(my_exception_name)); - deallocate_via_handler_v3 (const_cast<char*>(my_exception_info)); -} +//------------------------------------------------------------------------ +// task_group_context +//------------------------------------------------------------------------ -captured_exception* captured_exception::move () throw() { - captured_exception *e = (captured_exception*)allocate_via_handler_v3(sizeof(captured_exception)); - if ( e ) { - ::new (e) captured_exception(); - e->my_exception_name = my_exception_name; - e->my_exception_info = my_exception_info; - e->my_dynamic = true; - my_exception_name = my_exception_info = NULL; - } - return e; -} +void task_group_context_impl::destroy(d1::task_group_context& ctx) { + __TBB_ASSERT(!is_poisoned(ctx.my_context_list), nullptr); -void captured_exception::destroy () throw() { - __TBB_ASSERT ( my_dynamic, "Method destroy can be used only on objects created by clone or allocate" ); - if ( my_dynamic ) { - this->captured_exception::~captured_exception(); - deallocate_via_handler_v3 (this); + if (ctx.my_context_list != nullptr) { + __TBB_ASSERT(ctx.my_state.load(std::memory_order_relaxed) == d1::task_group_context::state::bound, nullptr); + // The owner can be destroyed at any moment. Access the associate data with caution. + ctx.my_context_list->remove(ctx.my_node); } -} + d1::cpu_ctl_env* ctl = reinterpret_cast<d1::cpu_ctl_env*>(&ctx.my_cpu_ctl_env); +#if _MSC_VER && _MSC_VER <= 1900 && !__INTEL_COMPILER + suppress_unused_warning(ctl); +#endif + ctl->~cpu_ctl_env(); -captured_exception* captured_exception::allocate ( const char* a_name, const char* info ) { - captured_exception *e = (captured_exception*)allocate_via_handler_v3( sizeof(captured_exception) ); - if ( e ) { - ::new (e) captured_exception(a_name, info); - e->my_dynamic = true; + auto exception = ctx.my_exception.load(std::memory_order_relaxed); + if (exception) { + exception->destroy(); } - return e; -} - -const char* captured_exception::name() const throw() { - return my_exception_name; -} - -const char* captured_exception::what() const throw() { - return my_exception_info; -} + ITT_STACK_DESTROY(ctx.my_itt_caller); + poison_pointer(ctx.my_parent); + poison_pointer(ctx.my_context_list); + poison_pointer(ctx.my_node.my_next_node); + poison_pointer(ctx.my_node.my_prev_node); + poison_pointer(ctx.my_exception); + poison_pointer(ctx.my_itt_caller); -//------------------------------------------------------------------------ -// tbb_exception_ptr -//------------------------------------------------------------------------ - -#if !TBB_USE_CAPTURED_EXCEPTION - -namespace internal { - -template<typename T> -tbb_exception_ptr* AllocateExceptionContainer( const T& src ) { - tbb_exception_ptr *eptr = (tbb_exception_ptr*)allocate_via_handler_v3( sizeof(tbb_exception_ptr) ); - if ( eptr ) - new (eptr) tbb_exception_ptr(src); - return eptr; + ctx.my_state.store(d1::task_group_context::state::dead, std::memory_order_release); } -tbb_exception_ptr* tbb_exception_ptr::allocate () { - return AllocateExceptionContainer( std::current_exception() ); +void task_group_context_impl::initialize(d1::task_group_context& ctx) { + ITT_TASK_GROUP(&ctx, ctx.my_name, nullptr); + + ctx.my_node.my_next_node = &ctx.my_node; + ctx.my_node.my_prev_node = &ctx.my_node; + ctx.my_cpu_ctl_env = 0; + ctx.my_cancellation_requested = 0; + ctx.my_may_have_children.store(0, std::memory_order_relaxed); + // Set the created state to bound at the first usage. + ctx.my_state.store(d1::task_group_context::state::created, std::memory_order_relaxed); + ctx.my_parent = nullptr; + ctx.my_context_list = nullptr; + ctx.my_exception.store(nullptr, std::memory_order_relaxed); + ctx.my_itt_caller = nullptr; + + static_assert(sizeof(d1::cpu_ctl_env) <= sizeof(ctx.my_cpu_ctl_env), "FPU settings storage does not fit to uint64_t"); + d1::cpu_ctl_env* ctl = new (&ctx.my_cpu_ctl_env) d1::cpu_ctl_env; + if (ctx.my_traits.fp_settings) + ctl->get_env(); } -tbb_exception_ptr* tbb_exception_ptr::allocate ( const tbb_exception& ) { - return AllocateExceptionContainer( std::current_exception() ); -} +void task_group_context_impl::register_with(d1::task_group_context& ctx, thread_data* td) { + __TBB_ASSERT(!is_poisoned(ctx.my_context_list), nullptr); + __TBB_ASSERT(td, nullptr); + ctx.my_context_list = td->my_context_list; -tbb_exception_ptr* tbb_exception_ptr::allocate ( captured_exception& src ) { - tbb_exception_ptr *res = AllocateExceptionContainer( src ); - src.destroy(); - return res; + ctx.my_context_list->push_front(ctx.my_node); } -void tbb_exception_ptr::destroy () throw() { - this->tbb_exception_ptr::~tbb_exception_ptr(); - deallocate_via_handler_v3 (this); -} +void task_group_context_impl::bind_to_impl(d1::task_group_context& ctx, thread_data* td) { + __TBB_ASSERT(!is_poisoned(ctx.my_context_list), nullptr); + __TBB_ASSERT(ctx.my_state.load(std::memory_order_relaxed) == d1::task_group_context::state::locked, "The context can be bound only under the lock."); + __TBB_ASSERT(!ctx.my_parent, "Parent is set before initial binding"); -} // namespace internal -#endif /* !TBB_USE_CAPTURED_EXCEPTION */ + ctx.my_parent = td->my_task_dispatcher->m_execute_data_ext.context; + __TBB_ASSERT(ctx.my_parent, nullptr); - -//------------------------------------------------------------------------ -// task_group_context -//------------------------------------------------------------------------ - -task_group_context::~task_group_context () { - if ( __TBB_load_relaxed(my_kind) == binding_completed ) { - if ( governor::is_set(my_owner) ) { - // Local update of the context list - uintptr_t local_count_snapshot = my_owner->my_context_state_propagation_epoch; - my_owner->my_local_ctx_list_update.store<relaxed>(1); - // Prevent load of nonlocal update flag from being hoisted before the - // store to local update flag. - atomic_fence(); - if ( my_owner->my_nonlocal_ctx_list_update.load<relaxed>() ) { - spin_mutex::scoped_lock lock(my_owner->my_context_list_mutex); - my_node.my_prev->my_next = my_node.my_next; - my_node.my_next->my_prev = my_node.my_prev; - my_owner->my_local_ctx_list_update.store<relaxed>(0); - } - else { - my_node.my_prev->my_next = my_node.my_next; - my_node.my_next->my_prev = my_node.my_prev; - // Release fence is necessary so that update of our neighbors in - // the context list was committed when possible concurrent destroyer - // proceeds after local update flag is reset by the following store. - my_owner->my_local_ctx_list_update.store<release>(0); - if ( local_count_snapshot != the_context_state_propagation_epoch ) { - // Another thread was propagating cancellation request when we removed - // ourselves from the list. We must ensure that it is not accessing us - // when this destructor finishes. We'll be able to acquire the lock - // below only after the other thread finishes with us. - spin_mutex::scoped_lock lock(my_owner->my_context_list_mutex); - } - } - } - else { - // Nonlocal update of the context list - // Synchronizes with generic_scheduler::cleanup_local_context_list() - // TODO: evaluate and perhaps relax, or add some lock instead - if ( internal::as_atomic(my_kind).fetch_and_store(dying) == detached ) { - my_node.my_prev->my_next = my_node.my_next; - my_node.my_next->my_prev = my_node.my_prev; - } - else { - //TODO: evaluate and perhaps relax - my_owner->my_nonlocal_ctx_list_update.fetch_and_increment<full_fence>(); - //TODO: evaluate and perhaps remove - spin_wait_until_eq( my_owner->my_local_ctx_list_update, 0u ); - my_owner->my_context_list_mutex.lock(); - my_node.my_prev->my_next = my_node.my_next; - my_node.my_next->my_prev = my_node.my_prev; - my_owner->my_context_list_mutex.unlock(); - //TODO: evaluate and perhaps relax - my_owner->my_nonlocal_ctx_list_update.fetch_and_decrement<full_fence>(); - } - } - } -#if __TBB_FP_CONTEXT - internal::punned_cast<cpu_ctl_env*>(&my_cpu_ctl_env)->~cpu_ctl_env(); -#endif - poison_value(my_version_and_traits); - if ( my_exception ) - my_exception->destroy(); - ITT_STACK(itt_caller != ITT_CALLER_NULL, caller_destroy, itt_caller); -} - -void task_group_context::init () { - __TBB_STATIC_ASSERT ( sizeof(my_version_and_traits) >= 4, "Layout of my_version_and_traits must be reconsidered on this platform" ); - __TBB_STATIC_ASSERT ( sizeof(task_group_context) == 2 * NFS_MaxLineSize, "Context class has wrong size - check padding and members alignment" ); - __TBB_ASSERT ( (uintptr_t(this) & (sizeof(my_cancellation_requested) - 1)) == 0, "Context is improperly aligned" ); - __TBB_ASSERT ( __TBB_load_relaxed(my_kind) == isolated || __TBB_load_relaxed(my_kind) == bound, "Context can be created only as isolated or bound" ); - my_parent = NULL; - my_cancellation_requested = 0; - my_exception = NULL; - my_owner = NULL; - my_state = 0; - itt_caller = ITT_CALLER_NULL; -#if __TBB_TASK_PRIORITY - my_priority = normalized_normal_priority; -#endif /* __TBB_TASK_PRIORITY */ -#if __TBB_FP_CONTEXT - __TBB_STATIC_ASSERT( sizeof(my_cpu_ctl_env) == sizeof(internal::uint64_t), "The reserved space for FPU settings are not equal sizeof(uint64_t)" ); - __TBB_STATIC_ASSERT( sizeof(cpu_ctl_env) <= sizeof(my_cpu_ctl_env), "FPU settings storage does not fit to uint64_t" ); - suppress_unused_warning( my_cpu_ctl_env.space ); - - cpu_ctl_env &ctl = *internal::punned_cast<cpu_ctl_env*>(&my_cpu_ctl_env); - new ( &ctl ) cpu_ctl_env; - if ( my_version_and_traits & fp_settings ) - ctl.get_env(); -#endif -} - -void task_group_context::register_with ( generic_scheduler *local_sched ) { - __TBB_ASSERT( local_sched, NULL ); - my_owner = local_sched; - // state propagation logic assumes new contexts are bound to head of the list - my_node.my_prev = &local_sched->my_context_list_head; - // Notify threads that may be concurrently destroying contexts registered - // in this scheduler's list that local list update is underway. - local_sched->my_local_ctx_list_update.store<relaxed>(1); - // Prevent load of global propagation epoch counter from being hoisted before - // speculative stores above, as well as load of nonlocal update flag from - // being hoisted before the store to local update flag. - atomic_fence(); - // Finalize local context list update - if ( local_sched->my_nonlocal_ctx_list_update.load<relaxed>() ) { - spin_mutex::scoped_lock lock(my_owner->my_context_list_mutex); - local_sched->my_context_list_head.my_next->my_prev = &my_node; - my_node.my_next = local_sched->my_context_list_head.my_next; - my_owner->my_local_ctx_list_update.store<relaxed>(0); - local_sched->my_context_list_head.my_next = &my_node; - } - else { - local_sched->my_context_list_head.my_next->my_prev = &my_node; - my_node.my_next = local_sched->my_context_list_head.my_next; - my_owner->my_local_ctx_list_update.store<release>(0); - // Thread-local list of contexts allows concurrent traversal by another thread - // while propagating state change. To ensure visibility of my_node's members - // to the concurrently traversing thread, the list's head is updated by means - // of store-with-release. - __TBB_store_with_release(local_sched->my_context_list_head.my_next, &my_node); - } -} - -void task_group_context::bind_to ( generic_scheduler *local_sched ) { - __TBB_ASSERT ( __TBB_load_relaxed(my_kind) == binding_required, "Already bound or isolated?" ); - __TBB_ASSERT ( !my_parent, "Parent is set before initial binding" ); - my_parent = local_sched->my_innermost_running_task->prefix().context; -#if __TBB_FP_CONTEXT // Inherit FPU settings only if the context has not captured FPU settings yet. - if ( !(my_version_and_traits & fp_settings) ) - copy_fp_settings(*my_parent); -#endif + if (!ctx.my_traits.fp_settings) + copy_fp_settings(ctx, *ctx.my_parent); // Condition below prevents unnecessary thrashing parent context's cache line - if ( !(my_parent->my_state & may_have_children) ) - my_parent->my_state |= may_have_children; // full fence is below - if ( my_parent->my_parent ) { + if (ctx.my_parent->my_may_have_children.load(std::memory_order_relaxed) != d1::task_group_context::may_have_children) { + ctx.my_parent->my_may_have_children.store(d1::task_group_context::may_have_children, std::memory_order_relaxed); // full fence is below + } + if (ctx.my_parent->my_parent) { // Even if this context were made accessible for state change propagation - // (by placing __TBB_store_with_release(s->my_context_list_head.my_next, &my_node) + // (by placing store_with_release(td->my_context_list_state.head.my_next, &ctx.my_node) // above), it still could be missed if state propagation from a grand-ancestor // was underway concurrently with binding. // Speculative propagation from the parent together with epoch counters @@ -282,208 +138,221 @@ void task_group_context::bind_to ( generic_scheduler *local_sched ) { // Acquire fence is necessary to prevent reordering subsequent speculative // loads of parent state data out of the scope where epoch counters comparison // can reliably validate it. - uintptr_t local_count_snapshot = __TBB_load_with_acquire( my_parent->my_owner->my_context_state_propagation_epoch ); + uintptr_t local_count_snapshot = ctx.my_parent->my_context_list->epoch.load(std::memory_order_acquire); // Speculative propagation of parent's state. The speculation will be // validated by the epoch counters check further on. - my_cancellation_requested = my_parent->my_cancellation_requested; -#if __TBB_TASK_PRIORITY - my_priority = my_parent->my_priority; -#endif /* __TBB_TASK_PRIORITY */ - register_with( local_sched ); // Issues full fence + ctx.my_cancellation_requested.store(ctx.my_parent->my_cancellation_requested.load(std::memory_order_relaxed), std::memory_order_relaxed); + register_with(ctx, td); // Issues full fence // If no state propagation was detected by the following condition, the above // full fence guarantees that the parent had correct state during speculative // propagation before the fence. Otherwise the propagation from parent is // repeated under the lock. - if ( local_count_snapshot != the_context_state_propagation_epoch ) { + if (local_count_snapshot != the_context_state_propagation_epoch.load(std::memory_order_relaxed)) { // Another thread may be propagating state change right now. So resort to lock. context_state_propagation_mutex_type::scoped_lock lock(the_context_state_propagation_mutex); - my_cancellation_requested = my_parent->my_cancellation_requested; -#if __TBB_TASK_PRIORITY - my_priority = my_parent->my_priority; -#endif /* __TBB_TASK_PRIORITY */ + ctx.my_cancellation_requested.store(ctx.my_parent->my_cancellation_requested.load(std::memory_order_relaxed), std::memory_order_relaxed); } - } - else { - register_with( local_sched ); // Issues full fence + } else { + register_with(ctx, td); // Issues full fence // As we do not have grand-ancestors, concurrent state propagation (if any) // may originate only from the parent context, and thus it is safe to directly // copy the state from it. - my_cancellation_requested = my_parent->my_cancellation_requested; -#if __TBB_TASK_PRIORITY - my_priority = my_parent->my_priority; -#endif /* __TBB_TASK_PRIORITY */ + ctx.my_cancellation_requested.store(ctx.my_parent->my_cancellation_requested.load(std::memory_order_relaxed), std::memory_order_relaxed); } - __TBB_store_relaxed(my_kind, binding_completed); } -template <typename T> -void task_group_context::propagate_task_group_state ( T task_group_context::*mptr_state, task_group_context& src, T new_state ) { - if (this->*mptr_state == new_state) { - // Nothing to do, whether descending from "src" or not, so no need to scan. - // Hopefully this happens often thanks to earlier invocations. - // This optimization is enabled by LIFO order in the context lists: - // - new contexts are bound to the beginning of lists; - // - descendants are newer than ancestors; - // - earlier invocations are therefore likely to "paint" long chains. - } - else if (this == &src) { - // This clause is disjunct from the traversal below, which skips src entirely. - // Note that src.*mptr_state is not necessarily still equal to new_state (another thread may have changed it again). - // Such interference is probably not frequent enough to aim for optimisation by writing new_state again (to make the other thread back down). - // Letting the other thread prevail may also be fairer. - } - else { - for ( task_group_context *ancestor = my_parent; ancestor != NULL; ancestor = ancestor->my_parent ) { - __TBB_ASSERT(internal::is_alive(ancestor->my_version_and_traits), "context tree was corrupted"); - if ( ancestor == &src ) { - for ( task_group_context *ctx = this; ctx != ancestor; ctx = ctx->my_parent ) - ctx->*mptr_state = new_state; - break; +void task_group_context_impl::bind_to(d1::task_group_context& ctx, thread_data* td) { + d1::task_group_context::state state = ctx.my_state.load(std::memory_order_acquire); + if (state <= d1::task_group_context::state::locked) { + if (state == d1::task_group_context::state::created && +#if defined(__INTEL_COMPILER) && __INTEL_COMPILER <= 1910 + ((std::atomic<typename std::underlying_type<d1::task_group_context::state>::type>&)ctx.my_state).compare_exchange_strong( + (typename std::underlying_type<d1::task_group_context::state>::type&)state, + (typename std::underlying_type<d1::task_group_context::state>::type)d1::task_group_context::state::locked) +#else + ctx.my_state.compare_exchange_strong(state, d1::task_group_context::state::locked) +#endif + ) { + // If we are in the outermost task dispatch loop of an external thread, then + // there is nothing to bind this context to, and we skip the binding part + // treating the context as isolated. + __TBB_ASSERT(td->my_task_dispatcher->m_execute_data_ext.context != nullptr, nullptr); + d1::task_group_context::state release_state{}; + if (td->my_task_dispatcher->m_execute_data_ext.context == td->my_arena->my_default_ctx || !ctx.my_traits.bound) { + if (!ctx.my_traits.fp_settings) { + copy_fp_settings(ctx, *td->my_arena->my_default_ctx); + } + release_state = d1::task_group_context::state::isolated; + } else { + bind_to_impl(ctx, td); + release_state = d1::task_group_context::state::bound; } + ITT_STACK_CREATE(ctx.my_itt_caller); + ctx.my_state.store(release_state, std::memory_order_release); } + spin_wait_while_eq(ctx.my_state, d1::task_group_context::state::locked); } + __TBB_ASSERT(ctx.my_state.load(std::memory_order_relaxed) != d1::task_group_context::state::created, nullptr); + __TBB_ASSERT(ctx.my_state.load(std::memory_order_relaxed) != d1::task_group_context::state::locked, nullptr); } -template <typename T> -void generic_scheduler::propagate_task_group_state ( T task_group_context::*mptr_state, task_group_context& src, T new_state ) { - spin_mutex::scoped_lock lock(my_context_list_mutex); - // Acquire fence is necessary to ensure that the subsequent node->my_next load - // returned the correct value in case it was just inserted in another thread. - // The fence also ensures visibility of the correct my_parent value. - context_list_node_t *node = __TBB_load_with_acquire(my_context_list_head.my_next); - while ( node != &my_context_list_head ) { - task_group_context &ctx = __TBB_get_object_ref(task_group_context, my_node, node); - if ( ctx.*mptr_state != new_state ) - ctx.propagate_task_group_state( mptr_state, src, new_state ); - node = node->my_next; - __TBB_ASSERT( is_alive(ctx.my_version_and_traits), "Local context list contains destroyed object" ); - } - // Sync up local propagation epoch with the global one. Release fence prevents - // reordering of possible store to *mptr_state after the sync point. - __TBB_store_with_release(my_context_state_propagation_epoch, the_context_state_propagation_epoch); -} - -template <typename T> -bool market::propagate_task_group_state ( T task_group_context::*mptr_state, task_group_context& src, T new_state ) { - if ( !(src.my_state & task_group_context::may_have_children) ) - return true; - // The whole propagation algorithm is under the lock in order to ensure correctness - // in case of concurrent state changes at the different levels of the context tree. - // See comment at the bottom of scheduler.cpp - context_state_propagation_mutex_type::scoped_lock lock(the_context_state_propagation_mutex); - if ( src.*mptr_state != new_state ) - // Another thread has concurrently changed the state. Back down. - return false; - // Advance global state propagation epoch - __TBB_FetchAndAddWrelease(&the_context_state_propagation_epoch, 1); - // Propagate to all workers and masters and sync up their local epochs with the global one - unsigned num_workers = my_first_unused_worker_idx; - for ( unsigned i = 0; i < num_workers; ++i ) { - generic_scheduler *s = my_workers[i]; - // If the worker is only about to be registered, skip it. - if ( s ) - s->propagate_task_group_state( mptr_state, src, new_state ); +void task_group_context_impl::propagate_task_group_state(d1::task_group_context& ctx, std::atomic<std::uint32_t> d1::task_group_context::* mptr_state, d1::task_group_context& src, std::uint32_t new_state) { + __TBB_ASSERT(!is_poisoned(ctx.my_context_list), nullptr); + /* 1. if ((ctx.*mptr_state).load(std::memory_order_relaxed) == new_state): + Nothing to do, whether descending from "src" or not, so no need to scan. + Hopefully this happens often thanks to earlier invocations. + This optimization is enabled by LIFO order in the context lists: + - new contexts are bound to the beginning of lists; + - descendants are newer than ancestors; + - earlier invocations are therefore likely to "paint" long chains. + 2. if (&ctx != &src): + This clause is disjunct from the traversal below, which skips src entirely. + Note that src.*mptr_state is not necessarily still equal to new_state (another thread may have changed it again). + Such interference is probably not frequent enough to aim for optimisation by writing new_state again (to make the other thread back down). + Letting the other thread prevail may also be fairer. + */ + if ((ctx.*mptr_state).load(std::memory_order_relaxed) != new_state && &ctx != &src) { + for (d1::task_group_context* ancestor = ctx.my_parent; ancestor != nullptr; ancestor = ancestor->my_parent) { + if (ancestor == &src) { + for (d1::task_group_context* c = &ctx; c != ancestor; c = c->my_parent) + (c->*mptr_state).store(new_state, std::memory_order_relaxed); + break; + } + } } - // Propagate to all master threads - // The whole propagation sequence is locked, thus no contention is expected - for( scheduler_list_type::iterator it = my_masters.begin(); it != my_masters.end(); it++ ) - it->propagate_task_group_state( mptr_state, src, new_state ); - return true; } -bool task_group_context::cancel_group_execution () { - __TBB_ASSERT ( my_cancellation_requested == 0 || my_cancellation_requested == 1, "Invalid cancellation state"); - if ( my_cancellation_requested || as_atomic(my_cancellation_requested).compare_and_swap(1, 0) ) { +bool task_group_context_impl::cancel_group_execution(d1::task_group_context& ctx) { + __TBB_ASSERT(!is_poisoned(ctx.my_context_list), nullptr); + __TBB_ASSERT(ctx.my_cancellation_requested.load(std::memory_order_relaxed) <= 1, "The cancellation state can be either 0 or 1"); + if (ctx.my_cancellation_requested.load(std::memory_order_relaxed) || ctx.my_cancellation_requested.exchange(1)) { // This task group and any descendants have already been canceled. - // (A newly added descendant would inherit its parent's my_cancellation_requested, + // (A newly added descendant would inherit its parent's ctx.my_cancellation_requested, // not missing out on any cancellation still being propagated, and a context cannot be uncanceled.) return false; } - governor::local_scheduler_weak()->my_market->propagate_task_group_state( &task_group_context::my_cancellation_requested, *this, (uintptr_t)1 ); + governor::get_thread_data()->my_arena->my_threading_control->propagate_task_group_state(&d1::task_group_context::my_cancellation_requested, ctx, uint32_t(1)); return true; } -bool task_group_context::is_group_execution_cancelled () const { - return my_cancellation_requested != 0; +bool task_group_context_impl::is_group_execution_cancelled(const d1::task_group_context& ctx) { + return ctx.my_cancellation_requested.load(std::memory_order_relaxed) != 0; } // IMPORTANT: It is assumed that this method is not used concurrently! -void task_group_context::reset () { +void task_group_context_impl::reset(d1::task_group_context& ctx) { + __TBB_ASSERT(!is_poisoned(ctx.my_context_list), nullptr); //! TODO: Add assertion that this context does not have children // No fences are necessary since this context can be accessed from another thread // only after stealing happened (which means necessary fences were used). - if ( my_exception ) { - my_exception->destroy(); - my_exception = NULL; + + auto exception = ctx.my_exception.load(std::memory_order_relaxed); + if (exception) { + exception->destroy(); + ctx.my_exception.store(nullptr, std::memory_order_relaxed); } - my_cancellation_requested = 0; + ctx.my_cancellation_requested = 0; } -#if __TBB_FP_CONTEXT // IMPORTANT: It is assumed that this method is not used concurrently! -void task_group_context::capture_fp_settings () { +void task_group_context_impl::capture_fp_settings(d1::task_group_context& ctx) { + __TBB_ASSERT(!is_poisoned(ctx.my_context_list), nullptr); //! TODO: Add assertion that this context does not have children // No fences are necessary since this context can be accessed from another thread // only after stealing happened (which means necessary fences were used). - cpu_ctl_env &ctl = *internal::punned_cast<cpu_ctl_env*>(&my_cpu_ctl_env); - if ( !(my_version_and_traits & fp_settings) ) { - new ( &ctl ) cpu_ctl_env; - my_version_and_traits |= fp_settings; + d1::cpu_ctl_env* ctl = reinterpret_cast<d1::cpu_ctl_env*>(&ctx.my_cpu_ctl_env); + if (!ctx.my_traits.fp_settings) { + ctl = new (&ctx.my_cpu_ctl_env) d1::cpu_ctl_env; + ctx.my_traits.fp_settings = true; } - ctl.get_env(); + ctl->get_env(); } -void task_group_context::copy_fp_settings( const task_group_context &src ) { - __TBB_ASSERT( !(my_version_and_traits & fp_settings), "The context already has FPU settings." ); - __TBB_ASSERT( src.my_version_and_traits & fp_settings, "The source context does not have FPU settings." ); +void task_group_context_impl::copy_fp_settings(d1::task_group_context& ctx, const d1::task_group_context& src) { + __TBB_ASSERT(!is_poisoned(ctx.my_context_list), nullptr); + __TBB_ASSERT(!ctx.my_traits.fp_settings, "The context already has FPU settings."); + __TBB_ASSERT(src.my_traits.fp_settings, "The source context does not have FPU settings."); - cpu_ctl_env &ctl = *internal::punned_cast<cpu_ctl_env*>(&my_cpu_ctl_env); - cpu_ctl_env &src_ctl = *internal::punned_cast<cpu_ctl_env*>(&src.my_cpu_ctl_env); - new (&ctl) cpu_ctl_env( src_ctl ); - my_version_and_traits |= fp_settings; -} -#endif /* __TBB_FP_CONTEXT */ - -void task_group_context::register_pending_exception () { - if ( my_cancellation_requested ) - return; -#if TBB_USE_EXCEPTIONS - try { - throw; - } TbbCatchAll( this ); -#endif /* TBB_USE_EXCEPTIONS */ + const d1::cpu_ctl_env* src_ctl = reinterpret_cast<const d1::cpu_ctl_env*>(&src.my_cpu_ctl_env); + new (&ctx.my_cpu_ctl_env) d1::cpu_ctl_env(*src_ctl); + ctx.my_traits.fp_settings = true; } -#if __TBB_TASK_PRIORITY -void task_group_context::set_priority ( priority_t prio ) { - __TBB_ASSERT( prio == priority_low || prio == priority_normal || prio == priority_high, "Invalid priority level value" ); - intptr_t p = normalize_priority(prio); - if ( my_priority == p && !(my_state & task_group_context::may_have_children)) - return; - my_priority = p; - internal::generic_scheduler* s = governor::local_scheduler_if_initialized(); - if ( !s || !s->my_arena || !s->my_market->propagate_task_group_state(&task_group_context::my_priority, *this, p) ) - return; - - //! TODO: the arena of the calling thread might be unrelated; - // need to find out the right arena for priority update. - // The executing status check only guarantees being inside some working arena. - if ( s->my_innermost_running_task->state() == task::executing ) - // Updating arena priority here does not eliminate necessity of checking each - // task priority and updating arena priority if necessary before the task execution. - // These checks will be necessary because: - // a) set_priority() may be invoked before any tasks from this task group are spawned; - // b) all spawned tasks from this task group are retrieved from the task pools. - // These cases create a time window when arena priority may be lowered. - s->my_market->update_arena_priority( *s->my_arena, p ); -} +/* + Comments: + +1. The premise of the cancellation support implementation is that cancellations are + not part of the hot path of the program execution. Therefore all changes in its + implementation in order to reduce the overhead of the cancellation control flow + should be done only in ways that do not increase overhead of the normal execution. + + In general, contexts are used by all threads and their descendants are created in + different threads as well. In order to minimize impact of the cross-thread tree + maintenance (first of all because of the synchronization), the tree of contexts + is split into pieces, each of which is handled by a single thread. Such pieces + are represented as lists of contexts, members of which are contexts that were + bound to their parents in the given thread. + + The context tree maintenance and cancellation propagation algorithms are designed + in such a manner that cross-thread access to a context list will take place only + when cancellation signal is sent (by user or when an exception happens), and + synchronization is necessary only then. Thus the normal execution flow (without + exceptions and cancellation) remains free from any synchronization done on + behalf of exception handling and cancellation support. + +2. Consider parallel cancellations at the different levels of the context tree: + + Ctx1 <- Cancelled by Thread1 |- Thread2 started processing + | | + Ctx2 |- Thread1 started processing + | T1 |- Thread2 finishes and syncs up local counters + Ctx3 <- Cancelled by Thread2 | + | |- Ctx5 is bound to Ctx2 + Ctx4 | + T2 |- Thread1 reaches Ctx2 + + Thread-propagator of each cancellation increments global counter. However the thread + propagating the cancellation from the outermost context (Thread1) may be the last + to finish. Which means that the local counters may be synchronized earlier (by Thread2, + at Time1) than it propagated cancellation into Ctx2 (at time Time2). If a new context + (Ctx5) is created and bound to Ctx2 between Time1 and Time2, checking its parent only + (Ctx2) may result in cancellation request being lost. + + This issue is solved by doing the whole propagation under the lock. + + If we need more concurrency while processing parallel cancellations, we could try + the following modification of the propagation algorithm: + + advance global counter and remember it + for each thread: + scan thread's list of contexts + for each thread: + sync up its local counter only if the global counter has not been changed + + However this version of the algorithm requires more analysis and verification. +*/ -priority_t task_group_context::priority () const { - return static_cast<priority_t>(priority_from_normalized_rep[my_priority]); +void __TBB_EXPORTED_FUNC initialize(d1::task_group_context& ctx) { + task_group_context_impl::initialize(ctx); +} +void __TBB_EXPORTED_FUNC destroy(d1::task_group_context& ctx) { + task_group_context_impl::destroy(ctx); +} +void __TBB_EXPORTED_FUNC reset(d1::task_group_context& ctx) { + task_group_context_impl::reset(ctx); +} +bool __TBB_EXPORTED_FUNC cancel_group_execution(d1::task_group_context& ctx) { + return task_group_context_impl::cancel_group_execution(ctx); +} +bool __TBB_EXPORTED_FUNC is_group_execution_cancelled(d1::task_group_context& ctx) { + return task_group_context_impl::is_group_execution_cancelled(ctx); +} +void __TBB_EXPORTED_FUNC capture_fp_settings(d1::task_group_context& ctx) { + task_group_context_impl::capture_fp_settings(ctx); } -#endif /* __TBB_TASK_PRIORITY */ - -#endif /* __TBB_TASK_GROUP_CONTEXT */ +} // namespace r1 +} // namespace detail } // namespace tbb + diff --git a/src/tbb/src/tbb/task_stream.h b/src/tbb/src/tbb/task_stream.h index 0118e1b8a..0aaace52b 100644 --- a/src/tbb/src/tbb/task_stream.h +++ b/src/tbb/src/tbb/task_stream.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2022 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,161 +12,275 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef _TBB_task_stream_H #define _TBB_task_stream_H -#include "tbb/tbb_stddef.h" +//! This file is a possible future replacement for the task_stream class implemented in +//! task_stream.h. It refactors the code and extends task_stream capabilities by moving lane +//! management during operations on caller side. Despite the fact that new implementation should not +//! affect performance of the original task stream, analysis on this subject was not made at the +//! time it was developed. In addition, it is not clearly seen at the moment that this container +//! would be suitable for critical tasks due to linear time complexity on its operations. + +#include "oneapi/tbb/detail/_utils.h" +#include "oneapi/tbb/cache_aligned_allocator.h" +#include "oneapi/tbb/mutex.h" + +#include "scheduler_common.h" +#include "misc.h" // for FastRandom + #include <deque> #include <climits> -#include "tbb/atomic.h" // for __TBB_Atomic* -#include "tbb/spin_mutex.h" -#include "tbb/tbb_allocator.h" -#include "scheduler_common.h" -#include "tbb_misc.h" // for FastRandom +#include <atomic> namespace tbb { -namespace internal { +namespace detail { +namespace r1 { //! Essentially, this is just a pair of a queue and a mutex to protect the queue. /** The reason std::pair is not used is that the code would look less clean if field names were replaced with 'first' and 'second'. **/ template< typename T, typename mutex_t > -struct queue_and_mutex { - typedef std::deque< T, tbb_allocator<T> > queue_base_t; +struct alignas(max_nfs_size) queue_and_mutex { + typedef std::deque< T, cache_aligned_allocator<T> > queue_base_t; - queue_base_t my_queue; - mutex_t my_mutex; - - queue_and_mutex () : my_queue(), my_mutex() {} - ~queue_and_mutex () {} + queue_base_t my_queue{}; + mutex_t my_mutex{}; }; -typedef uintptr_t population_t; +using population_t = uintptr_t; const population_t one = 1; -inline void set_one_bit( population_t& dest, int pos ) { - __TBB_ASSERT( pos>=0, NULL ); - __TBB_ASSERT( pos<int(sizeof(population_t)*CHAR_BIT), NULL ); - __TBB_AtomicOR( &dest, one<<pos ); +inline void set_one_bit( std::atomic<population_t>& dest, int pos ) { + __TBB_ASSERT( pos>=0, nullptr); + __TBB_ASSERT( pos<int(sizeof(population_t)*CHAR_BIT), nullptr); + dest.fetch_or( one<<pos ); } -inline void clear_one_bit( population_t& dest, int pos ) { - __TBB_ASSERT( pos>=0, NULL ); - __TBB_ASSERT( pos<int(sizeof(population_t)*CHAR_BIT), NULL ); - __TBB_AtomicAND( &dest, ~(one<<pos) ); +inline void clear_one_bit( std::atomic<population_t>& dest, int pos ) { + __TBB_ASSERT( pos>=0, nullptr); + __TBB_ASSERT( pos<int(sizeof(population_t)*CHAR_BIT), nullptr); + dest.fetch_and( ~(one<<pos) ); } inline bool is_bit_set( population_t val, int pos ) { - __TBB_ASSERT( pos>=0, NULL ); - __TBB_ASSERT( pos<int(sizeof(population_t)*CHAR_BIT), NULL ); + __TBB_ASSERT( pos>=0, nullptr); + __TBB_ASSERT( pos<int(sizeof(population_t)*CHAR_BIT), nullptr); return (val & (one<<pos)) != 0; } +struct random_lane_selector : +#if __INTEL_COMPILER == 1110 || __INTEL_COMPILER == 1500 + no_assign +#else + no_copy +#endif +{ + random_lane_selector( FastRandom& random ) : my_random( random ) {} + unsigned operator()( unsigned out_of ) const { + __TBB_ASSERT( ((out_of-1) & out_of) == 0, "number of lanes is not power of two." ); + return my_random.get() & (out_of-1); + } +private: + FastRandom& my_random; +}; + +struct lane_selector_base : +#if __INTEL_COMPILER == 1110 || __INTEL_COMPILER == 1500 + no_assign +#else + no_copy +#endif +{ + unsigned& my_previous; + lane_selector_base( unsigned& previous ) : my_previous( previous ) {} +}; + +struct subsequent_lane_selector : lane_selector_base { + subsequent_lane_selector( unsigned& previous ) : lane_selector_base( previous ) {} + unsigned operator()( unsigned out_of ) const { + __TBB_ASSERT( ((out_of-1) & out_of) == 0, "number of lanes is not power of two." ); + return (++my_previous &= out_of-1); + } +}; + +struct preceding_lane_selector : lane_selector_base { + preceding_lane_selector( unsigned& previous ) : lane_selector_base( previous ) {} + unsigned operator()( unsigned out_of ) const { + __TBB_ASSERT( ((out_of-1) & out_of) == 0, "number of lanes is not power of two." ); + return (--my_previous &= (out_of-1)); + } +}; + +//! Specializes from which side of the underlying container elements are retrieved. Method must be +//! called under corresponding mutex locked. +template<task_stream_accessor_type accessor> +class task_stream_accessor : no_copy { +protected: + using lane_t = queue_and_mutex <d1::task*, mutex>; + d1::task* get_item( lane_t::queue_base_t& queue ) { + d1::task* result = queue.front(); + queue.pop_front(); + return result; + } +}; + +template<> +class task_stream_accessor< back_nonnull_accessor > : no_copy { +protected: + using lane_t = queue_and_mutex <d1::task*, mutex>; + d1::task* get_item( lane_t::queue_base_t& queue ) { + d1::task* result = nullptr; + __TBB_ASSERT(!queue.empty(), nullptr); + // Isolated task can put zeros in queue see look_specific + do { + result = queue.back(); + queue.pop_back(); + } while ( !result && !queue.empty() ); + return result; + } +}; + //! The container for "fairness-oriented" aka "enqueued" tasks. -template<int Levels> -class task_stream : no_copy { - typedef queue_and_mutex <task*, spin_mutex> lane_t; - population_t population[Levels]; - padded<lane_t>* lanes[Levels]; - unsigned N; +template<task_stream_accessor_type accessor> +class task_stream : public task_stream_accessor< accessor > { + using lane_t = typename task_stream_accessor<accessor>::lane_t; + std::atomic<population_t> population{}; + lane_t* lanes{nullptr}; + unsigned N{}; public: - task_stream() : N() { - for(int level = 0; level < Levels; level++) { - population[level] = 0; - lanes[level] = NULL; - } - } + task_stream() = default; void initialize( unsigned n_lanes ) { const unsigned max_lanes = sizeof(population_t) * CHAR_BIT; - N = n_lanes>=max_lanes ? max_lanes : n_lanes>2 ? 1<<(__TBB_Log2(n_lanes-1)+1) : 2; - __TBB_ASSERT( N==max_lanes || N>=n_lanes && ((N-1)&N)==0, "number of lanes miscalculated"); - __TBB_ASSERT( N <= sizeof(population_t) * CHAR_BIT, NULL ); - for(int level = 0; level < Levels; level++) { - lanes[level] = new padded<lane_t>[N]; - __TBB_ASSERT( !population[level], NULL ); + N = n_lanes >= max_lanes ? max_lanes : n_lanes > 2 ? 1 << (tbb::detail::log2(n_lanes - 1) + 1) : 2; + __TBB_ASSERT( N == max_lanes || (N >= n_lanes && ((N - 1) & N) == 0), "number of lanes miscalculated" ); + __TBB_ASSERT( N <= sizeof(population_t) * CHAR_BIT, nullptr); + lanes = static_cast<lane_t*>(cache_aligned_allocate(sizeof(lane_t) * N)); + for (unsigned i = 0; i < N; ++i) { + new (lanes + i) lane_t; } + __TBB_ASSERT( !population.load(std::memory_order_relaxed), nullptr); } ~task_stream() { - for(int level = 0; level < Levels; level++) - if (lanes[level]) delete[] lanes[level]; + if (lanes) { + for (unsigned i = 0; i < N; ++i) { + lanes[i].~lane_t(); + } + cache_aligned_deallocate(lanes); + } } - //! Push a task into a lane. - void push( task* source, int level, FastRandom& random ) { - // Lane selection is random. Each thread should keep a separate seed value. - unsigned idx; - for( ; ; ) { - idx = random.get() & (N-1); - spin_mutex::scoped_lock lock; - if( lock.try_acquire(lanes[level][idx].my_mutex) ) { - lanes[level][idx].my_queue.push_back(source); - set_one_bit( population[level], idx ); //TODO: avoid atomic op if the bit is already set - break; - } + //! Push a task into a lane. Lane selection is performed by passed functor. + template<typename lane_selector_t> + void push(d1::task* source, const lane_selector_t& next_lane ) { + bool succeed = false; + unsigned lane = 0; + do { + lane = next_lane( /*out_of=*/N ); + __TBB_ASSERT( lane < N, "Incorrect lane index." ); + } while( ! (succeed = try_push( source, lane )) ); + } + + //! Try finding and popping a task using passed functor for lane selection. Last used lane is + //! updated inside lane selector. + template<typename lane_selector_t> + d1::task* pop( const lane_selector_t& next_lane ) { + d1::task* popped = nullptr; + unsigned lane = 0; + for (atomic_backoff b; !empty() && !popped; b.pause()) { + lane = next_lane( /*out_of=*/N); + __TBB_ASSERT(lane < N, "Incorrect lane index."); + popped = try_pop(lane); } + return popped; } - //! Try finding and popping a task. - task* pop( int level, unsigned& last_used_lane ) { - task* result = NULL; - // Lane selection is round-robin. Each thread should keep its last used lane. - unsigned idx = (last_used_lane+1)&(N-1); - for( ; population[level]; idx=(idx+1)&(N-1) ) { - if( is_bit_set( population[level], idx ) ) { - lane_t& lane = lanes[level][idx]; - spin_mutex::scoped_lock lock; + //! Try finding and popping a related task. + d1::task* pop_specific( unsigned& last_used_lane, isolation_type isolation ) { + d1::task* result = nullptr; + // Lane selection is round-robin in backward direction. + unsigned idx = last_used_lane & (N-1); + do { + if( is_bit_set( population.load(std::memory_order_relaxed), idx ) ) { + lane_t& lane = lanes[idx]; + mutex::scoped_lock lock; if( lock.try_acquire(lane.my_mutex) && !lane.my_queue.empty() ) { - result = lane.my_queue.front(); - lane.my_queue.pop_front(); + result = look_specific( lane.my_queue, isolation ); if( lane.my_queue.empty() ) - clear_one_bit( population[level], idx ); - break; + clear_one_bit( population, idx ); + if( result ) + break; } } - } + idx=(idx-1)&(N-1); + } while( !empty() && idx != last_used_lane ); last_used_lane = idx; return result; } //! Checks existence of a task. - bool empty(int level) { - return !population[level]; + bool empty() { + return !population.load(std::memory_order_relaxed); } - //! Destroys all remaining tasks in every lane. Returns the number of destroyed tasks. - /** Tasks are not executed, because it would potentially create more tasks at a late stage. - The scheduler is really expected to execute all tasks before task_stream destruction. */ - intptr_t drain() { - intptr_t result = 0; - for(int level = 0; level < Levels; level++) - for(unsigned i=0; i<N; ++i) { - lane_t& lane = lanes[level][i]; - spin_mutex::scoped_lock lock(lane.my_mutex); - for(lane_t::queue_base_t::iterator it=lane.my_queue.begin(); - it!=lane.my_queue.end(); ++it, ++result) - { - __TBB_ASSERT( is_bit_set( population[level], i ), NULL ); - task* t = *it; - tbb::task::destroy(*t); - } - lane.my_queue.clear(); - clear_one_bit( population[level], i ); - } +private: + //! Returns true on successful push, otherwise - false. + bool try_push(d1::task* source, unsigned lane_idx ) { + mutex::scoped_lock lock; + if( lock.try_acquire( lanes[lane_idx].my_mutex ) ) { + lanes[lane_idx].my_queue.push_back( source ); + set_one_bit( population, lane_idx ); // TODO: avoid atomic op if the bit is already set + return true; + } + return false; + } + + //! Returns pointer to task on successful pop, otherwise - nullptr. + d1::task* try_pop( unsigned lane_idx ) { + if( !is_bit_set( population.load(std::memory_order_relaxed), lane_idx ) ) + return nullptr; + d1::task* result = nullptr; + lane_t& lane = lanes[lane_idx]; + mutex::scoped_lock lock; + if( lock.try_acquire( lane.my_mutex ) && !lane.my_queue.empty() ) { + result = this->get_item( lane.my_queue ); + if( lane.my_queue.empty() ) + clear_one_bit( population, lane_idx ); + } return result; } + + // TODO: unify '*_specific' logic with 'pop' methods above + d1::task* look_specific( typename lane_t::queue_base_t& queue, isolation_type isolation ) { + __TBB_ASSERT( !queue.empty(), nullptr); + // TODO: add a worst-case performance test and consider an alternative container with better + // performance for isolation search. + typename lane_t::queue_base_t::iterator curr = queue.end(); + do { + // TODO: consider logic from get_task to simplify the code. + d1::task* result = *--curr; + if( result && task_accessor::isolation(*result) == isolation ) { + if( queue.end() - curr == 1 ) + queue.pop_back(); // a little of housekeeping along the way + else + *curr = nullptr; // grabbing task with the same isolation + // TODO: move one of the container's ends instead if the task has been found there + return result; + } + } while( curr != queue.begin() ); + return nullptr; + } + }; // task_stream -} // namespace internal +} // namespace r1 +} // namespace detail } // namespace tbb #endif /* _TBB_task_stream_H */ diff --git a/src/tbb/src/tbb/tbb_resource.rc b/src/tbb/src/tbb/tbb.rc similarity index 51% rename from src/tbb/src/tbb/tbb_resource.rc rename to src/tbb/src/tbb/tbb.rc index 7dc4fe6f1..57e9d391d 100644 --- a/src/tbb/src/tbb/tbb_resource.rc +++ b/src/tbb/src/tbb/tbb.rc @@ -1,4 +1,4 @@ -// Copyright (c) 2005-2017 Intel Corporation +// Copyright (c) 2005-2024 Intel Corporation // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -11,55 +11,28 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -// -// -// -// - -// Microsoft Visual C++ generated resource script. -// -#ifdef APSTUDIO_INVOKED -#ifndef APSTUDIO_READONLY_SYMBOLS -#define _APS_NO_MFC 1 -#define _APS_NEXT_RESOURCE_VALUE 102 -#define _APS_NEXT_COMMAND_VALUE 40001 -#define _APS_NEXT_CONTROL_VALUE 1001 -#define _APS_NEXT_SYMED_VALUE 101 -#endif -#endif -#define APSTUDIO_READONLY_SYMBOLS ///////////////////////////////////////////////////////////////////////////// // -// Generated from the TEXTINCLUDE 2 resource. +// Includes // #include <winresrc.h> -#define ENDL "\r\n" -#include "tbb_version.h" - -///////////////////////////////////////////////////////////////////////////// -#undef APSTUDIO_READONLY_SYMBOLS +#include "../../include/oneapi/tbb/version.h" ///////////////////////////////////////////////////////////////////////////// // Neutral resources -//#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_NEU) #ifdef _WIN32 LANGUAGE LANG_NEUTRAL, SUBLANG_NEUTRAL #pragma code_page(1252) #endif //_WIN32 -///////////////////////////////////////////////////////////////////////////// -// manifest integration -#ifdef TBB_MANIFEST -#include "winuser.h" -2 RT_MANIFEST tbbmanifest.exe.manifest -#endif - ///////////////////////////////////////////////////////////////////////////// // // Version // +#define TBB_VERNUMBERS TBB_VERSION_MAJOR,TBB_VERSION_MINOR,TBB_VERSION_PATCH +#define TBB_VERSION TBB_VERSION_STRING VS_VERSION_INFO VERSIONINFO FILEVERSION TBB_VERNUMBERS @@ -79,16 +52,16 @@ BEGIN BLOCK "000004b0" BEGIN VALUE "CompanyName", "Intel Corporation\0" - VALUE "FileDescription", "Intel(R) Threading Building Blocks library\0" + VALUE "FileDescription", "oneAPI Threading Building Blocks (oneTBB) library\0" VALUE "FileVersion", TBB_VERSION "\0" - VALUE "LegalCopyright", "Copyright 2005-2017 Intel Corporation. All Rights Reserved.\0" + VALUE "LegalCopyright", "Copyright 2005-2024 Intel Corporation. All Rights Reserved.\0" VALUE "LegalTrademarks", "\0" #ifndef TBB_USE_DEBUG - VALUE "OriginalFilename", "tbb.dll\0" + VALUE "OriginalFilename", "tbb12.dll\0" #else - VALUE "OriginalFilename", "tbb_debug.dll\0" + VALUE "OriginalFilename", "tbb12_debug.dll\0" #endif - VALUE "ProductName", "Intel(R) Threading Building Blocks for Windows\0" + VALUE "ProductName", "oneAPI Threading Building Blocks (oneTBB)\0" VALUE "ProductVersion", TBB_VERSION "\0" VALUE "PrivateBuild", "\0" VALUE "SpecialBuild", "\0" @@ -99,18 +72,3 @@ BEGIN VALUE "Translation", 0x0, 1200 END END - -//#endif // Neutral resources -///////////////////////////////////////////////////////////////////////////// - - -#ifndef APSTUDIO_INVOKED -///////////////////////////////////////////////////////////////////////////// -// -// Generated from the TEXTINCLUDE 3 resource. -// - - -///////////////////////////////////////////////////////////////////////////// -#endif // not APSTUDIO_INVOKED - diff --git a/src/tbb/src/tbb/tbb_assert_impl.h b/src/tbb/src/tbb/tbb_assert_impl.h deleted file mode 100644 index 6fcf5bf26..000000000 --- a/src/tbb/src/tbb/tbb_assert_impl.h +++ /dev/null @@ -1,103 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -// IMPORTANT: To use assertion handling in TBB, exactly one of the TBB source files -// should #include tbb_assert_impl.h thus instantiating assertion handling routines. -// The intent of putting it to a separate file is to allow some tests to use it -// as well in order to avoid dependency on the library. - -// include headers for required function declarations -#include <cstdlib> -#include <stdio.h> -#include <string.h> -#include <stdarg.h> -#if _MSC_VER -#include <crtdbg.h> -#endif - -#if _MSC_VER >= 1400 -#define __TBB_EXPORTED_FUNC __cdecl -#else -#define __TBB_EXPORTED_FUNC -#endif - -using namespace std; - -#if __TBBMALLOC_BUILD -namespace rml { namespace internal { -#else -namespace tbb { -#endif - //! Type for an assertion handler - typedef void(*assertion_handler_type)( const char* filename, int line, const char* expression, const char * comment ); - - static assertion_handler_type assertion_handler; - - assertion_handler_type __TBB_EXPORTED_FUNC set_assertion_handler( assertion_handler_type new_handler ) { - assertion_handler_type old_handler = assertion_handler; - assertion_handler = new_handler; - return old_handler; - } - - void __TBB_EXPORTED_FUNC assertion_failure( const char* filename, int line, const char* expression, const char* comment ) { - if( assertion_handler_type a = assertion_handler ) { - (*a)(filename,line,expression,comment); - } else { - static bool already_failed; - if( !already_failed ) { - already_failed = true; - fprintf( stderr, "Assertion %s failed on line %d of file %s\n", - expression, line, filename ); - if( comment ) - fprintf( stderr, "Detailed description: %s\n", comment ); -#if _MSC_VER && _DEBUG - if(1 == _CrtDbgReport(_CRT_ASSERT, filename, line, "tbb_debug.dll", "%s\r\n%s", expression, comment?comment:"")) - _CrtDbgBreak(); -#else - fflush(stderr); - abort(); -#endif - } - } - } - -#if defined(_MSC_VER)&&_MSC_VER<1400 -# define vsnprintf _vsnprintf -#endif - -#if !__TBBMALLOC_BUILD - namespace internal { - //! Report a runtime warning. - void __TBB_EXPORTED_FUNC runtime_warning( const char* format, ... ) - { - char str[1024]; memset(str, 0, 1024); - va_list args; va_start(args, format); - vsnprintf( str, 1024-1, format, args); - va_end(args); - fprintf( stderr, "TBB Warning: %s\n", str); - } - } // namespace internal -#endif - -#if __TBBMALLOC_BUILD -}} // namespaces rml::internal -#else -} // namespace tbb -#endif diff --git a/src/tbb/src/tbb/tbb_main.cpp b/src/tbb/src/tbb/tbb_main.cpp deleted file mode 100644 index 685d28ec3..000000000 --- a/src/tbb/src/tbb/tbb_main.cpp +++ /dev/null @@ -1,563 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#include "tbb/tbb_config.h" -#include "tbb/global_control.h" -#include "tbb_main.h" -#include "governor.h" -#include "market.h" -#include "tbb_misc.h" -#include "itt_notify.h" - -namespace tbb { -namespace internal { - -//------------------------------------------------------------------------ -// Begin shared data layout. -// The following global data items are mostly read-only after initialization. -//------------------------------------------------------------------------ - -//! Padding in order to prevent false sharing. -static const char _pad[NFS_MaxLineSize - sizeof(int)] = {}; - -//------------------------------------------------------------------------ -// governor data -basic_tls<uintptr_t> governor::theTLS; -unsigned governor::DefaultNumberOfThreads; -rml::tbb_factory governor::theRMLServerFactory; -bool governor::UsePrivateRML; -bool governor::is_speculation_enabled; -bool governor::is_rethrow_broken; - -//------------------------------------------------------------------------ -// market data -market* market::theMarket; -market::global_market_mutex_type market::theMarketMutex; - -//------------------------------------------------------------------------ -// One time initialization data - -//! Counter of references to global shared resources such as TLS. -atomic<int> __TBB_InitOnce::count; - -__TBB_atomic_flag __TBB_InitOnce::InitializationLock; - -//! Flag that is set to true after one-time initializations are done. -bool __TBB_InitOnce::InitializationDone; - -#if DO_ITT_NOTIFY - static bool ITT_Present; - static bool ITT_InitializationDone; -#endif - -#if !(_WIN32||_WIN64) || __TBB_SOURCE_DIRECTLY_INCLUDED - static __TBB_InitOnce __TBB_InitOnceHiddenInstance; -#endif - -//------------------------------------------------------------------------ -// generic_scheduler data - -//! Pointer to the scheduler factory function -generic_scheduler* (*AllocateSchedulerPtr)( market& ); - -#if __TBB_OLD_PRIMES_RNG -//! Table of primes used by fast random-number generator (FastRandom). -/** Also serves to keep anything else from being placed in the same - cache line as the global data items preceding it. */ -static const unsigned Primes[] = { - 0x9e3779b1, 0xffe6cc59, 0x2109f6dd, 0x43977ab5, - 0xba5703f5, 0xb495a877, 0xe1626741, 0x79695e6b, - 0xbc98c09f, 0xd5bee2b3, 0x287488f9, 0x3af18231, - 0x9677cd4d, 0xbe3a6929, 0xadc6a877, 0xdcf0674b, - 0xbe4d6fe9, 0x5f15e201, 0x99afc3fd, 0xf3f16801, - 0xe222cfff, 0x24ba5fdb, 0x0620452d, 0x79f149e3, - 0xc8b93f49, 0x972702cd, 0xb07dd827, 0x6c97d5ed, - 0x085a3d61, 0x46eb5ea7, 0x3d9910ed, 0x2e687b5b, - 0x29609227, 0x6eb081f1, 0x0954c4e1, 0x9d114db9, - 0x542acfa9, 0xb3e6bd7b, 0x0742d917, 0xe9f3ffa7, - 0x54581edb, 0xf2480f45, 0x0bb9288f, 0xef1affc7, - 0x85fa0ca7, 0x3ccc14db, 0xe6baf34b, 0x343377f7, - 0x5ca19031, 0xe6d9293b, 0xf0a9f391, 0x5d2e980b, - 0xfc411073, 0xc3749363, 0xb892d829, 0x3549366b, - 0x629750ad, 0xb98294e5, 0x892d9483, 0xc235baf3, - 0x3d2402a3, 0x6bdef3c9, 0xbec333cd, 0x40c9520f -}; - -//------------------------------------------------------------------------ -// End of shared data layout -//------------------------------------------------------------------------ - -//------------------------------------------------------------------------ -// Shared data accessors -//------------------------------------------------------------------------ - -unsigned GetPrime ( unsigned seed ) { - return Primes[seed%(sizeof(Primes)/sizeof(Primes[0]))]; -} -#endif //__TBB_OLD_PRIMES_RNG - -//------------------------------------------------------------------------ -// __TBB_InitOnce -//------------------------------------------------------------------------ - -void __TBB_InitOnce::add_ref() { - if( ++count==1 ) - governor::acquire_resources(); -} - -void __TBB_InitOnce::remove_ref() { - int k = --count; - __TBB_ASSERT(k>=0,"removed __TBB_InitOnce ref that was not added?"); - if( k==0 ) { - governor::release_resources(); - ITT_FINI_ITTLIB(); - } -} - -//------------------------------------------------------------------------ -// One-time Initializations -//------------------------------------------------------------------------ - -//! Defined in cache_aligned_allocator.cpp -void initialize_cache_aligned_allocator(); - -//! Defined in scheduler.cpp -void Scheduler_OneTimeInitialization ( bool itt_present ); - -#if DO_ITT_NOTIFY - -#if __TBB_ITT_STRUCTURE_API - -static __itt_domain *fgt_domain = NULL; - -struct resource_string { - const char *str; - __itt_string_handle *itt_str_handle; -}; - -// -// populate resource strings -// -#define TBB_STRING_RESOURCE( index_name, str ) { str, NULL }, -static resource_string strings_for_itt[] = { - #include "tbb/internal/_tbb_strings.h" - { "num_resource_strings", NULL } -}; -#undef TBB_STRING_RESOURCE - -static __itt_string_handle *ITT_get_string_handle(int idx) { - __TBB_ASSERT(idx >= 0, NULL); - return idx < NUM_STRINGS ? strings_for_itt[idx].itt_str_handle : NULL; -} - -static void ITT_init_domains() { - fgt_domain = __itt_domain_create( _T("tbb.flow") ); - fgt_domain->flags = 1; -} - -static void ITT_init_strings() { - for ( int i = 0; i < NUM_STRINGS; ++i ) { -#if _WIN32||_WIN64 - strings_for_itt[i].itt_str_handle = __itt_string_handle_createA( strings_for_itt[i].str ); -#else - strings_for_itt[i].itt_str_handle = __itt_string_handle_create( strings_for_itt[i].str ); -#endif - } -} - -static void ITT_init() { - ITT_init_domains(); - ITT_init_strings(); -} - -#endif // __TBB_ITT_STRUCTURE_API - -/** Thread-unsafe lazy one-time initialization of tools interop. - Used by both dummy handlers and general TBB one-time initialization routine. **/ -void ITT_DoUnsafeOneTimeInitialization () { - if ( !ITT_InitializationDone ) { - ITT_Present = (__TBB_load_ittnotify()!=0); -#if __TBB_ITT_STRUCTURE_API - if (ITT_Present) ITT_init(); -#endif - ITT_InitializationDone = true; - ITT_SYNC_CREATE(&market::theMarketMutex, SyncType_GlobalLock, SyncObj_SchedulerInitialization); - } -} - -/** Thread-safe lazy one-time initialization of tools interop. - Used by dummy handlers only. **/ -extern "C" -void ITT_DoOneTimeInitialization() { - __TBB_InitOnce::lock(); - ITT_DoUnsafeOneTimeInitialization(); - __TBB_InitOnce::unlock(); -} -#endif /* DO_ITT_NOTIFY */ - -//! Performs thread-safe lazy one-time general TBB initialization. -void DoOneTimeInitializations() { - suppress_unused_warning(_pad); - __TBB_InitOnce::lock(); - // No fence required for load of InitializationDone, because we are inside a critical section. - if( !__TBB_InitOnce::InitializationDone ) { - __TBB_InitOnce::add_ref(); - if( GetBoolEnvironmentVariable("TBB_VERSION") ) - PrintVersion(); - bool itt_present = false; -#if DO_ITT_NOTIFY - ITT_DoUnsafeOneTimeInitialization(); - itt_present = ITT_Present; -#endif /* DO_ITT_NOTIFY */ - initialize_cache_aligned_allocator(); - governor::initialize_rml_factory(); - Scheduler_OneTimeInitialization( itt_present ); - // Force processor groups support detection - governor::default_num_threads(); - // Dump version data - governor::print_version_info(); - PrintExtraVersionInfo( "Tools support", itt_present ? "enabled" : "disabled" ); - __TBB_InitOnce::InitializationDone = true; - } - __TBB_InitOnce::unlock(); -} - -#if (_WIN32||_WIN64) && !__TBB_SOURCE_DIRECTLY_INCLUDED -//! Windows "DllMain" that handles startup and shutdown of dynamic library. -extern "C" bool WINAPI DllMain( HANDLE /*hinstDLL*/, DWORD reason, LPVOID /*lpvReserved*/ ) { - switch( reason ) { - case DLL_PROCESS_ATTACH: - __TBB_InitOnce::add_ref(); - break; - case DLL_PROCESS_DETACH: - __TBB_InitOnce::remove_ref(); - // It is assumed that InitializationDone is not set after DLL_PROCESS_DETACH, - // and thus no race on InitializationDone is possible. - if( __TBB_InitOnce::initialization_done() ) { - // Remove reference that we added in DoOneTimeInitializations. - __TBB_InitOnce::remove_ref(); - } - break; - case DLL_THREAD_DETACH: - governor::terminate_auto_initialized_scheduler(); - break; - } - return true; -} -#endif /* (_WIN32||_WIN64) && !__TBB_SOURCE_DIRECTLY_INCLUDED */ - -void itt_store_pointer_with_release_v3( void* dst, void* src ) { - ITT_NOTIFY(sync_releasing, dst); - __TBB_store_with_release(*static_cast<void**>(dst),src); -} - -void* itt_load_pointer_with_acquire_v3( const void* src ) { - void* result = __TBB_load_with_acquire(*static_cast<void*const*>(src)); - ITT_NOTIFY(sync_acquired, const_cast<void*>(src)); - return result; -} - -#if DO_ITT_NOTIFY -void call_itt_notify_v5(int t, void *ptr) { - switch (t) { - case 0: ITT_NOTIFY(sync_prepare, ptr); break; - case 1: ITT_NOTIFY(sync_cancel, ptr); break; - case 2: ITT_NOTIFY(sync_acquired, ptr); break; - case 3: ITT_NOTIFY(sync_releasing, ptr); break; - } -} -#else -void call_itt_notify_v5(int /*t*/, void* /*ptr*/) {} -#endif - -#if __TBB_ITT_STRUCTURE_API - -#if DO_ITT_NOTIFY - -const __itt_id itt_null_id = {0, 0, 0}; - -static inline __itt_domain* get_itt_domain( itt_domain_enum idx ) { - return ( idx == ITT_DOMAIN_FLOW ) ? fgt_domain : NULL; -} - -static inline void itt_id_make(__itt_id *id, void* addr, unsigned long long extra) { - *id = __itt_id_make(addr, extra); -} - -static inline void itt_id_create(const __itt_domain *domain, __itt_id id) { - ITTNOTIFY_VOID_D1(id_create, domain, id); -} - -void itt_make_task_group_v7( itt_domain_enum domain, void *group, unsigned long long group_extra, - void *parent, unsigned long long parent_extra, string_index name_index ) { - if ( __itt_domain *d = get_itt_domain( domain ) ) { - __itt_id group_id = itt_null_id; - __itt_id parent_id = itt_null_id; - itt_id_make( &group_id, group, group_extra ); - itt_id_create( d, group_id ); - if ( parent ) { - itt_id_make( &parent_id, parent, parent_extra ); - } - __itt_string_handle *n = ITT_get_string_handle(name_index); - ITTNOTIFY_VOID_D3(task_group, d, group_id, parent_id, n); - } -} - -void itt_metadata_str_add_v7( itt_domain_enum domain, void *addr, unsigned long long addr_extra, - string_index key, const char *value ) { - if ( __itt_domain *d = get_itt_domain( domain ) ) { - __itt_id id = itt_null_id; - itt_id_make( &id, addr, addr_extra ); - __itt_string_handle *k = ITT_get_string_handle(key); - size_t value_length = strlen( value ); -#if _WIN32||_WIN64 - ITTNOTIFY_VOID_D4(metadata_str_addA, d, id, k, value, value_length); -#else - ITTNOTIFY_VOID_D4(metadata_str_add, d, id, k, value, value_length); -#endif - } -} - -void itt_relation_add_v7( itt_domain_enum domain, void *addr0, unsigned long long addr0_extra, - itt_relation relation, void *addr1, unsigned long long addr1_extra ) { - if ( __itt_domain *d = get_itt_domain( domain ) ) { - __itt_id id0 = itt_null_id; - __itt_id id1 = itt_null_id; - itt_id_make( &id0, addr0, addr0_extra ); - itt_id_make( &id1, addr1, addr1_extra ); - ITTNOTIFY_VOID_D3(relation_add, d, id0, (__itt_relation)relation, id1); - } -} - -void itt_task_begin_v7( itt_domain_enum domain, void *task, unsigned long long task_extra, - void *parent, unsigned long long parent_extra, string_index name_index ) { - if ( __itt_domain *d = get_itt_domain( domain ) ) { - __itt_id task_id = itt_null_id; - __itt_id parent_id = itt_null_id; - itt_id_make( &task_id, task, task_extra ); - if ( parent ) { - itt_id_make( &parent_id, parent, parent_extra ); - } - __itt_string_handle *n = ITT_get_string_handle(name_index); - ITTNOTIFY_VOID_D3(task_begin, d, task_id, parent_id, n ); - } -} - -void itt_task_end_v7( itt_domain_enum domain ) { - if ( __itt_domain *d = get_itt_domain( domain ) ) { - ITTNOTIFY_VOID_D0(task_end, d); - } -} - -void itt_region_begin_v9( itt_domain_enum domain, void *region, unsigned long long region_extra, - void *parent, unsigned long long parent_extra, string_index /* name_index */ ) { - if ( __itt_domain *d = get_itt_domain( domain ) ) { - __itt_id region_id = itt_null_id; - __itt_id parent_id = itt_null_id; - itt_id_make( ®ion_id, region, region_extra ); - if ( parent ) { - itt_id_make( &parent_id, parent, parent_extra ); - } - ITTNOTIFY_VOID_D3(region_begin, d, region_id, parent_id, NULL ); - } -} - -void itt_region_end_v9( itt_domain_enum domain, void *region, unsigned long long region_extra ) { - if ( __itt_domain *d = get_itt_domain( domain ) ) { - __itt_id region_id = itt_null_id; - itt_id_make( ®ion_id, region, region_extra ); - ITTNOTIFY_VOID_D1( region_end, d, region_id ); - } -} - -#else // DO_ITT_NOTIFY - -void itt_make_task_group_v7( itt_domain_enum domain, void *group, unsigned long long group_extra, - void *parent, unsigned long long parent_extra, string_index name_index ) { } - -void itt_metadata_str_add_v7( itt_domain_enum domain, void *addr, unsigned long long addr_extra, - string_index key, const char *value ) { } - -void itt_relation_add_v7( itt_domain_enum domain, void *addr0, unsigned long long addr0_extra, - itt_relation relation, void *addr1, unsigned long long addr1_extra ) { } - -void itt_task_begin_v7( itt_domain_enum domain, void *task, unsigned long long task_extra, - void * /*parent*/, unsigned long long /* parent_extra */, string_index /* name_index */ ) { } - -void itt_task_end_v7( itt_domain_enum domain ) { } - -void itt_region_begin_v9( itt_domain_enum domain, void *region, unsigned long long region_extra, - void *parent, unsigned long long parent_extra, string_index /* name_index */ ) { } - -void itt_region_end_v9( itt_domain_enum domain, void *region, unsigned long long region_extra ) { } - -#endif // DO_ITT_NOTIFY - -#endif // __TBB_ITT_STRUCTURE_API - -void* itt_load_pointer_v3( const void* src ) { - //TODO: replace this with __TBB_load_relaxed - void* result = *static_cast<void*const*>(src); - return result; -} - -void itt_set_sync_name_v3( void* obj, const tchar* name) { - ITT_SYNC_RENAME(obj, name); - suppress_unused_warning(obj, name); -} - - -class control_storage { - friend class tbb::interface9::global_control; -protected: - size_t my_active_value; - atomic<global_control*> my_head; - spin_mutex my_list_mutex; - - virtual size_t default_value() const = 0; - virtual void apply_active() const {} - virtual bool is_first_arg_preferred(size_t a, size_t b) const { - return a>b; // prefer max by default - } - virtual size_t active_value() const { - return my_head? my_active_value : default_value(); - } -}; - -class allowed_parallelism_control : public padded<control_storage> { - virtual size_t default_value() const __TBB_override { - return max(1U, governor::default_num_threads()); - } - virtual bool is_first_arg_preferred(size_t a, size_t b) const __TBB_override { - return a<b; // prefer min allowed parallelism - } - virtual void apply_active() const __TBB_override { - __TBB_ASSERT( my_active_value>=1, NULL ); - // -1 to take master into account - market::set_active_num_workers( my_active_value-1 ); - } - virtual size_t active_value() const __TBB_override { -/* Reading of my_active_value is not synchronized with possible updating - of my_head by other thread. It's ok, as value of my_active_value became - not invalid, just obsolete. */ - if (!my_head) - return default_value(); - // non-zero, if market is active - const size_t workers = market::max_num_workers(); - // We can't exceed market's maximal number of workers. - // +1 to take master into account - return workers? min(workers+1, my_active_value): my_active_value; - } -public: - size_t active_value_if_present() const { - return my_head? my_active_value : 0; - } -}; - -class stack_size_control : public padded<control_storage> { - virtual size_t default_value() const __TBB_override { - return tbb::internal::ThreadStackSize; - } - virtual void apply_active() const __TBB_override { -#if __TBB_WIN8UI_SUPPORT - __TBB_ASSERT( false, "For Windows Store* apps we must not set stack size" ); -#endif - } -}; - -static allowed_parallelism_control allowed_parallelism_ctl; -static stack_size_control stack_size_ctl; - -static control_storage *controls[] = {&allowed_parallelism_ctl, &stack_size_ctl}; - -unsigned market::app_parallelism_limit() { - return allowed_parallelism_ctl.active_value_if_present(); -} - -} // namespace internal - -namespace interface9 { - -using namespace internal; -using namespace tbb::internal; - -void global_control::internal_create() { - __TBB_ASSERT_RELEASE( my_param < global_control::parameter_max, NULL ); - control_storage *const c = controls[my_param]; - - spin_mutex::scoped_lock lock(c->my_list_mutex); - if (!c->my_head || c->is_first_arg_preferred(my_value, c->my_active_value)) { - c->my_active_value = my_value; - // to guarantee that apply_active() is called with current active value, - // calls it here and in internal_destroy() under my_list_mutex - c->apply_active(); - } - my_next = c->my_head; - // publish my_head, at this point my_active_value must be valid - c->my_head = this; -} - -void global_control::internal_destroy() { - global_control *prev = 0; - - __TBB_ASSERT_RELEASE( my_param < global_control::parameter_max, NULL ); - control_storage *const c = controls[my_param]; - __TBB_ASSERT( c->my_head, NULL ); - - // Concurrent reading and changing global parameter is possible. - // In this case, my_active_value may not match current state of parameters. - // This is OK because: - // 1) my_active_value is either current or previous - // 2) my_active_value is current on internal_destroy leave - spin_mutex::scoped_lock lock(c->my_list_mutex); - size_t new_active = (size_t)-1, old_active = c->my_active_value; - - if ( c->my_head != this ) - new_active = c->my_head->my_value; - else if ( c->my_head->my_next ) - new_active = c->my_head->my_next->my_value; - // if there is only one element, new_active will be set later - for ( global_control *curr = c->my_head; curr; prev = curr, curr = curr->my_next ) - if ( curr == this ) { - if ( prev ) - prev->my_next = my_next; - else - c->my_head = my_next; - } else - if (c->is_first_arg_preferred(curr->my_value, new_active)) - new_active = curr->my_value; - - if ( !c->my_head ) { - __TBB_ASSERT( new_active==(size_t)-1, NULL ); - new_active = c->default_value(); - } - if ( new_active != old_active ) { - c->my_active_value = new_active; - c->apply_active(); - } -} - -size_t global_control::active_value( int param ) { - __TBB_ASSERT_RELEASE( param < global_control::parameter_max, NULL ); - return controls[param]->active_value(); -} - -} // tbb::interface9 -} // namespace tbb diff --git a/src/tbb/src/tbb/tbb_misc.cpp b/src/tbb/src/tbb/tbb_misc.cpp deleted file mode 100644 index 5e389fec9..000000000 --- a/src/tbb/src/tbb/tbb_misc.cpp +++ /dev/null @@ -1,321 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -// Source file for miscellaneous entities that are infrequently referenced by -// an executing program. - -#include "tbb/tbb_stddef.h" -#include "tbb_assert_impl.h" // Out-of-line TBB assertion handling routines are instantiated here. -#include "tbb/tbb_exception.h" -#include "tbb/tbb_machine.h" -#include "tbb_misc.h" -#include "tbb_version.h" - -#include <cstdio> -#include <cstdlib> -#include <stdexcept> -#include <cstring> - -#if _WIN32||_WIN64 -#include "tbb/machine/windows_api.h" -#endif - -#define __TBB_STD_RETHROW_EXCEPTION_POSSIBLY_BROKEN \ - (__GLIBCXX__ && __TBB_GLIBCXX_VERSION>=40700 && __TBB_GLIBCXX_VERSION<60000 \ - && TBB_USE_EXCEPTIONS && !TBB_USE_CAPTURED_EXCEPTION) - -#if __TBB_STD_RETHROW_EXCEPTION_POSSIBLY_BROKEN -// GCC ABI declarations necessary for a workaround -#include <cxxabi.h> -#endif - -using namespace std; - -namespace tbb { - -const char* bad_last_alloc::what() const throw() { return "bad allocation in previous or concurrent attempt"; } -const char* improper_lock::what() const throw() { return "attempted recursive lock on critical section or non-recursive mutex"; } -const char* user_abort::what() const throw() { return "User-initiated abort has terminated this operation"; } -const char* invalid_multiple_scheduling::what() const throw() { return "The same task_handle object cannot be executed more than once"; } -const char* missing_wait::what() const throw() { return "wait() was not called on the structured_task_group"; } - -namespace internal { - -#if TBB_USE_EXCEPTIONS - #define DO_THROW(exc, init_args) throw exc init_args; -#else /* !TBB_USE_EXCEPTIONS */ - #define PRINT_ERROR_AND_ABORT(exc_name, msg) \ - fprintf (stderr, "Exception %s with message %s would've been thrown, " \ - "if exception handling were not disabled. Aborting.\n", exc_name, msg); \ - fflush(stderr); \ - abort(); - #define DO_THROW(exc, init_args) PRINT_ERROR_AND_ABORT(#exc, #init_args) -#endif /* !TBB_USE_EXCEPTIONS */ - - -/* The "what" should be fairly short, not more than about 128 characters. - Because we control all the call sites to handle_perror, it is pointless - to bullet-proof it for very long strings. - - Design note: ADR put this routine off to the side in tbb_misc.cpp instead of - Task.cpp because the throw generates a pathetic lot of code, and ADR wanted - this large chunk of code to be placed on a cold page. */ -void handle_perror( int error_code, const char* what ) { - char buf[256]; -#if _MSC_VER - #define snprintf _snprintf -#endif - int written = snprintf(buf, sizeof(buf), "%s: %s", what, strerror( error_code )); - // On overflow, the returned value exceeds sizeof(buf) (for GLIBC) or is negative (for MSVC). - __TBB_ASSERT_EX( written>0 && written<(int)sizeof(buf), "Error description is too long" ); - // Ensure that buffer ends in terminator. - buf[sizeof(buf)-1] = 0; -#if TBB_USE_EXCEPTIONS - throw runtime_error(buf); -#else - PRINT_ERROR_AND_ABORT( "runtime_error", buf); -#endif /* !TBB_USE_EXCEPTIONS */ -} - -#if _WIN32||_WIN64 -void handle_win_error( int error_code ) { - char buf[512]; -#if !__TBB_WIN8UI_SUPPORT - FormatMessageA( FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, - NULL, error_code, 0, buf, sizeof(buf), NULL ); -#else -//TODO: update with right replacement for FormatMessageA - sprintf_s((char*)&buf, 512, "error code %d", error_code); -#endif -#if TBB_USE_EXCEPTIONS - throw runtime_error(buf); -#else - PRINT_ERROR_AND_ABORT( "runtime_error", buf); -#endif /* !TBB_USE_EXCEPTIONS */ -} -#endif // _WIN32||_WIN64 - -void throw_bad_last_alloc_exception_v4() { - throw_exception_v4(eid_bad_last_alloc); -} - -void throw_exception_v4 ( exception_id eid ) { - __TBB_ASSERT ( eid > 0 && eid < eid_max, "Unknown exception ID" ); - switch ( eid ) { - case eid_bad_alloc: DO_THROW( bad_alloc, () ); - case eid_bad_last_alloc: DO_THROW( bad_last_alloc, () ); - case eid_nonpositive_step: DO_THROW( invalid_argument, ("Step must be positive") ); - case eid_out_of_range: DO_THROW( out_of_range, ("Index out of requested size range") ); - case eid_segment_range_error: DO_THROW( range_error, ("Index out of allocated segment slots") ); - case eid_index_range_error: DO_THROW( range_error, ("Index is not allocated") ); - case eid_missing_wait: DO_THROW( missing_wait, () ); - case eid_invalid_multiple_scheduling: DO_THROW( invalid_multiple_scheduling, () ); - case eid_improper_lock: DO_THROW( improper_lock, () ); - case eid_possible_deadlock: DO_THROW( runtime_error, ("Resource deadlock would occur") ); - case eid_operation_not_permitted: DO_THROW( runtime_error, ("Operation not permitted") ); - case eid_condvar_wait_failed: DO_THROW( runtime_error, ("Wait on condition variable failed") ); - case eid_invalid_load_factor: DO_THROW( out_of_range, ("Invalid hash load factor") ); - case eid_reserved: DO_THROW( out_of_range, ("[backward compatibility] Invalid number of buckets") ); - case eid_invalid_swap: DO_THROW( invalid_argument, ("swap() is invalid on non-equal allocators") ); - case eid_reservation_length_error: DO_THROW( length_error, ("reservation size exceeds permitted max size") ); - case eid_invalid_key: DO_THROW( out_of_range, ("invalid key") ); - case eid_user_abort: DO_THROW( user_abort, () ); - case eid_bad_tagged_msg_cast: DO_THROW( runtime_error, ("Illegal tagged_msg cast") ); -#if __TBB_SUPPORTS_WORKERS_WAITING_IN_TERMINATE - case eid_blocking_thread_join_impossible: DO_THROW( runtime_error, ("Blocking terminate failed") ); -#endif - default: break; - } -#if !TBB_USE_EXCEPTIONS && __APPLE__ - out_of_range e1(""); - length_error e2(""); - range_error e3(""); - invalid_argument e4(""); -#endif /* !TBB_USE_EXCEPTIONS && __APPLE__ */ -} - -#if __TBB_STD_RETHROW_EXCEPTION_POSSIBLY_BROKEN -// Runtime detection and workaround for the GCC bug 62258. -// The problem is that std::rethrow_exception() does not increment a counter -// of active exceptions, causing std::uncaught_exception() to return a wrong value. -// The code is created after, and roughly reflects, the workaround -// at https://gcc.gnu.org/bugzilla/attachment.cgi?id=34683 - -void fix_broken_rethrow() { - struct gcc_eh_data { - void * caughtExceptions; - unsigned int uncaughtExceptions; - }; - gcc_eh_data* eh_data = punned_cast<gcc_eh_data*>( abi::__cxa_get_globals() ); - ++eh_data->uncaughtExceptions; -} - -bool gcc_rethrow_exception_broken() { - bool is_broken; - __TBB_ASSERT( !std::uncaught_exception(), - "gcc_rethrow_exception_broken() must not be called when an exception is active" ); - try { - // Throw, catch, and rethrow an exception - try { - throw __TBB_GLIBCXX_VERSION; - } catch(...) { - std::rethrow_exception( std::current_exception() ); - } - } catch(...) { - // Check the bug presence - is_broken = std::uncaught_exception(); - } - if( is_broken ) fix_broken_rethrow(); - __TBB_ASSERT( !std::uncaught_exception(), NULL ); - return is_broken; -} -#else -void fix_broken_rethrow() {} -bool gcc_rethrow_exception_broken() { return false; } -#endif /* __TBB_STD_RETHROW_EXCEPTION_POSSIBLY_BROKEN */ - -#if __TBB_WIN8UI_SUPPORT -bool GetBoolEnvironmentVariable( const char * ) { return false;} -#else /* __TBB_WIN8UI_SUPPORT */ -bool GetBoolEnvironmentVariable( const char * name ) { - if( const char* s = getenv(name) ) - return strcmp(s,"0") != 0; - return false; -} -#endif /* __TBB_WIN8UI_SUPPORT */ - -/** The leading "\0" is here so that applying "strings" to the binary delivers a clean result. */ -static const char VersionString[] = "\0" TBB_VERSION_STRINGS; - -static bool PrintVersionFlag = false; - -void PrintVersion() { - PrintVersionFlag = true; - fputs(VersionString+1,stderr); -} - -void PrintExtraVersionInfo( const char* category, const char* format, ... ) { - if( PrintVersionFlag ) { - char str[1024]; memset(str, 0, 1024); - va_list args; va_start(args, format); - // Note: correct vsnprintf definition obtained from tbb_assert_impl.h - vsnprintf( str, 1024-1, format, args); - va_end(args); - fprintf(stderr, "TBB: %s\t%s\n", category, str ); - } -} - -void PrintRMLVersionInfo( void* arg, const char* server_info ) { - PrintExtraVersionInfo( server_info, (const char *)arg ); -} - -//! check for transaction support. -#if _MSC_VER -#include <intrin.h> // for __cpuid -#endif -bool cpu_has_speculation() { -#if __TBB_TSX_AVAILABLE -#if (__INTEL_COMPILER || __GNUC__ || _MSC_VER || __SUNPRO_CC) - bool result = false; - const int rtm_ebx_mask = 1<<11; -#if _MSC_VER - int info[4] = {0,0,0,0}; - const int reg_ebx = 1; - __cpuidex(info, 7, 0); - result = (info[reg_ebx] & rtm_ebx_mask)!=0; -#elif __GNUC__ || __SUNPRO_CC - int32_t reg_ebx = 0; - int32_t reg_eax = 7; - int32_t reg_ecx = 0; - __asm__ __volatile__ ( "movl %%ebx, %%esi\n" - "cpuid\n" - "movl %%ebx, %0\n" - "movl %%esi, %%ebx\n" - : "=a"(reg_ebx) : "0" (reg_eax), "c" (reg_ecx) : "esi", -#if __TBB_x86_64 - "ebx", -#endif - "edx" - ); - result = (reg_ebx & rtm_ebx_mask)!=0 ; -#endif - return result; -#else - #error Speculation detection not enabled for compiler -#endif /* __INTEL_COMPILER || __GNUC__ || _MSC_VER */ -#else /* __TBB_TSX_AVAILABLE */ - return false; -#endif /* __TBB_TSX_AVAILABLE */ -} - -} // namespace internal - -extern "C" int TBB_runtime_interface_version() { - return TBB_INTERFACE_VERSION; -} - -} // namespace tbb - -#if !__TBB_RML_STATIC -#if __TBB_x86_32 - -#include "tbb/atomic.h" - -// in MSVC environment, int64_t defined in tbb::internal namespace only (see tbb_stddef.h) -#if _MSC_VER -using tbb::internal::int64_t; -#endif - -//! Warn about 8-byte store that crosses a cache line. -extern "C" void __TBB_machine_store8_slow_perf_warning( volatile void *ptr ) { - // Report run-time warning unless we have already recently reported warning for that address. - const unsigned n = 4; - static tbb::atomic<void*> cache[n]; - static tbb::atomic<unsigned> k; - for( unsigned i=0; i<n; ++i ) - if( ptr==cache[i] ) - goto done; - cache[(k++)%n] = const_cast<void*>(ptr); - tbb::internal::runtime_warning( "atomic store on misaligned 8-byte location %p is slow", ptr ); -done:; -} - -//! Handle 8-byte store that crosses a cache line. -extern "C" void __TBB_machine_store8_slow( volatile void *ptr, int64_t value ) { - for( tbb::internal::atomic_backoff b;;b.pause() ) { - int64_t tmp = *(int64_t*)ptr; - if( __TBB_machine_cmpswp8(ptr,value,tmp)==tmp ) - break; - } -} - -#endif /* __TBB_x86_32 */ -#endif /* !__TBB_RML_STATIC */ - -#if __TBB_ipf -/* It was found that on IA-64 architecture inlining of __TBB_machine_lockbyte leads - to serious performance regression with ICC. So keep it out-of-line. - */ -extern "C" intptr_t __TBB_machine_lockbyte( volatile unsigned char& flag ) { - tbb::internal::atomic_backoff backoff; - while( !__TBB_TryLockByte(flag) ) backoff.pause(); - return 0; -} -#endif diff --git a/src/tbb/src/tbb/tbb_statistics.cpp b/src/tbb/src/tbb/tbb_statistics.cpp deleted file mode 100644 index 3656f9ab6..000000000 --- a/src/tbb/src/tbb/tbb_statistics.cpp +++ /dev/null @@ -1,187 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#include "tbb_statistics.h" - -#if __TBB_STATISTICS - -#include <climits> -#include <cstdarg> -#if __TBB_STATISTICS_STDOUT -#include <cstdio> -#endif - -#include "tbb/spin_mutex.h" - -namespace tbb { -namespace internal { - -//! Human readable titles of statistics groups defined by statistics_groups enum. -/** The order of this vector elements must correspond to the statistics_counters - structure layout. **/ -const char* StatGroupTitles[] = { - "task objects", "tasks executed", "stealing attempts", "task proxies", "arena", "market", "priority ops", "prio ops details" -}; - -//! Human readable titles of statistics elements defined by statistics_counters struct. -/** The order of this vector elements must correspond to the statistics_counters - structure layout (with NULLs interspersed to separate groups). **/ -const char* StatFieldTitles[] = { - /*task objects*/ "active", "freed", "big", NULL, - /*tasks executed*/ "total", "w/o spawn", NULL, - /*stealing attempts*/ "succeeded", "failed", "conflicts", "backoffs", NULL, - /*task proxies*/ "mailed", "revoked", "stolen", "bypassed", "ignored", NULL, - /*arena*/ "switches", "roundtrips", "avg.conc", "avg.allot", NULL, - /*market*/ "roundtrips", NULL, - /*priority ops*/ "ar.switch", "mkt.switch", "ar.reset", "ref.fixup", "avg.ar.pr", "avg.mkt.pr", NULL, - /*prio ops details*/ "winnows", "reloads", "orphaned", "winnowed", "reloaded", NULL -}; - -//! Class for logging statistics -/** There should be only one instance of this class. - Results are written to a file "statistics.txt" in tab-separated format. */ -class statistics_logger { -public: - statistics_logger () { - __TBB_ASSERT( sg_end - 1 == 1 << (sizeof(StatGroupTitles)/sizeof(*StatGroupTitles) - 1), NULL ); - - my_file = fopen("statistics.txt","w"); - if( !my_file ) - perror("fopen(\"statistics.txt\"\")"); - // Initialize groups dump layout info - group_start_field[0] = 0; - for ( size_t i = 0, j = 0; i < NumGroups; ++i, ++j ) { - __TBB_ASSERT( StatFieldTitles[j], "Empty group occurred" ); - while ( StatFieldTitles[j] ) - ++j; - group_start_field[i + 1] = j - i; // -i accounts for preceding NULL separators - } - __TBB_ASSERT( group_start_field[NumGroups] == statistics_counters::size(), - "Wrong number of elements in StatFieldTitles" ); - dump( "\n%-*s", IDColumnWidth, ""); - process_groups( &statistics_logger::print_group_title ); - dump( "%-*s", IDColumnWidth, "ID"); - process_groups( &statistics_logger::print_field_titles ); - } - - ~statistics_logger () { fclose(my_file); } - - void record( const statistics_counters& c, size_t id ) { - spin_mutex::scoped_lock lock(my_mutex); - counters_to_dump = &c; -#if __TBB_STATISTICS_TOTALS_ONLY - if ( id == arena_counters_total ) { - dump( "%-*s", IDColumnWidth, "Tot" ); - process_groups( &statistics_logger::print_field_values ); - } -#else /* !__TBB_STATISTICS_TOTALS_ONLY */ - const char* idString = NULL; - switch ( id ) { - case 0: - idString = "M"; break; - case workers_counters_total: - idString = "Wtot"; break; - case arena_counters_total: - idString = "Tot"; break; - default: - dump( "W%-*u", IDColumnWidth - 1, id ); - } - if ( idString ) - dump( "%-*s", IDColumnWidth, idString ); - process_groups( &statistics_logger::print_field_values ); -#endif /* !__TBB_STATISTICS_TOTALS_ONLY */ - } -private: - static const size_t IDColumnWidth = 5; - static const size_t StatisticsColumnWidth = 10; - static const size_t NumGroups = sizeof(StatGroupTitles)/sizeof(char*); - - //! File into which statistics are written. - FILE* my_file; - //! Mutex that serializes accesses to my_file - spin_mutex my_mutex; - //! Indices of the each group's first field in statistics_counters struct. - /** An extra element is used to track the total number of statistics fields. **/ - size_t group_start_field[NumGroups + 1]; - //! Currently processed set of counters. - const statistics_counters* counters_to_dump; - - static const size_t NumFields = sizeof(StatFieldTitles)/sizeof(*StatFieldTitles) - NumGroups; - bool averages_fields[NumFields]; - - void dump ( char const* fmt, ... ) { - va_list args; - if ( my_file ) { - va_start( args, fmt ); - vfprintf( my_file, fmt, args ); - va_end( args ); - } -#if __TBB_STATISTICS_STDOUT - va_start( args, fmt ); - vprintf( fmt, args ); - va_end( args ); -#endif - } - - void process_groups ( void (statistics_logger::*per_group_action)(size_t group_idx) ) { - for ( size_t i = 0, group_flag = 1; i < NumGroups; ++i, group_flag <<= 1 ) { - __TBB_ASSERT( group_flag < sg_end, "StatGroupTitles contents is incompatible with statistics_groups definition" ); - if ( __TBB_ActiveStatisticsGroups & group_flag ) - (this->*per_group_action)( i ); - } - dump( "\n" ); - } - - void print_group_title ( size_t group_idx ) { - dump( "%-*s", (group_start_field[group_idx + 1] - group_start_field[group_idx]) * (StatisticsColumnWidth + 1), - StatGroupTitles[group_idx] ); - } - - void print_field_titles ( size_t group_idx ) { - // +group_idx accounts for preceding NULL separators - size_t i = group_start_field[group_idx] + group_idx; - while ( StatFieldTitles[i] ) { - averages_fields[i - group_idx] = strncmp(StatFieldTitles[i], "avg.", 4) == 0; - dump( "%-*s ", StatisticsColumnWidth, StatFieldTitles[i++] ); - } - } - - void print_field_values ( size_t group_idx ) { - size_t begin = group_start_field[group_idx], - end = group_start_field[group_idx + 1]; - for ( size_t i = begin; i < end; ++i ) { - if ( averages_fields[i] ) - dump( "%-*.2f ", StatisticsColumnWidth, (double)counters_to_dump->field(i)/counters_to_dump->tasks_executed ); - else - dump( "%-*ld ", StatisticsColumnWidth, counters_to_dump->field(i) ); - } - } -}; // class statistics_logger - -static statistics_logger the_statistics; - -void dump_statistics ( const statistics_counters& c, size_t id ) { - the_statistics.record(c, id); -} - -} // namespace internal -} // namespace tbb - -#endif /* __TBB_STATISTICS */ diff --git a/src/tbb/src/tbb/tbb_statistics.h b/src/tbb/src/tbb/tbb_statistics.h deleted file mode 100644 index 11180a6ad..000000000 --- a/src/tbb/src/tbb/tbb_statistics.h +++ /dev/null @@ -1,240 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#ifndef _TBB_tbb_statistics_H -#define _TBB_tbb_statistics_H - -/** - This file defines parameters of the internal statistics collected by the TBB - library (currently by the task scheduler only). - - Statistics is accumulated separately in each thread and is dumped when - the scheduler instance associated with the given thread is destroyed. - For apps with multiple master threads or with the same master repeatedly - initializing and then deinitializing task scheduler this results in TBB - workers statistics getting inseparably mixed. - - Therefore statistics is accumulated in arena slots, and should be dumped - when arena is destroyed. This separates statistics collected for each - scheduler activity region in each master thread. - - With the current RML implementation (TBB 2.2, 3.0) to avoid complete loss of - statistics data during app shutdown (because of lazy workers deinitialization - logic) set __TBB_STATISTICS_EARLY_DUMP macro to write the statistics at the - moment a master thread deinitializes its scheduler. This may happen a little - earlier than the moment of arena destruction resulting in the following undesired - (though usually tolerable) effects: - - a few events related to unsuccessful stealing or thread pool activity may be lost, - - statistics may be substantially incomplete in case of FIFO tasks used in - the FAF mode. - - Macro __TBB_STATISTICS_STDOUT and global variable __TBB_ActiveStatisticsGroups - defined below can be used to configure the statistics output. - - To add new counter: - 1) Insert it into the appropriate group range in statistics_counters; - 2) Insert the corresponding field title into StatFieldTitles (preserving - relative order of the fields). - - To add new counters group: - 1) Insert new group bit flag into statistics_groups; - 2) Insert the new group title into StatGroupTitles (preserving - relative order of the groups). - 3) Add counter belonging to the new group as described above -**/ - -#include "tbb/tbb_stddef.h" - -#ifndef __TBB_STATISTICS -#define __TBB_STATISTICS 0 -#endif /* __TBB_STATISTICS */ - -#if __TBB_STATISTICS - -#include <string.h> // for memset - -//! Dump counters into stdout as well. -/** By default statistics counters are written to the file "statistics.txt" only. **/ -#define __TBB_STATISTICS_STDOUT 1 - -//! Dump only totals for all threads in the given arena -/** By default statistics counters for each arena slot are dumped separately, as - well as the subtotal for workers. **/ -#define __TBB_STATISTICS_TOTALS_ONLY 1 - -//! Dump statistics for an arena when its master completes -/** By default (when this macro is not set) the statistics is sent to output when - arena object is destroyed. But with the current lazy workers termination - logic default behavior may result in losing all statistics output. **/ -#define __TBB_STATISTICS_EARLY_DUMP 1 - -#define GATHER_STATISTIC(x) (x) - -namespace tbb { -namespace internal { - -//! Groups of statistics counters. -/** The order of enumerators must be the same as the order of the corresponding - field groups in the statistics_counters structure. **/ -enum statistics_groups { - sg_task_allocation = 0x01, - sg_task_execution = 0x02, - sg_stealing = 0x04, - sg_affinity = 0x08, - sg_arena = 0x10, - sg_market = 0x20, - sg_prio = 0x40, - sg_prio_ex = 0x80, - // List end marker. Insert new groups only before it. - sg_end -}; - -//! Groups of counters to output -const uintptr_t __TBB_ActiveStatisticsGroups = sg_task_execution | sg_stealing | sg_affinity | sg_arena | sg_market; - -//! A set of various statistics counters that are updated by the library on per thread basis. -/** All the fields must be of the same type (statistics_counters::counter_type). - This is necessary to allow reinterpreting this structure as an array. **/ -struct statistics_counters { - typedef long counter_type; - - // Group: sg_task_allocation - // Counters in this group can have negative values as the tasks migrate across - // threads while the associated counters are updated in the current thread only - // to avoid data races - - //! Number of tasks allocated and not yet destroyed - counter_type active_tasks; - //! Number of task corpses stored for future reuse - counter_type free_list_length; - //! Number of big tasks allocated during the run - /** To find total number of tasks malloc'd, compute (big_tasks+my_small_task_count) */ - counter_type big_tasks; - - // Group: sg_task_execution - - //! Number of tasks executed - counter_type tasks_executed; - //! Number of elided spawns - counter_type spawns_bypassed; - - // Group: sg_stealing - - //! Number of tasks successfully stolen - counter_type steals_committed; - //! Number of failed stealing attempts - counter_type steals_failed; - //! Number of failed attempts to lock victim's task pool - counter_type thieves_conflicts; - //! Number of times thief backed off because of the collision with the owner - counter_type thief_backoffs; - - // Group: sg_affinity - - //! Number of tasks received from mailbox - counter_type mails_received; - //! Number of affinitized tasks executed by the owner - /** Goes as "revoked" in statistics printout. **/ - counter_type proxies_executed; - //! Number of affinitized tasks intercepted by thieves - counter_type proxies_stolen; - //! Number of proxy bypasses by thieves during stealing - counter_type proxies_bypassed; - //! Number of affinitized tasks executed by the owner via scheduler bypass mechanism - counter_type affinity_ignored; - - // Group: sg_arena - - //! Number of times the state of arena switched between "full" and "empty" - counter_type gate_switches; - //! Number of times workers left an arena and returned into the market - counter_type arena_roundtrips; - // !Average concurrency level of this arena - counter_type avg_arena_concurrency; - //! Average assigned priority - counter_type avg_assigned_workers; - - // Group: sg_market - - //! Number of times workers left the market and returned into RML - counter_type market_roundtrips; - - // Group; sg_prio - - //! Number of arena priority switches - counter_type arena_prio_switches; - //! Number of market priority switches - counter_type market_prio_switches; - //! Number of arena priority switches - counter_type arena_prio_resets; - //! Number of reference priority source fixups to avoid deadlock - counter_type prio_ref_fixups; - //! Average arena priority - counter_type avg_arena_prio; - //! Average market priority - counter_type avg_market_prio; - - // Group; sg_prio_ex - - //! Number of times local task pools were winnowed - counter_type prio_winnowings; - //! Number of times secondary task pools were searched for top priority tasks - counter_type prio_reloads; - //! Number of times secondary task pools were abandoned by quitting workers - counter_type prio_orphanings; - //! Number of tasks offloaded into secondary task pools - counter_type prio_tasks_offloaded; - //! Number of tasks reloaded from secondary task pools - counter_type prio_tasks_reloaded; - - // Constructor and helpers - - statistics_counters() { reset(); } - - void reset () { memset( this, 0, sizeof(statistics_counters) ); } - - counter_type& field ( size_t index ) { return reinterpret_cast<counter_type*>(this)[index]; } - - const counter_type& field ( size_t index ) const { return reinterpret_cast<const counter_type*>(this)[index]; } - - static size_t size () { return sizeof(statistics_counters) / sizeof(counter_type); } - - const statistics_counters& operator += ( const statistics_counters& rhs ) { - for ( size_t i = 0; i < size(); ++i ) - field(i) += rhs.field(i); - return *this; - } -}; // statistics_counters - -static const size_t workers_counters_total = (size_t)-1; -static const size_t arena_counters_total = (size_t)-2; - -void dump_statistics ( const statistics_counters& c, size_t id ); - -} // namespace internal -} // namespace tbb - -#else /* !__TBB_STATISTICS */ - -#define GATHER_STATISTIC(x) ((void)0) - -#endif /* !__TBB_STATISTICS */ - -#endif /* _TBB_tbb_statistics_H */ diff --git a/src/tbb/src/tbb/tbb_thread.cpp b/src/tbb/src/tbb/tbb_thread.cpp deleted file mode 100644 index f64fb1f37..000000000 --- a/src/tbb/src/tbb/tbb_thread.cpp +++ /dev/null @@ -1,196 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#if _WIN32||_WIN64 -#include <process.h> // _beginthreadex() -#endif -#include <errno.h> -#include "tbb_misc.h" // handle_win_error() -#include "tbb/tbb_stddef.h" -#include "tbb/tbb_thread.h" -#include "tbb/tbb_allocator.h" -#include "tbb/global_control.h" // thread_stack_size -#include "governor.h" // default_num_threads() -#if __TBB_WIN8UI_SUPPORT -#include <thread> -#endif - -namespace tbb { -namespace internal { - -//! Allocate a closure -void* allocate_closure_v3( size_t size ) -{ - return allocate_via_handler_v3( size ); -} - -//! Free a closure allocated by allocate_closure_v3 -void free_closure_v3( void *ptr ) -{ - deallocate_via_handler_v3( ptr ); -} - -void tbb_thread_v3::join() -{ - if (!joinable()) - handle_perror( EINVAL, "tbb_thread::join" ); // Invalid argument - if (this_tbb_thread::get_id() == get_id()) - handle_perror( EDEADLK, "tbb_thread::join" ); // Resource deadlock avoided -#if _WIN32||_WIN64 -#if __TBB_WIN8UI_SUPPORT - std::thread* thread_tmp=(std::thread*)my_thread_id; - thread_tmp->join(); - delete thread_tmp; -#else // __TBB_WIN8UI_SUPPORT - DWORD status = WaitForSingleObjectEx( my_handle, INFINITE, FALSE ); - if ( status == WAIT_FAILED ) - handle_win_error( GetLastError() ); - BOOL close_stat = CloseHandle( my_handle ); - if ( close_stat == 0 ) - handle_win_error( GetLastError() ); - my_thread_id = 0; -#endif // __TBB_WIN8UI_SUPPORT -#else - int status = pthread_join( my_handle, NULL ); - if( status ) - handle_perror( status, "pthread_join" ); -#endif // _WIN32||_WIN64 - my_handle = 0; -} - -void tbb_thread_v3::detach() { - if (!joinable()) - handle_perror( EINVAL, "tbb_thread::detach" ); // Invalid argument -#if _WIN32||_WIN64 - BOOL status = CloseHandle( my_handle ); - if ( status == 0 ) - handle_win_error( GetLastError() ); - my_thread_id = 0; -#else - int status = pthread_detach( my_handle ); - if( status ) - handle_perror( status, "pthread_detach" ); -#endif // _WIN32||_WIN64 - my_handle = 0; -} - -void tbb_thread_v3::internal_start( __TBB_NATIVE_THREAD_ROUTINE_PTR(start_routine), - void* closure ) { -#if _WIN32||_WIN64 -#if __TBB_WIN8UI_SUPPORT - std::thread* thread_tmp=new std::thread(start_routine, closure); - my_handle = thread_tmp->native_handle(); -// TODO: to find out the way to find thread_id without GetThreadId and other -// desktop functions. -// Now tbb_thread does have its own thread_id that stores std::thread object - my_thread_id = (size_t)thread_tmp; -#else - unsigned thread_id; - // The return type of _beginthreadex is "uintptr_t" on new MS compilers, - // and 'unsigned long' on old MS compilers. uintptr_t works for both. - uintptr_t status = _beginthreadex( NULL, (unsigned)global_control::active_value(global_control::thread_stack_size), - start_routine, closure, 0, &thread_id ); - if( status==0 ) - handle_perror(errno,"__beginthreadex"); - else { - my_handle = (HANDLE)status; - my_thread_id = thread_id; - } -#endif -#else - pthread_t thread_handle; - int status; - pthread_attr_t stack_size; - status = pthread_attr_init( &stack_size ); - if( status ) - handle_perror( status, "pthread_attr_init" ); - status = pthread_attr_setstacksize( &stack_size, global_control::active_value(global_control::thread_stack_size) ); - if( status ) - handle_perror( status, "pthread_attr_setstacksize" ); - - status = pthread_create( &thread_handle, &stack_size, start_routine, closure ); - if( status ) - handle_perror( status, "pthread_create" ); - status = pthread_attr_destroy( &stack_size ); - if( status ) - handle_perror( status, "pthread_attr_destroy" ); - - my_handle = thread_handle; -#endif // _WIN32||_WIN64 -} - -unsigned tbb_thread_v3::hardware_concurrency() __TBB_NOEXCEPT(true) { - return governor::default_num_threads(); -} - -tbb_thread_v3::id thread_get_id_v3() { -#if _WIN32||_WIN64 - return tbb_thread_v3::id( GetCurrentThreadId() ); -#else - return tbb_thread_v3::id( pthread_self() ); -#endif // _WIN32||_WIN64 -} - -void move_v3( tbb_thread_v3& t1, tbb_thread_v3& t2 ) -{ - if (t1.joinable()) - t1.detach(); - t1.my_handle = t2.my_handle; - t2.my_handle = 0; -#if _WIN32||_WIN64 - t1.my_thread_id = t2.my_thread_id; - t2.my_thread_id = 0; -#endif // _WIN32||_WIN64 -} - -void thread_yield_v3() -{ - __TBB_Yield(); -} - -void thread_sleep_v3(const tick_count::interval_t &i) -{ -#if _WIN32||_WIN64 - tick_count t0 = tick_count::now(); - tick_count t1 = t0; - for(;;) { - double remainder = (i-(t1-t0)).seconds()*1e3; // milliseconds remaining to sleep - if( remainder<=0 ) break; - DWORD t = remainder>=INFINITE ? INFINITE-1 : DWORD(remainder); -#if !__TBB_WIN8UI_SUPPORT - Sleep( t ); -#else - std::chrono::milliseconds sleep_time( t ); - std::this_thread::sleep_for( sleep_time ); -#endif - t1 = tick_count::now(); - } -#else - struct timespec req; - double sec = i.seconds(); - - req.tv_sec = static_cast<long>(sec); - req.tv_nsec = static_cast<long>( (sec - req.tv_sec)*1e9 ); - nanosleep(&req, NULL); -#endif // _WIN32||_WIN64 -} - -} // internal -} // tbb diff --git a/src/tbb/src/tbb/tbb_version.h b/src/tbb/src/tbb/tbb_version.h deleted file mode 100644 index 89a1c5162..000000000 --- a/src/tbb/src/tbb/tbb_version.h +++ /dev/null @@ -1,119 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -// Please define version number in the file: -#include "tbb/tbb_stddef.h" - -// And don't touch anything below -#ifndef ENDL -#define ENDL "\n" -#endif -#include "version_string.ver" - -#ifndef __TBB_VERSION_STRINGS -#pragma message("Warning: version_string.ver isn't generated properly by version_info.sh script!") -// here is an example of macros value: -#define __TBB_VERSION_STRINGS \ -"TBB: BUILD_HOST\tUnknown\n" \ -"TBB: BUILD_ARCH\tUnknown\n" \ -"TBB: BUILD_OS\t\tUnknown\n" \ -"TBB: BUILD_CL\t\tUnknown\n" \ -"TBB: BUILD_COMPILER\tUnknown\n" \ -"TBB: BUILD_COMMAND\tUnknown\n" -#endif -#ifndef __TBB_DATETIME -#ifdef RC_INVOKED -#define __TBB_DATETIME "Unknown" -#else -#define __TBB_DATETIME __DATE__ __TIME__ -#endif -#endif - -#define __TBB_VERSION_NUMBER(N) #N ": VERSION\t\t" __TBB_STRING(TBB_VERSION_MAJOR.TBB_VERSION_MINOR) ENDL -#define __TBB_INTERFACE_VERSION_NUMBER(N) #N ": INTERFACE VERSION\t" __TBB_STRING(TBB_INTERFACE_VERSION) ENDL - -#define __TBB_VERSION_DATETIME(N) #N ": BUILD_DATE\t\t" __TBB_DATETIME ENDL -#ifndef TBB_USE_DEBUG - #define __TBB_VERSION_USE_DEBUG(N) #N ": TBB_USE_DEBUG\tundefined" ENDL -#elif TBB_USE_DEBUG==0 - #define __TBB_VERSION_USE_DEBUG(N) #N ": TBB_USE_DEBUG\t0" ENDL -#elif TBB_USE_DEBUG==1 - #define __TBB_VERSION_USE_DEBUG(N) #N ": TBB_USE_DEBUG\t1" ENDL -#elif TBB_USE_DEBUG==2 - #define __TBB_VERSION_USE_DEBUG(N) #N ": TBB_USE_DEBUG\t2" ENDL -#else - #error Unexpected value for TBB_USE_DEBUG -#endif - -/* Make __TBB_VERSION_USE_ASSERT and __TBB_VERSION_DO_NOTIFY empty for rc - * because rc from VS2005 crashed with fatal error RC10056 for too complex - * macros (for example, when __TBB_CPF_BUILD is enabled). - * All information is available in BUILD_COMMAND anyway. - */ - -#ifdef RC_INVOKED - #define __TBB_VERSION_USE_ASSERT(N) -#else // RC_INVOKED -#ifndef TBB_USE_ASSERT - #define __TBB_VERSION_USE_ASSERT(N) #N ": TBB_USE_ASSERT\tundefined" ENDL -#elif TBB_USE_ASSERT==0 - #define __TBB_VERSION_USE_ASSERT(N) #N ": TBB_USE_ASSERT\t0" ENDL -#elif TBB_USE_ASSERT==1 - #define __TBB_VERSION_USE_ASSERT(N) #N ": TBB_USE_ASSERT\t1" ENDL -#elif TBB_USE_ASSERT==2 - #define __TBB_VERSION_USE_ASSERT(N) #N ": TBB_USE_ASSERT\t2" ENDL -#else - #error Unexpected value for TBB_USE_ASSERT -#endif -#endif // RC_INVOKED - -#ifndef __TBB_CPF_BUILD - #define __TBB_VERSION_TBB_PREVIEW_BINARY(N) -#else - #define __TBB_VERSION_TBB_PREVIEW_BINARY(N) #N ": TBB_PREVIEW_BINARY\t1" ENDL -#endif - -#ifdef RC_INVOKED - #define __TBB_VERSION_DO_NOTIFY(N) -#else -#ifndef DO_ITT_NOTIFY - #define __TBB_VERSION_DO_NOTIFY(N) #N ": DO_ITT_NOTIFY\tundefined" ENDL -#elif DO_ITT_NOTIFY==1 - #define __TBB_VERSION_DO_NOTIFY(N) #N ": DO_ITT_NOTIFY\t1" ENDL -#elif DO_ITT_NOTIFY==0 - #define __TBB_VERSION_DO_NOTIFY(N) -#else - #error Unexpected value for DO_ITT_NOTIFY -#endif -#endif // RC_INVOKED - -#define TBB_VERSION_STRINGS_P(N) __TBB_VERSION_NUMBER(N) __TBB_INTERFACE_VERSION_NUMBER(N) __TBB_VERSION_DATETIME(N) __TBB_VERSION_STRINGS(N) __TBB_VERSION_USE_DEBUG(N) __TBB_VERSION_USE_ASSERT(N) __TBB_VERSION_TBB_PREVIEW_BINARY(N) __TBB_VERSION_DO_NOTIFY(N) - -#define TBB_VERSION_STRINGS TBB_VERSION_STRINGS_P(TBB) -#define TBBMALLOC_VERSION_STRINGS TBB_VERSION_STRINGS_P(TBBmalloc) - -// numbers -#ifndef __TBB_VERSION_YMD -#define __TBB_VERSION_YMD 0, 0 -#endif - -#define TBB_VERNUMBERS TBB_VERSION_MAJOR, TBB_VERSION_MINOR, __TBB_VERSION_YMD - -#define TBB_VERSION __TBB_STRING(TBB_VERNUMBERS) diff --git a/src/tbb/src/tbb/tcm.h b/src/tbb/src/tbb/tcm.h new file mode 100644 index 000000000..66ee18a2f --- /dev/null +++ b/src/tbb/src/tbb/tcm.h @@ -0,0 +1,174 @@ +/* + Copyright (c) 2023-2024 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef _TBB_tcm_H +#define _TBB_tcm_H + +#include <stdint.h> +#include <stdbool.h> + +#ifdef __cplusplus +extern "C" { +#endif + +// Support for the TCM API return value + +typedef enum _tcm_result_t { + TCM_RESULT_SUCCESS = 0x0, + TCM_RESULT_ERROR_INVALID_ARGUMENT = 0x78000004, + TCM_RESULT_ERROR_UNKNOWN = 0x7ffffffe +} tcm_result_t; + +// Support for permit states + +enum tcm_permit_states_t { + TCM_PERMIT_STATE_VOID, + TCM_PERMIT_STATE_INACTIVE, + TCM_PERMIT_STATE_PENDING, + TCM_PERMIT_STATE_IDLE, + TCM_PERMIT_STATE_ACTIVE +}; + +typedef uint8_t tcm_permit_state_t; + +// Support for permit flags + +typedef struct _tcm_permit_flags_t { + uint32_t stale : 1; + uint32_t rigid_concurrency : 1; + uint32_t exclusive : 1; + uint32_t request_as_inactive : 1; + uint32_t reserved : 28; +} tcm_permit_flags_t; + +typedef struct _tcm_callback_flags_t { + uint32_t new_concurrency : 1; + uint32_t new_state : 1; + uint32_t reserved : 30; +} tcm_callback_flags_t; + +// Support for cpu masks + +struct hwloc_bitmap_s; +typedef struct hwloc_bitmap_s* hwloc_bitmap_t; +typedef hwloc_bitmap_t tcm_cpu_mask_t; + +// Support for ids + +typedef uint64_t tcm_client_id_t; + +// Support for permits + +typedef struct _tcm_permit_t { + uint32_t* concurrencies; + tcm_cpu_mask_t* cpu_masks; + uint32_t size; + tcm_permit_state_t state; + tcm_permit_flags_t flags; +} tcm_permit_t; + +// Support for permit handle + +typedef struct tcm_permit_rep_t* tcm_permit_handle_t; + +// Support for constraints + +typedef int32_t tcm_numa_node_t; +typedef int32_t tcm_core_type_t; + +const int8_t tcm_automatic = -1; +const int8_t tcm_any = -2; + +#define TCM_PERMIT_REQUEST_CONSTRAINTS_INITIALIZER {tcm_automatic, tcm_automatic, NULL, \ + tcm_automatic, tcm_automatic, tcm_automatic} + +typedef struct _tcm_cpu_constraints_t { + int32_t min_concurrency; + int32_t max_concurrency; + tcm_cpu_mask_t mask; + tcm_numa_node_t numa_id; + tcm_core_type_t core_type_id; + int32_t threads_per_core; +} tcm_cpu_constraints_t; + +// Support for priorities + +enum tcm_request_priorities_t { + TCM_REQUEST_PRIORITY_LOW = (INT32_MAX / 4) * 1, + TCM_REQUEST_PRIORITY_NORMAL = (INT32_MAX / 4) * 2, + TCM_REQUEST_PRIORITY_HIGH = (INT32_MAX / 4) * 3 +}; + +typedef int32_t tcm_request_priority_t; + +// Support for requests + +#define TCM_PERMIT_REQUEST_INITIALIZER {tcm_automatic, tcm_automatic, \ + NULL, 0, TCM_REQUEST_PRIORITY_NORMAL, {}, {}} + +typedef struct _tcm_permit_request_t { + int32_t min_sw_threads; + int32_t max_sw_threads; + tcm_cpu_constraints_t* cpu_constraints; + uint32_t constraints_size; + tcm_request_priority_t priority; + tcm_permit_flags_t flags; + char reserved[4]; +} tcm_permit_request_t; + +// Support for client callback + +typedef tcm_result_t (*tcm_callback_t)(tcm_permit_handle_t p, void* callback_arg, tcm_callback_flags_t); + +#if _WIN32 + #define __TCM_EXPORT __declspec(dllexport) +#else + #define __TCM_EXPORT +#endif + + +__TCM_EXPORT tcm_result_t tcmConnect(tcm_callback_t callback, + tcm_client_id_t *client_id); +__TCM_EXPORT tcm_result_t tcmDisconnect(tcm_client_id_t client_id); + +__TCM_EXPORT tcm_result_t tcmRequestPermit(tcm_client_id_t client_id, + tcm_permit_request_t request, + void* callback_arg, + tcm_permit_handle_t* permit_handle, + tcm_permit_t* permit); + +__TCM_EXPORT tcm_result_t tcmGetPermitData(tcm_permit_handle_t permit_handle, + tcm_permit_t* permit); + +__TCM_EXPORT tcm_result_t tcmReleasePermit(tcm_permit_handle_t permit); + +__TCM_EXPORT tcm_result_t tcmIdlePermit(tcm_permit_handle_t permit_handle); + +__TCM_EXPORT tcm_result_t tcmDeactivatePermit(tcm_permit_handle_t permit_handle); + +__TCM_EXPORT tcm_result_t tcmActivatePermit(tcm_permit_handle_t permit_handle); + +__TCM_EXPORT tcm_result_t tcmRegisterThread(tcm_permit_handle_t permit_handle); + +__TCM_EXPORT tcm_result_t tcmUnregisterThread(); + +__TCM_EXPORT tcm_result_t tcmGetVersionInfo(char* buffer, uint32_t buffer_size); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif /* _TBB_tcm_H */ diff --git a/src/tbb/src/tbb/tcm_adaptor.cpp b/src/tbb/src/tbb/tcm_adaptor.cpp new file mode 100644 index 000000000..85ca125b4 --- /dev/null +++ b/src/tbb/src/tbb/tcm_adaptor.cpp @@ -0,0 +1,327 @@ +/* + Copyright (c) 2023-2024 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "oneapi/tbb/detail/_intrusive_list_node.h" +#include "oneapi/tbb/detail/_template_helpers.h" +#include "oneapi/tbb/task_arena.h" + +#include "pm_client.h" +#include "dynamic_link.h" +#include "misc.h" +#include "tcm.h" +#include "tcm_adaptor.h" + +#include <iostream> + +namespace tbb { +namespace detail { +namespace r1 { + +namespace { +#if __TBB_WEAK_SYMBOLS_PRESENT +#pragma weak tcmConnect +#pragma weak tcmDisconnect +#pragma weak tcmRequestPermit +#pragma weak tcmGetPermitData +#pragma weak tcmReleasePermit +#pragma weak tcmIdlePermit +#pragma weak tcmDeactivatePermit +#pragma weak tcmActivatePermit +#pragma weak tcmRegisterThread +#pragma weak tcmUnregisterThread +#pragma weak tcmGetVersionInfo +#endif /* __TBB_WEAK_SYMBOLS_PRESENT */ + +tcm_result_t(*tcm_connect)(tcm_callback_t callback, tcm_client_id_t* client_id){nullptr}; +tcm_result_t(*tcm_disconnect)(tcm_client_id_t client_id){ nullptr }; +tcm_result_t(*tcm_request_permit)(tcm_client_id_t client_id, tcm_permit_request_t request, + void* callback_arg, tcm_permit_handle_t* permit_handle, tcm_permit_t* permit){nullptr}; +tcm_result_t(*tcm_get_permit_data)(tcm_permit_handle_t permit_handle, tcm_permit_t* permit){nullptr}; +tcm_result_t(*tcm_release_permit)(tcm_permit_handle_t permit){nullptr}; +tcm_result_t(*tcm_idle_permit)(tcm_permit_handle_t permit_handle){nullptr}; +tcm_result_t(*tcm_deactivate_permit)(tcm_permit_handle_t permit_handle){nullptr}; +tcm_result_t(*tcm_activate_permit)(tcm_permit_handle_t permit_handle){nullptr}; +tcm_result_t(*tcm_register_thread)(tcm_permit_handle_t permit_handle){nullptr}; +tcm_result_t(*tcm_unregister_thread)(){nullptr}; +tcm_result_t (*tcm_get_version_info)(char* buffer, uint32_t buffer_size){nullptr}; + +static const dynamic_link_descriptor tcm_link_table[] = { + DLD(tcmConnect, tcm_connect), + DLD(tcmDisconnect, tcm_disconnect), + DLD(tcmRequestPermit, tcm_request_permit), + DLD(tcmGetPermitData, tcm_get_permit_data), + DLD(tcmReleasePermit, tcm_release_permit), + DLD(tcmIdlePermit, tcm_idle_permit), + DLD(tcmDeactivatePermit, tcm_deactivate_permit), + DLD(tcmActivatePermit, tcm_activate_permit), + DLD(tcmRegisterThread, tcm_register_thread), + DLD(tcmUnregisterThread, tcm_unregister_thread), + DLD(tcmGetVersionInfo, tcm_get_version_info) +}; + +#if TBB_USE_DEBUG +#define DEBUG_SUFFIX "_debug" +#else +#define DEBUG_SUFFIX +#endif /* TBB_USE_DEBUG */ + +#if _WIN32 || _WIN64 +#define LIBRARY_EXTENSION ".dll" +#define LIBRARY_PREFIX +#elif __unix__ +#define LIBRARY_EXTENSION ".so.1" +#define LIBRARY_PREFIX "lib" +#else +#define LIBRARY_EXTENSION +#define LIBRARY_PREFIX +#endif /* __unix__ */ + +#define TCMLIB_NAME LIBRARY_PREFIX "tcm" DEBUG_SUFFIX LIBRARY_EXTENSION + +static bool tcm_functions_loaded{ false }; +} + +class tcm_client : public pm_client { + using tcm_client_mutex_type = d1::mutex; +public: + tcm_client(tcm_adaptor& adaptor, arena& a) : pm_client(a), my_tcm_adaptor(adaptor) {} + + ~tcm_client() { + if (my_permit_handle) { + __TBB_ASSERT(tcm_release_permit, nullptr); + auto res = tcm_release_permit(my_permit_handle); + __TBB_ASSERT_EX(res == TCM_RESULT_SUCCESS, nullptr); + } + } + + int update_concurrency(uint32_t concurrency) { + return my_arena.update_concurrency(concurrency); + } + + unsigned priority_level() { + return my_arena.priority_level(); + } + + tcm_permit_request_t& permit_request() { + return my_permit_request; + } + + tcm_permit_handle_t& permit_handle() { + return my_permit_handle; + } + + void actualize_permit() { + __TBB_ASSERT(tcm_get_permit_data, nullptr); + int delta{}; + { + tcm_client_mutex_type::scoped_lock lock(my_permit_mutex); + + uint32_t new_concurrency{}; + tcm_permit_t new_permit{ &new_concurrency, nullptr, 1, TCM_PERMIT_STATE_VOID, {} }; + auto res = tcm_get_permit_data(my_permit_handle, &new_permit); + __TBB_ASSERT_EX(res == TCM_RESULT_SUCCESS, nullptr); + + // The permit has changed during the reading, so the callback will be invoked soon one more time and + // we can just skip this renegotiation iteration. + if (!new_permit.flags.stale) { + // If there is no other demand in TCM, the permit may still have granted concurrency but + // be in the deactivated state thus we enforce 0 allotment to preserve arena invariants. + delta = update_concurrency(new_permit.state != TCM_PERMIT_STATE_INACTIVE ? new_concurrency : 0); + } + } + if (delta) { + my_tcm_adaptor.notify_thread_request(delta); + } + } + + void request_permit(tcm_client_id_t client_id) { + __TBB_ASSERT(tcm_request_permit, nullptr); + + my_permit_request.max_sw_threads = max_workers(); + my_permit_request.min_sw_threads = my_permit_request.max_sw_threads == 0 ? 0 : min_workers(); + + if (my_permit_request.constraints_size > 0) { + my_permit_request.cpu_constraints->min_concurrency = my_permit_request.min_sw_threads; + my_permit_request.cpu_constraints->max_concurrency = my_permit_request.max_sw_threads; + } + + __TBB_ASSERT(my_permit_request.max_sw_threads >= my_permit_request.min_sw_threads, nullptr); + + tcm_result_t res = tcm_request_permit(client_id, my_permit_request, this, &my_permit_handle, nullptr); + __TBB_ASSERT_EX(res == TCM_RESULT_SUCCESS, nullptr); + } + + void deactivate_permit() { + __TBB_ASSERT(tcm_deactivate_permit, nullptr); + tcm_result_t res = tcm_deactivate_permit(my_permit_handle); + __TBB_ASSERT_EX(res == TCM_RESULT_SUCCESS, nullptr); + } + + void init(tcm_client_id_t client_id, d1::constraints& constraints) { + __TBB_ASSERT(tcm_request_permit, nullptr); + __TBB_ASSERT(tcm_deactivate_permit, nullptr); + + if (constraints.core_type != d1::task_arena::automatic || + constraints.numa_id != d1::task_arena::automatic || + constraints.max_threads_per_core != d1::task_arena::automatic) + { + my_permit_constraints.max_concurrency = constraints.max_concurrency; + my_permit_constraints.min_concurrency = 0; + my_permit_constraints.core_type_id = constraints.core_type; + my_permit_constraints.numa_id = constraints.numa_id; + my_permit_constraints.threads_per_core = constraints.max_threads_per_core; + + my_permit_request.cpu_constraints = &my_permit_constraints; + my_permit_request.constraints_size = 1; + } + + my_permit_request.min_sw_threads = 0; + my_permit_request.max_sw_threads = 0; + my_permit_request.flags.request_as_inactive = 1; + + tcm_result_t res = tcm_request_permit(client_id, my_permit_request, this, &my_permit_handle, nullptr); + __TBB_ASSERT_EX(res == TCM_RESULT_SUCCESS, nullptr); + + my_permit_request.flags.request_as_inactive = 0; + } + + void register_thread() override { + __TBB_ASSERT(tcm_register_thread, nullptr); + auto return_code = tcm_register_thread(my_permit_handle); + __TBB_ASSERT_EX(return_code == TCM_RESULT_SUCCESS, nullptr); + } + + void unregister_thread() override { + __TBB_ASSERT(tcm_unregister_thread, nullptr); + auto return_code = tcm_unregister_thread(); + __TBB_ASSERT_EX(return_code == TCM_RESULT_SUCCESS, nullptr); + } + +private: + tcm_cpu_constraints_t my_permit_constraints = TCM_PERMIT_REQUEST_CONSTRAINTS_INITIALIZER; + tcm_permit_request_t my_permit_request = TCM_PERMIT_REQUEST_INITIALIZER; + tcm_permit_handle_t my_permit_handle{}; + tcm_client_mutex_type my_permit_mutex; + tcm_adaptor& my_tcm_adaptor; +}; + +//------------------------------------------------------------------------ +// tcm_adaptor_impl +//------------------------------------------------------------------------ + +struct tcm_adaptor_impl { + using demand_mutex_type = d1::mutex; + demand_mutex_type my_demand_mutex; + tcm_client_id_t client_id{}; + + tcm_adaptor_impl(tcm_client_id_t id) : client_id(id) + {} +}; + +//------------------------------------------------------------------------ +// tcm_adaptor +//------------------------------------------------------------------------ + +tcm_result_t renegotiation_callback(tcm_permit_handle_t, void* client_ptr, tcm_callback_flags_t) { + __TBB_ASSERT(client_ptr, nullptr); + static_cast<tcm_client*>(client_ptr)->actualize_permit(); + return TCM_RESULT_SUCCESS; +} + +void tcm_adaptor::initialize() { + tcm_functions_loaded = dynamic_link(TCMLIB_NAME, tcm_link_table, /* tcm_link_table size = */ 11); +} + +bool tcm_adaptor::is_initialized() { + return tcm_functions_loaded; +} + +void tcm_adaptor::print_version() { + if (is_initialized()) { + __TBB_ASSERT(tcm_get_version_info, nullptr); + char buffer[1024]; + tcm_get_version_info(buffer, 1024); + std::fprintf(stderr, "%.*s", 1024, buffer); + } +} + +tcm_adaptor::tcm_adaptor() { + __TBB_ASSERT(tcm_connect, nullptr); + tcm_client_id_t client_id{}; + auto return_code = tcm_connect(renegotiation_callback, &client_id); + if (return_code == TCM_RESULT_SUCCESS) { + my_impl = make_cache_aligned_unique<tcm_adaptor_impl>(client_id); + } +} + +tcm_adaptor::~tcm_adaptor() { + if (my_impl) { + __TBB_ASSERT(tcm_disconnect, nullptr); + auto return_code = tcm_disconnect(my_impl->client_id); + __TBB_ASSERT_EX(return_code == TCM_RESULT_SUCCESS, nullptr); + my_impl = nullptr; + } +} + +bool tcm_adaptor::is_connected() { + return my_impl != nullptr; +} + +pm_client* tcm_adaptor::create_client(arena& a) { + return new (cache_aligned_allocate(sizeof(tcm_client))) tcm_client(*this, a); +} + +void tcm_adaptor::register_client(pm_client* c, d1::constraints& constraints) { + static_cast<tcm_client*>(c)->init(my_impl->client_id, constraints); +} + +void tcm_adaptor::unregister_and_destroy_client(pm_client& c) { + auto& client = static_cast<tcm_client&>(c); + + { + tcm_adaptor_impl::demand_mutex_type::scoped_lock lock(my_impl->my_demand_mutex); + client.~tcm_client(); + } + cache_aligned_deallocate(&client); +} + +void tcm_adaptor::set_active_num_workers(int) {} + + +void tcm_adaptor::adjust_demand(pm_client& c, int mandatory_delta, int workers_delta) { + __TBB_ASSERT(-1 <= mandatory_delta && mandatory_delta <= 1, nullptr); + + auto& client = static_cast<tcm_client&>(c); + { + tcm_adaptor_impl::demand_mutex_type::scoped_lock lock(my_impl->my_demand_mutex); + + // Update client's state + workers_delta = client.update_request(mandatory_delta, workers_delta); + if (workers_delta == 0) return; + + if (client.max_workers() == 0) { + client.deactivate_permit(); + } else { + client.request_permit(my_impl->client_id); + } + } + + client.actualize_permit(); +} + +} // namespace r1 +} // namespace detail +} // namespace tbb diff --git a/src/tbb/src/tbb/tcm_adaptor.h b/src/tbb/src/tbb/tcm_adaptor.h new file mode 100644 index 000000000..f9f4d5277 --- /dev/null +++ b/src/tbb/src/tbb/tcm_adaptor.h @@ -0,0 +1,63 @@ +/* + Copyright (c) 2023 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef _TBB_tcm_adaptor_H +#define _TBB_tcm_adaptor_H + +#include "scheduler_common.h" + +#include "permit_manager.h" +#include "pm_client.h" + +namespace tbb { +namespace detail { +namespace r1 { + +struct tcm_adaptor_impl; + +//------------------------------------------------------------------------ +// Class tcm_adaptor +//------------------------------------------------------------------------ + +class tcm_adaptor : public permit_manager { +public: + tcm_adaptor(); + ~tcm_adaptor(); + + pm_client* create_client(arena& a) override; + void register_client(pm_client* client, d1::constraints& constraints) override; + void unregister_and_destroy_client(pm_client& c) override; + + void set_active_num_workers(int soft_limit) override; + + void adjust_demand(pm_client& c, int mandatory_delta, int workers_delta) override; + + bool is_connected(); + + static void initialize(); + static bool is_initialized(); + static void print_version(); +private: + cache_aligned_unique_ptr<tcm_adaptor_impl> my_impl; + + friend class tcm_client; +}; // class tcm_adaptor + +} // namespace r1 +} // namespace detail +} // namespace tbb + +#endif /* _TBB_tcm_adaptor_H */ diff --git a/src/tbb/src/tbb/thread_control_monitor.h b/src/tbb/src/tbb/thread_control_monitor.h new file mode 100644 index 000000000..f9c3cacc9 --- /dev/null +++ b/src/tbb/src/tbb/thread_control_monitor.h @@ -0,0 +1,116 @@ +/* + Copyright (c) 2021-2023 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_thread_control_monitor_H +#define __TBB_thread_control_monitor_H + +#include "concurrent_monitor.h" +#include "scheduler_common.h" + +#include <atomic> + +namespace tbb { +namespace detail { +namespace r1 { + +struct market_context { + market_context() = default; + + market_context(std::uintptr_t first_addr, arena* a) : + my_uniq_addr(first_addr), my_arena_addr(a) + {} + + std::uintptr_t my_uniq_addr{0}; + arena* my_arena_addr{nullptr}; +}; + +#if __TBB_RESUMABLE_TASKS +class resume_node : public wait_node<market_context> { + using base_type = wait_node<market_context>; +public: + resume_node(market_context ctx, execution_data_ext& ed_ext, task_dispatcher& target) + : base_type(ctx), my_curr_dispatcher(ed_ext.task_disp), my_target_dispatcher(&target) + , my_suspend_point(my_curr_dispatcher->get_suspend_point()) + {} + + ~resume_node() override { + if (this->my_skipped_wakeup) { + spin_wait_until_eq(this->my_notify_calls, 1); + } + + poison_pointer(my_curr_dispatcher); + poison_pointer(my_target_dispatcher); + poison_pointer(my_suspend_point); + } + + void init() override { + base_type::init(); + } + + void wait() override { + my_curr_dispatcher->resume(*my_target_dispatcher); + __TBB_ASSERT(!this->my_is_in_list.load(std::memory_order_relaxed), "Still in the queue?"); + } + + void reset() override { + base_type::reset(); + spin_wait_until_eq(this->my_notify_calls, 1); + my_notify_calls.store(0, std::memory_order_relaxed); + } + + // notify is called (perhaps, concurrently) twice from: + // - concurrent_monitor::notify + // - post_resume_action::register_waiter + // The second notify is called after thread switches the stack + // (Because we can not call resume while the stack is occupied) + // We need calling resume only when both notifications are performed. + void notify() override { + if (++my_notify_calls == 2) { + r1::resume(my_suspend_point); + } + } + +private: + friend class thread_data; + friend struct suspend_point_type::resume_task; + task_dispatcher* my_curr_dispatcher; + task_dispatcher* my_target_dispatcher; + suspend_point_type* my_suspend_point; + std::atomic<int> my_notify_calls{0}; +}; +#endif // __TBB_RESUMABLE_TASKS + +class thread_control_monitor : public concurrent_monitor_base<market_context> { + using base_type = concurrent_monitor_base<market_context>; +public: + using base_type::base_type; + + ~thread_control_monitor() { + destroy(); + } + + /** per-thread descriptor for concurrent_monitor */ + using thread_context = sleep_node<market_context>; +#if __TBB_RESUMABLE_TASKS + using resume_context = resume_node; +#endif +}; + +} // namespace r1 +} // namespace detail +} // namespace tbb + +#endif // __TBB_thread_control_monitor_H diff --git a/src/tbb/src/tbb/thread_data.h b/src/tbb/src/tbb/thread_data.h new file mode 100644 index 000000000..422ec694e --- /dev/null +++ b/src/tbb/src/tbb/thread_data.h @@ -0,0 +1,262 @@ +/* + Copyright (c) 2020-2024 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_thread_data_H +#define __TBB_thread_data_H + +#include "oneapi/tbb/detail/_task.h" +#include "oneapi/tbb/task.h" + +#include "rml_base.h" // rml::job + +#include "scheduler_common.h" +#include "arena.h" +#include "concurrent_monitor.h" +#include "mailbox.h" +#include "misc.h" // FastRandom +#include "small_object_pool_impl.h" +#include "intrusive_list.h" + +#include <atomic> + +namespace tbb { +namespace detail { +namespace r1 { + +class task; +class arena_slot; +class task_group_context; +class task_dispatcher; +class thread_dispatcher_client; + +class context_list : public intrusive_list<d1::intrusive_list_node> { +public: + bool orphaned{false}; + + //! Last state propagation epoch known to this thread + /** Together with the_context_state_propagation_epoch constitute synchronization protocol + that keeps hot path of task group context construction destruction mostly + lock-free. + When local epoch equals the global one, the state of task group contexts + registered with this thread is consistent with that of the task group trees + they belong to. **/ + std::atomic<std::uintptr_t> epoch{}; + + //! Mutex protecting access to the list of task group contexts. + d1::mutex m_mutex{}; + + void destroy() { + this->~context_list(); + cache_aligned_deallocate(this); + } + + void remove(d1::intrusive_list_node& val) { + mutex::scoped_lock lock(m_mutex); + + intrusive_list<d1::intrusive_list_node>::remove(val); + + if (orphaned && empty()) { + lock.release(); + destroy(); + } + } + + void push_front(d1::intrusive_list_node& val) { + mutex::scoped_lock lock(m_mutex); + + intrusive_list<d1::intrusive_list_node>::push_front(val); + } + + void orphan() { + mutex::scoped_lock lock(m_mutex); + + orphaned = true; + if (empty()) { + lock.release(); + destroy(); + } + } +}; + +//------------------------------------------------------------------------ +// Thread Data +//------------------------------------------------------------------------ +class thread_data : public ::rml::job + , public d1::intrusive_list_node + , no_copy { +public: + thread_data(unsigned short index, bool is_worker) + : my_arena_index{ index } + , my_is_worker{ is_worker } + , my_is_registered { false } + , my_task_dispatcher{ nullptr } + , my_arena{ nullptr } + , my_last_client{ nullptr } + , my_arena_slot{} + , my_random{ this } + , my_last_observer{ nullptr } + , my_small_object_pool{new (cache_aligned_allocate(sizeof(small_object_pool_impl))) small_object_pool_impl{}} + , my_context_list(new (cache_aligned_allocate(sizeof(context_list))) context_list{}) +#if __TBB_RESUMABLE_TASKS + , my_post_resume_action{ task_dispatcher::post_resume_action::none } + , my_post_resume_arg{nullptr} +#endif /* __TBB_RESUMABLE_TASKS */ + { + ITT_SYNC_CREATE(&my_context_list->m_mutex, SyncType_Scheduler, SyncObj_ContextsList); + } + + ~thread_data() { + my_context_list->orphan(); + my_small_object_pool->destroy(); + poison_pointer(my_task_dispatcher); + poison_pointer(my_arena); + poison_pointer(my_arena_slot); + poison_pointer(my_last_observer); + poison_pointer(my_small_object_pool); + poison_pointer(my_context_list); +#if __TBB_RESUMABLE_TASKS + poison_pointer(my_post_resume_arg); +#endif /* __TBB_RESUMABLE_TASKS */ + } + + void attach_arena(arena& a, std::size_t index); + bool is_attached_to(arena*); + void attach_task_dispatcher(task_dispatcher&); + void detach_task_dispatcher(); + void enter_task_dispatcher(task_dispatcher& task_disp, std::uintptr_t stealing_threshold); + void leave_task_dispatcher(); + void propagate_task_group_state(std::atomic<uint32_t> d1::task_group_context::* mptr_state, d1::task_group_context& src, uint32_t new_state); + + //! Index of the arena slot the scheduler occupies now, or occupied last time + unsigned short my_arena_index; + + //! Indicates if the thread is created by RML + const bool my_is_worker; + + bool my_is_registered; + + //! The current task dipsatcher + task_dispatcher* my_task_dispatcher; + + //! The arena that I own (if external thread) or am servicing at the moment (if worker) + arena* my_arena; + + thread_dispatcher_client* my_last_client; + + //! Pointer to the slot in the arena we own at the moment + arena_slot* my_arena_slot; + + //! The mailbox (affinity mechanism) the current thread attached to + mail_inbox my_inbox; + + //! The random generator + FastRandom my_random; + + //! Last observer in the observers list processed on this slot + observer_proxy* my_last_observer; + + //! Pool of small object for fast task allocation + small_object_pool_impl* my_small_object_pool; + + context_list* my_context_list; +#if __TBB_RESUMABLE_TASKS + //! Suspends the current coroutine (task_dispatcher). + void suspend(void* suspend_callback, void* user_callback); + + //! Resumes the target task_dispatcher. + void resume(task_dispatcher& target); + + //! Set post resume action to perform after resume. + void set_post_resume_action(task_dispatcher::post_resume_action pra, void* arg) { + __TBB_ASSERT(my_post_resume_action == task_dispatcher::post_resume_action::none, "The Post resume action must not be set"); + __TBB_ASSERT(!my_post_resume_arg, "The post resume action must not have an argument"); + my_post_resume_action = pra; + my_post_resume_arg = arg; + } + + void clear_post_resume_action() { + my_post_resume_action = task_dispatcher::post_resume_action::none; + my_post_resume_arg = nullptr; + } + + //! The post resume action requested after the swap contexts. + task_dispatcher::post_resume_action my_post_resume_action; + + //! The post resume action argument. + void* my_post_resume_arg; +#endif /* __TBB_RESUMABLE_TASKS */ + + //! The default context + // TODO: consider using common default context because it is used only to simplify + // cancellation check. + d1::task_group_context my_default_context; +}; + +inline void thread_data::attach_arena(arena& a, std::size_t index) { + my_arena = &a; + my_arena_index = static_cast<unsigned short>(index); + my_arena_slot = a.my_slots + index; + // Read the current slot mail_outbox and attach it to the mail_inbox (remove inbox later maybe) + my_inbox.attach(my_arena->mailbox(index)); +} + +inline bool thread_data::is_attached_to(arena* a) { return my_arena == a; } + +inline void thread_data::attach_task_dispatcher(task_dispatcher& task_disp) { + __TBB_ASSERT(my_task_dispatcher == nullptr, nullptr); + __TBB_ASSERT(task_disp.m_thread_data == nullptr, nullptr); + task_disp.m_thread_data = this; + my_task_dispatcher = &task_disp; +} + +inline void thread_data::detach_task_dispatcher() { + __TBB_ASSERT(my_task_dispatcher != nullptr, nullptr); + __TBB_ASSERT(my_task_dispatcher->m_thread_data == this, nullptr); + my_task_dispatcher->m_thread_data = nullptr; + my_task_dispatcher = nullptr; +} + +inline void thread_data::enter_task_dispatcher(task_dispatcher& task_disp, std::uintptr_t stealing_threshold) { + task_disp.set_stealing_threshold(stealing_threshold); + attach_task_dispatcher(task_disp); +} + +inline void thread_data::leave_task_dispatcher() { + my_task_dispatcher->set_stealing_threshold(0); + detach_task_dispatcher(); +} + +inline void thread_data::propagate_task_group_state(std::atomic<std::uint32_t> d1::task_group_context::* mptr_state, d1::task_group_context& src, std::uint32_t new_state) { + mutex::scoped_lock lock(my_context_list->m_mutex); + // Acquire fence is necessary to ensure that the subsequent node->my_next load + // returned the correct value in case it was just inserted in another thread. + // The fence also ensures visibility of the correct ctx.my_parent value. + for (context_list::iterator it = my_context_list->begin(); it != my_context_list->end(); ++it) { + d1::task_group_context& ctx = __TBB_get_object_ref(d1::task_group_context, my_node, &(*it)); + if ((ctx.*mptr_state).load(std::memory_order_relaxed) != new_state) + task_group_context_impl::propagate_task_group_state(ctx, mptr_state, src, new_state); + } + // Sync up local propagation epoch with the global one. Release fence prevents + // reordering of possible store to *mptr_state after the sync point. + my_context_list->epoch.store(the_context_state_propagation_epoch.load(std::memory_order_relaxed), std::memory_order_release); +} + +} // namespace r1 +} // namespace detail +} // namespace tbb + +#endif // __TBB_thread_data_H + diff --git a/src/tbb/src/tbb/thread_dispatcher.cpp b/src/tbb/src/tbb/thread_dispatcher.cpp new file mode 100644 index 000000000..69a108d6f --- /dev/null +++ b/src/tbb/src/tbb/thread_dispatcher.cpp @@ -0,0 +1,236 @@ +/* + Copyright (c) 2022-2024 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "thread_dispatcher.h" +#include "threading_control.h" + +namespace tbb { +namespace detail { +namespace r1 { + +thread_dispatcher::thread_dispatcher(threading_control& tc, unsigned hard_limit, std::size_t stack_size) + : my_threading_control(tc) + , my_num_workers_hard_limit(hard_limit) + , my_stack_size(stack_size) +{ + my_server = governor::create_rml_server( *this ); + __TBB_ASSERT( my_server, "Failed to create RML server" ); +} + +thread_dispatcher::~thread_dispatcher() { + poison_pointer(my_server); +} + +thread_dispatcher_client* thread_dispatcher::select_next_client(thread_dispatcher_client* hint) { + unsigned next_client_priority_level = num_priority_levels; + if (hint) { + next_client_priority_level = hint->priority_level(); + } + + for (unsigned idx = 0; idx < next_client_priority_level; ++idx) { + if (!my_client_list[idx].empty()) { + return &*my_client_list[idx].begin(); + } + } + + return hint; +} + +thread_dispatcher_client* thread_dispatcher::create_client(arena& a) { + return new (cache_aligned_allocate(sizeof(thread_dispatcher_client))) thread_dispatcher_client(a, my_clients_aba_epoch); +} + + +void thread_dispatcher::register_client(thread_dispatcher_client* client) { + client_list_mutex_type::scoped_lock lock(my_list_mutex); + insert_client(*client); +} + +bool thread_dispatcher::try_unregister_client(thread_dispatcher_client* client, std::uint64_t aba_epoch, unsigned priority) { + __TBB_ASSERT(client, nullptr); + // we hold reference to the server, so market cannot be destroyed at any moment here + __TBB_ASSERT(!is_poisoned(my_server), nullptr); + my_list_mutex.lock(); + for (auto& it : my_client_list[priority]) { + if (client == &it) { + if (it.get_aba_epoch() == aba_epoch) { + // Client is alive + // Acquire my_references to sync with threads that just left the arena + // Pay attention that references should be read before workers_requested because + // if references is no zero some other thread might call adjust_demand and lead to + // a race over workers_requested + if (!client->references() && !client->has_request()) { + // Client is abandoned. Destroy it. + remove_client(*client); + ++my_clients_aba_epoch; + + my_list_mutex.unlock(); + destroy_client(client); + + return true; + } + } + break; + } + } + my_list_mutex.unlock(); + return false; +} + +void thread_dispatcher::destroy_client(thread_dispatcher_client* client) { + client->~thread_dispatcher_client(); + cache_aligned_deallocate(client); +} + +// Should be called under lock +void thread_dispatcher::insert_client(thread_dispatcher_client& client) { + __TBB_ASSERT(client.priority_level() < num_priority_levels, nullptr); + my_client_list[client.priority_level()].push_front(client); + + __TBB_ASSERT(!my_next_client || my_next_client->priority_level() < num_priority_levels, nullptr); + my_next_client = select_next_client(my_next_client); +} + +// Should be called under lock +void thread_dispatcher::remove_client(thread_dispatcher_client& client) { + __TBB_ASSERT(client.priority_level() < num_priority_levels, nullptr); + my_client_list[client.priority_level()].remove(client); + + if (my_next_client == &client) { + my_next_client = nullptr; + } + my_next_client = select_next_client(my_next_client); +} + +bool thread_dispatcher::is_client_alive(thread_dispatcher_client* client) { + if (!client) { + return false; + } + + // Still cannot access internals of the client since the object itself might be destroyed. + for (auto& priority_list : my_client_list) { + for (auto& c : priority_list) { + if (client == &c) { + return true; + } + } + } + return false; +} + +thread_dispatcher_client* thread_dispatcher::client_in_need(client_list_type* clients, thread_dispatcher_client* hint) { + // TODO: make sure client with higher priority returned only if there are available slots in it. + hint = select_next_client(hint); + if (!hint) { + return nullptr; + } + + client_list_type::iterator it = hint; + unsigned curr_priority_level = hint->priority_level(); + __TBB_ASSERT(it != clients[curr_priority_level].end(), nullptr); + do { + thread_dispatcher_client& t = *it; + if (++it == clients[curr_priority_level].end()) { + do { + ++curr_priority_level %= num_priority_levels; + } while (clients[curr_priority_level].empty()); + it = clients[curr_priority_level].begin(); + } + if (t.try_join()) { + return &t; + } + } while (it != hint); + return nullptr; +} + +thread_dispatcher_client* thread_dispatcher::client_in_need(thread_dispatcher_client* prev) { + client_list_mutex_type::scoped_lock lock(my_list_mutex, /*is_writer=*/false); + if (is_client_alive(prev)) { + return client_in_need(my_client_list, prev); + } + return client_in_need(my_client_list, my_next_client); +} + +bool thread_dispatcher::is_any_client_in_need() { + client_list_mutex_type::scoped_lock lock(my_list_mutex, /*is_writer=*/false); + for (auto& priority_list : my_client_list) { + for (auto& client : priority_list) { + if (client.is_joinable()) { + return true; + } + } + } + return false; +} + +void thread_dispatcher::adjust_job_count_estimate(int delta) { + my_server->adjust_job_count_estimate(delta); +} + +void thread_dispatcher::release(bool blocking_terminate) { + my_join_workers = blocking_terminate; + my_server->request_close_connection(); +} + +void thread_dispatcher::process(job& j) { + thread_data& td = static_cast<thread_data&>(j); + // td.my_last_client can be dead. Don't access it until client_in_need is called + thread_dispatcher_client* client = td.my_last_client; + for (int i = 0; i < 2; ++i) { + while ((client = client_in_need(client)) ) { + td.my_last_client = client; + client->process(td); + } + // Workers leave thread_dispatcher because there is no client in need. It can happen earlier than + // adjust_job_count_estimate() decreases my_slack and RML can put this thread to sleep. + // It might result in a busy-loop checking for my_slack<0 and calling this method instantly. + // the yield refines this spinning. + if ( !i ) { + yield(); + } + } +} + + +//! Used when RML asks for join mode during workers termination. +bool thread_dispatcher::must_join_workers() const { return my_join_workers; } + +//! Returns the requested stack size of worker threads. +std::size_t thread_dispatcher::worker_stack_size() const { return my_stack_size; } + +void thread_dispatcher::acknowledge_close_connection() { + my_threading_control.destroy(); +} + +::rml::job* thread_dispatcher::create_one_job() { + unsigned short index = ++my_first_unused_worker_idx; + __TBB_ASSERT(index > 0, nullptr); + ITT_THREAD_SET_NAME(_T("TBB Worker Thread")); + // index serves as a hint decreasing conflicts between workers when they migrate between arenas + thread_data* td = new (cache_aligned_allocate(sizeof(thread_data))) thread_data{ index, true }; + __TBB_ASSERT(index <= my_num_workers_hard_limit, nullptr); + my_threading_control.register_thread(*td); + return td; +} + +void thread_dispatcher::cleanup(job& j) { + my_threading_control.unregister_thread(static_cast<thread_data&>(j)); + governor::auto_terminate(&j); +} + +} // namespace r1 +} // namespace detail +} // namespace tbb diff --git a/src/tbb/src/tbb/thread_dispatcher.h b/src/tbb/src/tbb/thread_dispatcher.h new file mode 100644 index 000000000..e511e2b73 --- /dev/null +++ b/src/tbb/src/tbb/thread_dispatcher.h @@ -0,0 +1,107 @@ +/* + Copyright (c) 2022-2024 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef _TBB_thread_dispatcher_H +#define _TBB_thread_dispatcher_H + +#include "oneapi/tbb/detail/_config.h" +#include "oneapi/tbb/detail/_utils.h" +#include "oneapi/tbb/rw_mutex.h" +#include "oneapi/tbb/task_arena.h" + +#include "arena.h" +#include "governor.h" +#include "thread_data.h" +#include "rml_tbb.h" +#include "thread_dispatcher_client.h" + +namespace tbb { +namespace detail { +namespace r1 { + +class threading_control_impl; + +class thread_dispatcher : no_copy, rml::tbb_client { + using client_list_type = intrusive_list<thread_dispatcher_client>; + using client_list_mutex_type = d1::rw_mutex; +public: + thread_dispatcher(threading_control& tc, unsigned hard_limit, std::size_t stack_size); + ~thread_dispatcher(); + + thread_dispatcher_client* create_client(arena& a); + void register_client(thread_dispatcher_client* client); + bool try_unregister_client(thread_dispatcher_client* client, std::uint64_t aba_epoch, unsigned priority); + bool is_any_client_in_need(); + + void adjust_job_count_estimate(int delta); + void release(bool blocking_terminate); + void process(job& j) override; + //! Used when RML asks for join mode during workers termination. + bool must_join_workers() const; + //! Returns the requested stack size of worker threads. + std::size_t worker_stack_size() const; + +private: + version_type version () const override { return 0; } + unsigned max_job_count () const override { return my_num_workers_hard_limit; } + std::size_t min_stack_size () const override { return worker_stack_size(); } + void cleanup(job& j) override; + void acknowledge_close_connection() override; + ::rml::job* create_one_job() override; + + thread_dispatcher_client* select_next_client(thread_dispatcher_client* hint); + void destroy_client(thread_dispatcher_client* client); + void insert_client(thread_dispatcher_client& client); + void remove_client(thread_dispatcher_client& client); + bool is_client_alive(thread_dispatcher_client* client); + thread_dispatcher_client* client_in_need(client_list_type* clients, thread_dispatcher_client* hint); + thread_dispatcher_client* client_in_need(thread_dispatcher_client* prev); + + friend class threading_control_impl; + static constexpr unsigned num_priority_levels = d1::num_priority_levels; + client_list_mutex_type my_list_mutex; + client_list_type my_client_list[num_priority_levels]; + + thread_dispatcher_client* my_next_client{nullptr}; + + //! Shutdown mode + bool my_join_workers{false}; + + threading_control& my_threading_control; + + //! ABA prevention marker to assign to newly created clients + std::atomic<std::uint64_t> my_clients_aba_epoch{0}; + + //! Maximal number of workers allowed for use by the underlying resource manager + /** It can't be changed after thread_dispatcher creation. **/ + unsigned my_num_workers_hard_limit{0}; + + //! Stack size of worker threads + std::size_t my_stack_size{0}; + + //! First unused index of worker + /** Used to assign indices to the new workers coming from RML **/ + std::atomic<unsigned> my_first_unused_worker_idx{0}; + + //! Pointer to the RML server object that services this TBB instance. + rml::tbb_server* my_server{nullptr}; +}; + +} // namespace r1 +} // namespace detail +} // namespace tbb + +#endif // _TBB_thread_dispatcher_H diff --git a/src/tbb/src/tbb/thread_dispatcher_client.h b/src/tbb/src/tbb/thread_dispatcher_client.h new file mode 100644 index 000000000..f7c199cb8 --- /dev/null +++ b/src/tbb/src/tbb/thread_dispatcher_client.h @@ -0,0 +1,69 @@ +/* + Copyright (c) 2022-2024 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef _TBB_thread_dispatcher_client_H +#define _TBB_thread_dispatcher_client_H + +#include "oneapi/tbb/detail/_intrusive_list_node.h" +#include "arena.h" + +namespace tbb { +namespace detail { +namespace r1 { + +class thread_dispatcher_client : public d1::intrusive_list_node /* Need for list in thread pool */ { +public: + thread_dispatcher_client(arena& a, std::uint64_t aba_epoch) : my_arena(a), my_aba_epoch(aba_epoch) {} + + // Interface of communication with thread pool + bool try_join() { + return my_arena.try_join(); + } + + bool is_joinable() { + return my_arena.is_joinable(); + } + + void process(thread_data& td) { + my_arena.process(td); + } + + unsigned priority_level() { + return my_arena.priority_level(); + } + + std::uint64_t get_aba_epoch() { + return my_aba_epoch; + } + + unsigned references() { + return my_arena.references(); + } + + bool has_request() { + return my_arena.has_request(); + } + +private: + arena& my_arena; + std::uint64_t my_aba_epoch; +}; + +} // namespace r1 +} // namespace detail +} // namespace tbb + +#endif // _TBB_thread_dispatcher_client_H diff --git a/src/tbb/src/tbb/thread_request_serializer.cpp b/src/tbb/src/tbb/thread_request_serializer.cpp new file mode 100644 index 000000000..41cf51b0b --- /dev/null +++ b/src/tbb/src/tbb/thread_request_serializer.cpp @@ -0,0 +1,139 @@ +/* + Copyright (c) 2022-2024 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "misc.h" +#include "thread_request_serializer.h" + +namespace tbb { +namespace detail { +namespace r1 { + +thread_request_serializer::thread_request_serializer(thread_dispatcher& td, int soft_limit) + : my_thread_dispatcher(td) + , my_soft_limit(soft_limit) +{} + +void thread_request_serializer::update(int delta) { + constexpr std::uint64_t delta_mask = (pending_delta_base << 1) - 1; + constexpr std::uint64_t counter_value = delta_mask + 1; + + int prev_pending_delta = my_pending_delta.fetch_add(counter_value + delta); + + // There is a pseudo request aggregator, so only thread that see pending_delta_base in my_pending_delta + // Will enter to critical section and call adjust_job_count_estimate + if (prev_pending_delta == pending_delta_base) { + delta = int(my_pending_delta.exchange(pending_delta_base) & delta_mask) - int(pending_delta_base); + mutex_type::scoped_lock lock(my_mutex); + my_total_request.store(my_total_request.load(std::memory_order_relaxed) + delta, std::memory_order_relaxed); + delta = limit_delta(delta, my_soft_limit, my_total_request.load(std::memory_order_relaxed)); + my_thread_dispatcher.adjust_job_count_estimate(delta); + } +} + +void thread_request_serializer::set_active_num_workers(int soft_limit) { + mutex_type::scoped_lock lock(my_mutex); + int delta = soft_limit - my_soft_limit; + delta = limit_delta(delta, my_total_request.load(std::memory_order_relaxed), soft_limit); + my_thread_dispatcher.adjust_job_count_estimate(delta); + my_soft_limit = soft_limit; +} + +int thread_request_serializer::limit_delta(int delta, int limit, int new_value) { + // This method can be described with such pseudocode: + // bool above_limit = prev_value >= limit && new_value >= limit; + // bool below_limit = prev_value <= limit && new_value <= limit; + // enum request_type { ABOVE_LIMIT, CROSS_LIMIT, BELOW_LIMIT }; + // request = above_limit ? ABOVE_LIMIT : below_limit ? BELOW_LIMIT : CROSS_LIMIT; + + // switch (request) { + // case ABOVE_LIMIT: + // delta = 0; + // case CROSS_LIMIT: + // delta = delta > 0 ? limit - prev_value : new_value - limit; + // case BELOW_LIMIT: + // // No changes to delta + // } + + int prev_value = new_value - delta; + + // actual new_value and prev_value cannot exceed the limit + new_value = min(limit, new_value); + prev_value = min(limit, prev_value); + return new_value - prev_value; +} + + +thread_request_serializer_proxy::thread_request_serializer_proxy(thread_dispatcher& td, int soft_limit) : my_serializer(td, soft_limit) +{} + +void thread_request_serializer_proxy::register_mandatory_request(int mandatory_delta) { + if (mandatory_delta != 0) { + mutex_type::scoped_lock lock(my_mutex, /* is_write = */ false); + int prev_value = my_num_mandatory_requests.fetch_add(mandatory_delta); + + const bool should_try_enable = mandatory_delta > 0 && prev_value == 0; + const bool should_try_disable = mandatory_delta < 0 && prev_value == 1; + + if (should_try_enable) { + enable_mandatory_concurrency(lock); + } else if (should_try_disable) { + disable_mandatory_concurrency(lock); + } + } +} + +void thread_request_serializer_proxy::set_active_num_workers(int soft_limit) { + mutex_type::scoped_lock lock(my_mutex, /* is_write = */ true); + + if (soft_limit != 0) { + my_is_mandatory_concurrency_enabled = false; + } else if (my_num_mandatory_requests > 0) { + my_is_mandatory_concurrency_enabled = true; + soft_limit = 1; + } + + my_serializer.set_active_num_workers(soft_limit); +} + +int thread_request_serializer_proxy::num_workers_requested() { return my_serializer.num_workers_requested(); } + +void thread_request_serializer_proxy::update(int delta) { my_serializer.update(delta); } + +void thread_request_serializer_proxy::enable_mandatory_concurrency(mutex_type::scoped_lock& lock) { + lock.upgrade_to_writer(); + bool still_should_enable = my_num_mandatory_requests.load(std::memory_order_relaxed) > 0 && + !my_is_mandatory_concurrency_enabled && my_serializer.is_no_workers_avaliable(); + + if (still_should_enable) { + my_is_mandatory_concurrency_enabled = true; + my_serializer.set_active_num_workers(1); + } +} + +void thread_request_serializer_proxy::disable_mandatory_concurrency(mutex_type::scoped_lock& lock) { + lock.upgrade_to_writer(); + bool still_should_disable = my_num_mandatory_requests.load(std::memory_order_relaxed) <= 0 && + my_is_mandatory_concurrency_enabled && !my_serializer.is_no_workers_avaliable(); + + if (still_should_disable) { + my_is_mandatory_concurrency_enabled = false; + my_serializer.set_active_num_workers(0); + } +} + +} // r1 +} // detail +} // tbb diff --git a/src/tbb/src/tbb/thread_request_serializer.h b/src/tbb/src/tbb/thread_request_serializer.h new file mode 100644 index 000000000..9dc9799e1 --- /dev/null +++ b/src/tbb/src/tbb/thread_request_serializer.h @@ -0,0 +1,84 @@ +/* + Copyright (c) 2022-2024 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef _TBB_thread_serializer_handlers_H +#define _TBB_thread_serializer_handlers_H + +#include "oneapi/tbb/mutex.h" +#include "oneapi/tbb/rw_mutex.h" + +#include "thread_dispatcher.h" + +namespace tbb { +namespace detail { +namespace r1 { + +class thread_request_observer { +protected: + virtual ~thread_request_observer() {} +public: + virtual void update(int delta) = 0; +}; + + +class thread_request_serializer : public thread_request_observer { + using mutex_type = d1::mutex; +public: + thread_request_serializer(thread_dispatcher& td, int soft_limit); + void set_active_num_workers(int soft_limit); + int num_workers_requested() { return my_total_request.load(std::memory_order_relaxed); } + bool is_no_workers_avaliable() { return my_soft_limit == 0; } + +private: + friend class thread_request_serializer_proxy; + void update(int delta) override; + static int limit_delta(int delta, int limit, int new_value); + + thread_dispatcher& my_thread_dispatcher; + int my_soft_limit{ 0 }; + std::atomic<int> my_total_request{ 0 }; + // my_pending_delta is set to pending_delta_base to have ability to hold negative values + // consider increase base since thead number will be bigger than 1 << 15 + static constexpr std::uint64_t pending_delta_base = 1 << 15; + std::atomic<std::uint64_t> my_pending_delta{ pending_delta_base }; + mutex_type my_mutex; +}; + +// Handles mandatory concurrency i.e. enables worker threads for enqueue tasks +class thread_request_serializer_proxy : public thread_request_observer { + using mutex_type = d1::rw_mutex; +public: + thread_request_serializer_proxy(thread_dispatcher& td, int soft_limit); + void register_mandatory_request(int mandatory_delta); + void set_active_num_workers(int soft_limit); + int num_workers_requested(); + +private: + void update(int delta) override; + void enable_mandatory_concurrency(mutex_type::scoped_lock& lock); + void disable_mandatory_concurrency(mutex_type::scoped_lock& lock); + + std::atomic<int> my_num_mandatory_requests{0}; + bool my_is_mandatory_concurrency_enabled{false}; + thread_request_serializer my_serializer; + mutex_type my_mutex; +}; + +} // namespace r1 +} // namespace detail +} // namespace tbb + +#endif // _TBB_thread_serializer_handlers_H diff --git a/src/tbb/src/tbb/threading_control.cpp b/src/tbb/src/tbb/threading_control.cpp new file mode 100644 index 000000000..7a62b337f --- /dev/null +++ b/src/tbb/src/tbb/threading_control.cpp @@ -0,0 +1,411 @@ +/* + Copyright (c) 2022-2024 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "threading_control.h" +#include "permit_manager.h" +#include "market.h" +#include "tcm_adaptor.h" +#include "thread_dispatcher.h" +#include "governor.h" +#include "thread_dispatcher_client.h" + +namespace tbb { +namespace detail { +namespace r1 { + +// ---------------------------------------- threading_control_impl -------------------------------------------------------------- + +std::size_t global_control_active_value_unsafe(d1::global_control::parameter); + +std::pair<unsigned, unsigned> threading_control_impl::calculate_workers_limits() { + // Expecting that 4P is suitable for most applications. + // Limit to 2P for large thread number. + // TODO: ask RML for max concurrency and possibly correct hard_limit + unsigned factor = governor::default_num_threads() <= 128 ? 4 : 2; + + // The requested number of threads is intentionally not considered in + // computation of the hard limit, in order to separate responsibilities + // and avoid complicated interactions between global_control and task_scheduler_init. + // The threading control guarantees that at least 256 threads might be created. + unsigned workers_app_limit = global_control_active_value_unsafe(global_control::max_allowed_parallelism); + unsigned workers_hard_limit = max(max(factor * governor::default_num_threads(), 256u), workers_app_limit); + unsigned workers_soft_limit = calc_workers_soft_limit(workers_hard_limit); + + return std::make_pair(workers_soft_limit, workers_hard_limit); +} + +unsigned threading_control_impl::calc_workers_soft_limit(unsigned workers_hard_limit) { + unsigned workers_soft_limit{}; + unsigned soft_limit = global_control_active_value_unsafe(global_control::max_allowed_parallelism); + + // if user set no limits (yet), use default value + workers_soft_limit = soft_limit != 0 ? soft_limit - 1 : governor::default_num_threads() - 1; + + if (workers_soft_limit >= workers_hard_limit) { + workers_soft_limit = workers_hard_limit - 1; + } + + return workers_soft_limit; +} + +cache_aligned_unique_ptr<permit_manager> threading_control_impl::make_permit_manager(unsigned workers_soft_limit) { + if (tcm_adaptor::is_initialized()) { + auto tcm = make_cache_aligned_unique<tcm_adaptor>(); + if (tcm->is_connected()) { + return tcm; + } + } + return make_cache_aligned_unique<market>(workers_soft_limit); +} + +cache_aligned_unique_ptr<thread_dispatcher> threading_control_impl::make_thread_dispatcher(threading_control& tc, + unsigned workers_soft_limit, + unsigned workers_hard_limit) +{ + stack_size_type stack_size = global_control_active_value_unsafe(global_control::thread_stack_size); + + cache_aligned_unique_ptr<thread_dispatcher> td = + make_cache_aligned_unique<thread_dispatcher>(tc, workers_hard_limit, stack_size); + // This check relies on the fact that for shared RML default_concurrency == max_concurrency + if (!governor::UsePrivateRML && td->my_server->default_concurrency() < workers_soft_limit) { + runtime_warning("RML might limit the number of workers to %u while %u is requested.\n", + td->my_server->default_concurrency(), workers_soft_limit); + } + + return td; +} + +threading_control_impl::threading_control_impl(threading_control* tc) { + unsigned workers_soft_limit{}, workers_hard_limit{}; + std::tie(workers_soft_limit, workers_hard_limit) = calculate_workers_limits(); + + my_permit_manager = make_permit_manager(workers_soft_limit); + my_thread_dispatcher = make_thread_dispatcher(*tc, workers_soft_limit, workers_hard_limit); + my_thread_request_serializer = + make_cache_aligned_unique<thread_request_serializer_proxy>(*my_thread_dispatcher, workers_soft_limit); + my_permit_manager->set_thread_request_observer(*my_thread_request_serializer); + + my_cancellation_disseminator = make_cache_aligned_unique<cancellation_disseminator>(); + my_waiting_threads_monitor = make_cache_aligned_unique<thread_control_monitor>(); +} + +void threading_control_impl::release(bool blocking_terminate) { + my_thread_dispatcher->release(blocking_terminate); +} + +void threading_control_impl::set_active_num_workers(unsigned soft_limit) { + __TBB_ASSERT(soft_limit <= my_thread_dispatcher->my_num_workers_hard_limit, nullptr); + my_thread_request_serializer->set_active_num_workers(soft_limit); + my_permit_manager->set_active_num_workers(soft_limit); +} + +threading_control_client threading_control_impl::create_client(arena& a) { + pm_client* pm_client = my_permit_manager->create_client(a); + thread_dispatcher_client* td_client = my_thread_dispatcher->create_client(a); + + return threading_control_client{pm_client, td_client}; +} + +threading_control_impl::client_snapshot threading_control_impl::prepare_client_destruction(threading_control_client client) { + auto td_client = client.get_thread_dispatcher_client(); + return {td_client->get_aba_epoch(), td_client->priority_level(), td_client, client.get_pm_client()}; +} + +bool threading_control_impl::try_destroy_client(threading_control_impl::client_snapshot snapshot) { + if (my_thread_dispatcher->try_unregister_client(snapshot.my_td_client, snapshot.aba_epoch, snapshot.priority_level)) { + my_permit_manager->unregister_and_destroy_client(*snapshot.my_pm_client); + return true; + } + return false; +} + +void threading_control_impl::publish_client(threading_control_client tc_client, d1::constraints& constraints) { + my_permit_manager->register_client(tc_client.get_pm_client(), constraints); + my_thread_dispatcher->register_client(tc_client.get_thread_dispatcher_client()); +} + +void threading_control_impl::register_thread(thread_data& td) { + my_cancellation_disseminator->register_thread(td); +} +void threading_control_impl::unregister_thread(thread_data& td) { + my_cancellation_disseminator->unregister_thread(td); +} + +void threading_control_impl::propagate_task_group_state(std::atomic<uint32_t> d1::task_group_context::*mptr_state, + d1::task_group_context& src, uint32_t new_state) +{ + my_cancellation_disseminator->propagate_task_group_state(mptr_state, src, new_state); +} + +std::size_t threading_control_impl::worker_stack_size() { + return my_thread_dispatcher->worker_stack_size(); +} + +unsigned threading_control_impl::max_num_workers() { + return my_thread_dispatcher->my_num_workers_hard_limit; +} + +void threading_control_impl::adjust_demand(threading_control_client tc_client, int mandatory_delta, int workers_delta) { + auto& c = *tc_client.get_pm_client(); + my_thread_request_serializer->register_mandatory_request(mandatory_delta); + my_permit_manager->adjust_demand(c, mandatory_delta, workers_delta); +} + +bool threading_control_impl::is_any_other_client_active() { + return my_thread_request_serializer->num_workers_requested() > 0 ? my_thread_dispatcher->is_any_client_in_need() : false; +} + +thread_control_monitor& threading_control_impl::get_waiting_threads_monitor() { + return *my_waiting_threads_monitor; +} + +// ---------------------------------------- threading_control ------------------------------------------------------------------- + +// Defined in global_control.cpp +void global_control_lock(); +void global_control_unlock(); + +void threading_control::add_ref(bool is_public) { + ++my_ref_count; + if (is_public) { + my_public_ref_count++; + } +} + +bool threading_control::remove_ref(bool is_public) { + if (is_public) { + __TBB_ASSERT(g_threading_control == this, "Global threading control instance was destroyed prematurely?"); + __TBB_ASSERT(my_public_ref_count.load(std::memory_order_relaxed), nullptr); + --my_public_ref_count; + } + + bool is_last_ref = --my_ref_count == 0; + if (is_last_ref) { + __TBB_ASSERT(!my_public_ref_count.load(std::memory_order_relaxed), nullptr); + g_threading_control = nullptr; + } + + return is_last_ref; +} + +threading_control* threading_control::get_threading_control(bool is_public) { + threading_control* control = g_threading_control; + if (control) { + control->add_ref(is_public); + } + + return control; +} + +threading_control* threading_control::create_threading_control() { + // Global control should be locked before threading_control_impl + global_control_lock(); + + threading_control* thr_control{ nullptr }; + try_call([&] { + global_mutex_type::scoped_lock lock(g_threading_control_mutex); + + thr_control = get_threading_control(/*public = */ true); + if (thr_control == nullptr) { + thr_control = new (cache_aligned_allocate(sizeof(threading_control))) threading_control(/*public_ref = */ 1, /*private_ref = */ 1); + thr_control->my_pimpl = make_cache_aligned_unique<threading_control_impl>(thr_control); + + __TBB_InitOnce::add_ref(); + + if (global_control_active_value_unsafe(global_control::scheduler_handle)) { + ++thr_control->my_public_ref_count; + ++thr_control->my_ref_count; + } + + g_threading_control = thr_control; + } + }).on_exception([&] { + global_control_unlock(); + + cache_aligned_deleter deleter{}; + deleter(thr_control); + }); + + global_control_unlock(); + return thr_control; +} + +void threading_control::destroy () { + cache_aligned_deleter deleter; + deleter(this); + __TBB_InitOnce::remove_ref(); +} + +void threading_control::wait_last_reference(global_mutex_type::scoped_lock& lock) { + while (my_public_ref_count.load(std::memory_order_relaxed) == 1 && my_ref_count.load(std::memory_order_relaxed) > 1) { + lock.release(); + // To guarantee that request_close_connection() is called by the last external thread, we need to wait till all + // references are released. Re-read my_public_ref_count to limit waiting if new external threads are created. + // Theoretically, new private references to the threading control can be added during waiting making it potentially + // endless. + // TODO: revise why the weak scheduler needs threading control's pointer and try to remove this wait. + // Note that the threading control should know about its schedulers for cancellation/exception/priority propagation, + // see e.g. task_group_context::cancel_group_execution() + while (my_public_ref_count.load(std::memory_order_acquire) == 1 && my_ref_count.load(std::memory_order_acquire) > 1) { + yield(); + } + lock.acquire(g_threading_control_mutex); + } +} + +bool threading_control::release(bool is_public, bool blocking_terminate) { + bool do_release = false; + { + global_mutex_type::scoped_lock lock(g_threading_control_mutex); + if (blocking_terminate) { + __TBB_ASSERT(is_public, "Only an object with a public reference can request the blocking terminate"); + wait_last_reference(lock); + } + do_release = remove_ref(is_public); + } + + if (do_release) { + __TBB_ASSERT(!my_public_ref_count.load(std::memory_order_relaxed), "No public references must remain if we remove the threading control."); + // inform RML that blocking termination is required + my_pimpl->release(blocking_terminate); + return blocking_terminate; + } + return false; +} + +threading_control::threading_control(unsigned public_ref, unsigned ref) : my_public_ref_count(public_ref), my_ref_count(ref) +{} + +threading_control* threading_control::register_public_reference() { + threading_control* control{nullptr}; + global_mutex_type::scoped_lock lock(g_threading_control_mutex); + control = get_threading_control(/*public = */ true); + if (!control) { + // We are going to create threading_control_impl, we should acquire mutexes in right order + lock.release(); + control = create_threading_control(); + } + + return control; +} + +bool threading_control::unregister_public_reference(bool blocking_terminate) { + __TBB_ASSERT(g_threading_control, "Threading control should exist until last public reference"); + __TBB_ASSERT(g_threading_control->my_public_ref_count.load(std::memory_order_relaxed), nullptr); + return g_threading_control->release(/*public = */ true, /*blocking_terminate = */ blocking_terminate); +} + +threading_control_client threading_control::create_client(arena& a) { + { + global_mutex_type::scoped_lock lock(g_threading_control_mutex); + add_ref(/*public = */ false); + } + + return my_pimpl->create_client(a); +} + +void threading_control::publish_client(threading_control_client client, d1::constraints& constraints) { + return my_pimpl->publish_client(client, constraints); +} + +threading_control::client_snapshot threading_control::prepare_client_destruction(threading_control_client client) { + return my_pimpl->prepare_client_destruction(client); +} + +bool threading_control::try_destroy_client(threading_control::client_snapshot deleter) { + bool res = my_pimpl->try_destroy_client(deleter); + if (res) { + release(/*public = */ false, /*blocking_terminate = */ false); + } + return res; +} + +void threading_control::set_active_num_workers(unsigned soft_limit) { + threading_control* thr_control{nullptr}; + { + global_mutex_type::scoped_lock lock(g_threading_control_mutex); + thr_control = get_threading_control(/*public = */ false); + } + + if (thr_control != nullptr) { + thr_control->my_pimpl->set_active_num_workers(soft_limit); + thr_control->release(/*is_public=*/false, /*blocking_terminate=*/false); + } +} + +bool threading_control::is_present() { + global_mutex_type::scoped_lock lock(g_threading_control_mutex); + return g_threading_control != nullptr; +} + +bool threading_control::register_lifetime_control() { + global_mutex_type::scoped_lock lock(g_threading_control_mutex); + return get_threading_control(/*public = */ true) != nullptr; +} + +bool threading_control::unregister_lifetime_control(bool blocking_terminate) { + threading_control* thr_control{nullptr}; + { + global_mutex_type::scoped_lock lock(g_threading_control_mutex); + thr_control = g_threading_control; + } + + bool released{true}; + if (thr_control) { + released = thr_control->release(/*public = */ true, /*blocking_terminate = */ blocking_terminate); + } + + return released; +} + +void threading_control::register_thread(thread_data& td) { + my_pimpl->register_thread(td); +} + +void threading_control::unregister_thread(thread_data& td) { + my_pimpl->unregister_thread(td); +} + +void threading_control::propagate_task_group_state(std::atomic<uint32_t> d1::task_group_context::*mptr_state, + d1::task_group_context& src, uint32_t new_state) +{ + my_pimpl->propagate_task_group_state(mptr_state, src, new_state); +} + +std::size_t threading_control::worker_stack_size() { + return my_pimpl->worker_stack_size(); +} + +unsigned threading_control::max_num_workers() { + global_mutex_type::scoped_lock lock(g_threading_control_mutex); + return g_threading_control ? g_threading_control->my_pimpl->max_num_workers() : 0; +} + +void threading_control::adjust_demand(threading_control_client client, int mandatory_delta, int workers_delta) { + my_pimpl->adjust_demand(client, mandatory_delta, workers_delta); +} + +bool threading_control::is_any_other_client_active() { + return my_pimpl->is_any_other_client_active(); +} + +thread_control_monitor& threading_control::get_waiting_threads_monitor() { + return my_pimpl->get_waiting_threads_monitor(); +} + +} // r1 +} // detail +} // tbb diff --git a/src/tbb/src/tbb/threading_control.h b/src/tbb/src/tbb/threading_control.h new file mode 100644 index 000000000..7381b2978 --- /dev/null +++ b/src/tbb/src/tbb/threading_control.h @@ -0,0 +1,154 @@ +/* + Copyright (c) 2022-2024 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef _TBB_threading_control_H +#define _TBB_threading_control_H + +#include "oneapi/tbb/mutex.h" +#include "oneapi/tbb/global_control.h" + +#include "threading_control_client.h" +#include "intrusive_list.h" +#include "main.h" +#include "permit_manager.h" +#include "pm_client.h" +#include "thread_dispatcher.h" +#include "cancellation_disseminator.h" +#include "thread_request_serializer.h" +#include "scheduler_common.h" + +namespace tbb { +namespace detail { +namespace r1 { + +class arena; +class thread_data; + +class threading_control; + +class threading_control_impl { +public: + threading_control_impl(threading_control*); + +public: + void release(bool blocking_terminate); + + threading_control_client create_client(arena& a); + void publish_client(threading_control_client client, d1::constraints& constraints); + + struct client_snapshot { + std::uint64_t aba_epoch; + unsigned priority_level; + thread_dispatcher_client* my_td_client; + pm_client* my_pm_client; + }; + + client_snapshot prepare_client_destruction(threading_control_client client); + bool try_destroy_client(client_snapshot deleter); + + void register_thread(thread_data& td); + void unregister_thread(thread_data& td); + void propagate_task_group_state(std::atomic<uint32_t> d1::task_group_context::*mptr_state, + d1::task_group_context& src, uint32_t new_state); + + void set_active_num_workers(unsigned soft_limit); + std::size_t worker_stack_size(); + unsigned max_num_workers(); + + void adjust_demand(threading_control_client, int mandatory_delta, int workers_delta); + bool is_any_other_client_active(); + + thread_control_monitor& get_waiting_threads_monitor(); + +private: + static unsigned calc_workers_soft_limit(unsigned workers_hard_limit); + static std::pair<unsigned, unsigned> calculate_workers_limits(); + static cache_aligned_unique_ptr<permit_manager> make_permit_manager(unsigned workers_soft_limit); + static cache_aligned_unique_ptr<thread_dispatcher> make_thread_dispatcher(threading_control& control, + unsigned workers_soft_limit, + unsigned workers_hard_limit); + + // TODO: Consider allocation one chunk of memory and construct objects on it + cache_aligned_unique_ptr<permit_manager> my_permit_manager{nullptr}; + cache_aligned_unique_ptr<thread_dispatcher> my_thread_dispatcher{nullptr}; + cache_aligned_unique_ptr<thread_request_serializer_proxy> my_thread_request_serializer{nullptr}; + cache_aligned_unique_ptr<cancellation_disseminator> my_cancellation_disseminator{nullptr}; + cache_aligned_unique_ptr<thread_control_monitor> my_waiting_threads_monitor{nullptr}; +}; + + +class threading_control { + using global_mutex_type = d1::mutex; +public: + using client_snapshot = threading_control_impl::client_snapshot; + + static threading_control* register_public_reference(); + static bool unregister_public_reference(bool blocking_terminate); + + static bool is_present(); + static void set_active_num_workers(unsigned soft_limit); + static bool register_lifetime_control(); + static bool unregister_lifetime_control(bool blocking_terminate); + + threading_control_client create_client(arena& a); + void publish_client(threading_control_client client, d1::constraints& constraints); + client_snapshot prepare_client_destruction(threading_control_client client); + bool try_destroy_client(client_snapshot deleter); + + void register_thread(thread_data& td); + void unregister_thread(thread_data& td); + void propagate_task_group_state(std::atomic<uint32_t> d1::task_group_context::*mptr_state, + d1::task_group_context& src, uint32_t new_state); + + std::size_t worker_stack_size(); + static unsigned max_num_workers(); + + void adjust_demand(threading_control_client client, int mandatory_delta, int workers_delta); + bool is_any_other_client_active(); + + thread_control_monitor& get_waiting_threads_monitor(); + +private: + threading_control(unsigned public_ref, unsigned ref); + void add_ref(bool is_public); + bool remove_ref(bool is_public); + + static threading_control* get_threading_control(bool is_public); + static threading_control* create_threading_control(); + + bool release(bool is_public, bool blocking_terminate); + void wait_last_reference(global_mutex_type::scoped_lock& lock); + void destroy(); + + friend class thread_dispatcher; + + static threading_control* g_threading_control; + //! Mutex guarding creation/destruction of g_threading_control, insertions/deletions in my_arenas, and cancellation propagation + static global_mutex_type g_threading_control_mutex; + + cache_aligned_unique_ptr<threading_control_impl> my_pimpl{nullptr}; + //! Count of external threads attached + std::atomic<unsigned> my_public_ref_count{0}; + //! Reference count controlling threading_control object lifetime + std::atomic<unsigned> my_ref_count{0}; +}; + +} // r1 +} // detail +} // tbb + + +#endif // _TBB_threading_control_H diff --git a/src/tbb/src/tbb/threading_control_client.h b/src/tbb/src/tbb/threading_control_client.h new file mode 100644 index 000000000..4ff9359cf --- /dev/null +++ b/src/tbb/src/tbb/threading_control_client.h @@ -0,0 +1,58 @@ +/* + Copyright (c) 2022-2023 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef _TBB_threading_control_client_H +#define _TBB_threading_control_client_H + +#include "oneapi/tbb/detail/_assert.h" + +namespace tbb { +namespace detail { +namespace r1 { + +class pm_client; +class thread_dispatcher_client; + +class threading_control_client { +public: + threading_control_client() = default; + threading_control_client(const threading_control_client&) = default; + threading_control_client& operator=(const threading_control_client&) = default; + + threading_control_client(pm_client* p, thread_dispatcher_client* t) : my_pm_client(p), my_thread_dispatcher_client(t) { + __TBB_ASSERT(my_pm_client, nullptr); + __TBB_ASSERT(my_thread_dispatcher_client, nullptr); + } + + pm_client* get_pm_client() { + return my_pm_client; + } + + thread_dispatcher_client* get_thread_dispatcher_client() { + return my_thread_dispatcher_client; + } + +private: + pm_client* my_pm_client{nullptr}; + thread_dispatcher_client* my_thread_dispatcher_client{nullptr}; +}; + + +} +} +} + +#endif // _TBB_threading_control_client_H diff --git a/src/tbb/src/tbb/tls.h b/src/tbb/src/tbb/tls.h index fd4355e8c..e87a943aa 100644 --- a/src/tbb/src/tbb/tls.h +++ b/src/tbb/src/tbb/tls.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2022 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,40 +12,38 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef _TBB_tls_H #define _TBB_tls_H -#if USE_PTHREAD +#include "oneapi/tbb/detail/_config.h" + +#if __TBB_USE_POSIX #include <pthread.h> -#else /* assume USE_WINTHREAD */ -#include "tbb/machine/windows_api.h" +#else /* assume __TBB_USE_WINAPI */ +#include <windows.h> #endif namespace tbb { - -namespace internal { +namespace detail { +namespace r1 { typedef void (*tls_dtor_t)(void*); //! Basic cross-platform wrapper class for TLS operations. template <typename T> class basic_tls { -#if USE_PTHREAD +#if __TBB_USE_POSIX typedef pthread_key_t tls_key_t; public: - int create( tls_dtor_t dtor = NULL ) { + int create( tls_dtor_t dtor = nullptr ) { return pthread_key_create(&my_key, dtor); } int destroy() { return pthread_key_delete(my_key); } void set( T value ) { pthread_setspecific(my_key, (void*)value); } T get() { return (T)pthread_getspecific(my_key); } -#else /* USE_WINTHREAD */ +#else /* __TBB_USE_WINAPI */ typedef DWORD tls_key_t; public: #if !__TBB_WIN8UI_SUPPORT @@ -61,7 +59,7 @@ class basic_tls { T get() { return (T)TlsGetValue(my_key); } #else /*!__TBB_WIN8UI_SUPPORT*/ int create() { - tls_key_t tmp = FlsAlloc(NULL); + tls_key_t tmp = FlsAlloc(nullptr); if( tmp== (DWORD)0xFFFFFFFF ) return (DWORD)0xFFFFFFFF; my_key = tmp; @@ -71,54 +69,13 @@ class basic_tls { void set( T value ) { FlsSetValue(my_key, (LPVOID)value); } T get() { return (T)FlsGetValue(my_key); } #endif /* !__TBB_WIN8UI_SUPPORT */ -#endif /* USE_WINTHREAD */ +#endif /* __TBB_USE_WINAPI */ private: tls_key_t my_key; }; -//! More advanced TLS support template class. -/** It supports RAII and to some extent mimic __declspec(thread) variables. */ -template <typename T> -class tls : public basic_tls<T> { - typedef basic_tls<T> base; -public: - tls() { base::create(); } - ~tls() { base::destroy(); } - T operator=(T value) { base::set(value); return value; } - operator T() { return base::get(); } -}; - -template <typename T> -class tls<T*> : basic_tls<T*> { - typedef basic_tls<T*> base; - static void internal_dtor(void* ptr) { - if (ptr) delete (T*)ptr; - } - T* internal_get() { - T* result = base::get(); - if (!result) { - result = new T; - base::set(result); - } - return result; - } -public: - tls() { -#if USE_PTHREAD - base::create( internal_dtor ); -#else - base::create(); -#endif - } - ~tls() { base::destroy(); } - T* operator=(T* value) { base::set(value); return value; } - operator T*() { return internal_get(); } - T* operator->() { return internal_get(); } - T& operator*() { return *internal_get(); } -}; - -} // namespace internal - +} // namespace r1 +} // namespace detail } // namespace tbb #endif /* _TBB_tls_H */ diff --git a/src/tbb/src/tbb/tools_api/disable_warnings.h b/src/tbb/src/tbb/tools_api/disable_warnings.h index a2452bb69..c5999a158 100644 --- a/src/tbb/src/tbb/tools_api/disable_warnings.h +++ b/src/tbb/src/tbb/tools_api/disable_warnings.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2022 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,28 +12,28 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include "ittnotify_config.h" #if ITT_PLATFORM==ITT_PLATFORM_WIN -#pragma warning (disable: 593) /* parameter "XXXX" was set but never used */ -#pragma warning (disable: 344) /* typedef name has already been declared (with same type) */ -#pragma warning (disable: 174) /* expression has no effect */ -#pragma warning (disable: 4127) /* conditional expression is constant */ -#pragma warning (disable: 4306) /* conversion from '?' to '?' of greater size */ +#if defined _MSC_VER + +// #pragma warning (disable: 593) /* parameter "XXXX" was set but never used */ +// #pragma warning (disable: 344) /* typedef name has already been declared (with same type) */ +// #pragma warning (disable: 174) /* expression has no effect */ +// #pragma warning (disable: 4127) /* conditional expression is constant */ +// #pragma warning (disable: 4306) /* conversion from '?' to '?' of greater size */ + +#endif #endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ #if defined __INTEL_COMPILER -#pragma warning (disable: 869) /* parameter "XXXXX" was never referenced */ -#pragma warning (disable: 1418) /* external function definition with no prior declaration */ -#pragma warning (disable: 1419) /* external declaration in primary source file */ +// #pragma warning (disable: 869) /* parameter "XXXXX" was never referenced */ +// #pragma warning (disable: 1418) /* external function definition with no prior declaration */ +// #pragma warning (disable: 1419) /* external declaration in primary source file */ #endif /* __INTEL_COMPILER */ diff --git a/src/tbb/src/tbb/tools_api/ittnotify.h b/src/tbb/src/tbb/tools_api/ittnotify.h index 037588cee..eb1571dc8 100644 --- a/src/tbb/src/tbb/tools_api/ittnotify.h +++ b/src/tbb/src/tbb/tools_api/ittnotify.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2024 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef _ITTNOTIFY_H_ @@ -26,7 +22,8 @@ @brief Public User API functions and types @mainpage -The ITT API is used to annotate a user's program with additional information +The Instrumentation and Tracing Technology API (ITT API) is used to +annotate a user's program with additional information that can be used by correctness and performance tools. The user inserts calls in their program. Those calls generate information that is collected at runtime, and used by Intel(R) Threading Tools. @@ -104,6 +101,11 @@ The same ID may not be reused for different instances, unless a previous # define ITT_OS_FREEBSD 4 #endif /* ITT_OS_FREEBSD */ +#ifndef ITT_OS_OPENBSD +# define ITT_OS_OPENBSD 5 +#endif /* ITT_OS_OPENBSD */ + + #ifndef ITT_OS # if defined WIN32 || defined _WIN32 # define ITT_OS ITT_OS_WIN @@ -111,6 +113,8 @@ The same ID may not be reused for different instances, unless a previous # define ITT_OS ITT_OS_MAC # elif defined( __FreeBSD__ ) # define ITT_OS ITT_OS_FREEBSD +# elif defined( __OpenBSD__ ) +# define ITT_OS ITT_OS_OPENBSD # else # define ITT_OS ITT_OS_LINUX # endif @@ -132,6 +136,10 @@ The same ID may not be reused for different instances, unless a previous # define ITT_PLATFORM_FREEBSD 4 #endif /* ITT_PLATFORM_FREEBSD */ +#ifndef ITT_PLATFORM_OPENBSD +# define ITT_PLATFORM_OPENBSD 5 +#endif /* ITT_PLATFORM_OPENBSD */ + #ifndef ITT_PLATFORM # if ITT_OS==ITT_OS_WIN # define ITT_PLATFORM ITT_PLATFORM_WIN @@ -139,6 +147,8 @@ The same ID may not be reused for different instances, unless a previous # define ITT_PLATFORM ITT_PLATFORM_MAC # elif ITT_OS==ITT_OS_FREEBSD # define ITT_PLATFORM ITT_PLATFORM_FREEBSD +# elif ITT_OS==ITT_OS_OPENBSD +# define ITT_PLATFORM ITT_PLATFORM_OPENBSD # else # define ITT_PLATFORM ITT_PLATFORM_POSIX # endif @@ -158,17 +168,17 @@ The same ID may not be reused for different instances, unless a previous #endif /* UNICODE || _UNICODE */ #endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ -#ifndef CDECL +#ifndef ITTAPI_CDECL # if ITT_PLATFORM==ITT_PLATFORM_WIN -# define CDECL __cdecl +# define ITTAPI_CDECL __cdecl # else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ # if defined _M_IX86 || defined __i386__ -# define CDECL __attribute__ ((cdecl)) +# define ITTAPI_CDECL __attribute__ ((cdecl)) # else /* _M_IX86 || __i386__ */ -# define CDECL /* actual only on x86 platform */ +# define ITTAPI_CDECL /* actual only on x86 platform */ # endif /* _M_IX86 || __i386__ */ # endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ -#endif /* CDECL */ +#endif /* ITTAPI_CDECL */ #ifndef STDCALL # if ITT_PLATFORM==ITT_PLATFORM_WIN @@ -182,16 +192,21 @@ The same ID may not be reused for different instances, unless a previous # endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ #endif /* STDCALL */ -#define ITTAPI CDECL -#define LIBITTAPI CDECL +#define ITTAPI ITTAPI_CDECL +#define LIBITTAPI ITTAPI_CDECL /* TODO: Temporary for compatibility! */ -#define ITTAPI_CALL CDECL -#define LIBITTAPI_CALL CDECL +#define ITTAPI_CALL ITTAPI_CDECL +#define LIBITTAPI_CALL ITTAPI_CDECL #if ITT_PLATFORM==ITT_PLATFORM_WIN /* use __forceinline (VC++ specific) */ -#define ITT_INLINE __forceinline +#if defined(__MINGW32__) && !defined(__cplusplus) +#define ITT_INLINE static __inline__ __attribute__((__always_inline__,__gnu_inline__)) +#else +#define ITT_INLINE static __forceinline +#endif /* __MINGW32__ */ + #define ITT_INLINE_ATTRIBUTE /* nothing */ #else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ /* @@ -213,8 +228,7 @@ The same ID may not be reused for different instances, unless a previous # if ITT_PLATFORM==ITT_PLATFORM_WIN # pragma message("WARNING!!! Deprecated API is used. Please undefine INTEL_ITTNOTIFY_ENABLE_LEGACY macro") # else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ -// #warning usage leads to ICC's compilation error -// # warning "Deprecated API is used. Please undefine INTEL_ITTNOTIFY_ENABLE_LEGACY macro" +# warning "Deprecated API is used. Please undefine INTEL_ITTNOTIFY_ENABLE_LEGACY macro" # endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ # include "legacy/ittnotify.h" #endif /* INTEL_ITTNOTIFY_ENABLE_LEGACY */ @@ -253,20 +267,20 @@ The same ID may not be reused for different instances, unless a previous #define ITTNOTIFY_VOID(n) (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n) #define ITTNOTIFY_DATA(n) (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n) -#define ITTNOTIFY_VOID_D0(n,d) (!(d)->flags) ? (void)0 : (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n)(d) -#define ITTNOTIFY_VOID_D1(n,d,x) (!(d)->flags) ? (void)0 : (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n)(d,x) -#define ITTNOTIFY_VOID_D2(n,d,x,y) (!(d)->flags) ? (void)0 : (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n)(d,x,y) -#define ITTNOTIFY_VOID_D3(n,d,x,y,z) (!(d)->flags) ? (void)0 : (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n)(d,x,y,z) -#define ITTNOTIFY_VOID_D4(n,d,x,y,z,a) (!(d)->flags) ? (void)0 : (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n)(d,x,y,z,a) -#define ITTNOTIFY_VOID_D5(n,d,x,y,z,a,b) (!(d)->flags) ? (void)0 : (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n)(d,x,y,z,a,b) -#define ITTNOTIFY_VOID_D6(n,d,x,y,z,a,b,c) (!(d)->flags) ? (void)0 : (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n)(d,x,y,z,a,b,c) -#define ITTNOTIFY_DATA_D0(n,d) (!(d)->flags) ? 0 : (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n)(d) -#define ITTNOTIFY_DATA_D1(n,d,x) (!(d)->flags) ? 0 : (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n)(d,x) -#define ITTNOTIFY_DATA_D2(n,d,x,y) (!(d)->flags) ? 0 : (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n)(d,x,y) -#define ITTNOTIFY_DATA_D3(n,d,x,y,z) (!(d)->flags) ? 0 : (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n)(d,x,y,z) -#define ITTNOTIFY_DATA_D4(n,d,x,y,z,a) (!(d)->flags) ? 0 : (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n)(d,x,y,z,a) -#define ITTNOTIFY_DATA_D5(n,d,x,y,z,a,b) (!(d)->flags) ? 0 : (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n)(d,x,y,z,a,b) -#define ITTNOTIFY_DATA_D6(n,d,x,y,z,a,b,c) (!(d)->flags) ? 0 : (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n)(d,x,y,z,a,b,c) +#define ITTNOTIFY_VOID_D0(n,d) (d == NULL) ? (void)0 : (!(d)->flags) ? (void)0 : (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n)(d) +#define ITTNOTIFY_VOID_D1(n,d,x) (d == NULL) ? (void)0 : (!(d)->flags) ? (void)0 : (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n)(d,x) +#define ITTNOTIFY_VOID_D2(n,d,x,y) (d == NULL) ? (void)0 : (!(d)->flags) ? (void)0 : (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n)(d,x,y) +#define ITTNOTIFY_VOID_D3(n,d,x,y,z) (d == NULL) ? (void)0 : (!(d)->flags) ? (void)0 : (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n)(d,x,y,z) +#define ITTNOTIFY_VOID_D4(n,d,x,y,z,a) (d == NULL) ? (void)0 : (!(d)->flags) ? (void)0 : (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n)(d,x,y,z,a) +#define ITTNOTIFY_VOID_D5(n,d,x,y,z,a,b) (d == NULL) ? (void)0 : (!(d)->flags) ? (void)0 : (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n)(d,x,y,z,a,b) +#define ITTNOTIFY_VOID_D6(n,d,x,y,z,a,b,c) (d == NULL) ? (void)0 : (!(d)->flags) ? (void)0 : (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n)(d,x,y,z,a,b,c) +#define ITTNOTIFY_DATA_D0(n,d) (d == NULL) ? 0 : (!(d)->flags) ? 0 : (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n)(d) +#define ITTNOTIFY_DATA_D1(n,d,x) (d == NULL) ? 0 : (!(d)->flags) ? 0 : (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n)(d,x) +#define ITTNOTIFY_DATA_D2(n,d,x,y) (d == NULL) ? 0 : (!(d)->flags) ? 0 : (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n)(d,x,y) +#define ITTNOTIFY_DATA_D3(n,d,x,y,z) (d == NULL) ? 0 : (!(d)->flags) ? 0 : (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n)(d,x,y,z) +#define ITTNOTIFY_DATA_D4(n,d,x,y,z,a) (d == NULL) ? 0 : (!(d)->flags) ? 0 : (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n)(d,x,y,z,a) +#define ITTNOTIFY_DATA_D5(n,d,x,y,z,a,b) (d == NULL) ? 0 : (!(d)->flags) ? 0 : (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n)(d,x,y,z,a,b) +#define ITTNOTIFY_DATA_D6(n,d,x,y,z,a,b,c) (d == NULL) ? 0 : (!(d)->flags) ? 0 : (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n)(d,x,y,z,a,b,c) #ifdef ITT_STUB #undef ITT_STUB @@ -304,7 +318,7 @@ extern "C" { * only pauses tracing and analyzing memory access. * It does not pause tracing or analyzing threading APIs. * . - * - Intel(R) Parallel Amplifier and Intel(R) VTune(TM) Amplifier XE: + * Intel(R) VTune(TM) Profiler: * - Does continue to record when new threads are started. * . * - Other effects: @@ -319,35 +333,143 @@ void ITTAPI __itt_resume(void); /** @brief Detach collection */ void ITTAPI __itt_detach(void); +/** + * @enum __itt_collection_scope + * @brief Enumerator for collection scopes + */ +typedef enum { + __itt_collection_scope_host = 1 << 0, + __itt_collection_scope_offload = 1 << 1, + __itt_collection_scope_all = 0x7FFFFFFF +} __itt_collection_scope; + +/** @brief Pause scoped collection */ +void ITTAPI __itt_pause_scoped(__itt_collection_scope); +/** @brief Resume scoped collection */ +void ITTAPI __itt_resume_scoped(__itt_collection_scope); + /** @cond exclude_from_documentation */ #ifndef INTEL_NO_MACRO_BODY #ifndef INTEL_NO_ITTNOTIFY_API -ITT_STUBV(ITTAPI, void, pause, (void)) -ITT_STUBV(ITTAPI, void, resume, (void)) -ITT_STUBV(ITTAPI, void, detach, (void)) -#define __itt_pause ITTNOTIFY_VOID(pause) -#define __itt_pause_ptr ITTNOTIFY_NAME(pause) -#define __itt_resume ITTNOTIFY_VOID(resume) -#define __itt_resume_ptr ITTNOTIFY_NAME(resume) -#define __itt_detach ITTNOTIFY_VOID(detach) -#define __itt_detach_ptr ITTNOTIFY_NAME(detach) +ITT_STUBV(ITTAPI, void, pause, (void)) +ITT_STUBV(ITTAPI, void, pause_scoped, (__itt_collection_scope)) +ITT_STUBV(ITTAPI, void, resume, (void)) +ITT_STUBV(ITTAPI, void, resume_scoped, (__itt_collection_scope)) +ITT_STUBV(ITTAPI, void, detach, (void)) +#define __itt_pause ITTNOTIFY_VOID(pause) +#define __itt_pause_ptr ITTNOTIFY_NAME(pause) +#define __itt_pause_scoped ITTNOTIFY_VOID(pause_scoped) +#define __itt_pause_scoped_ptr ITTNOTIFY_NAME(pause_scoped) +#define __itt_resume ITTNOTIFY_VOID(resume) +#define __itt_resume_ptr ITTNOTIFY_NAME(resume) +#define __itt_resume_scoped ITTNOTIFY_VOID(resume_scoped) +#define __itt_resume_scoped_ptr ITTNOTIFY_NAME(resume_scoped) +#define __itt_detach ITTNOTIFY_VOID(detach) +#define __itt_detach_ptr ITTNOTIFY_NAME(detach) #else /* INTEL_NO_ITTNOTIFY_API */ #define __itt_pause() -#define __itt_pause_ptr 0 +#define __itt_pause_ptr 0 +#define __itt_pause_scoped(scope) +#define __itt_pause_scoped_ptr 0 #define __itt_resume() -#define __itt_resume_ptr 0 +#define __itt_resume_ptr 0 +#define __itt_resume_scoped(scope) +#define __itt_resume_scoped_ptr 0 #define __itt_detach() -#define __itt_detach_ptr 0 +#define __itt_detach_ptr 0 #endif /* INTEL_NO_ITTNOTIFY_API */ #else /* INTEL_NO_MACRO_BODY */ -#define __itt_pause_ptr 0 -#define __itt_resume_ptr 0 -#define __itt_detach_ptr 0 +#define __itt_pause_ptr 0 +#define __itt_pause_scoped_ptr 0 +#define __itt_resume_ptr 0 +#define __itt_resume_scoped_ptr 0 +#define __itt_detach_ptr 0 #endif /* INTEL_NO_MACRO_BODY */ /** @endcond */ /** @} control group */ /** @endcond */ +/** + * @defgroup Intel Processor Trace control + * API from this group provides control over collection and analysis of Intel Processor Trace (Intel PT) data + * Information about Intel Processor Trace technology can be found here (Volume 3 chapter 35): + * https://github.com/tpn/pdfs/blob/master/Intel%2064%20and%20IA-32%20Architectures%20Software%20Developer's%20Manual%20-%20Combined%20Volumes%201-4%20-%20May%202018%20(325462-sdm-vol-1-2abcd-3abcd).pdf + * Use this API to mark particular code regions for loading detailed performance statistics. + * This mode makes your analysis faster and more accurate. + * @{ +*/ +typedef unsigned char __itt_pt_region; + +/** + * @brief function saves a region name marked with Intel PT API and returns a region id. + * Only 7 names can be registered. Attempts to register more names will be ignored and a region id with auto names will be returned. + * For automatic naming of regions pass NULL as function parameter +*/ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +__itt_pt_region ITTAPI __itt_pt_region_createA(const char *name); +__itt_pt_region ITTAPI __itt_pt_region_createW(const wchar_t *name); +#if defined(UNICODE) || defined(_UNICODE) +# define __itt_pt_region_create __itt_pt_region_createW +#else /* UNICODE */ +# define __itt_pt_region_create __itt_pt_region_createA +#endif /* UNICODE */ +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +__itt_pt_region ITTAPI __itt_pt_region_create(const char *name); +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUB(ITTAPI, __itt_pt_region, pt_region_createA, (const char *name)) +ITT_STUB(ITTAPI, __itt_pt_region, pt_region_createW, (const wchar_t *name)) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUB(ITTAPI, __itt_pt_region, pt_region_create, (const char *name)) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_pt_region_createA ITTNOTIFY_DATA(pt_region_createA) +#define __itt_pt_region_createA_ptr ITTNOTIFY_NAME(pt_region_createA) +#define __itt_pt_region_createW ITTNOTIFY_DATA(pt_region_createW) +#define __itt_pt_region_createW_ptr ITTNOTIFY_NAME(pt_region_createW) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_pt_region_create ITTNOTIFY_DATA(pt_region_create) +#define __itt_pt_region_create_ptr ITTNOTIFY_NAME(pt_region_create) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#else /* INTEL_NO_ITTNOTIFY_API */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_pt_region_createA(name) (__itt_pt_region)0 +#define __itt_pt_region_createA_ptr 0 +#define __itt_pt_region_createW(name) (__itt_pt_region)0 +#define __itt_pt_region_createW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_pt_region_create(name) (__itt_pt_region)0 +#define __itt_pt_region_create_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_pt_region_createA_ptr 0 +#define __itt_pt_region_createW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_pt_region_create_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief function contains a special code pattern identified on the post-processing stage and + * marks the beginning of a code region targeted for Intel PT analysis + * @param[in] region - region id, 0 <= region < 8 +*/ +void __itt_mark_pt_region_begin(__itt_pt_region region); +/** + * @brief function contains a special code pattern identified on the post-processing stage and + * marks the end of a code region targeted for Intel PT analysis + * @param[in] region - region id, 0 <= region < 8 +*/ +void __itt_mark_pt_region_end(__itt_pt_region region); +/** @} Intel PT control group*/ + /** * @defgroup threads Threads * @ingroup public @@ -505,14 +627,26 @@ ITT_STUBV(ITTAPI, void, suppress_pop, (void)) /** @endcond */ /** - * @enum __itt_model_disable - * @brief Enumerator for the disable methods + * @enum __itt_suppress_mode + * @brief Enumerator for the suppressing modes */ typedef enum __itt_suppress_mode { __itt_unsuppress_range, __itt_suppress_range } __itt_suppress_mode_t; +/** + * @enum __itt_collection_state + * @brief Enumerator for collection state. + */ +typedef enum { + __itt_collection_uninitialized = 0, /* uninitialized */ + __itt_collection_init_fail = 1, /* failed to init */ + __itt_collection_collector_absent = 2, /* non work state collector is absent */ + __itt_collection_collector_exists = 3, /* work state collector exists */ + __itt_collection_init_successful = 4 /* success to init */ +} __itt_collection_state; + /** * @brief Mark a range of memory for error suppression or unsuppression for error types included in mask */ @@ -1460,7 +1594,7 @@ ITT_STUBV(ITTAPI, void, heap_allocate_end, (__itt_heap_function h, void** addr, /** @endcond */ /** - * @brief Record an free begin occurrence. + * @brief Record a free begin occurrence. */ void ITTAPI __itt_heap_free_begin(__itt_heap_function h, void* addr); @@ -1480,7 +1614,7 @@ ITT_STUBV(ITTAPI, void, heap_free_begin, (__itt_heap_function h, void* addr)) /** @endcond */ /** - * @brief Record an free end occurrence. + * @brief Record a free end occurrence. */ void ITTAPI __itt_heap_free_end(__itt_heap_function h, void* addr); @@ -1500,7 +1634,7 @@ ITT_STUBV(ITTAPI, void, heap_free_end, (__itt_heap_function h, void* addr)) /** @endcond */ /** - * @brief Record an reallocation begin occurrence. + * @brief Record a reallocation begin occurrence. */ void ITTAPI __itt_heap_reallocate_begin(__itt_heap_function h, void* addr, size_t new_size, int initialized); @@ -1520,7 +1654,7 @@ ITT_STUBV(ITTAPI, void, heap_reallocate_begin, (__itt_heap_function h, void* add /** @endcond */ /** - * @brief Record an reallocation end occurrence. + * @brief Record a reallocation end occurrence. */ void ITTAPI __itt_heap_reallocate_end(__itt_heap_function h, void* addr, void** new_addr, size_t new_size, int initialized); @@ -2248,63 +2382,10 @@ ITT_STUBV(ITTAPI, void, task_end_overlapped, (const __itt_domain *domain, __it /** @endcond */ /** @} tasks group */ -/** - * @defgroup counters Counters - * @ingroup public - * Counters are user-defined objects with a monotonically increasing - * value. Counter values are 64-bit unsigned integers. Counter values - * are tracked per-thread. Counters have names that can be displayed in - * the tools. - * @{ - */ - -/** - * @ingroup counters - * @brief Increment a counter by one. - * The first call with a given name creates a counter by that name and sets its - * value to zero on every thread. Successive calls increment the counter value - * on the thread on which the call is issued. - * @param[in] domain The domain controlling the call. Counter names are not domain specific. - * The domain argument is used only to enable or disable the API calls. - * @param[in] name The name of the counter - */ -void ITTAPI __itt_counter_inc_v3(const __itt_domain *domain, __itt_string_handle *name); - -/** - * @ingroup counters - * @brief Increment a counter by the value specified in delta. - * @param[in] domain The domain controlling the call. Counter names are not domain specific. - * The domain argument is used only to enable or disable the API calls. - * @param[in] name The name of the counter - * @param[in] delta The amount by which to increment the counter - */ -void ITTAPI __itt_counter_inc_delta_v3(const __itt_domain *domain, __itt_string_handle *name, unsigned long long delta); - -/** @cond exclude_from_documentation */ -#ifndef INTEL_NO_MACRO_BODY -#ifndef INTEL_NO_ITTNOTIFY_API -ITT_STUBV(ITTAPI, void, counter_inc_v3, (const __itt_domain *domain, __itt_string_handle *name)) -ITT_STUBV(ITTAPI, void, counter_inc_delta_v3, (const __itt_domain *domain, __itt_string_handle *name, unsigned long long delta)) -#define __itt_counter_inc_v3(d,x) ITTNOTIFY_VOID_D1(counter_inc_v3,d,x) -#define __itt_counter_inc_v3_ptr ITTNOTIFY_NAME(counter_inc_v3) -#define __itt_counter_inc_delta_v3(d,x,y) ITTNOTIFY_VOID_D2(counter_inc_delta_v3,d,x,y) -#define __itt_counter_inc_delta_v3_ptr ITTNOTIFY_NAME(counter_inc_delta_v3) -#else /* INTEL_NO_ITTNOTIFY_API */ -#define __itt_counter_inc_v3(domain,name) -#define __itt_counter_inc_v3_ptr 0 -#define __itt_counter_inc_delta_v3(domain,name,delta) -#define __itt_counter_inc_delta_v3_ptr 0 -#endif /* INTEL_NO_ITTNOTIFY_API */ -#else /* INTEL_NO_MACRO_BODY */ -#define __itt_counter_inc_v3_ptr 0 -#define __itt_counter_inc_delta_v3_ptr 0 -#endif /* INTEL_NO_MACRO_BODY */ -/** @endcond */ -/** @} counters group */ /** * @defgroup markers Markers - * Markers represent a single discreet event in time. Markers have a scope, + * Markers represent a single discrete event in time. Markers have a scope, * described by an enumerated type __itt_scope. Markers are created by * the API call __itt_marker. A marker instance can be given an ID for use in * adding metadata. @@ -2709,7 +2790,7 @@ ITT_STUB(ITTAPI, __itt_clock_domain*, clock_domain_create, (__itt_get_clock_info /** * @ingroup clockdomains - * @brief Recalculate clock domains frequences and clock base timestamps. + * @brief Recalculate clock domains frequencies and clock base timestamps. */ void ITTAPI __itt_clock_domain_reset(void); @@ -2836,89 +2917,454 @@ ITT_STUBV(ITTAPI, void, task_end_ex, (const __itt_domain *domain, __itt /** @endcond */ /** - * @ingroup markers - * @brief Create a marker instance. - * @param[in] domain The domain for this marker - * @param[in] clock_domain The clock domain controlling the execution of this call. - * @param[in] timestamp The user defined timestamp. - * @param[in] id The instance ID for this marker, or __itt_null - * @param[in] name The name for this marker - * @param[in] scope The scope for this marker + * @defgroup counters Counters + * @ingroup public + * Counters are user-defined objects with a monotonically increasing + * value. Counter values are 64-bit unsigned integers. + * Counters have names that can be displayed in + * the tools. + * @{ */ -void ITTAPI __itt_marker_ex(const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id id, __itt_string_handle *name, __itt_scope scope); +/** + * @brief opaque structure for counter identification + */ /** @cond exclude_from_documentation */ + +typedef struct ___itt_counter* __itt_counter; + +/** + * @brief Create an unsigned 64 bits integer counter with given name/domain + * + * After __itt_counter_create() is called, __itt_counter_inc(id), __itt_counter_inc_delta(id, delta), + * __itt_counter_set_value(id, value_ptr) or __itt_counter_set_value_ex(id, clock_domain, timestamp, value_ptr) + * can be used to change the value of the counter, where value_ptr is a pointer to an unsigned 64 bits integer + * + * The call is equal to __itt_counter_create_typed(name, domain, __itt_metadata_u64) + */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +__itt_counter ITTAPI __itt_counter_createA(const char *name, const char *domain); +__itt_counter ITTAPI __itt_counter_createW(const wchar_t *name, const wchar_t *domain); +#if defined(UNICODE) || defined(_UNICODE) +# define __itt_counter_create __itt_counter_createW +# define __itt_counter_create_ptr __itt_counter_createW_ptr +#else /* UNICODE */ +# define __itt_counter_create __itt_counter_createA +# define __itt_counter_create_ptr __itt_counter_createA_ptr +#endif /* UNICODE */ +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +__itt_counter ITTAPI __itt_counter_create(const char *name, const char *domain); +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + #ifndef INTEL_NO_MACRO_BODY #ifndef INTEL_NO_ITTNOTIFY_API -ITT_STUBV(ITTAPI, void, marker_ex, (const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id id, __itt_string_handle *name, __itt_scope scope)) -#define __itt_marker_ex(d,x,y,z,a,b) ITTNOTIFY_VOID_D5(marker_ex,d,x,y,z,a,b) -#define __itt_marker_ex_ptr ITTNOTIFY_NAME(marker_ex) +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUB(ITTAPI, __itt_counter, counter_createA, (const char *name, const char *domain)) +ITT_STUB(ITTAPI, __itt_counter, counter_createW, (const wchar_t *name, const wchar_t *domain)) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUB(ITTAPI, __itt_counter, counter_create, (const char *name, const char *domain)) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_counter_createA ITTNOTIFY_DATA(counter_createA) +#define __itt_counter_createA_ptr ITTNOTIFY_NAME(counter_createA) +#define __itt_counter_createW ITTNOTIFY_DATA(counter_createW) +#define __itt_counter_createW_ptr ITTNOTIFY_NAME(counter_createW) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_counter_create ITTNOTIFY_DATA(counter_create) +#define __itt_counter_create_ptr ITTNOTIFY_NAME(counter_create) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ #else /* INTEL_NO_ITTNOTIFY_API */ -#define __itt_marker_ex(domain,clock_domain,timestamp,id,name,scope) -#define __itt_marker_ex_ptr 0 +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_counter_createA(name, domain) +#define __itt_counter_createA_ptr 0 +#define __itt_counter_createW(name, domain) +#define __itt_counter_createW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_counter_create(name, domain) +#define __itt_counter_create_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ #endif /* INTEL_NO_ITTNOTIFY_API */ #else /* INTEL_NO_MACRO_BODY */ -#define __itt_marker_ex_ptr 0 +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_counter_createA_ptr 0 +#define __itt_counter_createW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_counter_create_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ #endif /* INTEL_NO_MACRO_BODY */ /** @endcond */ /** - * @ingroup clockdomain - * @brief Add a relation to the current task instance. - * The current task instance is the head of the relation. - * @param[in] domain The domain controlling this call - * @param[in] clock_domain The clock domain controlling the execution of this call. - * @param[in] timestamp The user defined timestamp. - * @param[in] relation The kind of relation - * @param[in] tail The ID for the tail of the relation + * @brief Increment the unsigned 64 bits integer counter value + * + * Calling this function to non-unsigned 64 bits integer counters has no effect */ -void ITTAPI __itt_relation_add_to_current_ex(const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_relation relation, __itt_id tail); +void ITTAPI __itt_counter_inc(__itt_counter id); +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, counter_inc, (__itt_counter id)) +#define __itt_counter_inc ITTNOTIFY_VOID(counter_inc) +#define __itt_counter_inc_ptr ITTNOTIFY_NAME(counter_inc) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_counter_inc(id) +#define __itt_counter_inc_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_counter_inc_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ /** - * @ingroup clockdomain - * @brief Add a relation between two instance identifiers. - * @param[in] domain The domain controlling this call - * @param[in] clock_domain The clock domain controlling the execution of this call. - * @param[in] timestamp The user defined timestamp. - * @param[in] head The ID for the head of the relation - * @param[in] relation The kind of relation - * @param[in] tail The ID for the tail of the relation + * @brief Increment the unsigned 64 bits integer counter value with x + * + * Calling this function to non-unsigned 64 bits integer counters has no effect */ -void ITTAPI __itt_relation_add_ex(const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id head, __itt_relation relation, __itt_id tail); +void ITTAPI __itt_counter_inc_delta(__itt_counter id, unsigned long long value); -/** @cond exclude_from_documentation */ #ifndef INTEL_NO_MACRO_BODY #ifndef INTEL_NO_ITTNOTIFY_API -ITT_STUBV(ITTAPI, void, relation_add_to_current_ex, (const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_relation relation, __itt_id tail)) -ITT_STUBV(ITTAPI, void, relation_add_ex, (const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id head, __itt_relation relation, __itt_id tail)) -#define __itt_relation_add_to_current_ex(d,x,y,z,a) ITTNOTIFY_VOID_D4(relation_add_to_current_ex,d,x,y,z,a) -#define __itt_relation_add_to_current_ex_ptr ITTNOTIFY_NAME(relation_add_to_current_ex) -#define __itt_relation_add_ex(d,x,y,z,a,b) ITTNOTIFY_VOID_D5(relation_add_ex,d,x,y,z,a,b) -#define __itt_relation_add_ex_ptr ITTNOTIFY_NAME(relation_add_ex) +ITT_STUBV(ITTAPI, void, counter_inc_delta, (__itt_counter id, unsigned long long value)) +#define __itt_counter_inc_delta ITTNOTIFY_VOID(counter_inc_delta) +#define __itt_counter_inc_delta_ptr ITTNOTIFY_NAME(counter_inc_delta) #else /* INTEL_NO_ITTNOTIFY_API */ -#define __itt_relation_add_to_current_ex(domain,clock_domain,timestame,relation,tail) -#define __itt_relation_add_to_current_ex_ptr 0 -#define __itt_relation_add_ex(domain,clock_domain,timestamp,head,relation,tail) -#define __itt_relation_add_ex_ptr 0 +#define __itt_counter_inc_delta(id, value) +#define __itt_counter_inc_delta_ptr 0 #endif /* INTEL_NO_ITTNOTIFY_API */ #else /* INTEL_NO_MACRO_BODY */ -#define __itt_relation_add_to_current_ex_ptr 0 -#define __itt_relation_add_ex_ptr 0 +#define __itt_counter_inc_delta_ptr 0 #endif /* INTEL_NO_MACRO_BODY */ /** @endcond */ -/** @cond exclude_from_documentation */ -typedef enum ___itt_track_group_type -{ - __itt_track_group_type_normal = 0 -} __itt_track_group_type; -/** @endcond */ +/** + * @brief Decrement the unsigned 64 bits integer counter value + * + * Calling this function to non-unsigned 64 bits integer counters has no effect + */ +void ITTAPI __itt_counter_dec(__itt_counter id); -/** @cond exclude_from_documentation */ -#pragma pack(push, 8) +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, counter_dec, (__itt_counter id)) +#define __itt_counter_dec ITTNOTIFY_VOID(counter_dec) +#define __itt_counter_dec_ptr ITTNOTIFY_NAME(counter_dec) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_counter_dec(id) +#define __itt_counter_dec_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_counter_dec_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** + * @brief Decrement the unsigned 64 bits integer counter value with x + * + * Calling this function to non-unsigned 64 bits integer counters has no effect + */ +void ITTAPI __itt_counter_dec_delta(__itt_counter id, unsigned long long value); -typedef struct ___itt_track_group -{ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, counter_dec_delta, (__itt_counter id, unsigned long long value)) +#define __itt_counter_dec_delta ITTNOTIFY_VOID(counter_dec_delta) +#define __itt_counter_dec_delta_ptr ITTNOTIFY_NAME(counter_dec_delta) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_counter_dec_delta(id, value) +#define __itt_counter_dec_delta_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_counter_dec_delta_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @ingroup counters + * @brief Increment a counter by one. + * The first call with a given name creates a counter by that name and sets its + * value to zero. Successive calls increment the counter value. + * @param[in] domain The domain controlling the call. Counter names are not domain specific. + * The domain argument is used only to enable or disable the API calls. + * @param[in] name The name of the counter + */ +void ITTAPI __itt_counter_inc_v3(const __itt_domain *domain, __itt_string_handle *name); + +/** + * @ingroup counters + * @brief Increment a counter by the value specified in delta. + * @param[in] domain The domain controlling the call. Counter names are not domain specific. + * The domain argument is used only to enable or disable the API calls. + * @param[in] name The name of the counter + * @param[in] delta The amount by which to increment the counter + */ +void ITTAPI __itt_counter_inc_delta_v3(const __itt_domain *domain, __itt_string_handle *name, unsigned long long delta); + +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, counter_inc_v3, (const __itt_domain *domain, __itt_string_handle *name)) +ITT_STUBV(ITTAPI, void, counter_inc_delta_v3, (const __itt_domain *domain, __itt_string_handle *name, unsigned long long delta)) +#define __itt_counter_inc_v3(d,x) ITTNOTIFY_VOID_D1(counter_inc_v3,d,x) +#define __itt_counter_inc_v3_ptr ITTNOTIFY_NAME(counter_inc_v3) +#define __itt_counter_inc_delta_v3(d,x,y) ITTNOTIFY_VOID_D2(counter_inc_delta_v3,d,x,y) +#define __itt_counter_inc_delta_v3_ptr ITTNOTIFY_NAME(counter_inc_delta_v3) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_counter_inc_v3(domain,name) +#define __itt_counter_inc_v3_ptr 0 +#define __itt_counter_inc_delta_v3(domain,name,delta) +#define __itt_counter_inc_delta_v3_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_counter_inc_v3_ptr 0 +#define __itt_counter_inc_delta_v3_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + + +/** + * @ingroup counters + * @brief Decrement a counter by one. + * The first call with a given name creates a counter by that name and sets its + * value to zero. Successive calls decrement the counter value. + * @param[in] domain The domain controlling the call. Counter names are not domain specific. + * The domain argument is used only to enable or disable the API calls. + * @param[in] name The name of the counter + */ +void ITTAPI __itt_counter_dec_v3(const __itt_domain *domain, __itt_string_handle *name); + +/** + * @ingroup counters + * @brief Decrement a counter by the value specified in delta. + * @param[in] domain The domain controlling the call. Counter names are not domain specific. + * The domain argument is used only to enable or disable the API calls. + * @param[in] name The name of the counter + * @param[in] delta The amount by which to decrement the counter + */ +void ITTAPI __itt_counter_dec_delta_v3(const __itt_domain *domain, __itt_string_handle *name, unsigned long long delta); + +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, counter_dec_v3, (const __itt_domain *domain, __itt_string_handle *name)) +ITT_STUBV(ITTAPI, void, counter_dec_delta_v3, (const __itt_domain *domain, __itt_string_handle *name, unsigned long long delta)) +#define __itt_counter_dec_v3(d,x) ITTNOTIFY_VOID_D1(counter_dec_v3,d,x) +#define __itt_counter_dec_v3_ptr ITTNOTIFY_NAME(counter_dec_v3) +#define __itt_counter_dec_delta_v3(d,x,y) ITTNOTIFY_VOID_D2(counter_dec_delta_v3,d,x,y) +#define __itt_counter_dec_delta_v3_ptr ITTNOTIFY_NAME(counter_dec_delta_v3) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_counter_dec_v3(domain,name) +#define __itt_counter_dec_v3_ptr 0 +#define __itt_counter_dec_delta_v3(domain,name,delta) +#define __itt_counter_dec_delta_v3_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_counter_dec_v3_ptr 0 +#define __itt_counter_dec_delta_v3_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** @} counters group */ + + +/** + * @brief Set the counter value + */ +void ITTAPI __itt_counter_set_value(__itt_counter id, void *value_ptr); + +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, counter_set_value, (__itt_counter id, void *value_ptr)) +#define __itt_counter_set_value ITTNOTIFY_VOID(counter_set_value) +#define __itt_counter_set_value_ptr ITTNOTIFY_NAME(counter_set_value) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_counter_set_value(id, value_ptr) +#define __itt_counter_set_value_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_counter_set_value_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Set the counter value + */ +void ITTAPI __itt_counter_set_value_ex(__itt_counter id, __itt_clock_domain *clock_domain, unsigned long long timestamp, void *value_ptr); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, counter_set_value_ex, (__itt_counter id, __itt_clock_domain *clock_domain, unsigned long long timestamp, void *value_ptr)) +#define __itt_counter_set_value_ex ITTNOTIFY_VOID(counter_set_value_ex) +#define __itt_counter_set_value_ex_ptr ITTNOTIFY_NAME(counter_set_value_ex) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_counter_set_value_ex(id, clock_domain, timestamp, value_ptr) +#define __itt_counter_set_value_ex_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_counter_set_value_ex_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Create a typed counter with given name/domain + * + * After __itt_counter_create_typed() is called, __itt_counter_inc(id), __itt_counter_inc_delta(id, delta), + * __itt_counter_set_value(id, value_ptr) or __itt_counter_set_value_ex(id, clock_domain, timestamp, value_ptr) + * can be used to change the value of the counter + */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +__itt_counter ITTAPI __itt_counter_create_typedA(const char *name, const char *domain, __itt_metadata_type type); +__itt_counter ITTAPI __itt_counter_create_typedW(const wchar_t *name, const wchar_t *domain, __itt_metadata_type type); +#if defined(UNICODE) || defined(_UNICODE) +# define __itt_counter_create_typed __itt_counter_create_typedW +# define __itt_counter_create_typed_ptr __itt_counter_create_typedW_ptr +#else /* UNICODE */ +# define __itt_counter_create_typed __itt_counter_create_typedA +# define __itt_counter_create_typed_ptr __itt_counter_create_typedA_ptr +#endif /* UNICODE */ +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +__itt_counter ITTAPI __itt_counter_create_typed(const char *name, const char *domain, __itt_metadata_type type); +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUB(ITTAPI, __itt_counter, counter_create_typedA, (const char *name, const char *domain, __itt_metadata_type type)) +ITT_STUB(ITTAPI, __itt_counter, counter_create_typedW, (const wchar_t *name, const wchar_t *domain, __itt_metadata_type type)) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUB(ITTAPI, __itt_counter, counter_create_typed, (const char *name, const char *domain, __itt_metadata_type type)) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_counter_create_typedA ITTNOTIFY_DATA(counter_create_typedA) +#define __itt_counter_create_typedA_ptr ITTNOTIFY_NAME(counter_create_typedA) +#define __itt_counter_create_typedW ITTNOTIFY_DATA(counter_create_typedW) +#define __itt_counter_create_typedW_ptr ITTNOTIFY_NAME(counter_create_typedW) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_counter_create_typed ITTNOTIFY_DATA(counter_create_typed) +#define __itt_counter_create_typed_ptr ITTNOTIFY_NAME(counter_create_typed) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#else /* INTEL_NO_ITTNOTIFY_API */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_counter_create_typedA(name, domain, type) +#define __itt_counter_create_typedA_ptr 0 +#define __itt_counter_create_typedW(name, domain, type) +#define __itt_counter_create_typedW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_counter_create_typed(name, domain, type) +#define __itt_counter_create_typed_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_counter_create_typedA_ptr 0 +#define __itt_counter_create_typedW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_counter_create_typed_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Destroy the counter identified by the pointer previously returned by __itt_counter_create() or + * __itt_counter_create_typed() + */ +void ITTAPI __itt_counter_destroy(__itt_counter id); + +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, counter_destroy, (__itt_counter id)) +#define __itt_counter_destroy ITTNOTIFY_VOID(counter_destroy) +#define __itt_counter_destroy_ptr ITTNOTIFY_NAME(counter_destroy) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_counter_destroy(id) +#define __itt_counter_destroy_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_counter_destroy_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** @} counters group */ + +/** + * @ingroup markers + * @brief Create a marker instance. + * @param[in] domain The domain for this marker + * @param[in] clock_domain The clock domain controlling the execution of this call. + * @param[in] timestamp The user defined timestamp. + * @param[in] id The instance ID for this marker, or __itt_null + * @param[in] name The name for this marker + * @param[in] scope The scope for this marker + */ +void ITTAPI __itt_marker_ex(const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id id, __itt_string_handle *name, __itt_scope scope); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, marker_ex, (const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id id, __itt_string_handle *name, __itt_scope scope)) +#define __itt_marker_ex(d,x,y,z,a,b) ITTNOTIFY_VOID_D5(marker_ex,d,x,y,z,a,b) +#define __itt_marker_ex_ptr ITTNOTIFY_NAME(marker_ex) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_marker_ex(domain,clock_domain,timestamp,id,name,scope) +#define __itt_marker_ex_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_marker_ex_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @ingroup clockdomain + * @brief Add a relation to the current task instance. + * The current task instance is the head of the relation. + * @param[in] domain The domain controlling this call + * @param[in] clock_domain The clock domain controlling the execution of this call. + * @param[in] timestamp The user defined timestamp. + * @param[in] relation The kind of relation + * @param[in] tail The ID for the tail of the relation + */ +void ITTAPI __itt_relation_add_to_current_ex(const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_relation relation, __itt_id tail); + +/** + * @ingroup clockdomain + * @brief Add a relation between two instance identifiers. + * @param[in] domain The domain controlling this call + * @param[in] clock_domain The clock domain controlling the execution of this call. + * @param[in] timestamp The user defined timestamp. + * @param[in] head The ID for the head of the relation + * @param[in] relation The kind of relation + * @param[in] tail The ID for the tail of the relation + */ +void ITTAPI __itt_relation_add_ex(const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id head, __itt_relation relation, __itt_id tail); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, relation_add_to_current_ex, (const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_relation relation, __itt_id tail)) +ITT_STUBV(ITTAPI, void, relation_add_ex, (const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id head, __itt_relation relation, __itt_id tail)) +#define __itt_relation_add_to_current_ex(d,x,y,z,a) ITTNOTIFY_VOID_D4(relation_add_to_current_ex,d,x,y,z,a) +#define __itt_relation_add_to_current_ex_ptr ITTNOTIFY_NAME(relation_add_to_current_ex) +#define __itt_relation_add_ex(d,x,y,z,a,b) ITTNOTIFY_VOID_D5(relation_add_ex,d,x,y,z,a,b) +#define __itt_relation_add_ex_ptr ITTNOTIFY_NAME(relation_add_ex) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_relation_add_to_current_ex(domain,clock_domain,timestame,relation,tail) +#define __itt_relation_add_to_current_ex_ptr 0 +#define __itt_relation_add_ex(domain,clock_domain,timestamp,head,relation,tail) +#define __itt_relation_add_ex_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_relation_add_to_current_ex_ptr 0 +#define __itt_relation_add_ex_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** @cond exclude_from_documentation */ +typedef enum ___itt_track_group_type +{ + __itt_track_group_type_normal = 0 +} __itt_track_group_type; +/** @endcond */ + +/** @cond exclude_from_documentation */ +#pragma pack(push, 8) + +typedef struct ___itt_track_group +{ __itt_string_handle* name; /*!< Name of the track group */ struct ___itt_track* track; /*!< List of child tracks */ __itt_track_group_type tgtype; /*!< Type of the track group */ @@ -3100,154 +3546,671 @@ ITT_STUB(LIBITTAPI, int, event_start, (__itt_event event)) #define __itt_event_start ITTNOTIFY_DATA(event_start) #define __itt_event_start_ptr ITTNOTIFY_NAME(event_start) #else /* INTEL_NO_ITTNOTIFY_API */ -#define __itt_event_start(event) (int)0 -#define __itt_event_start_ptr 0 +#define __itt_event_start(event) (int)0 +#define __itt_event_start_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_event_start_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Record an event end occurrence. + * @note It is optional if events do not have durations. + * @return __itt_err upon failure (invalid event id/user event feature not enabled) + */ +int LIBITTAPI __itt_event_end(__itt_event event); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUB(LIBITTAPI, int, event_end, (__itt_event event)) +#define __itt_event_end ITTNOTIFY_DATA(event_end) +#define __itt_event_end_ptr ITTNOTIFY_NAME(event_end) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_event_end(event) (int)0 +#define __itt_event_end_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_event_end_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** @} events group */ + + +/** + * @defgroup arrays Arrays Visualizer + * @ingroup public + * Visualize arrays + * @{ + */ + +/** + * @enum __itt_av_data_type + * @brief Defines types of arrays data (for C/C++ intrinsic types) + */ +typedef enum +{ + __itt_e_first = 0, + __itt_e_char = 0, /* 1-byte integer */ + __itt_e_uchar, /* 1-byte unsigned integer */ + __itt_e_int16, /* 2-byte integer */ + __itt_e_uint16, /* 2-byte unsigned integer */ + __itt_e_int32, /* 4-byte integer */ + __itt_e_uint32, /* 4-byte unsigned integer */ + __itt_e_int64, /* 8-byte integer */ + __itt_e_uint64, /* 8-byte unsigned integer */ + __itt_e_float, /* 4-byte floating */ + __itt_e_double, /* 8-byte floating */ + __itt_e_last = __itt_e_double +} __itt_av_data_type; + +/** + * @brief Save an array data to a file. + * Output format is defined by the file extension. The csv and bmp formats are supported (bmp - for 2-dimensional array only). + * @param[in] data - pointer to the array data + * @param[in] rank - the rank of the array + * @param[in] dimensions - pointer to an array of integers, which specifies the array dimensions. + * The size of dimensions must be equal to the rank + * @param[in] type - the type of the array, specified as one of the __itt_av_data_type values (for intrinsic types) + * @param[in] filePath - the file path; the output format is defined by the file extension + * @param[in] columnOrder - defines how the array is stored in the linear memory. + * It should be 1 for column-major order (e.g. in FORTRAN) or 0 - for row-major order (e.g. in C). + */ + +#if ITT_PLATFORM==ITT_PLATFORM_WIN +int ITTAPI __itt_av_saveA(void *data, int rank, const int *dimensions, int type, const char *filePath, int columnOrder); +int ITTAPI __itt_av_saveW(void *data, int rank, const int *dimensions, int type, const wchar_t *filePath, int columnOrder); +#if defined(UNICODE) || defined(_UNICODE) +# define __itt_av_save __itt_av_saveW +# define __itt_av_save_ptr __itt_av_saveW_ptr +#else /* UNICODE */ +# define __itt_av_save __itt_av_saveA +# define __itt_av_save_ptr __itt_av_saveA_ptr +#endif /* UNICODE */ +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +int ITTAPI __itt_av_save(void *data, int rank, const int *dimensions, int type, const char *filePath, int columnOrder); +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUB(ITTAPI, int, av_saveA, (void *data, int rank, const int *dimensions, int type, const char *filePath, int columnOrder)) +ITT_STUB(ITTAPI, int, av_saveW, (void *data, int rank, const int *dimensions, int type, const wchar_t *filePath, int columnOrder)) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUB(ITTAPI, int, av_save, (void *data, int rank, const int *dimensions, int type, const char *filePath, int columnOrder)) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_av_saveA ITTNOTIFY_DATA(av_saveA) +#define __itt_av_saveA_ptr ITTNOTIFY_NAME(av_saveA) +#define __itt_av_saveW ITTNOTIFY_DATA(av_saveW) +#define __itt_av_saveW_ptr ITTNOTIFY_NAME(av_saveW) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_av_save ITTNOTIFY_DATA(av_save) +#define __itt_av_save_ptr ITTNOTIFY_NAME(av_save) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#else /* INTEL_NO_ITTNOTIFY_API */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_av_saveA(name) +#define __itt_av_saveA_ptr 0 +#define __itt_av_saveW(name) +#define __itt_av_saveW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_av_save(name) +#define __itt_av_save_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_av_saveA_ptr 0 +#define __itt_av_saveW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_av_save_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +void ITTAPI __itt_enable_attach(void); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, enable_attach, (void)) +#define __itt_enable_attach ITTNOTIFY_VOID(enable_attach) +#define __itt_enable_attach_ptr ITTNOTIFY_NAME(enable_attach) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_enable_attach() +#define __itt_enable_attach_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_enable_attach_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** @cond exclude_from_gpa_documentation */ + +/** @} arrays group */ + +/** @endcond */ + +/** + * @brief Module load notification + * This API is used to report necessary information in case of bypassing default system loader. + * Notification should be done immediately after this module is loaded to process memory. + * @param[in] start_addr - module start address + * @param[in] end_addr - module end address + * @param[in] path - file system full path to the module + */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +void ITTAPI __itt_module_loadA(void *start_addr, void *end_addr, const char *path); +void ITTAPI __itt_module_loadW(void *start_addr, void *end_addr, const wchar_t *path); +#if defined(UNICODE) || defined(_UNICODE) +# define __itt_module_load __itt_module_loadW +# define __itt_module_load_ptr __itt_module_loadW_ptr +#else /* UNICODE */ +# define __itt_module_load __itt_module_loadA +# define __itt_module_load_ptr __itt_module_loadA_ptr +#endif /* UNICODE */ +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +void ITTAPI __itt_module_load(void *start_addr, void *end_addr, const char *path); +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUB(ITTAPI, void, module_loadA, (void *start_addr, void *end_addr, const char *path)) +ITT_STUB(ITTAPI, void, module_loadW, (void *start_addr, void *end_addr, const wchar_t *path)) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUB(ITTAPI, void, module_load, (void *start_addr, void *end_addr, const char *path)) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_module_loadA ITTNOTIFY_VOID(module_loadA) +#define __itt_module_loadA_ptr ITTNOTIFY_NAME(module_loadA) +#define __itt_module_loadW ITTNOTIFY_VOID(module_loadW) +#define __itt_module_loadW_ptr ITTNOTIFY_NAME(module_loadW) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_module_load ITTNOTIFY_VOID(module_load) +#define __itt_module_load_ptr ITTNOTIFY_NAME(module_load) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#else /* INTEL_NO_ITTNOTIFY_API */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_module_loadA(start_addr, end_addr, path) +#define __itt_module_loadA_ptr 0 +#define __itt_module_loadW(start_addr, end_addr, path) +#define __itt_module_loadW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_module_load(start_addr, end_addr, path) +#define __itt_module_load_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_module_loadA_ptr 0 +#define __itt_module_loadW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_module_load_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Report module unload + * This API is used to report necessary information in case of bypassing default system loader. + * Notification should be done just before the module is unloaded from process memory. + * @param[in] addr - base address of loaded module + */ +void ITTAPI __itt_module_unload(void *addr); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, module_unload, (void *addr)) +#define __itt_module_unload ITTNOTIFY_VOID(module_unload) +#define __itt_module_unload_ptr ITTNOTIFY_NAME(module_unload) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_module_unload(addr) +#define __itt_module_unload_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_module_unload_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** @cond exclude_from_documentation */ +typedef enum +{ + __itt_module_type_unknown = 0, + __itt_module_type_elf, + __itt_module_type_coff +} __itt_module_type; +/** @endcond */ + +/** @cond exclude_from_documentation */ +typedef enum +{ + itt_section_type_unknown, + itt_section_type_bss, /* notifies that the section contains uninitialized data. These are the relevant section types and the modules that contain them: + * ELF module: SHT_NOBITS section type + * COFF module: IMAGE_SCN_CNT_UNINITIALIZED_DATA section type + */ + itt_section_type_data, /* notifies that section contains initialized data. These are the relevant section types and the modules that contain them: + * ELF module: SHT_PROGBITS section type + * COFF module: IMAGE_SCN_CNT_INITIALIZED_DATA section type + */ + itt_section_type_text /* notifies that the section contains executable code. These are the relevant section types and the modules that contain them: + * ELF module: SHT_PROGBITS section type + * COFF module: IMAGE_SCN_CNT_CODE section type + */ +} __itt_section_type; +/** @endcond */ + +/** + * @hideinitializer + * @brief bit-mask, detects a section attribute that indicates whether a section can be executed as code: + * These are the relevant section attributes and the modules that contain them: + * ELF module: PF_X section attribute + * COFF module: IMAGE_SCN_MEM_EXECUTE attribute + */ +#define __itt_section_exec 0x20000000 + +/** + * @hideinitializer + * @brief bit-mask, detects a section attribute that indicates whether a section can be read. + * These are the relevant section attributes and the modules that contain them: + * ELF module: PF_R attribute + * COFF module: IMAGE_SCN_MEM_READ attribute + */ +#define __itt_section_read 0x40000000 + +/** + * @hideinitializer + * @brief bit-mask, detects a section attribute that indicates whether a section can be written to. + * These are the relevant section attributes and the modules that contain them: + * ELF module: PF_W attribute + * COFF module: IMAGE_SCN_MEM_WRITE attribute + */ +#define __itt_section_write 0x80000000 + +/** @cond exclude_from_documentation */ +#pragma pack(push, 8) + +typedef struct ___itt_section_info +{ + const char* name; /*!< Section name in UTF8 */ + __itt_section_type type; /*!< Section content and semantics description */ + size_t flags; /*!< Section bit flags that describe attributes using bit mask + * Zero if disabled, non-zero if enabled + */ + void* start_addr; /*!< Section load(relocated) start address */ + size_t size; /*!< Section file offset */ + size_t file_offset; /*!< Section size */ +} __itt_section_info; + +#pragma pack(pop) +/** @endcond */ + +/** @cond exclude_from_documentation */ +#pragma pack(push, 8) + +typedef struct ___itt_module_object +{ + unsigned int version; /*!< API version*/ + __itt_id module_id; /*!< Unique identifier. This is unchanged for sections that belong to the same module */ + __itt_module_type module_type; /*!< Binary module format */ + const char* module_name; /*!< Unique module name or path to module in UTF8 + * Contains module name when module_bufer and module_size exist + * Contains module path when module_bufer and module_size absent + * module_name remains the same for the certain module_id + */ + void* module_buffer; /*!< Module buffer content */ + size_t module_size; /*!< Module buffer size */ + /*!< If module_buffer and module_size exist, the binary module is dumped onto the system. + * If module_buffer and module_size do not exist, + * the binary module exists on the system already. + * The module_name parameter contains the path to the module. + */ + __itt_section_info* section_array; /*!< Reference to section information */ + size_t section_number; +} __itt_module_object; + +#pragma pack(pop) +/** @endcond */ + +/** + * @brief Load module content and its loaded(relocated) sections. + * This API is useful to save a module, or specify its location on the system and report information about loaded sections. + * The target module is saved on the system if module buffer content and size are available. + * If module buffer content and size are unavailable, the module name contains the path to the existing binary module. + * @param[in] module_obj - provides module and section information, along with unique module identifiers (name,module ID) + * which bind the binary module to particular sections. + */ +void ITTAPI __itt_module_load_with_sections(__itt_module_object* module_obj); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, module_load_with_sections, (__itt_module_object* module_obj)) +#define __itt_module_load_with_sections ITTNOTIFY_VOID(module_load_with_sections) +#define __itt_module_load_with_sections_ptr ITTNOTIFY_NAME(module_load_with_sections) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_module_load_with_sections(module_obj) +#define __itt_module_load_with_sections_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_module_load_with_sections_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Unload a module and its loaded(relocated) sections. + * This API notifies that the module and its sections were unloaded. + * @param[in] module_obj - provides module and sections information, along with unique module identifiers (name,module ID) + * which bind the binary module to particular sections. + */ +void ITTAPI __itt_module_unload_with_sections(__itt_module_object* module_obj); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, module_unload_with_sections, (__itt_module_object* module_obj)) +#define __itt_module_unload_with_sections ITTNOTIFY_VOID(module_unload_with_sections) +#define __itt_module_unload_with_sections_ptr ITTNOTIFY_NAME(module_unload_with_sections) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_module_unload_with_sections(module_obj) +#define __itt_module_unload_with_sections_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_module_unload_with_sections_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** @cond exclude_from_documentation */ +#pragma pack(push, 8) + +typedef struct ___itt_histogram +{ + const __itt_domain* domain; /*!< Domain of the histogram*/ + const char* nameA; /*!< Name of the histogram */ +#if defined(UNICODE) || defined(_UNICODE) + const wchar_t* nameW; +#else /* UNICODE || _UNICODE */ + void* nameW; +#endif /* UNICODE || _UNICODE */ + __itt_metadata_type x_type; /*!< Type of the histogram X axis */ + __itt_metadata_type y_type; /*!< Type of the histogram Y axis */ + int extra1; /*!< Reserved to the runtime */ + void* extra2; /*!< Reserved to the runtime */ + struct ___itt_histogram* next; +} __itt_histogram; + +#pragma pack(pop) +/** @endcond */ + +/** + * @brief Create a typed histogram instance with given name/domain. + * @param[in] domain The domain controlling the call. + * @param[in] name The name of the histogram. + * @param[in] x_type The type of the X axis in histogram (may be 0 to calculate batch statistics). + * @param[in] y_type The type of the Y axis in histogram. +*/ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +__itt_histogram* ITTAPI __itt_histogram_createA(const __itt_domain* domain, const char* name, __itt_metadata_type x_type, __itt_metadata_type y_type); +__itt_histogram* ITTAPI __itt_histogram_createW(const __itt_domain* domain, const wchar_t* name, __itt_metadata_type x_type, __itt_metadata_type y_type); +#if defined(UNICODE) || defined(_UNICODE) +# define __itt_histogram_create __itt_histogram_createW +# define __itt_histogram_create_ptr __itt_histogram_createW_ptr +#else /* UNICODE */ +# define __itt_histogram_create __itt_histogram_createA +# define __itt_histogram_create_ptr __itt_histogram_createA_ptr +#endif /* UNICODE */ +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +__itt_histogram* ITTAPI __itt_histogram_create(const __itt_domain* domain, const char* name, __itt_metadata_type x_type, __itt_metadata_type y_type); +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUB(ITTAPI, __itt_histogram*, histogram_createA, (const __itt_domain* domain, const char* name, __itt_metadata_type x_type, __itt_metadata_type y_type)) +ITT_STUB(ITTAPI, __itt_histogram*, histogram_createW, (const __itt_domain* domain, const wchar_t* name, __itt_metadata_type x_type, __itt_metadata_type y_type)) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUB(ITTAPI, __itt_histogram*, histogram_create, (const __itt_domain* domain, const char* name, __itt_metadata_type x_type, __itt_metadata_type y_type)) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_histogram_createA ITTNOTIFY_DATA(histogram_createA) +#define __itt_histogram_createA_ptr ITTNOTIFY_NAME(histogram_createA) +#define __itt_histogram_createW ITTNOTIFY_DATA(histogram_createW) +#define __itt_histogram_createW_ptr ITTNOTIFY_NAME(histogram_createW) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_histogram_create ITTNOTIFY_DATA(histogram_create) +#define __itt_histogram_create_ptr ITTNOTIFY_NAME(histogram_create) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#else /* INTEL_NO_ITTNOTIFY_API */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_histogram_createA(domain, name, x_type, y_type) (__itt_histogram*)0 +#define __itt_histogram_createA_ptr 0 +#define __itt_histogram_createW(domain, name, x_type, y_type) (__itt_histogram*)0 +#define __itt_histogram_createW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_histogram_create(domain, name, x_type, y_type) (__itt_histogram*)0 +#define __itt_histogram_create_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ #endif /* INTEL_NO_ITTNOTIFY_API */ #else /* INTEL_NO_MACRO_BODY */ -#define __itt_event_start_ptr 0 +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_histogram_createA_ptr 0 +#define __itt_histogram_createW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_histogram_create_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ #endif /* INTEL_NO_MACRO_BODY */ /** @endcond */ /** - * @brief Record an event end occurrence. - * @note It is optional if events do not have durations. - * @return __itt_err upon failure (invalid event id/user event feature not enabled) - */ -int LIBITTAPI __itt_event_end(__itt_event event); + * @brief Submit statistics for a histogram instance. + * @param[in] histogram Pointer to the histogram instance to which the histogram statistic is to be dumped. + * @param[in] length The number of elements in dumped axis data array. + * @param[in] x_data The X axis dumped data itself (may be NULL to calculate batch statistics). + * @param[in] y_data The Y axis dumped data itself. +*/ +void ITTAPI __itt_histogram_submit(__itt_histogram* histogram, size_t length, void* x_data, void* y_data); /** @cond exclude_from_documentation */ #ifndef INTEL_NO_MACRO_BODY #ifndef INTEL_NO_ITTNOTIFY_API -ITT_STUB(LIBITTAPI, int, event_end, (__itt_event event)) -#define __itt_event_end ITTNOTIFY_DATA(event_end) -#define __itt_event_end_ptr ITTNOTIFY_NAME(event_end) +ITT_STUBV(ITTAPI, void, histogram_submit, (__itt_histogram* histogram, size_t length, void* x_data, void* y_data)) +#define __itt_histogram_submit ITTNOTIFY_VOID(histogram_submit) +#define __itt_histogram_submit_ptr ITTNOTIFY_NAME(histogram_submit) #else /* INTEL_NO_ITTNOTIFY_API */ -#define __itt_event_end(event) (int)0 -#define __itt_event_end_ptr 0 +#define __itt_histogram_submit(histogram, length, x_data, y_data) +#define __itt_histogram_submit_ptr 0 #endif /* INTEL_NO_ITTNOTIFY_API */ #else /* INTEL_NO_MACRO_BODY */ -#define __itt_event_end_ptr 0 +#define __itt_histogram_submit_ptr 0 #endif /* INTEL_NO_MACRO_BODY */ -/** @endcond */ -/** @} events group */ - /** - * @defgroup arrays Arrays Visualizer - * @ingroup public - * Visualize arrays - * @{ - */ +* @brief function allows to obtain the current collection state at the moment +* @return collection state as a enum __itt_collection_state +*/ +__itt_collection_state __itt_get_collection_state(void); /** - * @enum __itt_av_data_type - * @brief Defines types of arrays data (for C/C++ intrinsic types) - */ -typedef enum -{ - __itt_e_first = 0, - __itt_e_char = 0, /* 1-byte integer */ - __itt_e_uchar, /* 1-byte unsigned integer */ - __itt_e_int16, /* 2-byte integer */ - __itt_e_uint16, /* 2-byte unsigned integer */ - __itt_e_int32, /* 4-byte integer */ - __itt_e_uint32, /* 4-byte unsigned integer */ - __itt_e_int64, /* 8-byte integer */ - __itt_e_uint64, /* 8-byte unsigned integer */ - __itt_e_float, /* 4-byte floating */ - __itt_e_double, /* 8-byte floating */ - __itt_e_last = __itt_e_double -} __itt_av_data_type; +* @brief function releases resources allocated by ITT API static part +* this API should be called from the library destructor +* @return void +*/ +void __itt_release_resources(void); +/** @endcond */ /** - * @brief Save an array data to a file. - * Output format is defined by the file extension. The csv and bmp formats are supported (bmp - for 2-dimensional array only). - * @param[in] data - pointer to the array data - * @param[in] rank - the rank of the array - * @param[in] dimensions - pointer to an array of integers, which specifies the array dimensions. - * The size of dimensions must be equal to the rank - * @param[in] type - the type of the array, specified as one of the __itt_av_data_type values (for intrinsic types) - * @param[in] filePath - the file path; the output format is defined by the file extension - * @param[in] columnOrder - defines how the array is stored in the linear memory. - * It should be 1 for column-major order (e.g. in FORTRAN) or 0 - for row-major order (e.g. in C). - */ - + * @brief Create a typed counter with given domain pointer, string name and counter type +*/ #if ITT_PLATFORM==ITT_PLATFORM_WIN -int ITTAPI __itt_av_saveA(void *data, int rank, const int *dimensions, int type, const char *filePath, int columnOrder); -int ITTAPI __itt_av_saveW(void *data, int rank, const int *dimensions, int type, const wchar_t *filePath, int columnOrder); +__itt_counter ITTAPI __itt_counter_createA_v3(const __itt_domain* domain, const char* name, __itt_metadata_type type); +__itt_counter ITTAPI __itt_counter_createW_v3(const __itt_domain* domain, const wchar_t* name, __itt_metadata_type type); #if defined(UNICODE) || defined(_UNICODE) -# define __itt_av_save __itt_av_saveW -# define __itt_av_save_ptr __itt_av_saveW_ptr +# define __itt_counter_create_v3 __itt_counter_createW_v3 +# define __itt_counter_create_v3_ptr __itt_counter_createW_v3_ptr #else /* UNICODE */ -# define __itt_av_save __itt_av_saveA -# define __itt_av_save_ptr __itt_av_saveA_ptr +# define __itt_counter_create_v3 __itt_counter_createA_v3 +# define __itt_counter_create_v3_ptr __itt_counter_createA_v3_ptr #endif /* UNICODE */ -#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ -int ITTAPI __itt_av_save(void *data, int rank, const int *dimensions, int type, const char *filePath, int columnOrder); +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +__itt_counter ITTAPI __itt_counter_create_v3(const __itt_domain* domain, const char* name, __itt_metadata_type type); #endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ -/** @cond exclude_from_documentation */ #ifndef INTEL_NO_MACRO_BODY #ifndef INTEL_NO_ITTNOTIFY_API #if ITT_PLATFORM==ITT_PLATFORM_WIN -ITT_STUB(ITTAPI, int, av_saveA, (void *data, int rank, const int *dimensions, int type, const char *filePath, int columnOrder)) -ITT_STUB(ITTAPI, int, av_saveW, (void *data, int rank, const int *dimensions, int type, const wchar_t *filePath, int columnOrder)) +ITT_STUB(ITTAPI, __itt_counter, counter_createA_v3, (const __itt_domain* domain, const char* name, __itt_metadata_type type)) +ITT_STUB(ITTAPI, __itt_counter, counter_createW_v3, (const __itt_domain* domain, const wchar_t* name, __itt_metadata_type type)) #else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ -ITT_STUB(ITTAPI, int, av_save, (void *data, int rank, const int *dimensions, int type, const char *filePath, int columnOrder)) +ITT_STUB(ITTAPI, __itt_counter, counter_create_v3, (const __itt_domain* domain, const char* name, __itt_metadata_type type)) #endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ #if ITT_PLATFORM==ITT_PLATFORM_WIN -#define __itt_av_saveA ITTNOTIFY_DATA(av_saveA) -#define __itt_av_saveA_ptr ITTNOTIFY_NAME(av_saveA) -#define __itt_av_saveW ITTNOTIFY_DATA(av_saveW) -#define __itt_av_saveW_ptr ITTNOTIFY_NAME(av_saveW) +#define __itt_counter_createA_v3 ITTNOTIFY_DATA(counter_createA_v3) +#define __itt_counter_createA_v3_ptr ITTNOTIFY_NAME(counter_createA_v3) +#define __itt_counter_createW_v3 ITTNOTIFY_DATA(counter_createW_v3) +#define __itt_counter_createW_v3_ptr ITTNOTIFY_NAME(counter_createW_v3) #else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ -#define __itt_av_save ITTNOTIFY_DATA(av_save) -#define __itt_av_save_ptr ITTNOTIFY_NAME(av_save) +#define __itt_counter_create_v3 ITTNOTIFY_DATA(counter_create_v3) +#define __itt_counter_create_v3_ptr ITTNOTIFY_NAME(counter_create_v3) #endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ #else /* INTEL_NO_ITTNOTIFY_API */ #if ITT_PLATFORM==ITT_PLATFORM_WIN -#define __itt_av_saveA(name) -#define __itt_av_saveA_ptr 0 -#define __itt_av_saveW(name) -#define __itt_av_saveW_ptr 0 +#define __itt_counter_createA_v3(domain, name, type) (__itt_counter)0 +#define __itt_counter_createA_v3_ptr 0 +#define __itt_counter_createW_v3(domain, name, type) (__itt_counter)0 +#define __itt_counter_create_typedW_ptr 0 #else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ -#define __itt_av_save(name) -#define __itt_av_save_ptr 0 +#define __itt_counter_create_v3(domain, name, type) (__itt_counter)0 +#define __itt_counter_create_v3_ptr 0 #endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ #endif /* INTEL_NO_ITTNOTIFY_API */ #else /* INTEL_NO_MACRO_BODY */ #if ITT_PLATFORM==ITT_PLATFORM_WIN -#define __itt_av_saveA_ptr 0 -#define __itt_av_saveW_ptr 0 +#define __itt_counter_createA_v3_ptr 0 +#define __itt_counter_createW_v3_ptr 0 #else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ -#define __itt_av_save_ptr 0 +#define __itt_counter_create_v3_ptr 0 #endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ #endif /* INTEL_NO_MACRO_BODY */ /** @endcond */ -void ITTAPI __itt_enable_attach(void); +/** + * @brief Set the counter value api + */ +void ITTAPI __itt_counter_set_value_v3(__itt_counter counter, void *value_ptr); -/** @cond exclude_from_documentation */ #ifndef INTEL_NO_MACRO_BODY #ifndef INTEL_NO_ITTNOTIFY_API -ITT_STUBV(ITTAPI, void, enable_attach, (void)) -#define __itt_enable_attach ITTNOTIFY_VOID(enable_attach) -#define __itt_enable_attach_ptr ITTNOTIFY_NAME(enable_attach) +ITT_STUBV(ITTAPI, void, counter_set_value_v3, (__itt_counter counter, void *value_ptr)) +#define __itt_counter_set_value_v3 ITTNOTIFY_VOID(counter_set_value_v3) +#define __itt_counter_set_value_v3_ptr ITTNOTIFY_NAME(counter_set_value_v3) #else /* INTEL_NO_ITTNOTIFY_API */ -#define __itt_enable_attach() -#define __itt_enable_attach_ptr 0 +#define __itt_counter_set_value_v3(counter, value_ptr) +#define __itt_counter_set_value_v3_ptr 0 #endif /* INTEL_NO_ITTNOTIFY_API */ #else /* INTEL_NO_MACRO_BODY */ -#define __itt_enable_attach_ptr 0 +#define __itt_counter_set_value_v3_ptr 0 #endif /* INTEL_NO_MACRO_BODY */ /** @endcond */ -/** @cond exclude_from_gpa_documentation */ +/** + * @brief describes the type of context metadata +*/ +typedef enum { + __itt_context_unknown = 0, /*!< Undefined type */ + __itt_context_nameA, /*!< ASCII string char* type */ + __itt_context_nameW, /*!< Unicode string wchar_t* type */ + __itt_context_deviceA, /*!< ASCII string char* type */ + __itt_context_deviceW, /*!< Unicode string wchar_t* type */ + __itt_context_unitsA, /*!< ASCII string char* type */ + __itt_context_unitsW, /*!< Unicode string wchar_t* type */ + __itt_context_pci_addrA, /*!< ASCII string char* type */ + __itt_context_pci_addrW, /*!< Unicode string wchar_t* type */ + __itt_context_tid, /*!< Unsigned 64-bit integer type */ + __itt_context_max_val, /*!< Unsigned 64-bit integer type */ + __itt_context_bandwidth_flag, /*!< Unsigned 64-bit integer type */ + __itt_context_latency_flag, /*!< Unsigned 64-bit integer type */ + __itt_context_occupancy_flag, /*!< Unsigned 64-bit integer type */ + __itt_context_on_thread_flag, /*!< Unsigned 64-bit integer type */ + __itt_context_is_abs_val_flag, /*!< Unsigned 64-bit integer type */ + __itt_context_cpu_instructions_flag, /*!< Unsigned 64-bit integer type */ + __itt_context_cpu_cycles_flag /*!< Unsigned 64-bit integer type */ +} __itt_context_type; -/** @} arrays group */ +#if defined(UNICODE) || defined(_UNICODE) +# define __itt_context_name __itt_context_nameW +# define __itt_context_device __itt_context_deviceW +# define __itt_context_units __itt_context_unitsW +# define __itt_context_pci_addr __itt_context_pci_addrW +#else /* UNICODE || _UNICODE */ +# define __itt_context_name __itt_context_nameA +# define __itt_context_device __itt_context_deviceA +# define __itt_context_units __itt_context_unitsA +# define __itt_context_pci_addr __itt_context_pci_addrA +#endif /* UNICODE || _UNICODE */ + +/** @cond exclude_from_documentation */ +#pragma pack(push, 8) + +typedef struct ___itt_context_metadata +{ + __itt_context_type type; /*!< Type of the context metadata value */ + void* value; /*!< Pointer to context metadata value itself */ +} __itt_context_metadata; + +#pragma pack(pop) +/** @endcond */ + +/** @cond exclude_from_documentation */ +#pragma pack(push, 8) + +typedef struct ___itt_counter_metadata +{ + __itt_counter counter; /*!< Associated context metadata counter */ + __itt_context_type type; /*!< Type of the context metadata value */ + const char* str_valueA; /*!< String context metadata value */ +#if defined(UNICODE) || defined(_UNICODE) + const wchar_t* str_valueW; +#else /* UNICODE || _UNICODE */ + void* str_valueW; +#endif /* UNICODE || _UNICODE */ + unsigned long long value; /*!< Numeric context metadata value */ + int extra1; /*!< Reserved to the runtime */ + void* extra2; /*!< Reserved to the runtime */ + struct ___itt_counter_metadata* next; +} __itt_counter_metadata; +#pragma pack(pop) /** @endcond */ +/** + * @brief Bind context metadata to counter instance + * @param[in] counter Pointer to the counter instance to which the context metadata is to be associated. + * @param[in] length The number of elements in context metadata array. + * @param[in] metadata The context metadata itself. +*/ +void ITTAPI __itt_bind_context_metadata_to_counter(__itt_counter counter, size_t length, __itt_context_metadata* metadata); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, bind_context_metadata_to_counter, (__itt_counter counter, size_t length, __itt_context_metadata* metadata)) +#define __itt_bind_context_metadata_to_counter ITTNOTIFY_VOID(bind_context_metadata_to_counter) +#define __itt_bind_context_metadata_to_counter_ptr ITTNOTIFY_NAME(bind_context_metadata_to_counter) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_bind_context_metadata_to_counter(counter, length, metadata) +#define __itt_bind_context_metadata_to_counter_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_bind_context_metadata_to_counter_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ #ifdef __cplusplus } @@ -3561,130 +4524,7 @@ ITT_STUB(ITTAPI, int, mark_global_off, (__itt_mark_type mt)) * Counters group * @{ */ -/** - * @brief opaque structure for counter identification - */ -typedef struct ___itt_counter *__itt_counter; - -/** - * @brief Create a counter with given name/domain for the calling thread - * - * After __itt_counter_create() is called, __itt_counter_inc() / __itt_counter_inc_delta() can be used - * to increment the counter on any thread - */ -#if ITT_PLATFORM==ITT_PLATFORM_WIN -__itt_counter ITTAPI __itt_counter_createA(const char *name, const char *domain); -__itt_counter ITTAPI __itt_counter_createW(const wchar_t *name, const wchar_t *domain); -#if defined(UNICODE) || defined(_UNICODE) -# define __itt_counter_create __itt_counter_createW -# define __itt_counter_create_ptr __itt_counter_createW_ptr -#else /* UNICODE */ -# define __itt_counter_create __itt_counter_createA -# define __itt_counter_create_ptr __itt_counter_createA_ptr -#endif /* UNICODE */ -#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ -__itt_counter ITTAPI __itt_counter_create(const char *name, const char *domain); -#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ - -/** @cond exclude_from_documentation */ -#ifndef INTEL_NO_MACRO_BODY -#ifndef INTEL_NO_ITTNOTIFY_API -#if ITT_PLATFORM==ITT_PLATFORM_WIN -ITT_STUB(ITTAPI, __itt_counter, counter_createA, (const char *name, const char *domain)) -ITT_STUB(ITTAPI, __itt_counter, counter_createW, (const wchar_t *name, const wchar_t *domain)) -#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ -ITT_STUB(ITTAPI, __itt_counter, counter_create, (const char *name, const char *domain)) -#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ -#if ITT_PLATFORM==ITT_PLATFORM_WIN -#define __itt_counter_createA ITTNOTIFY_DATA(counter_createA) -#define __itt_counter_createA_ptr ITTNOTIFY_NAME(counter_createA) -#define __itt_counter_createW ITTNOTIFY_DATA(counter_createW) -#define __itt_counter_createW_ptr ITTNOTIFY_NAME(counter_createW) -#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ -#define __itt_counter_create ITTNOTIFY_DATA(counter_create) -#define __itt_counter_create_ptr ITTNOTIFY_NAME(counter_create) -#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ -#else /* INTEL_NO_ITTNOTIFY_API */ -#if ITT_PLATFORM==ITT_PLATFORM_WIN -#define __itt_counter_createA(name, domain) -#define __itt_counter_createA_ptr 0 -#define __itt_counter_createW(name, domain) -#define __itt_counter_createW_ptr 0 -#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ -#define __itt_counter_create(name, domain) -#define __itt_counter_create_ptr 0 -#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ -#endif /* INTEL_NO_ITTNOTIFY_API */ -#else /* INTEL_NO_MACRO_BODY */ -#if ITT_PLATFORM==ITT_PLATFORM_WIN -#define __itt_counter_createA_ptr 0 -#define __itt_counter_createW_ptr 0 -#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ -#define __itt_counter_create_ptr 0 -#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ -#endif /* INTEL_NO_MACRO_BODY */ -/** @endcond */ - -/** - * @brief Destroy the counter identified by the pointer previously returned by __itt_counter_create() - */ -void ITTAPI __itt_counter_destroy(__itt_counter id); - -/** @cond exclude_from_documentation */ -#ifndef INTEL_NO_MACRO_BODY -#ifndef INTEL_NO_ITTNOTIFY_API -ITT_STUBV(ITTAPI, void, counter_destroy, (__itt_counter id)) -#define __itt_counter_destroy ITTNOTIFY_VOID(counter_destroy) -#define __itt_counter_destroy_ptr ITTNOTIFY_NAME(counter_destroy) -#else /* INTEL_NO_ITTNOTIFY_API */ -#define __itt_counter_destroy(id) -#define __itt_counter_destroy_ptr 0 -#endif /* INTEL_NO_ITTNOTIFY_API */ -#else /* INTEL_NO_MACRO_BODY */ -#define __itt_counter_destroy_ptr 0 -#endif /* INTEL_NO_MACRO_BODY */ -/** @endcond */ - -/** - * @brief Increment the counter value - */ -void ITTAPI __itt_counter_inc(__itt_counter id); - -/** @cond exclude_from_documentation */ -#ifndef INTEL_NO_MACRO_BODY -#ifndef INTEL_NO_ITTNOTIFY_API -ITT_STUBV(ITTAPI, void, counter_inc, (__itt_counter id)) -#define __itt_counter_inc ITTNOTIFY_VOID(counter_inc) -#define __itt_counter_inc_ptr ITTNOTIFY_NAME(counter_inc) -#else /* INTEL_NO_ITTNOTIFY_API */ -#define __itt_counter_inc(id) -#define __itt_counter_inc_ptr 0 -#endif /* INTEL_NO_ITTNOTIFY_API */ -#else /* INTEL_NO_MACRO_BODY */ -#define __itt_counter_inc_ptr 0 -#endif /* INTEL_NO_MACRO_BODY */ -/** @endcond */ - -/** - * @brief Increment the counter value with x - */ -void ITTAPI __itt_counter_inc_delta(__itt_counter id, unsigned long long value); -/** @cond exclude_from_documentation */ -#ifndef INTEL_NO_MACRO_BODY -#ifndef INTEL_NO_ITTNOTIFY_API -ITT_STUBV(ITTAPI, void, counter_inc_delta, (__itt_counter id, unsigned long long value)) -#define __itt_counter_inc_delta ITTNOTIFY_VOID(counter_inc_delta) -#define __itt_counter_inc_delta_ptr ITTNOTIFY_NAME(counter_inc_delta) -#else /* INTEL_NO_ITTNOTIFY_API */ -#define __itt_counter_inc_delta(id, value) -#define __itt_counter_inc_delta_ptr 0 -#endif /* INTEL_NO_ITTNOTIFY_API */ -#else /* INTEL_NO_MACRO_BODY */ -#define __itt_counter_inc_delta_ptr 0 -#endif /* INTEL_NO_MACRO_BODY */ -/** @endcond */ -/** @} counters group */ /** * @defgroup stitch Stack Stitching @@ -3719,7 +4559,7 @@ ITT_STUB(ITTAPI, __itt_caller, stack_caller_create, (void)) /** @endcond */ /** - * @brief Destroy the inforamtion about stitch point identified by the pointer previously returned by __itt_stack_caller_create() + * @brief Destroy the information about stitch point identified by the pointer previously returned by __itt_stack_caller_create() */ void ITTAPI __itt_stack_caller_destroy(__itt_caller id); @@ -3790,7 +4630,7 @@ typedef enum __itt_error_code { __itt_error_success = 0, /*!< no error */ __itt_error_no_module = 1, /*!< module can't be loaded */ - /* %1$s -- library name; win: %2$d -- system error code; unx: %2$s -- system error message. */ + /* %1$s -- library name; win: %2$d -- system error code; unix: %2$s -- system error message. */ __itt_error_no_symbol = 2, /*!< symbol not found */ /* %1$s -- library name, %2$s -- symbol name. */ __itt_error_unknown_group = 3, /*!< unknown group specified */ diff --git a/src/tbb/src/tbb/tools_api/ittnotify_config.h b/src/tbb/src/tbb/tools_api/ittnotify_config.h index 1f90963bc..001d42e0e 100644 --- a/src/tbb/src/tbb/tools_api/ittnotify_config.h +++ b/src/tbb/src/tbb/tools_api/ittnotify_config.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2024 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef _ITTNOTIFY_CONFIG_H_ @@ -38,6 +34,10 @@ # define ITT_OS_FREEBSD 4 #endif /* ITT_OS_FREEBSD */ +#ifndef ITT_OS_OPENBSD +# define ITT_OS_OPENBSD 5 +#endif /* ITT_OS_OPENBSD */ + #ifndef ITT_OS # if defined WIN32 || defined _WIN32 # define ITT_OS ITT_OS_WIN @@ -45,6 +45,8 @@ # define ITT_OS ITT_OS_MAC # elif defined( __FreeBSD__ ) # define ITT_OS ITT_OS_FREEBSD +# elif defined( __OpenBSD__ ) +# define ITT_OS ITT_OS_OPENBSD # else # define ITT_OS ITT_OS_LINUX # endif @@ -66,6 +68,10 @@ # define ITT_PLATFORM_FREEBSD 4 #endif /* ITT_PLATFORM_FREEBSD */ +#ifndef ITT_PLATFORM_OPENBSD +# define ITT_PLATFORM_OPENBSD 5 +#endif /* ITT_PLATFORM_OPENBSD */ + #ifndef ITT_PLATFORM # if ITT_OS==ITT_OS_WIN # define ITT_PLATFORM ITT_PLATFORM_WIN @@ -73,6 +79,8 @@ # define ITT_PLATFORM ITT_PLATFORM_MAC # elif ITT_OS==ITT_OS_FREEBSD # define ITT_PLATFORM ITT_PLATFORM_FREEBSD +# elif ITT_OS==ITT_OS_OPENBSD +# define ITT_PLATFORM ITT_PLATFORM_OPENBSD # else # define ITT_PLATFORM ITT_PLATFORM_POSIX # endif @@ -92,17 +100,17 @@ #endif /* UNICODE || _UNICODE */ #endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ -#ifndef CDECL +#ifndef ITTAPI_CDECL # if ITT_PLATFORM==ITT_PLATFORM_WIN -# define CDECL __cdecl +# define ITTAPI_CDECL __cdecl # else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ # if defined _M_IX86 || defined __i386__ -# define CDECL __attribute__ ((cdecl)) +# define ITTAPI_CDECL __attribute__ ((cdecl)) # else /* _M_IX86 || __i386__ */ -# define CDECL /* actual only on x86 platform */ +# define ITTAPI_CDECL /* actual only on x86 platform */ # endif /* _M_IX86 || __i386__ */ # endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ -#endif /* CDECL */ +#endif /* ITTAPI_CDECL */ #ifndef STDCALL # if ITT_PLATFORM==ITT_PLATFORM_WIN @@ -116,16 +124,21 @@ # endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ #endif /* STDCALL */ -#define ITTAPI CDECL -#define LIBITTAPI CDECL +#define ITTAPI ITTAPI_CDECL +#define LIBITTAPI ITTAPI_CDECL /* TODO: Temporary for compatibility! */ -#define ITTAPI_CALL CDECL -#define LIBITTAPI_CALL CDECL +#define ITTAPI_CALL ITTAPI_CDECL +#define LIBITTAPI_CALL ITTAPI_CDECL #if ITT_PLATFORM==ITT_PLATFORM_WIN /* use __forceinline (VC++ specific) */ -#define ITT_INLINE __forceinline +#if defined(__MINGW32__) && !defined(__cplusplus) +#define ITT_INLINE static __inline__ __attribute__((__always_inline__,__gnu_inline__)) +#else +#define ITT_INLINE static __forceinline +#endif /* __MINGW32__ */ + #define ITT_INLINE_ATTRIBUTE /* nothing */ #else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ /* @@ -151,6 +164,10 @@ # define ITT_ARCH_IA32E 2 #endif /* ITT_ARCH_IA32E */ +#ifndef ITT_ARCH_IA64 +# define ITT_ARCH_IA64 3 +#endif /* ITT_ARCH_IA64 */ + #ifndef ITT_ARCH_ARM # define ITT_ARCH_ARM 4 #endif /* ITT_ARCH_ARM */ @@ -159,6 +176,26 @@ # define ITT_ARCH_PPC64 5 #endif /* ITT_ARCH_PPC64 */ +#ifndef ITT_ARCH_ARM64 +# define ITT_ARCH_ARM64 6 +#endif /* ITT_ARCH_ARM64 */ + +#ifndef ITT_ARCH_LOONGARCH64 +# define ITT_ARCH_LOONGARCH64 7 +#endif /* ITT_ARCH_LOONGARCH64 */ + +#ifndef ITT_ARCH_S390X +# define ITT_ARCH_S390X 8 +#endif /* ITT_ARCH_S390X */ + +#ifndef ITT_ARCH_HPPA +# define ITT_ARCH_HPPA 9 +#endif /* ITT_ARCH_HPPA */ + +#ifndef ITT_ARCH_RISCV64 +# define ITT_ARCH_RISCV64 10 +#endif /* ITT_ARCH_RISCV64 */ + #ifndef ITT_ARCH # if defined _M_IX86 || defined __i386__ # define ITT_ARCH ITT_ARCH_IA32 @@ -166,11 +203,22 @@ # define ITT_ARCH ITT_ARCH_IA32E # elif defined _M_IA64 || defined __ia64__ # define ITT_ARCH ITT_ARCH_IA64 -# elif defined _M_ARM || __arm__ +# elif defined _M_ARM || defined __arm__ # define ITT_ARCH ITT_ARCH_ARM +# elif defined __aarch64__ +# define ITT_ARCH ITT_ARCH_ARM64 # elif defined __powerpc64__ # define ITT_ARCH ITT_ARCH_PPC64 +# elif defined __loongarch__ +# define ITT_ARCH ITT_ARCH_LOONGARCH64 +# elif defined __s390__ || defined __s390x__ +# define ITT_ARCH ITT_ARCH_S390X +# elif defined __hppa__ +# define ITT_ARCH ITT_ARCH_HPPA +# elif defined __riscv && __riscv_xlen == 64 +# define ITT_ARCH ITT_ARCH_RISCV64 # endif + #endif #ifdef __cplusplus @@ -196,10 +244,10 @@ #define ITT_MAGIC { 0xED, 0xAB, 0xAB, 0xEC, 0x0D, 0xEE, 0xDA, 0x30 } /* Replace with snapshot date YYYYMMDD for promotion build. */ -#define API_VERSION_BUILD 20111111 +#define API_VERSION_BUILD 20230630 #ifndef API_VERSION_NUM -#define API_VERSION_NUM 0.0.0 +#define API_VERSION_NUM 3.24.4 #endif /* API_VERSION_NUM */ #define API_VERSION "ITT-API-Version " ITT_TO_STR(API_VERSION_NUM) \ @@ -211,7 +259,11 @@ typedef HMODULE lib_t; typedef DWORD TIDT; typedef CRITICAL_SECTION mutex_t; +#ifdef __cplusplus +#define MUTEX_INITIALIZER {} +#else #define MUTEX_INITIALIZER { 0 } +#endif #define strong_alias(name, aliasname) /* empty for Windows */ #else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ #include <dlfcn.h> @@ -239,13 +291,13 @@ typedef pthread_mutex_t mutex_t; #define __itt_mutex_init(mutex) InitializeCriticalSection(mutex) #define __itt_mutex_lock(mutex) EnterCriticalSection(mutex) #define __itt_mutex_unlock(mutex) LeaveCriticalSection(mutex) +#define __itt_mutex_destroy(mutex) DeleteCriticalSection(mutex) #define __itt_load_lib(name) LoadLibraryA(name) #define __itt_unload_lib(handle) FreeLibrary(handle) #define __itt_system_error() (int)GetLastError() #define __itt_fstrcmp(s1, s2) lstrcmpA(s1, s2) #define __itt_fstrnlen(s, l) strnlen_s(s, l) #define __itt_fstrcpyn(s1, b, s2, l) strncpy_s(s1, b, s2, l) -#define __itt_fstrdup(s) _strdup(s) #define __itt_thread_id() GetCurrentThreadId() #define __itt_thread_yield() SwitchToThread() #ifndef ITT_SIMPLE_INIT @@ -255,7 +307,18 @@ ITT_INLINE long __itt_interlocked_increment(volatile long* ptr) { return InterlockedIncrement(ptr); } +ITT_INLINE long +__itt_interlocked_compare_exchange(volatile long* ptr, long exchange, long comperand) ITT_INLINE_ATTRIBUTE; +ITT_INLINE long +__itt_interlocked_compare_exchange(volatile long* ptr, long exchange, long comperand) +{ + return InterlockedCompareExchange(ptr, exchange, comperand); +} #endif /* ITT_SIMPLE_INIT */ + +#define DL_SYMBOLS (1) +#define PTHREAD_SYMBOLS (1) + #else /* ITT_PLATFORM!=ITT_PLATFORM_WIN */ #define __itt_get_proc(lib, name) dlsym(lib, name) #define __itt_mutex_init(mutex) {\ @@ -280,6 +343,7 @@ ITT_INLINE long __itt_interlocked_increment(volatile long* ptr) } #define __itt_mutex_lock(mutex) pthread_mutex_lock(mutex) #define __itt_mutex_unlock(mutex) pthread_mutex_unlock(mutex) +#define __itt_mutex_destroy(mutex) pthread_mutex_destroy(mutex) #define __itt_load_lib(name) dlopen(name, RTLD_LAZY) #define __itt_unload_lib(handle) dlclose(handle) #define __itt_system_error() errno @@ -294,17 +358,25 @@ ITT_INLINE long __itt_interlocked_increment(volatile long* ptr) #ifdef SDL_STRNCPY_S #define __itt_fstrcpyn(s1, b, s2, l) SDL_STRNCPY_S(s1, b, s2, l) #else -#define __itt_fstrcpyn(s1, b, s2, l) memcpy(s1, s2, l) +#define __itt_fstrcpyn(s1, b, s2, l) { \ + if (b > 0) { \ + /* 'volatile' is used to suppress the warning that a destination */ \ + /* bound depends on the length of the source. */ \ + volatile size_t num_to_copy = (size_t)(b - 1) < (size_t)(l) ? \ + (size_t)(b - 1) : (size_t)(l); \ + strncpy(s1, s2, num_to_copy); \ + s1[num_to_copy] = 0; \ + } \ +} #endif /* SDL_STRNCPY_S */ -#define __itt_fstrdup(s) strdup(s) #define __itt_thread_id() pthread_self() #define __itt_thread_yield() sched_yield() #if ITT_ARCH==ITT_ARCH_IA64 #ifdef __INTEL_COMPILER #define __TBB_machine_fetchadd4(addr, val) __fetchadd4_acq((void *)addr, val) #else /* __INTEL_COMPILER */ -/* TODO: Add Support for not Intel compilers for IA-64 architecture */ +#define __TBB_machine_fetchadd4(addr, val) __sync_fetch_and_add(addr, val) #endif /* __INTEL_COMPILER */ #elif ITT_ARCH==ITT_ARCH_IA32 || ITT_ARCH==ITT_ARCH_IA32E /* ITT_ARCH!=ITT_ARCH_IA64 */ ITT_INLINE long @@ -313,12 +385,12 @@ ITT_INLINE long __TBB_machine_fetchadd4(volatile void* ptr, long addend) { long result; __asm__ __volatile__("lock\nxadd %0,%1" - : "=r"(result),"=m"(*(int*)ptr) - : "0"(addend), "m"(*(int*)ptr) + : "=r"(result),"=m"(*(volatile int*)ptr) + : "0"(addend), "m"(*(volatile int*)ptr) : "memory"); return result; } -#elif ITT_ARCH==ITT_ARCH_ARM || ITT_ARCH==ITT_ARCH_PPC64 +#else #define __TBB_machine_fetchadd4(addr, val) __sync_fetch_and_add(addr, val) #endif /* ITT_ARCH==ITT_ARCH_IA64 */ #ifndef ITT_SIMPLE_INIT @@ -328,13 +400,46 @@ ITT_INLINE long __itt_interlocked_increment(volatile long* ptr) { return __TBB_machine_fetchadd4(ptr, 1) + 1L; } +ITT_INLINE long +__itt_interlocked_compare_exchange(volatile long* ptr, long exchange, long comperand) ITT_INLINE_ATTRIBUTE; +ITT_INLINE long +__itt_interlocked_compare_exchange(volatile long* ptr, long exchange, long comperand) +{ + return __sync_val_compare_and_swap(ptr, exchange, comperand); +} #endif /* ITT_SIMPLE_INIT */ + +void* dlopen(const char*, int) __attribute__((weak)); +void* dlsym(void*, const char*) __attribute__((weak)); +int dlclose(void*) __attribute__((weak)); +#define DL_SYMBOLS (dlopen && dlsym && dlclose) + +int pthread_mutex_init(pthread_mutex_t*, const pthread_mutexattr_t*) __attribute__((weak)); +int pthread_mutex_lock(pthread_mutex_t*) __attribute__((weak)); +int pthread_mutex_unlock(pthread_mutex_t*) __attribute__((weak)); +int pthread_mutex_destroy(pthread_mutex_t*) __attribute__((weak)); +int pthread_mutexattr_init(pthread_mutexattr_t*) __attribute__((weak)); +int pthread_mutexattr_settype(pthread_mutexattr_t*, int) __attribute__((weak)); +int pthread_mutexattr_destroy(pthread_mutexattr_t*) __attribute__((weak)); +pthread_t pthread_self(void) __attribute__((weak)); +#define PTHREAD_SYMBOLS (pthread_mutex_init && pthread_mutex_lock && pthread_mutex_unlock && pthread_mutex_destroy && pthread_mutexattr_init && pthread_mutexattr_settype && pthread_mutexattr_destroy && pthread_self) + #endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ -typedef enum { - __itt_collection_normal = 0, - __itt_collection_paused = 1 -} __itt_collection_state; +/* strdup() is not included into C99 which results in a compiler warning about + * implicitly declared symbol. To avoid the issue strdup is implemented + * manually. + */ +#define ITT_STRDUP_MAX_STRING_SIZE 4096 +#define __itt_fstrdup(s, new_s) do { \ + if (s != NULL) { \ + size_t s_len = __itt_fstrnlen(s, ITT_STRDUP_MAX_STRING_SIZE); \ + new_s = (char *)malloc(s_len + 1); \ + if (new_s != NULL) { \ + __itt_fstrcpyn(new_s, s_len + 1, s, s_len); \ + } \ + } \ +} while(0) typedef enum { __itt_thread_normal = 0, @@ -377,8 +482,33 @@ typedef struct ___itt_api_info __itt_group_id group; } __itt_api_info; +typedef struct __itt_counter_info +{ + const char* nameA; /*!< Copy of original name in ASCII. */ +#if defined(UNICODE) || defined(_UNICODE) + const wchar_t* nameW; /*!< Copy of original name in UNICODE. */ +#else /* UNICODE || _UNICODE */ + void* nameW; +#endif /* UNICODE || _UNICODE */ + const char* domainA; /*!< Copy of original name in ASCII. */ +#if defined(UNICODE) || defined(_UNICODE) + const wchar_t* domainW; /*!< Copy of original name in UNICODE. */ +#else /* UNICODE || _UNICODE */ + void* domainW; +#endif /* UNICODE || _UNICODE */ + int type; + long index; + int extra1; /*!< Reserved to the runtime */ + void* extra2; /*!< Reserved to the runtime */ + struct __itt_counter_info* next; +} __itt_counter_info_t; + struct ___itt_domain; struct ___itt_string_handle; +struct ___itt_histogram; +struct ___itt_counter_metadata; + +#include "ittnotify.h" typedef struct ___itt_global { @@ -400,6 +530,10 @@ typedef struct ___itt_global struct ___itt_domain* domain_list; struct ___itt_string_handle* string_list; __itt_collection_state state; + __itt_counter_info_t* counter_list; + unsigned int ipt_collect_events; + struct ___itt_histogram* histogram_list; + struct ___itt_counter_metadata* counter_metadata_list; } __itt_global; #pragma pack(pop) @@ -425,7 +559,9 @@ typedef struct ___itt_global h = (__itt_thread_info*)malloc(sizeof(__itt_thread_info)); \ if (h != NULL) { \ h->tid = t; \ - h->nameA = n ? __itt_fstrdup(n) : NULL; \ + char *n_copy = NULL; \ + __itt_fstrdup(n, n_copy); \ + h->nameA = n_copy; \ h->nameW = NULL; \ h->state = s; \ h->extra1 = 0; /* reserved */ \ @@ -458,7 +594,9 @@ typedef struct ___itt_global h = (__itt_domain*)malloc(sizeof(__itt_domain)); \ if (h != NULL) { \ h->flags = 1; /* domain is enabled by default */ \ - h->nameA = name ? __itt_fstrdup(name) : NULL; \ + char *name_copy = NULL; \ + __itt_fstrdup(name, name_copy); \ + h->nameA = name_copy; \ h->nameW = NULL; \ h->extra1 = 0; /* reserved */ \ h->extra2 = NULL; /* reserved */ \ @@ -488,7 +626,9 @@ typedef struct ___itt_global #define NEW_STRING_HANDLE_A(gptr,h,h_tail,name) { \ h = (__itt_string_handle*)malloc(sizeof(__itt_string_handle)); \ if (h != NULL) { \ - h->strA = name ? __itt_fstrdup(name) : NULL; \ + char *name_copy = NULL; \ + __itt_fstrdup(name, name_copy); \ + h->strA = name_copy; \ h->strW = NULL; \ h->extra1 = 0; /* reserved */ \ h->extra2 = NULL; /* reserved */ \ @@ -500,4 +640,136 @@ typedef struct ___itt_global } \ } +#define NEW_COUNTER_W(gptr,h,h_tail,name,domain,type) { \ + h = (__itt_counter_info_t*)malloc(sizeof(__itt_counter_info_t)); \ + if (h != NULL) { \ + h->nameA = NULL; \ + h->nameW = name ? _wcsdup(name) : NULL; \ + h->domainA = NULL; \ + h->domainW = domain ? _wcsdup(domain) : NULL; \ + h->type = type; \ + h->index = 0; \ + h->next = NULL; \ + if (h_tail == NULL) \ + (gptr)->counter_list = h; \ + else \ + h_tail->next = h; \ + } \ +} + +#define NEW_COUNTER_A(gptr,h,h_tail,name,domain,type) { \ + h = (__itt_counter_info_t*)malloc(sizeof(__itt_counter_info_t)); \ + if (h != NULL) { \ + char *name_copy = NULL; \ + __itt_fstrdup(name, name_copy); \ + h->nameA = name_copy; \ + h->nameW = NULL; \ + char *domain_copy = NULL; \ + __itt_fstrdup(domain, domain_copy); \ + h->domainA = domain_copy; \ + h->domainW = NULL; \ + h->type = type; \ + h->index = 0; \ + h->next = NULL; \ + if (h_tail == NULL) \ + (gptr)->counter_list = h; \ + else \ + h_tail->next = h; \ + } \ +} + +#define NEW_HISTOGRAM_W(gptr,h,h_tail,domain,name,x_type,y_type) { \ + h = (__itt_histogram*)malloc(sizeof(__itt_histogram)); \ + if (h != NULL) { \ + h->domain = domain; \ + h->nameA = NULL; \ + h->nameW = name ? _wcsdup(name) : NULL; \ + h->x_type = x_type; \ + h->y_type = y_type; \ + h->extra1 = 0; \ + h->extra2 = NULL; \ + h->next = NULL; \ + if (h_tail == NULL) \ + (gptr)->histogram_list = h; \ + else \ + h_tail->next = h; \ + } \ +} + +#define NEW_HISTOGRAM_A(gptr,h,h_tail,domain,name,x_type,y_type) { \ + h = (__itt_histogram*)malloc(sizeof(__itt_histogram)); \ + if (h != NULL) { \ + h->domain = domain; \ + char *name_copy = NULL; \ + __itt_fstrdup(name, name_copy); \ + h->nameA = name_copy; \ + h->nameW = NULL; \ + h->x_type = x_type; \ + h->y_type = y_type; \ + h->extra1 = 0; \ + h->extra2 = NULL; \ + h->next = NULL; \ + if (h_tail == NULL) \ + (gptr)->histogram_list = h; \ + else \ + h_tail->next = h; \ + } \ +} + +#define NEW_COUNTER_METADATA_NUM(gptr,h,h_tail,counter,type,value) { \ + h = (__itt_counter_metadata*)malloc(sizeof(__itt_counter_metadata)); \ + if (h != NULL) { \ + h->counter = counter; \ + h->type = type; \ + h->str_valueA = NULL; \ + h->str_valueW = NULL; \ + h->value = value; \ + h->extra1 = 0; \ + h->extra2 = NULL; \ + h->next = NULL; \ + if (h_tail == NULL) \ + (gptr)->counter_metadata_list = h; \ + else \ + h_tail->next = h; \ + } \ +} + +#define NEW_COUNTER_METADATA_STR_A(gptr,h,h_tail,counter,type,str_valueA) { \ + h = (__itt_counter_metadata*)malloc(sizeof(__itt_counter_metadata)); \ + if (h != NULL) { \ + h->counter = counter; \ + h->type = type; \ + char *str_value_copy = NULL; \ + __itt_fstrdup(str_valueA, str_value_copy); \ + h->str_valueA = str_value_copy; \ + h->str_valueW = NULL; \ + h->value = 0; \ + h->extra1 = 0; \ + h->extra2 = NULL; \ + h->next = NULL; \ + if (h_tail == NULL) \ + (gptr)->counter_metadata_list = h; \ + else \ + h_tail->next = h; \ + } \ +} + +#define NEW_COUNTER_METADATA_STR_W(gptr,h,h_tail,counter,type,str_valueW) { \ + h = (__itt_counter_metadata*)malloc(sizeof(__itt_counter_metadata)); \ + if (h != NULL) { \ + h->counter = counter; \ + h->type = type; \ + h->str_valueA = NULL; \ + h->str_valueW = str_valueW ? _wcsdup(str_valueW) : NULL; \ + h->value = 0; \ + h->extra1 = 0; \ + h->extra2 = NULL; \ + h->next = NULL; \ + if (h_tail == NULL) \ + (gptr)->counter_metadata_list = h; \ + else \ + h_tail->next = h; \ + } \ +} + #endif /* _ITTNOTIFY_CONFIG_H_ */ diff --git a/src/tbb/src/tbb/tools_api/ittnotify_static.c b/src/tbb/src/tbb/tools_api/ittnotify_static.c index c08674590..c3a53bf0a 100644 --- a/src/tbb/src/tbb/tools_api/ittnotify_static.c +++ b/src/tbb/src/tbb/tools_api/ittnotify_static.c @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2024 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,16 +12,16 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ +#define INTEL_NO_MACRO_BODY +#define INTEL_ITTNOTIFY_API_PRIVATE #include "ittnotify_config.h" #if ITT_PLATFORM==ITT_PLATFORM_WIN +#if !defined(PATH_MAX) #define PATH_MAX 512 +#endif #else /* ITT_PLATFORM!=ITT_PLATFORM_WIN */ #include <limits.h> #include <dlfcn.h> @@ -32,20 +32,56 @@ #include <stdarg.h> #include <string.h> -#define INTEL_NO_MACRO_BODY -#define INTEL_ITTNOTIFY_API_PRIVATE #include "ittnotify.h" #include "legacy/ittnotify.h" #include "disable_warnings.h" -static const char api_version[] = API_VERSION "\0\n@(#) $Revision: 413915 $\n"; +static const char api_version[] = API_VERSION "\0\n@(#) $Revision$\n"; #define _N_(n) ITT_JOIN(INTEL_ITTNOTIFY_PREFIX,n) +#ifndef HAS_CPP_ATTR +#if defined(__cplusplus) && defined(__has_cpp_attribute) +#define HAS_CPP_ATTR(X) __has_cpp_attribute(X) +#else +#define HAS_CPP_ATTR(X) 0 +#endif +#endif + +#ifndef HAS_C_ATTR +#if defined(__STDC__) && defined(__has_c_attribute) +#define HAS_C_ATTR(X) __has_c_attribute(X) +#else +#define HAS_C_ATTR(X) 0 +#endif +#endif + +#ifndef HAS_GNU_ATTR +#if defined(__has_attribute) +#define HAS_GNU_ATTR(X) __has_attribute(X) +#else +#define HAS_GNU_ATTR(X) 0 +#endif +#endif + +#ifndef ITT_ATTRIBUTE_FALLTHROUGH +#if (HAS_CPP_ATTR(fallthrough) || HAS_C_ATTR(fallthrough)) && (__cplusplus >= 201703L || _MSVC_LANG >= 201703L) +#define ITT_ATTRIBUTE_FALLTHROUGH [[fallthrough]] +#elif HAS_CPP_ATTR(gnu::fallthrough) +#define ITT_ATTRIBUTE_FALLTHROUGH [[gnu::fallthrough]] +#elif HAS_CPP_ATTR(clang::fallthrough) +#define ITT_ATTRIBUTE_FALLTHROUGH [[clang::fallthrough]] +#elif HAS_GNU_ATTR(fallthrough) && !__INTEL_COMPILER +#define ITT_ATTRIBUTE_FALLTHROUGH __attribute__((fallthrough)) +#else +#define ITT_ATTRIBUTE_FALLTHROUGH +#endif +#endif + #if ITT_OS==ITT_OS_WIN static const char* ittnotify_lib_name = "libittnotify.dll"; -#elif ITT_OS==ITT_OS_LINUX || ITT_OS==ITT_OS_FREEBSD +#elif ITT_OS==ITT_OS_LINUX || ITT_OS==ITT_OS_FREEBSD|| ITT_OS==ITT_OS_OPENBSD static const char* ittnotify_lib_name = "libittnotify.so"; #elif ITT_OS==ITT_OS_MAC static const char* ittnotify_lib_name = "libittnotify.dylib"; @@ -97,9 +133,11 @@ static const char* ittnotify_lib_name = "libittnotify.dylib"; #endif /* LIB_VAR_NAME */ #define ITT_MUTEX_INIT_AND_LOCK(p) { \ + if (PTHREAD_SYMBOLS) \ + { \ if (!p.mutex_initialized) \ { \ - if (__itt_interlocked_increment(&p.atomic_counter) == 1) \ + if (__itt_interlocked_compare_exchange(&p.atomic_counter, 1, 0) == 0) \ { \ __itt_mutex_init(&p.mutex); \ p.mutex_initialized = 1; \ @@ -109,9 +147,24 @@ static const char* ittnotify_lib_name = "libittnotify.dylib"; __itt_thread_yield(); \ } \ __itt_mutex_lock(&p.mutex); \ + } \ } -const int _N_(err) = 0; +#define ITT_MUTEX_DESTROY(p) { \ + if (PTHREAD_SYMBOLS) \ + { \ + if (p.mutex_initialized) \ + { \ + if (__itt_interlocked_compare_exchange(&p.atomic_counter, 0, 1) == 1) \ + { \ + __itt_mutex_destroy(&p.mutex); \ + p.mutex_initialized = 0; \ + } \ + } \ + } \ +} + +#define ITT_MODULE_OBJECT_VERSION 1 typedef int (__itt_init_ittlib_t)(const char*, __itt_group_id); @@ -131,6 +184,8 @@ static __itt_fini_ittlib_t* __itt_fini_ittlib_ptr = _N_(fini_ittlib); #define __itt_fini_ittlib_name __itt_fini_ittlib_ptr #endif /* __itt_fini_ittlib_name */ +extern __itt_global _N_(_ittapi_global); + /* building pointers to imported funcs */ #undef ITT_STUBV #undef ITT_STUB @@ -140,7 +195,8 @@ typedef type api ITT_JOIN(_N_(name),_t) args; \ ITT_EXTERN_C_BEGIN ITT_JOIN(_N_(name),_t)* ITTNOTIFY_NAME(name) = ITT_VERSIONIZE(ITT_JOIN(_N_(name),_init)); ITT_EXTERN_C_END \ static type api ITT_VERSIONIZE(ITT_JOIN(_N_(name),_init)) args \ { \ - __itt_init_ittlib_name(NULL, __itt_group_all); \ + if (!_N_(_ittapi_global).api_initialized && _N_(_ittapi_global).thread_list == NULL) \ + __itt_init_ittlib_name(NULL, __itt_group_all); \ if (ITTNOTIFY_NAME(name) && ITTNOTIFY_NAME(name) != ITT_VERSIONIZE(ITT_JOIN(_N_(name),_init))) \ return ITTNOTIFY_NAME(name) params; \ else \ @@ -153,7 +209,8 @@ typedef type api ITT_JOIN(_N_(name),_t) args; \ ITT_EXTERN_C_BEGIN ITT_JOIN(_N_(name),_t)* ITTNOTIFY_NAME(name) = ITT_VERSIONIZE(ITT_JOIN(_N_(name),_init)); ITT_EXTERN_C_END \ static type api ITT_VERSIONIZE(ITT_JOIN(_N_(name),_init)) args \ { \ - __itt_init_ittlib_name(NULL, __itt_group_all); \ + if (!_N_(_ittapi_global).api_initialized && _N_(_ittapi_global).thread_list == NULL) \ + __itt_init_ittlib_name(NULL, __itt_group_all); \ if (ITTNOTIFY_NAME(name) && ITTNOTIFY_NAME(name) != ITT_VERSIONIZE(ITT_JOIN(_N_(name),_init))) \ ITTNOTIFY_NAME(name) params; \ else \ @@ -199,8 +256,10 @@ static __itt_group_alias group_alias[] = { #pragma pack(pop) #if ITT_PLATFORM==ITT_PLATFORM_WIN +#if _MSC_VER #pragma warning(push) #pragma warning(disable: 4054) /* warning C4054: 'type cast' : from function pointer 'XXX' to data pointer 'void *' */ +#endif #endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ static __itt_api_info api_list[] = { @@ -222,11 +281,11 @@ static __itt_api_info api_list[] = { }; #if ITT_PLATFORM==ITT_PLATFORM_WIN +#if _MSC_VER #pragma warning(pop) +#endif #endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ -static const char dll_path[PATH_MAX] = { 0 }; - /* static part descriptor which handles. all notification api attributes. */ __itt_global _N_(_ittapi_global) = { ITT_MAGIC, /* identification info */ @@ -237,18 +296,23 @@ __itt_global _N_(_ittapi_global) = { MUTEX_INITIALIZER, /* mutex */ NULL, /* dynamic library handle */ NULL, /* error_handler */ - (const char**)&dll_path, /* dll_path_ptr */ + NULL, /* dll_path_ptr */ (__itt_api_info*)&api_list, /* api_list_ptr */ NULL, /* next __itt_global */ NULL, /* thread_list */ NULL, /* domain_list */ NULL, /* string_list */ - __itt_collection_normal /* collection state */ + __itt_collection_uninitialized, /* collection state */ + NULL, /* counter_list */ + 0, /* ipt_collect_events */ + NULL, /* histogram_list */ + NULL /* counter_metadata_list */ }; typedef void (__itt_api_init_t)(__itt_global*, __itt_group_id); typedef void (__itt_api_fini_t)(__itt_global*); +static __itt_domain dummy_domain; /* ========================================================================= */ #ifdef ITT_NOTIFY_EXT_REPORT @@ -256,11 +320,14 @@ ITT_EXTERN_C void _N_(error_handler)(__itt_error_code, va_list args); #endif /* ITT_NOTIFY_EXT_REPORT */ #if ITT_PLATFORM==ITT_PLATFORM_WIN +#if _MSC_VER #pragma warning(push) #pragma warning(disable: 4055) /* warning C4055: 'type cast' : from data pointer 'void *' to function pointer 'XXX' */ +#endif #endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ -static void __itt_report_error_impl(int code, ...) { +static void __itt_report_error(int code, ...) +{ va_list args; va_start(args, code); if (_N_(_ittapi_global).error_handler != NULL) @@ -269,18 +336,17 @@ static void __itt_report_error_impl(int code, ...) { handler((__itt_error_code)code, args); } #ifdef ITT_NOTIFY_EXT_REPORT - _N_(error_handler)(code, args); + _N_(error_handler)((__itt_error_code)code, args); #endif /* ITT_NOTIFY_EXT_REPORT */ va_end(args); } -//va_start cannot take enum (__itt_error_code) on clang, so it is necessary to transform it to int -#define __itt_report_error(code, ...) \ - __itt_report_error_impl((int)code,__VA_ARGS__) - +static int __itt_is_collector_available(void); #if ITT_PLATFORM==ITT_PLATFORM_WIN +#if _MSC_VER #pragma warning(pop) +#endif #endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ #if ITT_PLATFORM==ITT_PLATFORM_WIN @@ -301,16 +367,24 @@ static __itt_domain* ITTAPI ITT_VERSIONIZE(ITT_JOIN(_N_(domain_createW),_init))( __itt_mutex_unlock(&_N_(_ittapi_global).mutex); return ITTNOTIFY_NAME(domain_createW)(name); } + else + { + __itt_mutex_unlock(&_N_(_ittapi_global).mutex); + return &dummy_domain; + } } - for (h_tail = NULL, h = _N_(_ittapi_global).domain_list; h != NULL; h_tail = h, h = h->next) - { - if (h->nameW != NULL && !wcscmp(h->nameW, name)) break; - } - if (h == NULL) + if (__itt_is_collector_available()) { - NEW_DOMAIN_W(&_N_(_ittapi_global),h,h_tail,name); + for (h_tail = NULL, h = _N_(_ittapi_global).domain_list; h != NULL; h_tail = h, h = h->next) + { + if (h->nameW != NULL && !wcscmp(h->nameW, name)) break; + } + if (h == NULL) + { + NEW_DOMAIN_W(&_N_(_ittapi_global), h, h_tail, name); + } } - __itt_mutex_unlock(&_N_(_ittapi_global).mutex); + if (PTHREAD_SYMBOLS) __itt_mutex_unlock(&_N_(_ittapi_global).mutex); return h; } @@ -338,21 +412,65 @@ static __itt_domain* ITTAPI ITT_VERSIONIZE(ITT_JOIN(_N_(domain_create),_init))(c #else if (ITTNOTIFY_NAME(domain_create) && ITTNOTIFY_NAME(domain_create) != ITT_VERSIONIZE(ITT_JOIN(_N_(domain_create),_init))) { - __itt_mutex_unlock(&_N_(_ittapi_global).mutex); + if (PTHREAD_SYMBOLS) __itt_mutex_unlock(&_N_(_ittapi_global).mutex); return ITTNOTIFY_NAME(domain_create)(name); } #endif + else + { +#if ITT_PLATFORM==ITT_PLATFORM_WIN + __itt_mutex_unlock(&_N_(_ittapi_global).mutex); +#else + if (PTHREAD_SYMBOLS) __itt_mutex_unlock(&_N_(_ittapi_global).mutex); +#endif + return &dummy_domain; + } } - for (h_tail = NULL, h = _N_(_ittapi_global).domain_list; h != NULL; h_tail = h, h = h->next) + if (__itt_is_collector_available()) { - if (h->nameA != NULL && !__itt_fstrcmp(h->nameA, name)) break; + for (h_tail = NULL, h = _N_(_ittapi_global).domain_list; h != NULL; h_tail = h, h = h->next) + { + if (h->nameA != NULL && !__itt_fstrcmp(h->nameA, name)) break; + } + if (h == NULL) + { + NEW_DOMAIN_A(&_N_(_ittapi_global), h, h_tail, name); + } } - if (h == NULL) + if (PTHREAD_SYMBOLS) __itt_mutex_unlock(&_N_(_ittapi_global).mutex); + return h; +} + +static void ITTAPI ITT_VERSIONIZE(ITT_JOIN(_N_(module_load_with_sections),_init))(__itt_module_object* module_obj) +{ + if (!_N_(_ittapi_global).api_initialized && _N_(_ittapi_global).thread_list == NULL) { - NEW_DOMAIN_A(&_N_(_ittapi_global),h,h_tail,name); + __itt_init_ittlib_name(NULL, __itt_group_all); + } + if (ITTNOTIFY_NAME(module_load_with_sections) && ITTNOTIFY_NAME(module_load_with_sections) != ITT_VERSIONIZE(ITT_JOIN(_N_(module_load_with_sections),_init))) + { + if(module_obj != NULL) + { + module_obj->version = ITT_MODULE_OBJECT_VERSION; + ITTNOTIFY_NAME(module_load_with_sections)(module_obj); + } + } +} + +static void ITTAPI ITT_VERSIONIZE(ITT_JOIN(_N_(module_unload_with_sections),_init))(__itt_module_object* module_obj) +{ + if (!_N_(_ittapi_global).api_initialized && _N_(_ittapi_global).thread_list == NULL) + { + __itt_init_ittlib_name(NULL, __itt_group_all); + } + if (ITTNOTIFY_NAME(module_unload_with_sections) && ITTNOTIFY_NAME(module_unload_with_sections) != ITT_VERSIONIZE(ITT_JOIN(_N_(module_unload_with_sections),_init))) + { + if(module_obj != NULL) + { + module_obj->version = ITT_MODULE_OBJECT_VERSION; + ITTNOTIFY_NAME(module_unload_with_sections)(module_obj); + } } - __itt_mutex_unlock(&_N_(_ittapi_global).mutex); - return h; } #if ITT_PLATFORM==ITT_PLATFORM_WIN @@ -373,14 +491,22 @@ static __itt_string_handle* ITTAPI ITT_VERSIONIZE(ITT_JOIN(_N_(string_handle_cre __itt_mutex_unlock(&_N_(_ittapi_global).mutex); return ITTNOTIFY_NAME(string_handle_createW)(name); } + else + { + __itt_mutex_unlock(&_N_(_ittapi_global).mutex); + return NULL; + } } - for (h_tail = NULL, h = _N_(_ittapi_global).string_list; h != NULL; h_tail = h, h = h->next) - { - if (h->strW != NULL && !wcscmp(h->strW, name)) break; - } - if (h == NULL) + if (__itt_is_collector_available()) { - NEW_STRING_HANDLE_W(&_N_(_ittapi_global),h,h_tail,name); + for (h_tail = NULL, h = _N_(_ittapi_global).string_list; h != NULL; h_tail = h, h = h->next) + { + if (h->strW != NULL && !wcscmp(h->strW, name)) break; + } + if (h == NULL) + { + NEW_STRING_HANDLE_W(&_N_(_ittapi_global), h, h_tail, name); + } } __itt_mutex_unlock(&_N_(_ittapi_global).mutex); return h; @@ -410,23 +536,484 @@ static __itt_string_handle* ITTAPI ITT_VERSIONIZE(ITT_JOIN(_N_(string_handle_cre #else if (ITTNOTIFY_NAME(string_handle_create) && ITTNOTIFY_NAME(string_handle_create) != ITT_VERSIONIZE(ITT_JOIN(_N_(string_handle_create),_init))) { - __itt_mutex_unlock(&_N_(_ittapi_global).mutex); + if (PTHREAD_SYMBOLS) __itt_mutex_unlock(&_N_(_ittapi_global).mutex); return ITTNOTIFY_NAME(string_handle_create)(name); } #endif + else + { +#if ITT_PLATFORM==ITT_PLATFORM_WIN + __itt_mutex_unlock(&_N_(_ittapi_global).mutex); +#else + if (PTHREAD_SYMBOLS) __itt_mutex_unlock(&_N_(_ittapi_global).mutex); +#endif + return NULL; + } + } + if (__itt_is_collector_available()) + { + for (h_tail = NULL, h = _N_(_ittapi_global).string_list; h != NULL; h_tail = h, h = h->next) + { + if (h->strA != NULL && !__itt_fstrcmp(h->strA, name)) break; + } + if (h == NULL) + { + NEW_STRING_HANDLE_A(&_N_(_ittapi_global), h, h_tail, name); + } + } + if (PTHREAD_SYMBOLS) __itt_mutex_unlock(&_N_(_ittapi_global).mutex); + return h; +} + +#if ITT_PLATFORM==ITT_PLATFORM_WIN +static __itt_counter ITTAPI ITT_VERSIONIZE(ITT_JOIN(_N_(counter_createW),_init))(const wchar_t *name, const wchar_t *domain) +{ + __itt_counter_info_t *h_tail = NULL, *h = NULL; + __itt_metadata_type type = __itt_metadata_u64; + + if (name == NULL) + { + return NULL; + } + + ITT_MUTEX_INIT_AND_LOCK(_N_(_ittapi_global)); + if (_N_(_ittapi_global).api_initialized) + { + if (ITTNOTIFY_NAME(counter_createW) && ITTNOTIFY_NAME(counter_createW) != ITT_VERSIONIZE(ITT_JOIN(_N_(counter_createW),_init))) + { + __itt_mutex_unlock(&_N_(_ittapi_global).mutex); + return ITTNOTIFY_NAME(counter_createW)(name, domain); + } + else + { + __itt_mutex_unlock(&_N_(_ittapi_global).mutex); + return NULL; + } + } + if (__itt_is_collector_available()) + { + for (h_tail = NULL, h = _N_(_ittapi_global).counter_list; h != NULL; h_tail = h, h = h->next) + { + if (h->nameW != NULL && h->type == (int)type && !wcscmp(h->nameW, name) && ((h->domainW == NULL && domain == NULL) || + (h->domainW != NULL && domain != NULL && !wcscmp(h->domainW, domain)))) break; + + } + if (h == NULL) + { + NEW_COUNTER_W(&_N_(_ittapi_global), h, h_tail, name, domain, type); + } + } + __itt_mutex_unlock(&_N_(_ittapi_global).mutex); + return (__itt_counter)h; +} + +static __itt_counter ITTAPI ITT_VERSIONIZE(ITT_JOIN(_N_(counter_createA),_init))(const char *name, const char *domain) +#else /* ITT_PLATFORM!=ITT_PLATFORM_WIN */ +static __itt_counter ITTAPI ITT_VERSIONIZE(ITT_JOIN(_N_(counter_create),_init))(const char *name, const char *domain) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +{ + __itt_counter_info_t *h_tail = NULL, *h = NULL; + __itt_metadata_type type = __itt_metadata_u64; + + if (name == NULL) + { + return NULL; + } + + ITT_MUTEX_INIT_AND_LOCK(_N_(_ittapi_global)); + if (_N_(_ittapi_global).api_initialized) + { +#if ITT_PLATFORM==ITT_PLATFORM_WIN + if (ITTNOTIFY_NAME(counter_createA) && ITTNOTIFY_NAME(counter_createA) != ITT_VERSIONIZE(ITT_JOIN(_N_(counter_createA),_init))) + { + __itt_mutex_unlock(&_N_(_ittapi_global).mutex); + return ITTNOTIFY_NAME(counter_createA)(name, domain); + } +#else + if (ITTNOTIFY_NAME(counter_create) && ITTNOTIFY_NAME(counter_create) != ITT_VERSIONIZE(ITT_JOIN(_N_(counter_create),_init))) + { + if (PTHREAD_SYMBOLS) __itt_mutex_unlock(&_N_(_ittapi_global).mutex); + return ITTNOTIFY_NAME(counter_create)(name, domain); + } +#endif + else + { +#if ITT_PLATFORM==ITT_PLATFORM_WIN + __itt_mutex_unlock(&_N_(_ittapi_global).mutex); +#else + if (PTHREAD_SYMBOLS) __itt_mutex_unlock(&_N_(_ittapi_global).mutex); +#endif + return NULL; + } } - for (h_tail = NULL, h = _N_(_ittapi_global).string_list; h != NULL; h_tail = h, h = h->next) + if (__itt_is_collector_available()) { - if (h->strA != NULL && !__itt_fstrcmp(h->strA, name)) break; + for (h_tail = NULL, h = _N_(_ittapi_global).counter_list; h != NULL; h_tail = h, h = h->next) + { + if (h->nameA != NULL && h->type == (int)type && !__itt_fstrcmp(h->nameA, name) && ((h->domainA == NULL && domain == NULL) || + (h->domainA != NULL && domain != NULL && !__itt_fstrcmp(h->domainA, domain)))) break; + } + if (h == NULL) + { + NEW_COUNTER_A(&_N_(_ittapi_global), h, h_tail, name, domain, type); + } } - if (h == NULL) + if (PTHREAD_SYMBOLS) __itt_mutex_unlock(&_N_(_ittapi_global).mutex); + return (__itt_counter)h; +} + +#if ITT_PLATFORM==ITT_PLATFORM_WIN +static __itt_counter ITTAPI ITT_VERSIONIZE(ITT_JOIN(_N_(counter_create_typedW),_init))(const wchar_t *name, const wchar_t *domain, __itt_metadata_type type) +{ + __itt_counter_info_t *h_tail = NULL, *h = NULL; + + if (name == NULL) { - NEW_STRING_HANDLE_A(&_N_(_ittapi_global),h,h_tail,name); + return NULL; + } + + ITT_MUTEX_INIT_AND_LOCK(_N_(_ittapi_global)); + if (_N_(_ittapi_global).api_initialized) + { + if (ITTNOTIFY_NAME(counter_create_typedW) && ITTNOTIFY_NAME(counter_create_typedW) != ITT_VERSIONIZE(ITT_JOIN(_N_(counter_create_typedW),_init))) + { + __itt_mutex_unlock(&_N_(_ittapi_global).mutex); + return ITTNOTIFY_NAME(counter_create_typedW)(name, domain, type); + } + else + { + __itt_mutex_unlock(&_N_(_ittapi_global).mutex); + return NULL; + } + } + if (__itt_is_collector_available()) + { + for (h_tail = NULL, h = _N_(_ittapi_global).counter_list; h != NULL; h_tail = h, h = h->next) + { + if (h->nameW != NULL && h->type == (int)type && !wcscmp(h->nameW, name) && ((h->domainW == NULL && domain == NULL) || + (h->domainW != NULL && domain != NULL && !wcscmp(h->domainW, domain)))) break; + + } + if (h == NULL) + { + NEW_COUNTER_W(&_N_(_ittapi_global), h, h_tail, name, domain, type); + } } __itt_mutex_unlock(&_N_(_ittapi_global).mutex); - return h; + return (__itt_counter)h; +} + +static __itt_counter ITTAPI ITT_VERSIONIZE(ITT_JOIN(_N_(counter_create_typedA),_init))(const char *name, const char *domain, __itt_metadata_type type) +#else /* ITT_PLATFORM!=ITT_PLATFORM_WIN */ +static __itt_counter ITTAPI ITT_VERSIONIZE(ITT_JOIN(_N_(counter_create_typed),_init))(const char *name, const char *domain, __itt_metadata_type type) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +{ + __itt_counter_info_t *h_tail = NULL, *h = NULL; + + if (name == NULL) + { + return NULL; + } + + ITT_MUTEX_INIT_AND_LOCK(_N_(_ittapi_global)); + if (_N_(_ittapi_global).api_initialized) + { +#if ITT_PLATFORM==ITT_PLATFORM_WIN + if (ITTNOTIFY_NAME(counter_create_typedA) && ITTNOTIFY_NAME(counter_create_typedA) != ITT_VERSIONIZE(ITT_JOIN(_N_(counter_create_typedA),_init))) + { + __itt_mutex_unlock(&_N_(_ittapi_global).mutex); + return ITTNOTIFY_NAME(counter_create_typedA)(name, domain, type); + } +#else + if (ITTNOTIFY_NAME(counter_create_typed) && ITTNOTIFY_NAME(counter_create_typed) != ITT_VERSIONIZE(ITT_JOIN(_N_(counter_create_typed),_init))) + { + if (PTHREAD_SYMBOLS) __itt_mutex_unlock(&_N_(_ittapi_global).mutex); + return ITTNOTIFY_NAME(counter_create_typed)(name, domain, type); + } +#endif + else + { +#if ITT_PLATFORM==ITT_PLATFORM_WIN + __itt_mutex_unlock(&_N_(_ittapi_global).mutex); +#else + if (PTHREAD_SYMBOLS) __itt_mutex_unlock(&_N_(_ittapi_global).mutex); +#endif + return NULL; + } + } + if (__itt_is_collector_available()) + { + for (h_tail = NULL, h = _N_(_ittapi_global).counter_list; h != NULL; h_tail = h, h = h->next) + { + if (h->nameA != NULL && h->type == (int)type && !__itt_fstrcmp(h->nameA, name) && ((h->domainA == NULL && domain == NULL) || + (h->domainA != NULL && domain != NULL && !__itt_fstrcmp(h->domainA, domain)))) break; + } + if (h == NULL) + { + NEW_COUNTER_A(&_N_(_ittapi_global), h, h_tail, name, domain, type); + } + } + if (PTHREAD_SYMBOLS) __itt_mutex_unlock(&_N_(_ittapi_global).mutex); + return (__itt_counter)h; +} + +#if ITT_PLATFORM==ITT_PLATFORM_WIN +static __itt_histogram* ITTAPI ITT_VERSIONIZE(ITT_JOIN(_N_(histogram_createW),_init))(const __itt_domain* domain, const wchar_t* name, __itt_metadata_type x_type, __itt_metadata_type y_type) +{ + __itt_histogram *h_tail = NULL, *h = NULL; + + if (domain == NULL || name == NULL) + { + return NULL; + } + + ITT_MUTEX_INIT_AND_LOCK(_N_(_ittapi_global)); + if (_N_(_ittapi_global).api_initialized) + { + if (ITTNOTIFY_NAME(histogram_createW) && ITTNOTIFY_NAME(histogram_createW) != ITT_VERSIONIZE(ITT_JOIN(_N_(histogram_createW),_init))) + { + __itt_mutex_unlock(&_N_(_ittapi_global).mutex); + return ITTNOTIFY_NAME(histogram_createW)(domain, name, x_type, y_type); + } + else + { + __itt_mutex_unlock(&_N_(_ittapi_global).mutex); + return NULL; + } + } + if (__itt_is_collector_available()) + { + for (h_tail = NULL, h = _N_(_ittapi_global).histogram_list; h != NULL; h_tail = h, h = h->next) + { + if (h->domain == NULL) continue; + else if (h->domain == domain && h->nameW != NULL && !wcscmp(h->nameW, name)) break; + } + if (h == NULL) + { + NEW_HISTOGRAM_W(&_N_(_ittapi_global), h, h_tail, domain, name, x_type, y_type); + } + } + __itt_mutex_unlock(&_N_(_ittapi_global).mutex); + return (__itt_histogram*)h; +} + +static __itt_histogram* ITTAPI ITT_VERSIONIZE(ITT_JOIN(_N_(histogram_createA),_init))(const __itt_domain* domain, const char* name, __itt_metadata_type x_type, __itt_metadata_type y_type) +#else /* ITT_PLATFORM!=ITT_PLATFORM_WIN */ +static __itt_histogram* ITTAPI ITT_VERSIONIZE(ITT_JOIN(_N_(histogram_create),_init))(const __itt_domain* domain, const char* name, __itt_metadata_type x_type, __itt_metadata_type y_type) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +{ + __itt_histogram *h_tail = NULL, *h = NULL; + + if (domain == NULL || name == NULL) + { + return NULL; + } + + ITT_MUTEX_INIT_AND_LOCK(_N_(_ittapi_global)); + if (_N_(_ittapi_global).api_initialized) + { +#if ITT_PLATFORM==ITT_PLATFORM_WIN + if (ITTNOTIFY_NAME(histogram_createA) && ITTNOTIFY_NAME(histogram_createA) != ITT_VERSIONIZE(ITT_JOIN(_N_(histogram_createA),_init))) + { + __itt_mutex_unlock(&_N_(_ittapi_global).mutex); + return ITTNOTIFY_NAME(histogram_createA)(domain, name, x_type, y_type); + } +#else + if (ITTNOTIFY_NAME(histogram_create) && ITTNOTIFY_NAME(histogram_create) != ITT_VERSIONIZE(ITT_JOIN(_N_(histogram_create),_init))) + { + if (PTHREAD_SYMBOLS) __itt_mutex_unlock(&_N_(_ittapi_global).mutex); + return ITTNOTIFY_NAME(histogram_create)(domain, name, x_type, y_type); + } +#endif + else + { +#if ITT_PLATFORM==ITT_PLATFORM_WIN + __itt_mutex_unlock(&_N_(_ittapi_global).mutex); +#else + if (PTHREAD_SYMBOLS) __itt_mutex_unlock(&_N_(_ittapi_global).mutex); +#endif + return NULL; + } + } + if (__itt_is_collector_available()) + { + for (h_tail = NULL, h = _N_(_ittapi_global).histogram_list; h != NULL; h_tail = h, h = h->next) + { + if (h->domain == NULL) continue; + else if (h->domain == domain && h->nameA != NULL && !__itt_fstrcmp(h->nameA, name)) break; + } + if (h == NULL) + { + NEW_HISTOGRAM_A(&_N_(_ittapi_global), h, h_tail, domain, name, x_type, y_type); + } + } + if (PTHREAD_SYMBOLS) __itt_mutex_unlock(&_N_(_ittapi_global).mutex); + return (__itt_histogram*)h; } +#if ITT_PLATFORM==ITT_PLATFORM_WIN +static __itt_counter ITTAPI ITT_VERSIONIZE(ITT_JOIN(_N_(counter_createW_v3),_init))(const __itt_domain* domain, const wchar_t* name, __itt_metadata_type type) +{ + __itt_counter_info_t *h_tail = NULL, *h = NULL; + + if (name == NULL || domain == NULL) + { + return NULL; + } + + ITT_MUTEX_INIT_AND_LOCK(_N_(_ittapi_global)); + if (_N_(_ittapi_global).api_initialized) + { + if (ITTNOTIFY_NAME(counter_createW_v3) && ITTNOTIFY_NAME(counter_createW_v3) != ITT_VERSIONIZE(ITT_JOIN(_N_(counter_createW_v3),_init))) + { + __itt_mutex_unlock(&_N_(_ittapi_global).mutex); + return ITTNOTIFY_NAME(counter_createW_v3)(domain, name, type); + } + else + { + __itt_mutex_unlock(&_N_(_ittapi_global).mutex); + return NULL; + } + } + if (__itt_is_collector_available()) + { + for (h_tail = NULL, h = _N_(_ittapi_global).counter_list; h != NULL; h_tail = h, h = h->next) + { + if (h->nameW != NULL && h->type == (int)type && !wcscmp(h->nameW, name) && ((h->domainW == NULL && domain->nameW == NULL) || + (h->domainW != NULL && domain->nameW != NULL && !wcscmp(h->domainW, domain->nameW)))) break; + + } + if (h == NULL) + { + NEW_COUNTER_W(&_N_(_ittapi_global),h,h_tail,name,domain->nameW,type); + } + } + __itt_mutex_unlock(&_N_(_ittapi_global).mutex); + return (__itt_counter)h; +} + +static __itt_counter ITTAPI ITT_VERSIONIZE(ITT_JOIN(_N_(counter_createA_v3),_init))(const __itt_domain* domain, const char* name, __itt_metadata_type type) +#else /* ITT_PLATFORM!=ITT_PLATFORM_WIN */ +static __itt_counter ITTAPI ITT_VERSIONIZE(ITT_JOIN(_N_(counter_create_v3),_init))(const __itt_domain* domain, const char* name, __itt_metadata_type type) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +{ + __itt_counter_info_t *h_tail = NULL, *h = NULL; + + if (name == NULL || domain == NULL) + { + return NULL; + } + + ITT_MUTEX_INIT_AND_LOCK(_N_(_ittapi_global)); + if (_N_(_ittapi_global).api_initialized) + { +#if ITT_PLATFORM==ITT_PLATFORM_WIN + if (ITTNOTIFY_NAME(counter_createA_v3) && ITTNOTIFY_NAME(counter_createA_v3) != ITT_VERSIONIZE(ITT_JOIN(_N_(counter_createA_v3),_init))) + { + __itt_mutex_unlock(&_N_(_ittapi_global).mutex); + return ITTNOTIFY_NAME(counter_createA_v3)(domain, name, type); + } +#else + if (ITTNOTIFY_NAME(counter_create_v3) && ITTNOTIFY_NAME(counter_create_v3) != ITT_VERSIONIZE(ITT_JOIN(_N_(counter_create_v3),_init))) + { + if (PTHREAD_SYMBOLS) __itt_mutex_unlock(&_N_(_ittapi_global).mutex); + return ITTNOTIFY_NAME(counter_create_v3)(domain, name, type); + } +#endif + else + { +#if ITT_PLATFORM==ITT_PLATFORM_WIN + __itt_mutex_unlock(&_N_(_ittapi_global).mutex); +#else + if (PTHREAD_SYMBOLS) __itt_mutex_unlock(&_N_(_ittapi_global).mutex); +#endif + return NULL; + } + } + if (__itt_is_collector_available()) + { + for (h_tail = NULL, h = _N_(_ittapi_global).counter_list; h != NULL; h_tail = h, h = h->next) + { + if (h->nameA != NULL && h->type == (int)type && !__itt_fstrcmp(h->nameA, name) && ((h->domainA == NULL && domain->nameA == NULL) || + (h->domainA != NULL && domain->nameA != NULL && !__itt_fstrcmp(h->domainA, domain->nameA)))) break; + } + if (h == NULL) + { + NEW_COUNTER_A(&_N_(_ittapi_global),h,h_tail,name,domain->nameA,type); + } + } + if (PTHREAD_SYMBOLS) __itt_mutex_unlock(&_N_(_ittapi_global).mutex); + return (__itt_counter)h; +} + +static void ITTAPI ITT_VERSIONIZE(ITT_JOIN(_N_(bind_context_metadata_to_counter),_init))(__itt_counter counter, size_t length, __itt_context_metadata* metadata) +{ + __itt_counter_metadata *h_tail = NULL, *h = NULL; + + if (counter == NULL || length == 0 || metadata == NULL) + { + return; + } + + ITT_MUTEX_INIT_AND_LOCK(_N_(_ittapi_global)); + if (_N_(_ittapi_global).api_initialized) + { + if (ITTNOTIFY_NAME(bind_context_metadata_to_counter) && ITTNOTIFY_NAME(bind_context_metadata_to_counter) != ITT_VERSIONIZE(ITT_JOIN(_N_(bind_context_metadata_to_counter),_init))) + { + if (PTHREAD_SYMBOLS) __itt_mutex_unlock(&_N_(_ittapi_global).mutex); + ITTNOTIFY_NAME(bind_context_metadata_to_counter)(counter, length, metadata); + } + else + { +#if ITT_PLATFORM==ITT_PLATFORM_WIN + __itt_mutex_unlock(&_N_(_ittapi_global).mutex); +#else + if (PTHREAD_SYMBOLS) __itt_mutex_unlock(&_N_(_ittapi_global).mutex); +#endif + return; + } + } + if (__itt_is_collector_available()) + { + size_t item; + char* str_valueA = NULL; +#if ITT_PLATFORM==ITT_PLATFORM_WIN + wchar_t* str_valueW = NULL; +#endif + unsigned long long value = 0; + __itt_context_type type = __itt_context_unknown; + + for (item = 0; item < length; item++) + { + type = metadata[item].type; + for (h_tail = NULL, h = _N_(_ittapi_global).counter_metadata_list; h != NULL; h_tail = h, h = h->next) + { + if (h->counter != NULL && h->counter == counter && h->type == type) break; + } + if (h == NULL && counter != NULL && type != __itt_context_unknown) + { + if (type == __itt_context_nameA || type == __itt_context_deviceA || type == __itt_context_unitsA || type == __itt_context_pci_addrA) + { + str_valueA = (char*)(metadata[item].value); + NEW_COUNTER_METADATA_STR_A(&_N_(_ittapi_global),h,h_tail,counter,type,str_valueA); + } +#if ITT_PLATFORM==ITT_PLATFORM_WIN + else if (type == __itt_context_nameW || type == __itt_context_deviceW || type == __itt_context_unitsW || type == __itt_context_pci_addrW) + { + str_valueW = (wchar_t*)(metadata[item].value); + NEW_COUNTER_METADATA_STR_W(&_N_(_ittapi_global),h,h_tail,counter,type,str_valueW); + } +#endif + else if (type >= __itt_context_tid && type <= __itt_context_cpu_cycles_flag) + { + value = *(unsigned long long*)(metadata[item].value); + NEW_COUNTER_METADATA_NUM(&_N_(_ittapi_global),h,h_tail,counter,type,value); + } + } + } + } + if (PTHREAD_SYMBOLS) __itt_mutex_unlock(&_N_(_ittapi_global).mutex); +} /* -------------------------------------------------------------------------- */ static void ITTAPI ITT_VERSIONIZE(ITT_JOIN(_N_(pause),_init))(void) @@ -439,10 +1026,6 @@ static void ITTAPI ITT_VERSIONIZE(ITT_JOIN(_N_(pause),_init))(void) { ITTNOTIFY_NAME(pause)(); } - else - { - _N_(_ittapi_global).state = __itt_collection_paused; - } } static void ITTAPI ITT_VERSIONIZE(ITT_JOIN(_N_(resume),_init))(void) @@ -455,9 +1038,29 @@ static void ITTAPI ITT_VERSIONIZE(ITT_JOIN(_N_(resume),_init))(void) { ITTNOTIFY_NAME(resume)(); } - else +} + +static void ITTAPI ITT_VERSIONIZE(ITT_JOIN(_N_(pause_scoped),_init))(__itt_collection_scope scope) +{ + if (!_N_(_ittapi_global).api_initialized && _N_(_ittapi_global).thread_list == NULL) + { + __itt_init_ittlib_name(NULL, __itt_group_all); + } + if (ITTNOTIFY_NAME(pause_scoped) && ITTNOTIFY_NAME(pause_scoped) != ITT_VERSIONIZE(ITT_JOIN(_N_(pause_scoped),_init))) { - _N_(_ittapi_global).state = __itt_collection_normal; + ITTNOTIFY_NAME(pause_scoped)(scope); + } +} + +static void ITTAPI ITT_VERSIONIZE(ITT_JOIN(_N_(resume_scoped),_init))(__itt_collection_scope scope) +{ + if (!_N_(_ittapi_global).api_initialized && _N_(_ittapi_global).thread_list == NULL) + { + __itt_init_ittlib_name(NULL, __itt_group_all); + } + if (ITTNOTIFY_NAME(resume_scoped) && ITTNOTIFY_NAME(resume_scoped) != ITT_VERSIONIZE(ITT_JOIN(_N_(resume_scoped),_init))) + { + ITTNOTIFY_NAME(resume_scoped)(scope); } } @@ -607,7 +1210,7 @@ static const char* __itt_fsplit(const char* s, const char* sep, const char** out /* This function return value of env variable that placed into static buffer. * !!! The same static buffer is used for subsequent calls. !!! - * This was done to aviod dynamic allocation for few calls. + * This was done to avoid dynamic allocation for few calls. * Actually we need this function only four times. */ static const char* __itt_get_env_var(const char* name) @@ -631,7 +1234,7 @@ static const char* __itt_get_env_var(const char* name) } else { - /* If environment variable is empty, GetEnvirornmentVariables() + /* If environment variable is empty, GetEnvironmentVariables() * returns zero (number of characters (not including terminating null), * and GetLastError() returns ERROR_SUCCESS. */ DWORD err = GetLastError(); @@ -798,8 +1401,9 @@ static __itt_group_id __itt_get_groups(void) const char* chunk; while ((group_str = __itt_fsplit(group_str, ",; ", &chunk, &len)) != NULL) { - __itt_fstrcpyn(gr, sizeof(gr) - 1, chunk, len + 1); - gr[__itt_min(len, (int)(sizeof(gr) - 1))] = 0; + int min_len = __itt_min(len, (int)(sizeof(gr) - 1)); + __itt_fstrcpyn(gr, sizeof(gr) - 1, chunk, min_len); + gr[min_len] = 0; for (i = 0; group_list[i].name != NULL; i++) { @@ -829,6 +1433,7 @@ static __itt_group_id __itt_get_groups(void) return res; } + #undef __itt_min static int __itt_lib_version(lib_t lib) @@ -845,7 +1450,7 @@ static int __itt_lib_version(lib_t lib) /* It's not used right now! Comment it out to avoid warnings. static void __itt_reinit_all_pointers(void) { - int i; + register int i; // Fill all pointers with initial stubs for (i = 0; _N_(_ittapi_global).api_list_ptr[i].name != NULL; i++) *_N_(_ittapi_global).api_list_ptr[i].func_ptr = _N_(_ittapi_global).api_list_ptr[i].init_func; @@ -855,15 +1460,32 @@ static void __itt_reinit_all_pointers(void) static void __itt_nullify_all_pointers(void) { int i; - /* Nulify all pointers except domain_create and string_handle_create */ + /* Nulify all pointers except domain_create, string_handle_create and counter_create */ for (i = 0; _N_(_ittapi_global).api_list_ptr[i].name != NULL; i++) *_N_(_ittapi_global).api_list_ptr[i].func_ptr = _N_(_ittapi_global).api_list_ptr[i].null_func; } +static int __itt_is_collector_available(void) +{ + int is_available; + + ITT_MUTEX_INIT_AND_LOCK(_N_(_ittapi_global)); + if (_N_(_ittapi_global).state == __itt_collection_uninitialized) + { + _N_(_ittapi_global).state = (NULL == __itt_get_lib_name()) ? __itt_collection_collector_absent : __itt_collection_collector_exists; + } + is_available = (_N_(_ittapi_global).state == __itt_collection_collector_exists || + _N_(_ittapi_global).state == __itt_collection_init_successful); + __itt_mutex_unlock(&_N_(_ittapi_global).mutex); + return is_available; +} + #if ITT_PLATFORM==ITT_PLATFORM_WIN +#if _MSC_VER #pragma warning(push) #pragma warning(disable: 4054) /* warning C4054: 'type cast' : from function pointer 'XXX' to data pointer 'void *' */ #pragma warning(disable: 4055) /* warning C4055: 'type cast' : from data pointer 'void *' to function pointer 'XXX' */ +#endif #endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ ITT_EXTERN_C void _N_(fini_ittlib)(void) @@ -873,12 +1495,12 @@ ITT_EXTERN_C void _N_(fini_ittlib)(void) if (_N_(_ittapi_global).api_initialized) { - __itt_mutex_lock(&_N_(_ittapi_global).mutex); + ITT_MUTEX_INIT_AND_LOCK(_N_(_ittapi_global)); if (_N_(_ittapi_global).api_initialized) { if (current_thread == 0) { - current_thread = __itt_thread_id(); + if (PTHREAD_SYMBOLS) current_thread = __itt_thread_id(); if (_N_(_ittapi_global).lib != NULL) { __itt_api_fini_ptr = (__itt_api_fini_t*)(size_t)__itt_get_proc(_N_(_ittapi_global).lib, "__itt_api_fini"); @@ -899,10 +1521,82 @@ ITT_EXTERN_C void _N_(fini_ittlib)(void) current_thread = 0; } } - __itt_mutex_unlock(&_N_(_ittapi_global).mutex); + if (PTHREAD_SYMBOLS) __itt_mutex_unlock(&_N_(_ittapi_global).mutex); } } +/* !!! this function should be called under mutex lock !!! */ +static void __itt_free_allocated_resources(void) +{ + __itt_string_handle* current_string = _N_(_ittapi_global).string_list; + while (current_string != NULL) + { + __itt_string_handle* tmp = current_string->next; + free((char*)current_string->strA); +#if ITT_PLATFORM==ITT_PLATFORM_WIN + free((wchar_t*)current_string->strW); +#endif + free(current_string); + current_string = tmp; + } + _N_(_ittapi_global).string_list = NULL; + + __itt_domain* current_domain = _N_(_ittapi_global).domain_list; + while (current_domain != NULL) + { + __itt_domain* tmp = current_domain->next; + free((char*)current_domain->nameA); +#if ITT_PLATFORM==ITT_PLATFORM_WIN + free((wchar_t*)current_domain->nameW); +#endif + free(current_domain); + current_domain = tmp; + } + _N_(_ittapi_global).domain_list = NULL; + + __itt_counter_info_t* current_couter = _N_(_ittapi_global).counter_list; + while (current_couter != NULL) + { + __itt_counter_info_t* tmp = current_couter->next; + free((char*)current_couter->nameA); + free((char*)current_couter->domainA); +#if ITT_PLATFORM==ITT_PLATFORM_WIN + free((wchar_t*)current_couter->nameW); + free((wchar_t*)current_couter->domainW); +#endif + free(current_couter); + current_couter = tmp; + } + _N_(_ittapi_global).counter_list = NULL; + + __itt_histogram* current_histogram = _N_(_ittapi_global).histogram_list; + while (current_histogram != NULL) + { + __itt_histogram* tmp = current_histogram->next; + free((char*)current_histogram->nameA); +#if ITT_PLATFORM==ITT_PLATFORM_WIN + free((wchar_t*)current_histogram->nameW); +#endif + free(current_histogram); + current_histogram = tmp; + } + _N_(_ittapi_global).histogram_list = NULL; + + + __itt_counter_metadata* current_counter_metadata = _N_(_ittapi_global).counter_metadata_list; + while (current_counter_metadata != NULL) + { + __itt_counter_metadata* tmp = current_counter_metadata->next; + free((char*)current_counter_metadata->str_valueA); +#if ITT_PLATFORM==ITT_PLATFORM_WIN + free((wchar_t*)current_counter_metadata->str_valueW); +#endif + free(current_counter_metadata); + current_counter_metadata = tmp; + } + _N_(_ittapi_global).counter_metadata_list = NULL; +} + ITT_EXTERN_C int _N_(init_ittlib)(const char* lib_name, __itt_group_id init_groups) { int i; @@ -922,24 +1616,27 @@ ITT_EXTERN_C int _N_(init_ittlib)(const char* lib_name, __itt_group_id init_grou { if (current_thread == 0) { - current_thread = __itt_thread_id(); + if (PTHREAD_SYMBOLS) current_thread = __itt_thread_id(); if (lib_name == NULL) { lib_name = __itt_get_lib_name(); } groups = __itt_get_groups(); - if (groups != __itt_group_none || lib_name != NULL) + if (DL_SYMBOLS && (groups != __itt_group_none || lib_name != NULL)) { _N_(_ittapi_global).lib = __itt_load_lib((lib_name == NULL) ? ittnotify_lib_name : lib_name); if (_N_(_ittapi_global).lib != NULL) { + _N_(_ittapi_global).state = __itt_collection_init_successful; __itt_api_init_t* __itt_api_init_ptr; int lib_version = __itt_lib_version(_N_(_ittapi_global).lib); - switch (lib_version) { + switch (lib_version) + { case 0: groups = __itt_group_legacy; + ITT_ATTRIBUTE_FALLTHROUGH; case 1: /* Fill all pointers from dynamic library */ for (i = 0; _N_(_ittapi_global).api_list_ptr[i].name != NULL; i++) @@ -992,17 +1689,22 @@ ITT_EXTERN_C int _N_(init_ittlib)(const char* lib_name, __itt_group_id init_grou } else { + _N_(_ittapi_global).state = __itt_collection_init_fail; + __itt_free_allocated_resources(); __itt_nullify_all_pointers(); + + __itt_report_error(__itt_error_no_module, lib_name, #if ITT_PLATFORM==ITT_PLATFORM_WIN - int error = __itt_system_error(); + __itt_system_error() #else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ - const char* error = dlerror(); + dlerror() #endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ - __itt_report_error(__itt_error_no_module, lib_name, error); + ); } } else { + _N_(_ittapi_global).state = __itt_collection_collector_absent; __itt_nullify_all_pointers(); } _N_(_ittapi_global).api_initialized = 1; @@ -1013,7 +1715,7 @@ ITT_EXTERN_C int _N_(init_ittlib)(const char* lib_name, __itt_group_id init_grou } #ifndef ITT_SIMPLE_INIT - __itt_mutex_unlock(&_N_(_ittapi_global).mutex); + if (PTHREAD_SYMBOLS) __itt_mutex_unlock(&_N_(_ittapi_global).mutex); #endif /* ITT_SIMPLE_INIT */ } @@ -1037,6 +1739,69 @@ ITT_EXTERN_C __itt_error_handler_t* _N_(set_error_handler)(__itt_error_handler_t } #if ITT_PLATFORM==ITT_PLATFORM_WIN +#if _MSC_VER #pragma warning(pop) +#endif #endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +/** __itt_mark_pt_region functions marks region of interest + * region parameter defines different regions. + * 0 <= region < 8 */ + +#if defined(ITT_API_IPT_SUPPORT) && (ITT_PLATFORM==ITT_PLATFORM_WIN || ITT_PLATFORM==ITT_PLATFORM_POSIX) && !defined(__ANDROID__) +void __itt_pt_mark(__itt_pt_region region); +void __itt_pt_mark_event(__itt_pt_region region); +#endif + +ITT_EXTERN_C void _N_(mark_pt_region_begin)(__itt_pt_region region) +{ +#if defined(ITT_API_IPT_SUPPORT) && (ITT_PLATFORM==ITT_PLATFORM_WIN || ITT_PLATFORM==ITT_PLATFORM_POSIX) && !defined(__ANDROID__) + if (_N_(_ittapi_global).ipt_collect_events == 1) + { + __itt_pt_mark_event(2*region); + } + else + { + __itt_pt_mark(2*region); + } +#else + (void)region; +#endif +} + +ITT_EXTERN_C void _N_(mark_pt_region_end)(__itt_pt_region region) +{ +#if defined(ITT_API_IPT_SUPPORT) && (ITT_PLATFORM==ITT_PLATFORM_WIN || ITT_PLATFORM==ITT_PLATFORM_POSIX) && !defined(__ANDROID__) + if (_N_(_ittapi_global).ipt_collect_events == 1) + { + __itt_pt_mark_event(2*region + 1); + } + else + { + __itt_pt_mark(2*region + 1); + } +#else + (void)region; +#endif +} + +ITT_EXTERN_C __itt_collection_state (_N_(get_collection_state))(void) +{ + if (!_N_(_ittapi_global).api_initialized && _N_(_ittapi_global).thread_list == NULL) + { + __itt_init_ittlib_name(NULL, __itt_group_all); + } + return _N_(_ittapi_global).state; +} + +/* !!! should be called from the library destructor !!! + * this function destroys the mutex and frees resources + * allocated by ITT API static part + */ +ITT_EXTERN_C void (_N_(release_resources))(void) +{ + ITT_MUTEX_INIT_AND_LOCK(_N_(_ittapi_global)); + __itt_free_allocated_resources(); + if (PTHREAD_SYMBOLS) __itt_mutex_unlock(&_N_(_ittapi_global).mutex); + ITT_MUTEX_DESTROY(_N_(_ittapi_global)); +} diff --git a/src/tbb/src/tbb/tools_api/ittnotify_static.h b/src/tbb/src/tbb/tools_api/ittnotify_static.h index 6907b835f..7f5729140 100644 --- a/src/tbb/src/tbb/tools_api/ittnotify_static.h +++ b/src/tbb/src/tbb/tools_api/ittnotify_static.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2023 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include "ittnotify_config.h" @@ -43,6 +39,9 @@ ITT_STUB(ITTAPI, __itt_domain*, domain_createW, (const wchar_t *name), (ITT_FORM ITT_STUB(ITTAPI, __itt_domain*, domain_create, (const char *name), (ITT_FORMAT name), domain_create, __itt_group_structure, "\"%s\"") #endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUBV(ITTAPI, void, module_load_with_sections, (__itt_module_object* module_obj), (ITT_FORMAT module_obj), module_load_with_sections, __itt_group_module, "%p") +ITT_STUBV(ITTAPI, void, module_unload_with_sections, (__itt_module_object* module_obj), (ITT_FORMAT module_obj), module_unload_with_sections, __itt_group_module, "%p") + #if ITT_PLATFORM==ITT_PLATFORM_WIN ITT_STUB(ITTAPI, __itt_string_handle*, string_handle_createA, (const char *name), (ITT_FORMAT name), string_handle_createA, __itt_group_structure, "\"%s\"") ITT_STUB(ITTAPI, __itt_string_handle*, string_handle_createW, (const wchar_t *name), (ITT_FORMAT name), string_handle_createW, __itt_group_structure, "\"%S\"") @@ -50,8 +49,25 @@ ITT_STUB(ITTAPI, __itt_string_handle*, string_handle_createW, (const wchar_t *na ITT_STUB(ITTAPI, __itt_string_handle*, string_handle_create, (const char *name), (ITT_FORMAT name), string_handle_create, __itt_group_structure, "\"%s\"") #endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUB(ITTAPI, __itt_counter, counter_createA, (const char *name, const char *domain), (ITT_FORMAT name, domain), counter_createA, __itt_group_counter, "\"%s\", \"%s\"") +ITT_STUB(ITTAPI, __itt_counter, counter_createW, (const wchar_t *name, const wchar_t *domain), (ITT_FORMAT name, domain), counter_createW, __itt_group_counter, "\"%s\", \"%s\"") +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUB(ITTAPI, __itt_counter, counter_create, (const char *name, const char *domain), (ITT_FORMAT name, domain), counter_create, __itt_group_counter, "\"%s\", \"%s\"") +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUB(ITTAPI, __itt_counter, counter_create_typedA, (const char *name, const char *domain, __itt_metadata_type type), (ITT_FORMAT name, domain, type), counter_create_typedA, __itt_group_counter, "\"%s\", \"%s\", %d") +ITT_STUB(ITTAPI, __itt_counter, counter_create_typedW, (const wchar_t *name, const wchar_t *domain, __itt_metadata_type type), (ITT_FORMAT name, domain, type), counter_create_typedW, __itt_group_counter, "\"%s\", \"%s\", %d") +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUB(ITTAPI, __itt_counter, counter_create_typed, (const char *name, const char *domain, __itt_metadata_type type), (ITT_FORMAT name, domain, type), counter_create_typed, __itt_group_counter, "\"%s\", \"%s\", %d") +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + + ITT_STUBV(ITTAPI, void, pause, (void), (ITT_NO_PARAMS), pause, __itt_group_control | __itt_group_legacy, "no args") ITT_STUBV(ITTAPI, void, resume, (void), (ITT_NO_PARAMS), resume, __itt_group_control | __itt_group_legacy, "no args") +ITT_STUBV(ITTAPI, void, pause_scoped, (__itt_collection_scope scope), (ITT_FORMAT scope), pause_scoped, __itt_group_control, "%d") +ITT_STUBV(ITTAPI, void, resume_scoped, (__itt_collection_scope scope), (ITT_FORMAT scope), resume_scoped, __itt_group_control, "%d") #if ITT_PLATFORM==ITT_PLATFORM_WIN ITT_STUBV(ITTAPI, void, thread_set_nameA, (const char *name), (ITT_FORMAT name), thread_set_nameA, __itt_group_thread, "\"%s\"") @@ -68,6 +84,23 @@ ITT_STUB(LIBITTAPI, int, thr_name_setW, (const wchar_t *name, int namelen), (IT ITT_STUB(LIBITTAPI, int, thr_name_set, (const char *name, int namelen), (ITT_FORMAT name, namelen), thr_name_set, __itt_group_thread | __itt_group_legacy, "\"%s\", %d") #endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ ITT_STUBV(LIBITTAPI, void, thr_ignore, (void), (ITT_NO_PARAMS), thr_ignore, __itt_group_thread | __itt_group_legacy, "no args") + +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUB(ITTAPI, __itt_histogram*, histogram_createA, (const __itt_domain* domain, const char* name, __itt_metadata_type x_type, __itt_metadata_type y_type), (ITT_FORMAT domain, name, x_type, y_type), histogram_createA, __itt_group_structure, "%p, \"%s\", %d, %d") +ITT_STUB(ITTAPI, __itt_histogram*, histogram_createW, (const __itt_domain* domain, const wchar_t* name, __itt_metadata_type x_type, __itt_metadata_type y_type), (ITT_FORMAT domain, name, x_type, y_type), histogram_createW, __itt_group_structure, "%p, \"%s\", %d, %d") +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUB(ITTAPI, __itt_histogram*, histogram_create, (const __itt_domain* domain, const char* name, __itt_metadata_type x_type, __itt_metadata_type y_type), (ITT_FORMAT domain, name, x_type, y_type), histogram_create, __itt_group_structure, "%p, \"%s\", %d, %d") +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + + #if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUB(ITTAPI, __itt_counter, counter_createA_v3, (const __itt_domain* domain, const char *name, __itt_metadata_type type), (ITT_FORMAT domain, name, type), counter_createA_v3, __itt_group_counter, "%p, \"%s\", %d") +ITT_STUB(ITTAPI, __itt_counter, counter_createW_v3, (const __itt_domain* domain, const wchar_t *name, __itt_metadata_type type), (ITT_FORMAT domain, name, type), counter_createW_v3, __itt_group_counter, "%p, \"%s\", %d") +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUB(ITTAPI, __itt_counter, counter_create_v3, (const __itt_domain* domain, const char *name, __itt_metadata_type type), (ITT_FORMAT domain, name, type), counter_create_v3, __itt_group_counter, "%p, \"%s\", %d") +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + +ITT_STUBV(ITTAPI, void, bind_context_metadata_to_counter, (__itt_counter counter, size_t length, __itt_context_metadata* metadata), (ITT_FORMAT counter, length, metadata), bind_context_metadata_to_counter, __itt_group_structure, "%p, %lu, %p") + #endif /* __ITT_INTERNAL_BODY */ ITT_STUBV(ITTAPI, void, enable_attach, (void), (ITT_NO_PARAMS), enable_attach, __itt_group_all, "no args") @@ -179,6 +212,8 @@ ITT_STUBV(ITTAPI, void, task_end, (const __itt_domain *domain), ITT_STUBV(ITTAPI, void, counter_inc_v3, (const __itt_domain *domain, __itt_string_handle *name), (ITT_FORMAT domain, name), counter_inc_v3, __itt_group_structure, "%p, %p") ITT_STUBV(ITTAPI, void, counter_inc_delta_v3, (const __itt_domain *domain, __itt_string_handle *name, unsigned long long value), (ITT_FORMAT domain, name, value), counter_inc_delta_v3, __itt_group_structure, "%p, %p, %lu") +ITT_STUBV(ITTAPI, void, counter_dec_v3, (const __itt_domain *domain, __itt_string_handle *name), (ITT_FORMAT domain, name), counter_dec_v3, __itt_group_structure, "%p, %p") +ITT_STUBV(ITTAPI, void, counter_dec_delta_v3, (const __itt_domain *domain, __itt_string_handle *name, unsigned long long value), (ITT_FORMAT domain, name, value), counter_dec_delta_v3, __itt_group_structure, "%p, %p, %lu") ITT_STUBV(ITTAPI, void, marker, (const __itt_domain *domain, __itt_id id, __itt_string_handle *name, __itt_scope scope), (ITT_FORMAT domain, id, name, scope), marker, __itt_group_structure, "%p, %lu, %p, %d") @@ -241,21 +276,24 @@ ITT_STUB(ITTAPI, __itt_frame, frame_createW, (const wchar_t *domain), (ITT_FORMA #else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ ITT_STUB(ITTAPI, __itt_frame, frame_create, (const char *domain), (ITT_FORMAT domain), frame_create, __itt_group_frame, "\"%s\"") #endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ -#endif /* __ITT_INTERNAL_BODY */ -ITT_STUBV(ITTAPI, void, frame_begin, (__itt_frame frame), (ITT_FORMAT frame), frame_begin, __itt_group_frame, "%p") -ITT_STUBV(ITTAPI, void, frame_end, (__itt_frame frame), (ITT_FORMAT frame), frame_end, __itt_group_frame, "%p") -#ifndef __ITT_INTERNAL_BODY #if ITT_PLATFORM==ITT_PLATFORM_WIN -ITT_STUB(ITTAPI, __itt_counter, counter_createA, (const char *name, const char *domain), (ITT_FORMAT name, domain), counter_createA, __itt_group_counter, "\"%s\", \"%s\"") -ITT_STUB(ITTAPI, __itt_counter, counter_createW, (const wchar_t *name, const wchar_t *domain), (ITT_FORMAT name, domain), counter_createW, __itt_group_counter, "\"%s\", \"%s\"") -#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ -ITT_STUB(ITTAPI, __itt_counter, counter_create, (const char *name, const char *domain), (ITT_FORMAT name, domain), counter_create, __itt_group_counter, "\"%s\", \"%s\"") +ITT_STUB(ITTAPI, __itt_pt_region, pt_region_createA, (const char *name), (ITT_FORMAT name), pt_region_createA, __itt_group_structure, "\"%s\"") +ITT_STUB(ITTAPI, __itt_pt_region, pt_region_createW, (const wchar_t *name), (ITT_FORMAT name), pt_region_createW, __itt_group_structure, "\"%S\"") +#else /* ITT_PLATFORM!=ITT_PLATFORM_WIN */ +ITT_STUB(ITTAPI, __itt_pt_region, pt_region_create, (const char *name), (ITT_FORMAT name), pt_region_create, __itt_group_structure, "\"%s\"") #endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ #endif /* __ITT_INTERNAL_BODY */ -ITT_STUBV(ITTAPI, void, counter_destroy, (__itt_counter id), (ITT_FORMAT id), counter_destroy, __itt_group_counter, "%p") -ITT_STUBV(ITTAPI, void, counter_inc, (__itt_counter id), (ITT_FORMAT id), counter_inc, __itt_group_counter, "%p") -ITT_STUBV(ITTAPI, void, counter_inc_delta, (__itt_counter id, unsigned long long value), (ITT_FORMAT id, value), counter_inc_delta, __itt_group_counter, "%p, %lu") +ITT_STUBV(ITTAPI, void, frame_begin, (__itt_frame frame), (ITT_FORMAT frame), frame_begin, __itt_group_frame, "%p") +ITT_STUBV(ITTAPI, void, frame_end, (__itt_frame frame), (ITT_FORMAT frame), frame_end, __itt_group_frame, "%p") + +ITT_STUBV(ITTAPI, void, counter_destroy, (__itt_counter id), (ITT_FORMAT id), counter_destroy, __itt_group_counter, "%p") +ITT_STUBV(ITTAPI, void, counter_inc, (__itt_counter id), (ITT_FORMAT id), counter_inc, __itt_group_counter, "%p") +ITT_STUBV(ITTAPI, void, counter_inc_delta, (__itt_counter id, unsigned long long value), (ITT_FORMAT id, value), counter_inc_delta, __itt_group_counter, "%p, %lu") +ITT_STUBV(ITTAPI, void, counter_dec, (__itt_counter id), (ITT_FORMAT id), counter_dec, __itt_group_counter, "%p") +ITT_STUBV(ITTAPI, void, counter_dec_delta, (__itt_counter id, unsigned long long value), (ITT_FORMAT id, value), counter_dec_delta, __itt_group_counter, "%p, %lu") +ITT_STUBV(ITTAPI, void, counter_set_value, (__itt_counter id, void *value_ptr), (ITT_FORMAT id, value_ptr), counter_set_value, __itt_group_counter, "%p, %p") +ITT_STUBV(ITTAPI, void, counter_set_value_ex, (__itt_counter id, __itt_clock_domain *clock_domain, unsigned long long timestamp, void *value_ptr), (ITT_FORMAT id, clock_domain, timestamp, value_ptr), counter_set_value_ex, __itt_group_counter, "%p, %p, %llu, %p") #ifndef __ITT_INTERNAL_BODY #if ITT_PLATFORM==ITT_PLATFORM_WIN @@ -325,4 +363,16 @@ ITT_STUB(ITTAPI, int, av_save, (void *data, int rank, const int *dimensions, in #endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ #endif /* __ITT_INTERNAL_BODY */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUBV(ITTAPI, void, module_loadA, (void *start_addr, void* end_addr, const char *path), (ITT_FORMAT start_addr, end_addr, path), module_loadA, __itt_group_module, "%p, %p, %p") +ITT_STUBV(ITTAPI, void, module_loadW, (void *start_addr, void* end_addr, const wchar_t *path), (ITT_FORMAT start_addr, end_addr, path), module_loadW, __itt_group_module, "%p, %p, %p") +#else /* ITT_PLATFORM!=ITT_PLATFORM_WIN */ +ITT_STUBV(ITTAPI, void, module_load, (void *start_addr, void *end_addr, const char *path), (ITT_FORMAT start_addr, end_addr, path), module_load, __itt_group_module, "%p, %p, %p") +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUBV(ITTAPI, void, module_unload, (void *start_addr), (ITT_FORMAT start_addr), module_unload, __itt_group_module, "%p") + +ITT_STUBV(ITTAPI, void, histogram_submit, (__itt_histogram* histogram, size_t length, void* x_data, void* y_data), (ITT_FORMAT histogram, length, x_data, y_data), histogram_submit, __itt_group_structure, "%p, %lu, %p, %p") + +ITT_STUBV(ITTAPI, void, counter_set_value_v3, (__itt_counter counter, void *value_ptr), (ITT_FORMAT counter, value_ptr), counter_set_value_v3, __itt_group_counter, "%p, %p") + #endif /* __ITT_INTERNAL_INIT */ diff --git a/src/tbb/src/tbb/tools_api/ittnotify_types.h b/src/tbb/src/tbb/tools_api/ittnotify_types.h index 83cb0ddfc..1c0fded40 100644 --- a/src/tbb/src/tbb/tools_api/ittnotify_types.h +++ b/src/tbb/src/tbb/tools_api/ittnotify_types.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2022 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef _ITTNOTIFY_TYPES_H_ @@ -23,25 +19,26 @@ typedef enum ___itt_group_id { - __itt_group_none = 0, - __itt_group_legacy = 1<<0, - __itt_group_control = 1<<1, - __itt_group_thread = 1<<2, - __itt_group_mark = 1<<3, - __itt_group_sync = 1<<4, - __itt_group_fsync = 1<<5, - __itt_group_jit = 1<<6, - __itt_group_model = 1<<7, - __itt_group_splitter_min = 1<<7, - __itt_group_counter = 1<<8, - __itt_group_frame = 1<<9, - __itt_group_stitch = 1<<10, - __itt_group_heap = 1<<11, - __itt_group_splitter_max = 1<<12, - __itt_group_structure = 1<<12, - __itt_group_suppress = 1<<13, - __itt_group_arrays = 1<<14, - __itt_group_all = -1 + __itt_group_none = 0, + __itt_group_legacy = 1<<0, + __itt_group_control = 1<<1, + __itt_group_thread = 1<<2, + __itt_group_mark = 1<<3, + __itt_group_sync = 1<<4, + __itt_group_fsync = 1<<5, + __itt_group_jit = 1<<6, + __itt_group_model = 1<<7, + __itt_group_splitter_min = 1<<7, + __itt_group_counter = 1<<8, + __itt_group_frame = 1<<9, + __itt_group_stitch = 1<<10, + __itt_group_heap = 1<<11, + __itt_group_splitter_max = 1<<12, + __itt_group_structure = 1<<12, + __itt_group_suppress = 1<<13, + __itt_group_arrays = 1<<14, + __itt_group_module = 1<<15, + __itt_group_all = -1 } __itt_group_id; #pragma pack(push, 8) @@ -71,6 +68,7 @@ typedef struct ___itt_group_list { __itt_group_structure, "structure" }, \ { __itt_group_suppress, "suppress" }, \ { __itt_group_arrays, "arrays" }, \ + { __itt_group_module, "module" }, \ { __itt_group_none, NULL } \ } diff --git a/src/tbb/src/tbb/tools_api/legacy/ittnotify.h b/src/tbb/src/tbb/tools_api/legacy/ittnotify.h index e15734a4e..837bc4800 100644 --- a/src/tbb/src/tbb/tools_api/legacy/ittnotify.h +++ b/src/tbb/src/tbb/tools_api/legacy/ittnotify.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2024 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef _LEGACY_ITTNOTIFY_H_ @@ -43,6 +39,10 @@ # define ITT_OS_FREEBSD 4 #endif /* ITT_OS_FREEBSD */ +#ifndef ITT_OS_OPENBSD +# define ITT_OS_OPENBSD 5 +#endif /* ITT_OS_OPENBSD */ + #ifndef ITT_OS # if defined WIN32 || defined _WIN32 # define ITT_OS ITT_OS_WIN @@ -50,6 +50,8 @@ # define ITT_OS ITT_OS_MAC # elif defined( __FreeBSD__ ) # define ITT_OS ITT_OS_FREEBSD +# elif defined( __OpenBSD__ ) +# define ITT_OS ITT_OS_OPENBSD # else # define ITT_OS ITT_OS_LINUX # endif @@ -71,6 +73,10 @@ # define ITT_PLATFORM_FREEBSD 4 #endif /* ITT_PLATFORM_FREEBSD */ +#ifndef ITT_PLATFORM_OPENBSD +# define ITT_PLATFORM_OPENBSD 5 +#endif /* ITT_PLATFORM_OPENBSD */ + #ifndef ITT_PLATFORM # if ITT_OS==ITT_OS_WIN # define ITT_PLATFORM ITT_PLATFORM_WIN @@ -78,6 +84,8 @@ # define ITT_PLATFORM ITT_PLATFORM_MAC # elif ITT_OS==ITT_OS_FREEBSD # define ITT_PLATFORM ITT_PLATFORM_FREEBSD +# elif ITT_OS==ITT_OS_OPENBSD +# define ITT_PLATFORM ITT_PLATFORM_OPENBSD # else # define ITT_PLATFORM ITT_PLATFORM_POSIX # endif @@ -97,40 +105,45 @@ #endif /* UNICODE || _UNICODE */ #endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ -#ifndef CDECL +#ifndef ITTAPI_CDECL # if ITT_PLATFORM==ITT_PLATFORM_WIN -# define CDECL __cdecl +# define ITTAPI_CDECL __cdecl # else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ -# if defined _M_IX86 || defined __i386__ -# define CDECL __attribute__ ((cdecl)) +# if defined _M_IX86 || defined __i386__ +# define ITTAPI_CDECL __attribute__ ((cdecl)) # else /* _M_IX86 || __i386__ */ -# define CDECL /* actual only on x86 platform */ +# define ITTAPI_CDECL /* actual only on x86 platform */ # endif /* _M_IX86 || __i386__ */ # endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ -#endif /* CDECL */ +#endif /* ITTAPI_CDECL */ #ifndef STDCALL # if ITT_PLATFORM==ITT_PLATFORM_WIN # define STDCALL __stdcall # else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ # if defined _M_IX86 || defined __i386__ -# define STDCALL __attribute__ ((stdcall)) +# define STDCALL __attribute__ ((stdcall)) # else /* _M_IX86 || __i386__ */ # define STDCALL /* supported only on x86 platform */ # endif /* _M_IX86 || __i386__ */ # endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ #endif /* STDCALL */ -#define ITTAPI CDECL -#define LIBITTAPI CDECL +#define ITTAPI ITTAPI_CDECL +#define LIBITTAPI ITTAPI_CDECL /* TODO: Temporary for compatibility! */ -#define ITTAPI_CALL CDECL -#define LIBITTAPI_CALL CDECL +#define ITTAPI_CALL ITTAPI_CDECL +#define LIBITTAPI_CALL ITTAPI_CDECL #if ITT_PLATFORM==ITT_PLATFORM_WIN /* use __forceinline (VC++ specific) */ -#define ITT_INLINE __forceinline +#if defined(__MINGW32__) && !defined(__cplusplus) +#define ITT_INLINE static __inline__ __attribute__((__always_inline__,__gnu_inline__)) +#else +#define ITT_INLINE static __forceinline +#endif /* __MINGW32__ */ + #define ITT_INLINE_ATTRIBUTE /* nothing */ #else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ /* @@ -182,20 +195,20 @@ #define ITTNOTIFY_VOID(n) (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n) #define ITTNOTIFY_DATA(n) (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n) -#define ITTNOTIFY_VOID_D0(n,d) (!(d)->flags) ? (void)0 : (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n)(d) -#define ITTNOTIFY_VOID_D1(n,d,x) (!(d)->flags) ? (void)0 : (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n)(d,x) -#define ITTNOTIFY_VOID_D2(n,d,x,y) (!(d)->flags) ? (void)0 : (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n)(d,x,y) -#define ITTNOTIFY_VOID_D3(n,d,x,y,z) (!(d)->flags) ? (void)0 : (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n)(d,x,y,z) -#define ITTNOTIFY_VOID_D4(n,d,x,y,z,a) (!(d)->flags) ? (void)0 : (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n)(d,x,y,z,a) -#define ITTNOTIFY_VOID_D5(n,d,x,y,z,a,b) (!(d)->flags) ? (void)0 : (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n)(d,x,y,z,a,b) -#define ITTNOTIFY_VOID_D6(n,d,x,y,z,a,b,c) (!(d)->flags) ? (void)0 : (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n)(d,x,y,z,a,b,c) -#define ITTNOTIFY_DATA_D0(n,d) (!(d)->flags) ? 0 : (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n)(d) -#define ITTNOTIFY_DATA_D1(n,d,x) (!(d)->flags) ? 0 : (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n)(d,x) -#define ITTNOTIFY_DATA_D2(n,d,x,y) (!(d)->flags) ? 0 : (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n)(d,x,y) -#define ITTNOTIFY_DATA_D3(n,d,x,y,z) (!(d)->flags) ? 0 : (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n)(d,x,y,z) -#define ITTNOTIFY_DATA_D4(n,d,x,y,z,a) (!(d)->flags) ? 0 : (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n)(d,x,y,z,a) -#define ITTNOTIFY_DATA_D5(n,d,x,y,z,a,b) (!(d)->flags) ? 0 : (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n)(d,x,y,z,a,b) -#define ITTNOTIFY_DATA_D6(n,d,x,y,z,a,b,c) (!(d)->flags) ? 0 : (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n)(d,x,y,z,a,b,c) +#define ITTNOTIFY_VOID_D0(n,d) (d == NULL) ? (void)0 : (!(d)->flags) ? (void)0 : (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n)(d) +#define ITTNOTIFY_VOID_D1(n,d,x) (d == NULL) ? (void)0 : (!(d)->flags) ? (void)0 : (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n)(d,x) +#define ITTNOTIFY_VOID_D2(n,d,x,y) (d == NULL) ? (void)0 : (!(d)->flags) ? (void)0 : (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n)(d,x,y) +#define ITTNOTIFY_VOID_D3(n,d,x,y,z) (d == NULL) ? (void)0 : (!(d)->flags) ? (void)0 : (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n)(d,x,y,z) +#define ITTNOTIFY_VOID_D4(n,d,x,y,z,a) (d == NULL) ? (void)0 : (!(d)->flags) ? (void)0 : (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n)(d,x,y,z,a) +#define ITTNOTIFY_VOID_D5(n,d,x,y,z,a,b) (d == NULL) ? (void)0 : (!(d)->flags) ? (void)0 : (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n)(d,x,y,z,a,b) +#define ITTNOTIFY_VOID_D6(n,d,x,y,z,a,b,c) (d == NULL) ? (void)0 : (!(d)->flags) ? (void)0 : (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n)(d,x,y,z,a,b,c) +#define ITTNOTIFY_DATA_D0(n,d) (d == NULL) ? 0 : (!(d)->flags) ? 0 : (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n)(d) +#define ITTNOTIFY_DATA_D1(n,d,x) (d == NULL) ? 0 : (!(d)->flags) ? 0 : (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n)(d,x) +#define ITTNOTIFY_DATA_D2(n,d,x,y) (d == NULL) ? 0 : (!(d)->flags) ? 0 : (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n)(d,x,y) +#define ITTNOTIFY_DATA_D3(n,d,x,y,z) (d == NULL) ? 0 : (!(d)->flags) ? 0 : (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n)(d,x,y,z) +#define ITTNOTIFY_DATA_D4(n,d,x,y,z,a) (d == NULL) ? 0 : (!(d)->flags) ? 0 : (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n)(d,x,y,z,a) +#define ITTNOTIFY_DATA_D5(n,d,x,y,z,a,b) (d == NULL) ? 0 : (!(d)->flags) ? 0 : (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n)(d,x,y,z,a,b) +#define ITTNOTIFY_DATA_D6(n,d,x,y,z,a,b,c) (d == NULL) ? 0 : (!(d)->flags) ? 0 : (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n)(d,x,y,z,a,b,c) #ifdef ITT_STUB #undef ITT_STUB @@ -232,7 +245,7 @@ extern "C" { * only pauses tracing and analyzing memory access. * It does not pause tracing or analyzing threading APIs. * . - * - Intel(R) Parallel Amplifier and Intel(R) VTune(TM) Amplifier XE: + * - Intel(R) VTune(TM) Profiler: * - Does continue to record when new threads are started. * . * - Other effects: @@ -968,9 +981,9 @@ ITT_STUB(ITTAPI, __itt_frame, frame_create, (const char *domain)) #endif /* INTEL_NO_MACRO_BODY */ /** @endcond */ -/** @brief Record an frame begin occurrence. */ +/** @brief Record a frame begin occurrence. */ void ITTAPI __itt_frame_begin(__itt_frame frame); -/** @brief Record an frame end occurrence. */ +/** @brief Record a frame end occurrence. */ void ITTAPI __itt_frame_end (__itt_frame frame); /** @cond exclude_from_documentation */ diff --git a/src/tbb/src/tbb/version.cpp b/src/tbb/src/tbb/version.cpp new file mode 100644 index 000000000..ca113372f --- /dev/null +++ b/src/tbb/src/tbb/version.cpp @@ -0,0 +1,26 @@ +/* + Copyright (c) 2020-2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "oneapi/tbb/version.h" + +extern "C" int TBB_runtime_interface_version() { + return TBB_INTERFACE_VERSION; +} + +extern "C" const char* TBB_runtime_version() { + static const char version_str[] = TBB_VERSION_STRING; + return version_str; +} diff --git a/src/tbb/src/tbb/waiters.h b/src/tbb/src/tbb/waiters.h new file mode 100644 index 000000000..8ed431f85 --- /dev/null +++ b/src/tbb/src/tbb/waiters.h @@ -0,0 +1,214 @@ +/* + Copyright (c) 2005-2024 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef _TBB_waiters_H +#define _TBB_waiters_H + +#include "oneapi/tbb/detail/_task.h" +#include "scheduler_common.h" +#include "arena.h" +#include "threading_control.h" + +namespace tbb { +namespace detail { +namespace r1 { + +inline d1::task* get_self_recall_task(arena_slot& slot); + +class waiter_base { +public: + waiter_base(arena& a, int yields_multiplier = 1) : my_arena(a), my_backoff(int(a.my_num_slots), yields_multiplier) {} + + bool pause() { + if (my_backoff.pause()) { + my_arena.out_of_work(); + return true; + } + + return false; + } + + void reset_wait() { + my_backoff.reset_wait(); + } + +protected: + arena& my_arena; + stealing_loop_backoff my_backoff; +}; + +class outermost_worker_waiter : public waiter_base { +public: + using waiter_base::waiter_base; + + bool continue_execution(arena_slot& slot, d1::task*& t) const { + __TBB_ASSERT(t == nullptr, nullptr); + + if (is_worker_should_leave(slot)) { + if (!governor::hybrid_cpu()) { + static constexpr std::chrono::microseconds worker_wait_leave_duration(1000); + static_assert(worker_wait_leave_duration > std::chrono::steady_clock::duration(1), "Clock resolution is not enough for measured interval."); + + for (auto t1 = std::chrono::steady_clock::now(), t2 = t1; + std::chrono::duration_cast<std::chrono::microseconds>(t2 - t1) < worker_wait_leave_duration; + t2 = std::chrono::steady_clock::now()) + { + if (!my_arena.is_empty() && !my_arena.is_recall_requested()) { + return true; + } + + if (my_arena.my_threading_control->is_any_other_client_active()) { + break; + } + d0::yield(); + } + } + // Leave dispatch loop + return false; + } + + t = get_self_recall_task(slot); + return true; + } + + void pause(arena_slot&) { + waiter_base::pause(); + } + + + d1::wait_context* wait_ctx() { + return nullptr; + } + + static bool postpone_execution(d1::task&) { + return false; + } + +private: + using base_type = waiter_base; + + bool is_worker_should_leave(arena_slot& slot) const { + bool is_top_priority_arena = my_arena.is_top_priority(); + bool is_task_pool_empty = slot.task_pool.load(std::memory_order_relaxed) == EmptyTaskPool; + + if (is_top_priority_arena) { + // Worker in most priority arena do not leave arena, until all work in task_pool is done + if (is_task_pool_empty && my_arena.is_recall_requested()) { + return true; + } + } else { + if (my_arena.is_recall_requested()) { + // If worker has work in task pool, we must notify other threads, + // because can appear missed wake up of other threads + if (!is_task_pool_empty) { + my_arena.advertise_new_work<arena::wakeup>(); + } + return true; + } + } + + return false; + } +}; + +class sleep_waiter : public waiter_base { +protected: + using waiter_base::waiter_base; + + template <typename Pred> + void sleep(std::uintptr_t uniq_tag, Pred wakeup_condition) { + my_arena.get_waiting_threads_monitor().wait<thread_control_monitor::thread_context>(wakeup_condition, + market_context{uniq_tag, &my_arena}); + reset_wait(); + } +}; + +class external_waiter : public sleep_waiter { +public: + external_waiter(arena& a, d1::wait_context& wo) + : sleep_waiter(a, /*yields_multiplier*/10), my_wait_ctx(wo) + {} + + bool continue_execution(arena_slot& slot, d1::task*& t) const { + __TBB_ASSERT(t == nullptr, nullptr); + if (!my_wait_ctx.continue_execution()) + return false; + t = get_self_recall_task(slot); + return true; + } + + void pause(arena_slot&) { + if (!sleep_waiter::pause()) { + return; + } + + auto wakeup_condition = [&] { return !my_arena.is_empty() || !my_wait_ctx.continue_execution(); }; + + sleep(std::uintptr_t(&my_wait_ctx), wakeup_condition); + } + + d1::wait_context* wait_ctx() { + return &my_wait_ctx; + } + + static bool postpone_execution(d1::task&) { + return false; + } + +private: + d1::wait_context& my_wait_ctx; +}; + +#if __TBB_RESUMABLE_TASKS + +class coroutine_waiter : public sleep_waiter { +public: + using sleep_waiter::sleep_waiter; + + bool continue_execution(arena_slot& slot, d1::task*& t) const { + __TBB_ASSERT(t == nullptr, nullptr); + t = get_self_recall_task(slot); + return true; + } + + void pause(arena_slot& slot) { + if (!sleep_waiter::pause()) { + return; + } + + suspend_point_type* sp = slot.default_task_dispatcher().m_suspend_point; + + auto wakeup_condition = [&] { return !my_arena.is_empty() || sp->m_is_owner_recalled.load(std::memory_order_relaxed); }; + + sleep(std::uintptr_t(sp), wakeup_condition); + } + + d1::wait_context* wait_ctx() { + return nullptr; + } + + static bool postpone_execution(d1::task& t) { + return task_accessor::is_resume_task(t); + } +}; + +#endif // __TBB_RESUMABLE_TASKS + +} // namespace r1 +} // namespace detail +} // namespace tbb + +#endif // _TBB_waiters_H diff --git a/src/tbb/src/tbb/win32-tbb-export.def b/src/tbb/src/tbb/win32-tbb-export.def deleted file mode 100644 index 9f1b7dcaa..000000000 --- a/src/tbb/src/tbb/win32-tbb-export.def +++ /dev/null @@ -1,28 +0,0 @@ -; Copyright (c) 2005-2017 Intel Corporation -; -; Licensed under the Apache License, Version 2.0 (the "License"); -; you may not use this file except in compliance with the License. -; You may obtain a copy of the License at -; -; http://www.apache.org/licenses/LICENSE-2.0 -; -; Unless required by applicable law or agreed to in writing, software -; distributed under the License is distributed on an "AS IS" BASIS, -; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -; See the License for the specific language governing permissions and -; limitations under the License. -; -; -; -; - -EXPORTS - -#define __TBB_SYMBOL( sym ) sym -#if _M_ARM -#include "winrt-tbb-export.lst" -#else -#include "win32-tbb-export.lst" -#endif - - diff --git a/src/tbb/src/tbb/win32-tbb-export.lst b/src/tbb/src/tbb/win32-tbb-export.lst deleted file mode 100644 index 35b1833a1..000000000 --- a/src/tbb/src/tbb/win32-tbb-export.lst +++ /dev/null @@ -1,344 +0,0 @@ -; Copyright (c) 2005-2017 Intel Corporation -; -; Licensed under the Apache License, Version 2.0 (the "License"); -; you may not use this file except in compliance with the License. -; You may obtain a copy of the License at -; -; http://www.apache.org/licenses/LICENSE-2.0 -; -; Unless required by applicable law or agreed to in writing, software -; distributed under the License is distributed on an "AS IS" BASIS, -; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -; See the License for the specific language governing permissions and -; limitations under the License. -; -; -; -; - -#include "tbb/tbb_config.h" - -// Assembly-language support that is called directly by clients -// __TBB_SYMBOL( __TBB_machine_cmpswp1 ) -// __TBB_SYMBOL( __TBB_machine_cmpswp2 ) -// __TBB_SYMBOL( __TBB_machine_cmpswp4 ) -__TBB_SYMBOL( __TBB_machine_cmpswp8 ) -// __TBB_SYMBOL( __TBB_machine_fetchadd1 ) -// __TBB_SYMBOL( __TBB_machine_fetchadd2 ) -// __TBB_SYMBOL( __TBB_machine_fetchadd4 ) -__TBB_SYMBOL( __TBB_machine_fetchadd8 ) -// __TBB_SYMBOL( __TBB_machine_fetchstore1 ) -// __TBB_SYMBOL( __TBB_machine_fetchstore2 ) -// __TBB_SYMBOL( __TBB_machine_fetchstore4 ) -__TBB_SYMBOL( __TBB_machine_fetchstore8 ) -__TBB_SYMBOL( __TBB_machine_store8 ) -__TBB_SYMBOL( __TBB_machine_load8 ) -__TBB_SYMBOL( __TBB_machine_trylockbyte ) -__TBB_SYMBOL( __TBB_machine_try_lock_elided ) -__TBB_SYMBOL( __TBB_machine_unlock_elided ) -__TBB_SYMBOL( __TBB_machine_is_in_transaction ) - -// cache_aligned_allocator.cpp -__TBB_SYMBOL( ?NFS_Allocate@internal@tbb@@YAPAXIIPAX@Z ) -__TBB_SYMBOL( ?NFS_GetLineSize@internal@tbb@@YAIXZ ) -__TBB_SYMBOL( ?NFS_Free@internal@tbb@@YAXPAX@Z ) -__TBB_SYMBOL( ?allocate_via_handler_v3@internal@tbb@@YAPAXI@Z ) -__TBB_SYMBOL( ?deallocate_via_handler_v3@internal@tbb@@YAXPAX@Z ) -__TBB_SYMBOL( ?is_malloc_used_v3@internal@tbb@@YA_NXZ ) - -// task.cpp v3 -__TBB_SYMBOL( ?allocate@allocate_additional_child_of_proxy@internal@tbb@@QBEAAVtask@3@I@Z ) -__TBB_SYMBOL( ?allocate@allocate_child_proxy@internal@tbb@@QBEAAVtask@3@I@Z ) -__TBB_SYMBOL( ?allocate@allocate_continuation_proxy@internal@tbb@@QBEAAVtask@3@I@Z ) -__TBB_SYMBOL( ?allocate@allocate_root_proxy@internal@tbb@@SAAAVtask@3@I@Z ) -__TBB_SYMBOL( ?destroy@task_base@internal@interface5@tbb@@SAXAAVtask@4@@Z ) -__TBB_SYMBOL( ?free@allocate_additional_child_of_proxy@internal@tbb@@QBEXAAVtask@3@@Z ) -__TBB_SYMBOL( ?free@allocate_child_proxy@internal@tbb@@QBEXAAVtask@3@@Z ) -__TBB_SYMBOL( ?free@allocate_continuation_proxy@internal@tbb@@QBEXAAVtask@3@@Z ) -__TBB_SYMBOL( ?free@allocate_root_proxy@internal@tbb@@SAXAAVtask@3@@Z ) -__TBB_SYMBOL( ?internal_set_ref_count@task@tbb@@AAEXH@Z ) -__TBB_SYMBOL( ?internal_decrement_ref_count@task@tbb@@AAEHXZ ) -__TBB_SYMBOL( ?is_owned_by_current_thread@task@tbb@@QBE_NXZ ) -__TBB_SYMBOL( ?note_affinity@task@tbb@@UAEXG@Z ) -__TBB_SYMBOL( ?resize@affinity_partitioner_base_v3@internal@tbb@@AAEXI@Z ) -__TBB_SYMBOL( ?self@task@tbb@@SAAAV12@XZ ) -__TBB_SYMBOL( ?spawn_and_wait_for_all@task@tbb@@QAEXAAVtask_list@2@@Z ) -__TBB_SYMBOL( ?default_num_threads@task_scheduler_init@tbb@@SAHXZ ) -__TBB_SYMBOL( ?initialize@task_scheduler_init@tbb@@QAEXHI@Z ) -__TBB_SYMBOL( ?initialize@task_scheduler_init@tbb@@QAEXH@Z ) -__TBB_SYMBOL( ?terminate@task_scheduler_init@tbb@@QAEXXZ ) -__TBB_SYMBOL( ?internal_blocking_terminate@task_scheduler_init@tbb@@AAE_N_N@Z ) -#if __TBB_SCHEDULER_OBSERVER -__TBB_SYMBOL( ?observe@task_scheduler_observer_v3@internal@tbb@@QAEX_N@Z ) -#endif /* __TBB_SCHEDULER_OBSERVER */ - -/* arena.cpp */ -__TBB_SYMBOL( ?internal_max_concurrency@task_arena_base@internal@interface7@tbb@@KAHPBVtask_arena@34@@Z ) -__TBB_SYMBOL( ?internal_current_slot@task_arena_base@internal@interface7@tbb@@KAHXZ ) -__TBB_SYMBOL( ?internal_initialize@task_arena_base@internal@interface7@tbb@@IAEXXZ ) -__TBB_SYMBOL( ?internal_terminate@task_arena_base@internal@interface7@tbb@@IAEXXZ ) -__TBB_SYMBOL( ?internal_attach@task_arena_base@internal@interface7@tbb@@IAEXXZ ) -__TBB_SYMBOL( ?internal_enqueue@task_arena_base@internal@interface7@tbb@@IBEXAAVtask@4@H@Z ) -__TBB_SYMBOL( ?internal_execute@task_arena_base@internal@interface7@tbb@@IBEXAAVdelegate_base@234@@Z ) -__TBB_SYMBOL( ?internal_wait@task_arena_base@internal@interface7@tbb@@IBEXXZ ) -#if __TBB_TASK_ISOLATION -__TBB_SYMBOL( ?isolate_within_arena@internal@interface7@tbb@@YAXAAVdelegate_base@123@H@Z ) -#endif /* __TBB_TASK_ISOLATION */ - -#if !TBB_NO_LEGACY -// task_v2.cpp -__TBB_SYMBOL( ?destroy@task@tbb@@QAEXAAV12@@Z ) -#endif - -// exception handling support -#if __TBB_TASK_GROUP_CONTEXT -__TBB_SYMBOL( ?allocate@allocate_root_with_context_proxy@internal@tbb@@QBEAAVtask@3@I@Z ) -__TBB_SYMBOL( ?free@allocate_root_with_context_proxy@internal@tbb@@QBEXAAVtask@3@@Z ) -__TBB_SYMBOL( ?change_group@task@tbb@@QAEXAAVtask_group_context@2@@Z ) -__TBB_SYMBOL( ?is_group_execution_cancelled@task_group_context@tbb@@QBE_NXZ ) -__TBB_SYMBOL( ?cancel_group_execution@task_group_context@tbb@@QAE_NXZ ) -__TBB_SYMBOL( ?reset@task_group_context@tbb@@QAEXXZ ) -__TBB_SYMBOL( ?capture_fp_settings@task_group_context@tbb@@QAEXXZ ) -__TBB_SYMBOL( ?init@task_group_context@tbb@@IAEXXZ ) -__TBB_SYMBOL( ?register_pending_exception@task_group_context@tbb@@QAEXXZ ) -__TBB_SYMBOL( ??1task_group_context@tbb@@QAE@XZ ) -#if __TBB_TASK_PRIORITY -__TBB_SYMBOL( ?set_priority@task_group_context@tbb@@QAEXW4priority_t@2@@Z ) -__TBB_SYMBOL( ?priority@task_group_context@tbb@@QBE?AW4priority_t@2@XZ ) -#endif /* __TBB_TASK_PRIORITY */ -__TBB_SYMBOL( ?name@captured_exception@tbb@@UBEPBDXZ ) -__TBB_SYMBOL( ?what@captured_exception@tbb@@UBEPBDXZ ) -__TBB_SYMBOL( ??1captured_exception@tbb@@UAE@XZ ) -__TBB_SYMBOL( ?move@captured_exception@tbb@@UAEPAV12@XZ ) -__TBB_SYMBOL( ?destroy@captured_exception@tbb@@UAEXXZ ) -__TBB_SYMBOL( ?set@captured_exception@tbb@@QAEXPBD0@Z ) -__TBB_SYMBOL( ?clear@captured_exception@tbb@@QAEXXZ ) -#endif /* __TBB_TASK_GROUP_CONTEXT */ - -// Symbols for exceptions thrown from TBB -__TBB_SYMBOL( ?throw_bad_last_alloc_exception_v4@internal@tbb@@YAXXZ ) -__TBB_SYMBOL( ?throw_exception_v4@internal@tbb@@YAXW4exception_id@12@@Z ) -__TBB_SYMBOL( ?what@bad_last_alloc@tbb@@UBEPBDXZ ) -__TBB_SYMBOL( ?what@missing_wait@tbb@@UBEPBDXZ ) -__TBB_SYMBOL( ?what@invalid_multiple_scheduling@tbb@@UBEPBDXZ ) -__TBB_SYMBOL( ?what@improper_lock@tbb@@UBEPBDXZ ) -__TBB_SYMBOL( ?what@user_abort@tbb@@UBEPBDXZ ) - -// tbb_misc.cpp -__TBB_SYMBOL( ?assertion_failure@tbb@@YAXPBDH00@Z ) -__TBB_SYMBOL( ?get_initial_auto_partitioner_divisor@internal@tbb@@YAIXZ ) -__TBB_SYMBOL( ?handle_perror@internal@tbb@@YAXHPBD@Z ) -__TBB_SYMBOL( ?set_assertion_handler@tbb@@YAP6AXPBDH00@ZP6AX0H00@Z@Z ) -__TBB_SYMBOL( ?runtime_warning@internal@tbb@@YAXPBDZZ ) -__TBB_SYMBOL( TBB_runtime_interface_version ) - -// tbb_main.cpp -__TBB_SYMBOL( ?itt_load_pointer_with_acquire_v3@internal@tbb@@YAPAXPBX@Z ) -__TBB_SYMBOL( ?itt_store_pointer_with_release_v3@internal@tbb@@YAXPAX0@Z ) -__TBB_SYMBOL( ?call_itt_notify_v5@internal@tbb@@YAXHPAX@Z ) -__TBB_SYMBOL( ?itt_set_sync_name_v3@internal@tbb@@YAXPAXPB_W@Z ) -__TBB_SYMBOL( ?itt_load_pointer_v3@internal@tbb@@YAPAXPBX@Z ) -#if __TBB_ITT_STRUCTURE_API -__TBB_SYMBOL( ?itt_make_task_group_v7@internal@tbb@@YAXW4itt_domain_enum@12@PAX_K12W4string_index@12@@Z ) -__TBB_SYMBOL( ?itt_metadata_str_add_v7@internal@tbb@@YAXW4itt_domain_enum@12@PAX_KW4string_index@12@PBD@Z ) -__TBB_SYMBOL( ?itt_relation_add_v7@internal@tbb@@YAXW4itt_domain_enum@12@PAX_KW4itt_relation@12@12@Z ) -__TBB_SYMBOL( ?itt_task_begin_v7@internal@tbb@@YAXW4itt_domain_enum@12@PAX_K12W4string_index@12@@Z ) -__TBB_SYMBOL( ?itt_task_end_v7@internal@tbb@@YAXW4itt_domain_enum@12@@Z ) -__TBB_SYMBOL( ?itt_region_begin_v9@internal@tbb@@YAXW4itt_domain_enum@12@PAX_K12W4string_index@12@@Z ) -__TBB_SYMBOL( ?itt_region_end_v9@internal@tbb@@YAXW4itt_domain_enum@12@PAX_K@Z ) -#endif - -// pipeline.cpp -__TBB_SYMBOL( ??0pipeline@tbb@@QAE@XZ ) -__TBB_SYMBOL( ??1filter@tbb@@UAE@XZ ) -__TBB_SYMBOL( ??1pipeline@tbb@@UAE@XZ ) -__TBB_SYMBOL( ??_7pipeline@tbb@@6B@ ) -__TBB_SYMBOL( ?add_filter@pipeline@tbb@@QAEXAAVfilter@2@@Z ) -__TBB_SYMBOL( ?clear@pipeline@tbb@@QAEXXZ ) -__TBB_SYMBOL( ?inject_token@pipeline@tbb@@AAEXAAVtask@2@@Z ) -__TBB_SYMBOL( ?run@pipeline@tbb@@QAEXI@Z ) -#if __TBB_TASK_GROUP_CONTEXT -__TBB_SYMBOL( ?run@pipeline@tbb@@QAEXIAAVtask_group_context@2@@Z ) -#endif -__TBB_SYMBOL( ?process_item@thread_bound_filter@tbb@@QAE?AW4result_type@12@XZ ) -__TBB_SYMBOL( ?try_process_item@thread_bound_filter@tbb@@QAE?AW4result_type@12@XZ ) -__TBB_SYMBOL( ?set_end_of_input@filter@tbb@@IAEXXZ ) - -// queuing_rw_mutex.cpp -__TBB_SYMBOL( ?internal_construct@queuing_rw_mutex@tbb@@QAEXXZ ) -__TBB_SYMBOL( ?acquire@scoped_lock@queuing_rw_mutex@tbb@@QAEXAAV23@_N@Z ) -__TBB_SYMBOL( ?downgrade_to_reader@scoped_lock@queuing_rw_mutex@tbb@@QAE_NXZ ) -__TBB_SYMBOL( ?release@scoped_lock@queuing_rw_mutex@tbb@@QAEXXZ ) -__TBB_SYMBOL( ?upgrade_to_writer@scoped_lock@queuing_rw_mutex@tbb@@QAE_NXZ ) -__TBB_SYMBOL( ?try_acquire@scoped_lock@queuing_rw_mutex@tbb@@QAE_NAAV23@_N@Z ) - -// reader_writer_lock.cpp -__TBB_SYMBOL( ?try_lock_read@reader_writer_lock@interface5@tbb@@QAE_NXZ ) -__TBB_SYMBOL( ?try_lock@reader_writer_lock@interface5@tbb@@QAE_NXZ ) -__TBB_SYMBOL( ?unlock@reader_writer_lock@interface5@tbb@@QAEXXZ ) -__TBB_SYMBOL( ?lock_read@reader_writer_lock@interface5@tbb@@QAEXXZ ) -__TBB_SYMBOL( ?lock@reader_writer_lock@interface5@tbb@@QAEXXZ ) -__TBB_SYMBOL( ?internal_construct@reader_writer_lock@interface5@tbb@@AAEXXZ ) -__TBB_SYMBOL( ?internal_destroy@reader_writer_lock@interface5@tbb@@AAEXXZ ) -__TBB_SYMBOL( ?internal_construct@scoped_lock@reader_writer_lock@interface5@tbb@@AAEXAAV234@@Z ) -__TBB_SYMBOL( ?internal_destroy@scoped_lock@reader_writer_lock@interface5@tbb@@AAEXXZ ) -__TBB_SYMBOL( ?internal_construct@scoped_lock_read@reader_writer_lock@interface5@tbb@@AAEXAAV234@@Z ) -__TBB_SYMBOL( ?internal_destroy@scoped_lock_read@reader_writer_lock@interface5@tbb@@AAEXXZ ) - -#if !TBB_NO_LEGACY -// spin_rw_mutex.cpp v2 -__TBB_SYMBOL( ?internal_acquire_reader@spin_rw_mutex@tbb@@CAXPAV12@@Z ) -__TBB_SYMBOL( ?internal_acquire_writer@spin_rw_mutex@tbb@@CA_NPAV12@@Z ) -__TBB_SYMBOL( ?internal_downgrade@spin_rw_mutex@tbb@@CAXPAV12@@Z ) -__TBB_SYMBOL( ?internal_itt_releasing@spin_rw_mutex@tbb@@CAXPAV12@@Z ) -__TBB_SYMBOL( ?internal_release_reader@spin_rw_mutex@tbb@@CAXPAV12@@Z ) -__TBB_SYMBOL( ?internal_release_writer@spin_rw_mutex@tbb@@CAXPAV12@@Z ) -__TBB_SYMBOL( ?internal_upgrade@spin_rw_mutex@tbb@@CA_NPAV12@@Z ) -__TBB_SYMBOL( ?internal_try_acquire_writer@spin_rw_mutex@tbb@@CA_NPAV12@@Z ) -__TBB_SYMBOL( ?internal_try_acquire_reader@spin_rw_mutex@tbb@@CA_NPAV12@@Z ) -#endif - -// spin_rw_mutex v3 -__TBB_SYMBOL( ?internal_construct@spin_rw_mutex_v3@tbb@@AAEXXZ ) -__TBB_SYMBOL( ?internal_upgrade@spin_rw_mutex_v3@tbb@@AAE_NXZ ) -__TBB_SYMBOL( ?internal_downgrade@spin_rw_mutex_v3@tbb@@AAEXXZ ) -__TBB_SYMBOL( ?internal_acquire_reader@spin_rw_mutex_v3@tbb@@AAEXXZ ) -__TBB_SYMBOL( ?internal_acquire_writer@spin_rw_mutex_v3@tbb@@AAE_NXZ ) -__TBB_SYMBOL( ?internal_release_reader@spin_rw_mutex_v3@tbb@@AAEXXZ ) -__TBB_SYMBOL( ?internal_release_writer@spin_rw_mutex_v3@tbb@@AAEXXZ ) -__TBB_SYMBOL( ?internal_try_acquire_reader@spin_rw_mutex_v3@tbb@@AAE_NXZ ) -__TBB_SYMBOL( ?internal_try_acquire_writer@spin_rw_mutex_v3@tbb@@AAE_NXZ ) - -// x86_rtm_rw_mutex.cpp -__TBB_SYMBOL( ?internal_construct@x86_rtm_rw_mutex@internal@interface8@tbb@@AAEXXZ ) -__TBB_SYMBOL( ?internal_release@x86_rtm_rw_mutex@internal@interface8@tbb@@AAEXAAVscoped_lock@1234@@Z ) -__TBB_SYMBOL( ?internal_acquire_writer@x86_rtm_rw_mutex@internal@interface8@tbb@@AAEXAAVscoped_lock@1234@_N@Z ) -__TBB_SYMBOL( ?internal_acquire_reader@x86_rtm_rw_mutex@internal@interface8@tbb@@AAEXAAVscoped_lock@1234@_N@Z ) -__TBB_SYMBOL( ?internal_upgrade@x86_rtm_rw_mutex@internal@interface8@tbb@@AAE_NAAVscoped_lock@1234@@Z ) -__TBB_SYMBOL( ?internal_downgrade@x86_rtm_rw_mutex@internal@interface8@tbb@@AAE_NAAVscoped_lock@1234@@Z ) -__TBB_SYMBOL( ?internal_try_acquire_writer@x86_rtm_rw_mutex@internal@interface8@tbb@@AAE_NAAVscoped_lock@1234@@Z ) - -// spin_mutex.cpp -__TBB_SYMBOL( ?internal_construct@spin_mutex@tbb@@QAEXXZ ) -__TBB_SYMBOL( ?internal_acquire@scoped_lock@spin_mutex@tbb@@AAEXAAV23@@Z ) -__TBB_SYMBOL( ?internal_release@scoped_lock@spin_mutex@tbb@@AAEXXZ ) -__TBB_SYMBOL( ?internal_try_acquire@scoped_lock@spin_mutex@tbb@@AAE_NAAV23@@Z ) - -// mutex.cpp -__TBB_SYMBOL( ?internal_acquire@scoped_lock@mutex@tbb@@AAEXAAV23@@Z ) -__TBB_SYMBOL( ?internal_release@scoped_lock@mutex@tbb@@AAEXXZ ) -__TBB_SYMBOL( ?internal_try_acquire@scoped_lock@mutex@tbb@@AAE_NAAV23@@Z ) -__TBB_SYMBOL( ?internal_construct@mutex@tbb@@AAEXXZ ) -__TBB_SYMBOL( ?internal_destroy@mutex@tbb@@AAEXXZ ) - -// recursive_mutex.cpp -__TBB_SYMBOL( ?internal_acquire@scoped_lock@recursive_mutex@tbb@@AAEXAAV23@@Z ) -__TBB_SYMBOL( ?internal_release@scoped_lock@recursive_mutex@tbb@@AAEXXZ ) -__TBB_SYMBOL( ?internal_try_acquire@scoped_lock@recursive_mutex@tbb@@AAE_NAAV23@@Z ) -__TBB_SYMBOL( ?internal_construct@recursive_mutex@tbb@@AAEXXZ ) -__TBB_SYMBOL( ?internal_destroy@recursive_mutex@tbb@@AAEXXZ ) - -// queuing_mutex.cpp -__TBB_SYMBOL( ?internal_construct@queuing_mutex@tbb@@QAEXXZ ) -__TBB_SYMBOL( ?acquire@scoped_lock@queuing_mutex@tbb@@QAEXAAV23@@Z ) -__TBB_SYMBOL( ?release@scoped_lock@queuing_mutex@tbb@@QAEXXZ ) -__TBB_SYMBOL( ?try_acquire@scoped_lock@queuing_mutex@tbb@@QAE_NAAV23@@Z ) - -// critical_section.cpp -__TBB_SYMBOL( ?internal_construct@critical_section_v4@internal@tbb@@QAEXXZ ) - -#if !TBB_NO_LEGACY -// concurrent_hash_map.cpp -__TBB_SYMBOL( ?internal_grow_predicate@hash_map_segment_base@internal@tbb@@QBE_NXZ ) - -// concurrent_queue.cpp v2 -__TBB_SYMBOL( ?advance@concurrent_queue_iterator_base@internal@tbb@@IAEXXZ ) -__TBB_SYMBOL( ?assign@concurrent_queue_iterator_base@internal@tbb@@IAEXABV123@@Z ) -__TBB_SYMBOL( ?internal_size@concurrent_queue_base@internal@tbb@@IBEHXZ ) -__TBB_SYMBOL( ??0concurrent_queue_base@internal@tbb@@IAE@I@Z ) -__TBB_SYMBOL( ??0concurrent_queue_iterator_base@internal@tbb@@IAE@ABVconcurrent_queue_base@12@@Z ) -__TBB_SYMBOL( ??1concurrent_queue_base@internal@tbb@@MAE@XZ ) -__TBB_SYMBOL( ??1concurrent_queue_iterator_base@internal@tbb@@IAE@XZ ) -__TBB_SYMBOL( ?internal_pop@concurrent_queue_base@internal@tbb@@IAEXPAX@Z ) -__TBB_SYMBOL( ?internal_pop_if_present@concurrent_queue_base@internal@tbb@@IAE_NPAX@Z ) -__TBB_SYMBOL( ?internal_push@concurrent_queue_base@internal@tbb@@IAEXPBX@Z ) -__TBB_SYMBOL( ?internal_push_if_not_full@concurrent_queue_base@internal@tbb@@IAE_NPBX@Z ) -__TBB_SYMBOL( ?internal_set_capacity@concurrent_queue_base@internal@tbb@@IAEXHI@Z ) -#endif - -// concurrent_queue v3 -__TBB_SYMBOL( ??1concurrent_queue_iterator_base_v3@internal@tbb@@IAE@XZ ) -__TBB_SYMBOL( ??0concurrent_queue_iterator_base_v3@internal@tbb@@IAE@ABVconcurrent_queue_base_v3@12@@Z ) -__TBB_SYMBOL( ??0concurrent_queue_iterator_base_v3@internal@tbb@@IAE@ABVconcurrent_queue_base_v3@12@I@Z ) -__TBB_SYMBOL( ?advance@concurrent_queue_iterator_base_v3@internal@tbb@@IAEXXZ ) -__TBB_SYMBOL( ?assign@concurrent_queue_iterator_base_v3@internal@tbb@@IAEXABV123@@Z ) -__TBB_SYMBOL( ??0concurrent_queue_base_v3@internal@tbb@@IAE@I@Z ) -__TBB_SYMBOL( ??1concurrent_queue_base_v3@internal@tbb@@MAE@XZ ) -__TBB_SYMBOL( ?internal_pop@concurrent_queue_base_v3@internal@tbb@@IAEXPAX@Z ) -__TBB_SYMBOL( ?internal_pop_if_present@concurrent_queue_base_v3@internal@tbb@@IAE_NPAX@Z ) -__TBB_SYMBOL( ?internal_abort@concurrent_queue_base_v3@internal@tbb@@IAEXXZ ) -__TBB_SYMBOL( ?internal_push@concurrent_queue_base_v3@internal@tbb@@IAEXPBX@Z ) -__TBB_SYMBOL( ?internal_push_move@concurrent_queue_base_v8@internal@tbb@@IAEXPBX@Z ) -__TBB_SYMBOL( ?internal_push_if_not_full@concurrent_queue_base_v3@internal@tbb@@IAE_NPBX@Z ) -__TBB_SYMBOL( ?internal_push_move_if_not_full@concurrent_queue_base_v8@internal@tbb@@IAE_NPBX@Z ) -__TBB_SYMBOL( ?internal_size@concurrent_queue_base_v3@internal@tbb@@IBEHXZ ) -__TBB_SYMBOL( ?internal_empty@concurrent_queue_base_v3@internal@tbb@@IBE_NXZ ) -__TBB_SYMBOL( ?internal_set_capacity@concurrent_queue_base_v3@internal@tbb@@IAEXHI@Z ) -__TBB_SYMBOL( ?internal_finish_clear@concurrent_queue_base_v3@internal@tbb@@IAEXXZ ) -__TBB_SYMBOL( ?internal_throw_exception@concurrent_queue_base_v3@internal@tbb@@IBEXXZ ) -__TBB_SYMBOL( ?assign@concurrent_queue_base_v3@internal@tbb@@IAEXABV123@@Z ) -__TBB_SYMBOL( ?move_content@concurrent_queue_base_v8@internal@tbb@@IAEXAAV123@@Z ) - -#if !TBB_NO_LEGACY -// concurrent_vector.cpp v2 -__TBB_SYMBOL( ?internal_assign@concurrent_vector_base@internal@tbb@@IAEXABV123@IP6AXPAXI@ZP6AX1PBXI@Z4@Z ) -__TBB_SYMBOL( ?internal_capacity@concurrent_vector_base@internal@tbb@@IBEIXZ ) -__TBB_SYMBOL( ?internal_clear@concurrent_vector_base@internal@tbb@@IAEXP6AXPAXI@Z_N@Z ) -__TBB_SYMBOL( ?internal_copy@concurrent_vector_base@internal@tbb@@IAEXABV123@IP6AXPAXPBXI@Z@Z ) -__TBB_SYMBOL( ?internal_grow_by@concurrent_vector_base@internal@tbb@@IAEIIIP6AXPAXI@Z@Z ) -__TBB_SYMBOL( ?internal_grow_to_at_least@concurrent_vector_base@internal@tbb@@IAEXIIP6AXPAXI@Z@Z ) -__TBB_SYMBOL( ?internal_push_back@concurrent_vector_base@internal@tbb@@IAEPAXIAAI@Z ) -__TBB_SYMBOL( ?internal_reserve@concurrent_vector_base@internal@tbb@@IAEXIII@Z ) -#endif - -// concurrent_vector v3 -__TBB_SYMBOL( ??1concurrent_vector_base_v3@internal@tbb@@IAE@XZ ) -__TBB_SYMBOL( ?internal_assign@concurrent_vector_base_v3@internal@tbb@@IAEXABV123@IP6AXPAXI@ZP6AX1PBXI@Z4@Z ) -__TBB_SYMBOL( ?internal_capacity@concurrent_vector_base_v3@internal@tbb@@IBEIXZ ) -__TBB_SYMBOL( ?internal_clear@concurrent_vector_base_v3@internal@tbb@@IAEIP6AXPAXI@Z@Z ) -__TBB_SYMBOL( ?internal_copy@concurrent_vector_base_v3@internal@tbb@@IAEXABV123@IP6AXPAXPBXI@Z@Z ) -__TBB_SYMBOL( ?internal_grow_by@concurrent_vector_base_v3@internal@tbb@@IAEIIIP6AXPAXPBXI@Z1@Z ) -__TBB_SYMBOL( ?internal_grow_to_at_least@concurrent_vector_base_v3@internal@tbb@@IAEXIIP6AXPAXPBXI@Z1@Z ) -__TBB_SYMBOL( ?internal_push_back@concurrent_vector_base_v3@internal@tbb@@IAEPAXIAAI@Z ) -__TBB_SYMBOL( ?internal_reserve@concurrent_vector_base_v3@internal@tbb@@IAEXIII@Z ) -__TBB_SYMBOL( ?internal_compact@concurrent_vector_base_v3@internal@tbb@@IAEPAXIPAXP6AX0I@ZP6AX0PBXI@Z@Z ) -__TBB_SYMBOL( ?internal_swap@concurrent_vector_base_v3@internal@tbb@@IAEXAAV123@@Z ) -__TBB_SYMBOL( ?internal_throw_exception@concurrent_vector_base_v3@internal@tbb@@IBEXI@Z ) -__TBB_SYMBOL( ?internal_resize@concurrent_vector_base_v3@internal@tbb@@IAEXIIIPBXP6AXPAXI@ZP6AX10I@Z@Z ) -__TBB_SYMBOL( ?internal_grow_to_at_least_with_result@concurrent_vector_base_v3@internal@tbb@@IAEIIIP6AXPAXPBXI@Z1@Z ) - -// tbb_thread -__TBB_SYMBOL( ?join@tbb_thread_v3@internal@tbb@@QAEXXZ ) -__TBB_SYMBOL( ?detach@tbb_thread_v3@internal@tbb@@QAEXXZ ) -__TBB_SYMBOL( ?internal_start@tbb_thread_v3@internal@tbb@@AAEXP6GIPAX@Z0@Z ) -__TBB_SYMBOL( ?allocate_closure_v3@internal@tbb@@YAPAXI@Z ) -__TBB_SYMBOL( ?free_closure_v3@internal@tbb@@YAXPAX@Z ) -__TBB_SYMBOL( ?hardware_concurrency@tbb_thread_v3@internal@tbb@@SAIXZ ) -__TBB_SYMBOL( ?thread_yield_v3@internal@tbb@@YAXXZ ) -__TBB_SYMBOL( ?thread_sleep_v3@internal@tbb@@YAXABVinterval_t@tick_count@2@@Z ) -__TBB_SYMBOL( ?move_v3@internal@tbb@@YAXAAVtbb_thread_v3@12@0@Z ) -__TBB_SYMBOL( ?thread_get_id_v3@internal@tbb@@YA?AVid@tbb_thread_v3@12@XZ ) - -// condition_variable -__TBB_SYMBOL( ?internal_initialize_condition_variable@internal@interface5@tbb@@YAXAATcondvar_impl_t@123@@Z ) -__TBB_SYMBOL( ?internal_condition_variable_wait@internal@interface5@tbb@@YA_NAATcondvar_impl_t@123@PAVmutex@3@PBVinterval_t@tick_count@3@@Z ) -__TBB_SYMBOL( ?internal_condition_variable_notify_one@internal@interface5@tbb@@YAXAATcondvar_impl_t@123@@Z ) -__TBB_SYMBOL( ?internal_condition_variable_notify_all@internal@interface5@tbb@@YAXAATcondvar_impl_t@123@@Z ) -__TBB_SYMBOL( ?internal_destroy_condition_variable@internal@interface5@tbb@@YAXAATcondvar_impl_t@123@@Z ) - -// global parameter -__TBB_SYMBOL( ?active_value@global_control@interface9@tbb@@CAIH@Z ) -__TBB_SYMBOL( ?internal_create@global_control@interface9@tbb@@AAEXXZ ) -__TBB_SYMBOL( ?internal_destroy@global_control@interface9@tbb@@AAEXXZ ) - -#undef __TBB_SYMBOL diff --git a/src/tbb/src/tbb/win64-gcc-tbb-export.lst b/src/tbb/src/tbb/win64-gcc-tbb-export.lst deleted file mode 100644 index 46cb2985b..000000000 --- a/src/tbb/src/tbb/win64-gcc-tbb-export.lst +++ /dev/null @@ -1,384 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#include "tbb/tbb_config.h" - -/* cache_aligned_allocator.cpp */ -__TBB_SYMBOL( _ZN3tbb8internal12NFS_AllocateEyyPv ) // MODIFIED LINUX ENTRY -__TBB_SYMBOL( _ZN3tbb8internal15NFS_GetLineSizeEv ) -__TBB_SYMBOL( _ZN3tbb8internal8NFS_FreeEPv ) -__TBB_SYMBOL( _ZN3tbb8internal23allocate_via_handler_v3Ey ) // MODIFIED LINUX ENTRY -__TBB_SYMBOL( _ZN3tbb8internal25deallocate_via_handler_v3EPv ) -__TBB_SYMBOL( _ZN3tbb8internal17is_malloc_used_v3Ev ) - -/* task.cpp v3 */ -__TBB_SYMBOL( _ZN3tbb4task13note_affinityEt ) -__TBB_SYMBOL( _ZN3tbb4task22internal_set_ref_countEi ) -__TBB_SYMBOL( _ZN3tbb4task28internal_decrement_ref_countEv ) -__TBB_SYMBOL( _ZN3tbb4task22spawn_and_wait_for_allERNS_9task_listE ) -__TBB_SYMBOL( _ZN3tbb4task4selfEv ) -__TBB_SYMBOL( _ZN3tbb10interface58internal9task_base7destroyERNS_4taskE ) -__TBB_SYMBOL( _ZNK3tbb4task26is_owned_by_current_threadEv ) -__TBB_SYMBOL( _ZN3tbb8internal19allocate_root_proxy4freeERNS_4taskE ) -__TBB_SYMBOL( _ZN3tbb8internal19allocate_root_proxy8allocateEy ) // MODIFIED LINUX ENTRY -__TBB_SYMBOL( _ZN3tbb8internal28affinity_partitioner_base_v36resizeEj ) -__TBB_SYMBOL( _ZNK3tbb8internal20allocate_child_proxy4freeERNS_4taskE ) -__TBB_SYMBOL( _ZNK3tbb8internal20allocate_child_proxy8allocateEy ) // MODIFIED LINUX ENTRY -__TBB_SYMBOL( _ZNK3tbb8internal27allocate_continuation_proxy4freeERNS_4taskE ) -__TBB_SYMBOL( _ZNK3tbb8internal27allocate_continuation_proxy8allocateEy ) // MODIFIED LINUX ENTRY -__TBB_SYMBOL( _ZNK3tbb8internal34allocate_additional_child_of_proxy4freeERNS_4taskE ) -__TBB_SYMBOL( _ZNK3tbb8internal34allocate_additional_child_of_proxy8allocateEy ) // MODIFIED LINUX ENTRY -__TBB_SYMBOL( _ZTIN3tbb4taskE ) -__TBB_SYMBOL( _ZTSN3tbb4taskE ) -__TBB_SYMBOL( _ZTVN3tbb4taskE ) -__TBB_SYMBOL( _ZN3tbb19task_scheduler_init19default_num_threadsEv ) -__TBB_SYMBOL( _ZN3tbb19task_scheduler_init10initializeEiy ) // MODIFIED LINUX ENTRY -__TBB_SYMBOL( _ZN3tbb19task_scheduler_init10initializeEi ) -__TBB_SYMBOL( _ZN3tbb19task_scheduler_init9terminateEv ) -__TBB_SYMBOL( _ZN3tbb19task_scheduler_init27internal_blocking_terminateEb ) -#if __TBB_SCHEDULER_OBSERVER -__TBB_SYMBOL( _ZN3tbb8internal26task_scheduler_observer_v37observeEb ) -#endif /* __TBB_SCHEDULER_OBSERVER */ -__TBB_SYMBOL( _ZN3tbb10empty_task7executeEv ) -__TBB_SYMBOL( _ZN3tbb10empty_taskD0Ev ) -__TBB_SYMBOL( _ZN3tbb10empty_taskD1Ev ) -__TBB_SYMBOL( _ZTIN3tbb10empty_taskE ) -__TBB_SYMBOL( _ZTSN3tbb10empty_taskE ) -__TBB_SYMBOL( _ZTVN3tbb10empty_taskE ) - -/* arena.cpp */ -__TBB_SYMBOL( _ZN3tbb10interface78internal15task_arena_base19internal_initializeEv ) -__TBB_SYMBOL( _ZN3tbb10interface78internal15task_arena_base18internal_terminateEv ) -__TBB_SYMBOL( _ZN3tbb10interface78internal15task_arena_base15internal_attachEv ) -__TBB_SYMBOL( _ZNK3tbb10interface78internal15task_arena_base16internal_enqueueERNS_4taskEx ) -__TBB_SYMBOL( _ZNK3tbb10interface78internal15task_arena_base16internal_executeERNS1_13delegate_baseE ) -__TBB_SYMBOL( _ZNK3tbb10interface78internal15task_arena_base13internal_waitEv ) -__TBB_SYMBOL( _ZN3tbb10interface78internal15task_arena_base21internal_current_slotEv ) -__TBB_SYMBOL( _ZN3tbb10interface78internal15task_arena_base24internal_max_concurrencyEPKNS0_10task_arenaE ) -#if __TBB_TASK_ISOLATION -__TBB_SYMBOL( _ZN3tbb10interface78internal20isolate_within_arenaERNS1_13delegate_baseEx ) -#endif /* __TBB_TASK_ISOLATION */ - -#if !TBB_NO_LEGACY -/* task_v2.cpp */ -__TBB_SYMBOL( _ZN3tbb4task7destroyERS0_ ) -#endif /* !TBB_NO_LEGACY */ - -/* Exception handling in task scheduler */ -#if __TBB_TASK_GROUP_CONTEXT -__TBB_SYMBOL( _ZNK3tbb8internal32allocate_root_with_context_proxy8allocateEy ) // MODIFIED LINUX ENTRY -__TBB_SYMBOL( _ZNK3tbb8internal32allocate_root_with_context_proxy4freeERNS_4taskE ) -__TBB_SYMBOL( _ZN3tbb4task12change_groupERNS_18task_group_contextE ) -__TBB_SYMBOL( _ZNK3tbb18task_group_context28is_group_execution_cancelledEv ) -__TBB_SYMBOL( _ZN3tbb18task_group_context22cancel_group_executionEv ) -__TBB_SYMBOL( _ZN3tbb18task_group_context26register_pending_exceptionEv ) -__TBB_SYMBOL( _ZN3tbb18task_group_context5resetEv ) -__TBB_SYMBOL( _ZN3tbb18task_group_context19capture_fp_settingsEv ) -__TBB_SYMBOL( _ZN3tbb18task_group_context4initEv ) -__TBB_SYMBOL( _ZN3tbb18task_group_contextD1Ev ) -__TBB_SYMBOL( _ZN3tbb18task_group_contextD2Ev ) -#if __TBB_TASK_PRIORITY -__TBB_SYMBOL( _ZN3tbb18task_group_context12set_priorityENS_10priority_tE ) -__TBB_SYMBOL( _ZNK3tbb18task_group_context8priorityEv ) -#endif /* __TBB_TASK_PRIORITY */ -__TBB_SYMBOL( _ZNK3tbb18captured_exception4nameEv ) -__TBB_SYMBOL( _ZNK3tbb18captured_exception4whatEv ) -__TBB_SYMBOL( _ZN3tbb18captured_exception10throw_selfEv ) -__TBB_SYMBOL( _ZN3tbb18captured_exception3setEPKcS2_ ) -__TBB_SYMBOL( _ZN3tbb18captured_exception4moveEv ) -__TBB_SYMBOL( _ZN3tbb18captured_exception5clearEv ) -__TBB_SYMBOL( _ZN3tbb18captured_exception7destroyEv ) -__TBB_SYMBOL( _ZN3tbb18captured_exception8allocateEPKcS2_ ) -__TBB_SYMBOL( _ZN3tbb18captured_exceptionD0Ev ) -__TBB_SYMBOL( _ZN3tbb18captured_exceptionD1Ev ) -__TBB_SYMBOL( _ZN3tbb18captured_exceptionD2Ev ) -__TBB_SYMBOL( _ZTIN3tbb18captured_exceptionE ) -__TBB_SYMBOL( _ZTSN3tbb18captured_exceptionE ) -__TBB_SYMBOL( _ZTVN3tbb18captured_exceptionE ) -__TBB_SYMBOL( _ZN3tbb13tbb_exceptionD2Ev ) -__TBB_SYMBOL( _ZTIN3tbb13tbb_exceptionE ) -__TBB_SYMBOL( _ZTSN3tbb13tbb_exceptionE ) -__TBB_SYMBOL( _ZTVN3tbb13tbb_exceptionE ) -#endif /* __TBB_TASK_GROUP_CONTEXT */ - -/* Symbols for exceptions thrown from TBB */ -__TBB_SYMBOL( _ZN3tbb8internal33throw_bad_last_alloc_exception_v4Ev ) -__TBB_SYMBOL( _ZN3tbb8internal18throw_exception_v4ENS0_12exception_idE ) -__TBB_SYMBOL( _ZN3tbb14bad_last_allocD0Ev ) -__TBB_SYMBOL( _ZN3tbb14bad_last_allocD1Ev ) -__TBB_SYMBOL( _ZNK3tbb14bad_last_alloc4whatEv ) -__TBB_SYMBOL( _ZTIN3tbb14bad_last_allocE ) -__TBB_SYMBOL( _ZTSN3tbb14bad_last_allocE ) -__TBB_SYMBOL( _ZTVN3tbb14bad_last_allocE ) -__TBB_SYMBOL( _ZN3tbb12missing_waitD0Ev ) -__TBB_SYMBOL( _ZN3tbb12missing_waitD1Ev ) -__TBB_SYMBOL( _ZNK3tbb12missing_wait4whatEv ) -__TBB_SYMBOL( _ZTIN3tbb12missing_waitE ) -__TBB_SYMBOL( _ZTSN3tbb12missing_waitE ) -__TBB_SYMBOL( _ZTVN3tbb12missing_waitE ) -__TBB_SYMBOL( _ZN3tbb27invalid_multiple_schedulingD0Ev ) -__TBB_SYMBOL( _ZN3tbb27invalid_multiple_schedulingD1Ev ) -__TBB_SYMBOL( _ZNK3tbb27invalid_multiple_scheduling4whatEv ) -__TBB_SYMBOL( _ZTIN3tbb27invalid_multiple_schedulingE ) -__TBB_SYMBOL( _ZTSN3tbb27invalid_multiple_schedulingE ) -__TBB_SYMBOL( _ZTVN3tbb27invalid_multiple_schedulingE ) -__TBB_SYMBOL( _ZN3tbb13improper_lockD0Ev ) -__TBB_SYMBOL( _ZN3tbb13improper_lockD1Ev ) -__TBB_SYMBOL( _ZNK3tbb13improper_lock4whatEv ) -__TBB_SYMBOL( _ZTIN3tbb13improper_lockE ) -__TBB_SYMBOL( _ZTSN3tbb13improper_lockE ) -__TBB_SYMBOL( _ZTVN3tbb13improper_lockE ) -__TBB_SYMBOL( _ZN3tbb10user_abortD0Ev ) -__TBB_SYMBOL( _ZN3tbb10user_abortD1Ev ) -__TBB_SYMBOL( _ZNK3tbb10user_abort4whatEv ) -__TBB_SYMBOL( _ZTIN3tbb10user_abortE ) -__TBB_SYMBOL( _ZTSN3tbb10user_abortE ) -__TBB_SYMBOL( _ZTVN3tbb10user_abortE ) - -/* tbb_misc.cpp */ -__TBB_SYMBOL( _ZN3tbb17assertion_failureEPKciS1_S1_ ) -__TBB_SYMBOL( _ZN3tbb21set_assertion_handlerEPFvPKciS1_S1_E ) -__TBB_SYMBOL( _ZN3tbb8internal36get_initial_auto_partitioner_divisorEv ) -__TBB_SYMBOL( _ZN3tbb8internal13handle_perrorEiPKc ) -__TBB_SYMBOL( _ZN3tbb8internal15runtime_warningEPKcz ) -__TBB_SYMBOL( TBB_runtime_interface_version ) - -/* tbb_main.cpp */ -__TBB_SYMBOL( _ZN3tbb8internal32itt_load_pointer_with_acquire_v3EPKv ) -__TBB_SYMBOL( _ZN3tbb8internal33itt_store_pointer_with_release_v3EPvS1_ ) -__TBB_SYMBOL( _ZN3tbb8internal18call_itt_notify_v5EiPv ) -__TBB_SYMBOL( _ZN3tbb8internal20itt_set_sync_name_v3EPvPKc ) -__TBB_SYMBOL( _ZN3tbb8internal19itt_load_pointer_v3EPKv ) - -/* pipeline.cpp */ -__TBB_SYMBOL( _ZTIN3tbb6filterE ) -__TBB_SYMBOL( _ZTSN3tbb6filterE ) -__TBB_SYMBOL( _ZTVN3tbb6filterE ) -__TBB_SYMBOL( _ZN3tbb6filterD2Ev ) -__TBB_SYMBOL( _ZN3tbb8pipeline10add_filterERNS_6filterE ) -__TBB_SYMBOL( _ZN3tbb8pipeline12inject_tokenERNS_4taskE ) -__TBB_SYMBOL( _ZN3tbb8pipeline13remove_filterERNS_6filterE ) -__TBB_SYMBOL( _ZN3tbb8pipeline3runEy ) // MODIFIED LINUX ENTRY -#if __TBB_TASK_GROUP_CONTEXT -__TBB_SYMBOL( _ZN3tbb8pipeline3runEyRNS_18task_group_contextE ) // MODIFIED LINUX ENTRY -#endif -__TBB_SYMBOL( _ZN3tbb8pipeline5clearEv ) -__TBB_SYMBOL( _ZN3tbb19thread_bound_filter12process_itemEv ) -__TBB_SYMBOL( _ZN3tbb19thread_bound_filter16try_process_itemEv ) -__TBB_SYMBOL( _ZTIN3tbb8pipelineE ) -__TBB_SYMBOL( _ZTSN3tbb8pipelineE ) -__TBB_SYMBOL( _ZTVN3tbb8pipelineE ) -__TBB_SYMBOL( _ZN3tbb8pipelineC1Ev ) -__TBB_SYMBOL( _ZN3tbb8pipelineC2Ev ) -__TBB_SYMBOL( _ZN3tbb8pipelineD0Ev ) -__TBB_SYMBOL( _ZN3tbb8pipelineD1Ev ) -__TBB_SYMBOL( _ZN3tbb8pipelineD2Ev ) -__TBB_SYMBOL( _ZN3tbb6filter16set_end_of_inputEv ) - -/* queuing_rw_mutex.cpp */ -__TBB_SYMBOL( _ZN3tbb16queuing_rw_mutex18internal_constructEv ) -__TBB_SYMBOL( _ZN3tbb16queuing_rw_mutex11scoped_lock17upgrade_to_writerEv ) -__TBB_SYMBOL( _ZN3tbb16queuing_rw_mutex11scoped_lock19downgrade_to_readerEv ) -__TBB_SYMBOL( _ZN3tbb16queuing_rw_mutex11scoped_lock7acquireERS0_b ) -__TBB_SYMBOL( _ZN3tbb16queuing_rw_mutex11scoped_lock7releaseEv ) -__TBB_SYMBOL( _ZN3tbb16queuing_rw_mutex11scoped_lock11try_acquireERS0_b ) - -/* reader_writer_lock.cpp */ -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock11scoped_lock16internal_destroyEv ) -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock11scoped_lock18internal_constructERS1_ ) -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock13try_lock_readEv ) -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock16scoped_lock_read16internal_destroyEv ) -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock16scoped_lock_read18internal_constructERS1_ ) -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock16internal_destroyEv ) -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock18internal_constructEv ) -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock4lockEv ) -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock6unlockEv ) -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock8try_lockEv ) -__TBB_SYMBOL( _ZN3tbb10interface518reader_writer_lock9lock_readEv ) - -#if !TBB_NO_LEGACY -/* spin_rw_mutex.cpp v2 */ -__TBB_SYMBOL( _ZN3tbb13spin_rw_mutex16internal_upgradeEPS0_ ) -__TBB_SYMBOL( _ZN3tbb13spin_rw_mutex22internal_itt_releasingEPS0_ ) -__TBB_SYMBOL( _ZN3tbb13spin_rw_mutex23internal_acquire_readerEPS0_ ) -__TBB_SYMBOL( _ZN3tbb13spin_rw_mutex23internal_acquire_writerEPS0_ ) -__TBB_SYMBOL( _ZN3tbb13spin_rw_mutex18internal_downgradeEPS0_ ) -__TBB_SYMBOL( _ZN3tbb13spin_rw_mutex23internal_release_readerEPS0_ ) -__TBB_SYMBOL( _ZN3tbb13spin_rw_mutex23internal_release_writerEPS0_ ) -__TBB_SYMBOL( _ZN3tbb13spin_rw_mutex27internal_try_acquire_readerEPS0_ ) -__TBB_SYMBOL( _ZN3tbb13spin_rw_mutex27internal_try_acquire_writerEPS0_ ) -#endif - -// x86_rtm_rw_mutex.cpp -__TBB_SYMBOL( _ZN3tbb10interface88internal16x86_rtm_rw_mutex18internal_constructEv ) -__TBB_SYMBOL( _ZN3tbb10interface88internal16x86_rtm_rw_mutex23internal_acquire_writerERNS2_11scoped_lockEb ) -__TBB_SYMBOL( _ZN3tbb10interface88internal16x86_rtm_rw_mutex27internal_try_acquire_writerERNS2_11scoped_lockE ) -__TBB_SYMBOL( _ZN3tbb10interface88internal16x86_rtm_rw_mutex23internal_acquire_readerERNS2_11scoped_lockEb ) -__TBB_SYMBOL( _ZN3tbb10interface88internal16x86_rtm_rw_mutex16internal_releaseERNS2_11scoped_lockE ) -__TBB_SYMBOL( _ZN3tbb10interface88internal16x86_rtm_rw_mutex16internal_upgradeERNS2_11scoped_lockE ) -__TBB_SYMBOL( _ZN3tbb10interface88internal16x86_rtm_rw_mutex18internal_downgradeERNS2_11scoped_lockE ) - -/* spin_rw_mutex v3 */ -__TBB_SYMBOL( _ZN3tbb16spin_rw_mutex_v318internal_constructEv ) -__TBB_SYMBOL( _ZN3tbb16spin_rw_mutex_v316internal_upgradeEv ) -__TBB_SYMBOL( _ZN3tbb16spin_rw_mutex_v318internal_downgradeEv ) -__TBB_SYMBOL( _ZN3tbb16spin_rw_mutex_v323internal_acquire_readerEv ) -__TBB_SYMBOL( _ZN3tbb16spin_rw_mutex_v323internal_acquire_writerEv ) -__TBB_SYMBOL( _ZN3tbb16spin_rw_mutex_v323internal_release_readerEv ) -__TBB_SYMBOL( _ZN3tbb16spin_rw_mutex_v323internal_release_writerEv ) -__TBB_SYMBOL( _ZN3tbb16spin_rw_mutex_v327internal_try_acquire_readerEv ) -__TBB_SYMBOL( _ZN3tbb16spin_rw_mutex_v327internal_try_acquire_writerEv ) - -/* spin_mutex.cpp */ -__TBB_SYMBOL( _ZN3tbb10spin_mutex11scoped_lock16internal_acquireERS0_ ) -__TBB_SYMBOL( _ZN3tbb10spin_mutex11scoped_lock16internal_releaseEv ) -__TBB_SYMBOL( _ZN3tbb10spin_mutex11scoped_lock20internal_try_acquireERS0_ ) -__TBB_SYMBOL( _ZN3tbb10spin_mutex18internal_constructEv ) - -/* mutex.cpp */ -__TBB_SYMBOL( _ZN3tbb5mutex11scoped_lock16internal_acquireERS0_ ) -__TBB_SYMBOL( _ZN3tbb5mutex11scoped_lock16internal_releaseEv ) -__TBB_SYMBOL( _ZN3tbb5mutex11scoped_lock20internal_try_acquireERS0_ ) -__TBB_SYMBOL( _ZN3tbb5mutex16internal_destroyEv ) -__TBB_SYMBOL( _ZN3tbb5mutex18internal_constructEv ) - -/* recursive_mutex.cpp */ -__TBB_SYMBOL( _ZN3tbb15recursive_mutex11scoped_lock16internal_acquireERS0_ ) -__TBB_SYMBOL( _ZN3tbb15recursive_mutex11scoped_lock16internal_releaseEv ) -__TBB_SYMBOL( _ZN3tbb15recursive_mutex11scoped_lock20internal_try_acquireERS0_ ) -__TBB_SYMBOL( _ZN3tbb15recursive_mutex16internal_destroyEv ) -__TBB_SYMBOL( _ZN3tbb15recursive_mutex18internal_constructEv ) - -/* QueuingMutex.cpp */ -__TBB_SYMBOL( _ZN3tbb13queuing_mutex18internal_constructEv ) -__TBB_SYMBOL( _ZN3tbb13queuing_mutex11scoped_lock7acquireERS0_ ) -__TBB_SYMBOL( _ZN3tbb13queuing_mutex11scoped_lock7releaseEv ) -__TBB_SYMBOL( _ZN3tbb13queuing_mutex11scoped_lock11try_acquireERS0_ ) - -/* critical_section.cpp */ -__TBB_SYMBOL( _ZN3tbb8internal19critical_section_v418internal_constructEv ) - -#if !TBB_NO_LEGACY -/* concurrent_hash_map */ -__TBB_SYMBOL( _ZNK3tbb8internal21hash_map_segment_base23internal_grow_predicateEv ) - -/* concurrent_queue.cpp v2 */ -__TBB_SYMBOL( _ZN3tbb8internal21concurrent_queue_base12internal_popEPv ) -__TBB_SYMBOL( _ZN3tbb8internal21concurrent_queue_base13internal_pushEPKv ) -__TBB_SYMBOL( _ZN3tbb8internal21concurrent_queue_base21internal_set_capacityExy ) // MODIFIED LINUX ENTRY -__TBB_SYMBOL( _ZN3tbb8internal21concurrent_queue_base23internal_pop_if_presentEPv ) -__TBB_SYMBOL( _ZN3tbb8internal21concurrent_queue_base25internal_push_if_not_fullEPKv ) -__TBB_SYMBOL( _ZN3tbb8internal21concurrent_queue_baseC2Ey ) // MODIFIED LINUX ENTRY -__TBB_SYMBOL( _ZN3tbb8internal21concurrent_queue_baseD2Ev ) -__TBB_SYMBOL( _ZTIN3tbb8internal21concurrent_queue_baseE ) -__TBB_SYMBOL( _ZTSN3tbb8internal21concurrent_queue_baseE ) -__TBB_SYMBOL( _ZTVN3tbb8internal21concurrent_queue_baseE ) -__TBB_SYMBOL( _ZN3tbb8internal30concurrent_queue_iterator_base6assignERKS1_ ) -__TBB_SYMBOL( _ZN3tbb8internal30concurrent_queue_iterator_base7advanceEv ) -__TBB_SYMBOL( _ZN3tbb8internal30concurrent_queue_iterator_baseC2ERKNS0_21concurrent_queue_baseE ) -__TBB_SYMBOL( _ZN3tbb8internal30concurrent_queue_iterator_baseD2Ev ) -__TBB_SYMBOL( _ZNK3tbb8internal21concurrent_queue_base13internal_sizeEv ) -#endif - -/* concurrent_queue v3 */ -/* constructors */ -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v3C2Ey ) // MODIFIED LINUX ENTRY -__TBB_SYMBOL( _ZN3tbb8internal33concurrent_queue_iterator_base_v3C2ERKNS0_24concurrent_queue_base_v3E ) -__TBB_SYMBOL( _ZN3tbb8internal33concurrent_queue_iterator_base_v3C2ERKNS0_24concurrent_queue_base_v3Ey ) // MODIFIED LINUX ENTRY -/* destructors */ -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v3D2Ev ) -__TBB_SYMBOL( _ZN3tbb8internal33concurrent_queue_iterator_base_v3D2Ev ) -/* typeinfo */ -__TBB_SYMBOL( _ZTIN3tbb8internal24concurrent_queue_base_v3E ) -__TBB_SYMBOL( _ZTSN3tbb8internal24concurrent_queue_base_v3E ) -/* vtable */ -__TBB_SYMBOL( _ZTVN3tbb8internal24concurrent_queue_base_v3E ) -/* methods */ -__TBB_SYMBOL( _ZN3tbb8internal33concurrent_queue_iterator_base_v36assignERKS1_ ) -__TBB_SYMBOL( _ZN3tbb8internal33concurrent_queue_iterator_base_v37advanceEv ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v313internal_pushEPKv ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v818internal_push_moveEPKv ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v325internal_push_if_not_fullEPKv ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v830internal_push_move_if_not_fullEPKv ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v312internal_popEPv ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v323internal_pop_if_presentEPv ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v314internal_abortEv ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v321internal_finish_clearEv ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v321internal_set_capacityExy ) // MODIFIED LINUX ENTRY -__TBB_SYMBOL( _ZNK3tbb8internal24concurrent_queue_base_v313internal_sizeEv ) -__TBB_SYMBOL( _ZNK3tbb8internal24concurrent_queue_base_v314internal_emptyEv ) -__TBB_SYMBOL( _ZNK3tbb8internal24concurrent_queue_base_v324internal_throw_exceptionEv ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v36assignERKS1_ ) -__TBB_SYMBOL( _ZN3tbb8internal24concurrent_queue_base_v812move_contentERS1_ ) - - -#if !TBB_NO_LEGACY -/* concurrent_vector.cpp v2 */ -__TBB_SYMBOL( _ZN3tbb8internal22concurrent_vector_base13internal_copyERKS1_yPFvPvPKvyE ) // MODIFIED LINUX ENTRY -__TBB_SYMBOL( _ZN3tbb8internal22concurrent_vector_base14internal_clearEPFvPvyEb ) // MODIFIED LINUX ENTRY -__TBB_SYMBOL( _ZN3tbb8internal22concurrent_vector_base15internal_assignERKS1_yPFvPvyEPFvS4_PKvyESA_ ) // MODIFIED LINUX ENTRY -__TBB_SYMBOL( _ZN3tbb8internal22concurrent_vector_base16internal_grow_byEyyPFvPvyE ) // MODIFIED LINUX ENTRY -__TBB_SYMBOL( _ZN3tbb8internal22concurrent_vector_base16internal_reserveEyyy ) // MODIFIED LINUX ENTRY -__TBB_SYMBOL( _ZN3tbb8internal22concurrent_vector_base18internal_push_backEyRy ) // MODIFIED LINUX ENTRY -__TBB_SYMBOL( _ZN3tbb8internal22concurrent_vector_base25internal_grow_to_at_leastEyyPFvPvyE ) // MODIFIED LINUX ENTRY -__TBB_SYMBOL( _ZNK3tbb8internal22concurrent_vector_base17internal_capacityEv ) -#endif - -/* concurrent_vector v3 */ -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v313internal_copyERKS1_yPFvPvPKvyE ) // MODIFIED LINUX ENTRY -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v314internal_clearEPFvPvyE ) // MODIFIED LINUX ENTRY -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v315internal_assignERKS1_yPFvPvyEPFvS4_PKvyESA_ ) // MODIFIED LINUX ENTRY -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v316internal_grow_byEyyPFvPvPKvyES4_ ) // MODIFIED LINUX ENTRY -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v316internal_reserveEyyy ) // MODIFIED LINUX ENTRY -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v318internal_push_backEyRy ) // MODIFIED LINUX ENTRY -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v325internal_grow_to_at_leastEyyPFvPvPKvyES4_ ) // MODIFIED LINUX ENTRY -__TBB_SYMBOL( _ZNK3tbb8internal25concurrent_vector_base_v317internal_capacityEv ) -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v316internal_compactEyPvPFvS2_yEPFvS2_PKvyE ) // MODIFIED LINUX ENTRY -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v313internal_swapERS1_ ) -__TBB_SYMBOL( _ZNK3tbb8internal25concurrent_vector_base_v324internal_throw_exceptionEy ) // MODIFIED LINUX ENTRY -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v3D2Ev ) -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v315internal_resizeEyyyPKvPFvPvyEPFvS4_S3_yE ) // MODIFIED LINUX ENTRY -__TBB_SYMBOL( _ZN3tbb8internal25concurrent_vector_base_v337internal_grow_to_at_least_with_resultEyyPFvPvPKvyES4_ ) // MODIFIED LINUX ENTRY - -/* tbb_thread */ -__TBB_SYMBOL( _ZN3tbb8internal13tbb_thread_v320hardware_concurrencyEv ) -__TBB_SYMBOL( _ZN3tbb8internal13tbb_thread_v36detachEv ) -__TBB_SYMBOL( _ZN3tbb8internal16thread_get_id_v3Ev ) -__TBB_SYMBOL( _ZN3tbb8internal15free_closure_v3EPv ) -__TBB_SYMBOL( _ZN3tbb8internal13tbb_thread_v34joinEv ) -__TBB_SYMBOL( _ZN3tbb8internal13tbb_thread_v314internal_startEPFjPvES2_ ) // MODIFIED LINUX ENTRY -__TBB_SYMBOL( _ZN3tbb8internal19allocate_closure_v3Ey ) // MODIFIED LINUX ENTRY -__TBB_SYMBOL( _ZN3tbb8internal7move_v3ERNS0_13tbb_thread_v3ES2_ ) -__TBB_SYMBOL( _ZN3tbb8internal15thread_yield_v3Ev ) -__TBB_SYMBOL( _ZN3tbb8internal15thread_sleep_v3ERKNS_10tick_count10interval_tE ) - -/* condition_variable */ -__TBB_SYMBOL( _ZN3tbb10interface58internal32internal_condition_variable_waitERNS1_14condvar_impl_tEPNS_5mutexEPKNS_10tick_count10interval_tE ) -__TBB_SYMBOL( _ZN3tbb10interface58internal35internal_destroy_condition_variableERNS1_14condvar_impl_tE ) -__TBB_SYMBOL( _ZN3tbb10interface58internal38internal_condition_variable_notify_allERNS1_14condvar_impl_tE ) -__TBB_SYMBOL( _ZN3tbb10interface58internal38internal_condition_variable_notify_oneERNS1_14condvar_impl_tE ) -__TBB_SYMBOL( _ZN3tbb10interface58internal38internal_initialize_condition_variableERNS1_14condvar_impl_tE ) - -// global parameter -__TBB_SYMBOL( _ZN3tbb10interface914global_control12active_valueEi ) -__TBB_SYMBOL( _ZN3tbb10interface914global_control15internal_createEv ) -__TBB_SYMBOL( _ZN3tbb10interface914global_control16internal_destroyEv ) - -#undef __TBB_SYMBOL diff --git a/src/tbb/src/tbb/win64-tbb-export.lst b/src/tbb/src/tbb/win64-tbb-export.lst deleted file mode 100644 index 3bca01a9f..000000000 --- a/src/tbb/src/tbb/win64-tbb-export.lst +++ /dev/null @@ -1,339 +0,0 @@ -; Copyright (c) 2005-2017 Intel Corporation -; -; Licensed under the Apache License, Version 2.0 (the "License"); -; you may not use this file except in compliance with the License. -; You may obtain a copy of the License at -; -; http://www.apache.org/licenses/LICENSE-2.0 -; -; Unless required by applicable law or agreed to in writing, software -; distributed under the License is distributed on an "AS IS" BASIS, -; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -; See the License for the specific language governing permissions and -; limitations under the License. -; -; -; -; - -// This file is organized with a section for each .cpp file. -// Each of these sections is in alphabetical order. - -#include "tbb/tbb_config.h" - -// Assembly-language support that is called directly by clients -__TBB_SYMBOL( __TBB_machine_cmpswp1 ) -__TBB_SYMBOL( __TBB_machine_fetchadd1 ) -__TBB_SYMBOL( __TBB_machine_fetchstore1 ) -__TBB_SYMBOL( __TBB_machine_cmpswp2 ) -__TBB_SYMBOL( __TBB_machine_fetchadd2 ) -__TBB_SYMBOL( __TBB_machine_fetchstore2 ) -__TBB_SYMBOL( __TBB_machine_pause ) -__TBB_SYMBOL( __TBB_machine_try_lock_elided ) -__TBB_SYMBOL( __TBB_machine_unlock_elided ) -__TBB_SYMBOL( __TBB_machine_is_in_transaction ) - -// cache_aligned_allocator.cpp -__TBB_SYMBOL( ?NFS_Allocate@internal@tbb@@YAPEAX_K0PEAX@Z ) -__TBB_SYMBOL( ?NFS_GetLineSize@internal@tbb@@YA_KXZ ) -__TBB_SYMBOL( ?NFS_Free@internal@tbb@@YAXPEAX@Z ) -__TBB_SYMBOL( ?allocate_via_handler_v3@internal@tbb@@YAPEAX_K@Z ) -__TBB_SYMBOL( ?deallocate_via_handler_v3@internal@tbb@@YAXPEAX@Z ) -__TBB_SYMBOL( ?is_malloc_used_v3@internal@tbb@@YA_NXZ ) - - -// task.cpp v3 -__TBB_SYMBOL( ?resize@affinity_partitioner_base_v3@internal@tbb@@AEAAXI@Z ) -__TBB_SYMBOL( ?allocate@allocate_additional_child_of_proxy@internal@tbb@@QEBAAEAVtask@3@_K@Z ) -__TBB_SYMBOL( ?allocate@allocate_child_proxy@internal@tbb@@QEBAAEAVtask@3@_K@Z ) -__TBB_SYMBOL( ?allocate@allocate_continuation_proxy@internal@tbb@@QEBAAEAVtask@3@_K@Z ) -__TBB_SYMBOL( ?allocate@allocate_root_proxy@internal@tbb@@SAAEAVtask@3@_K@Z ) -__TBB_SYMBOL( ?destroy@task_base@internal@interface5@tbb@@SAXAEAVtask@4@@Z ) -__TBB_SYMBOL( ?free@allocate_additional_child_of_proxy@internal@tbb@@QEBAXAEAVtask@3@@Z ) -__TBB_SYMBOL( ?free@allocate_child_proxy@internal@tbb@@QEBAXAEAVtask@3@@Z ) -__TBB_SYMBOL( ?free@allocate_continuation_proxy@internal@tbb@@QEBAXAEAVtask@3@@Z ) -__TBB_SYMBOL( ?free@allocate_root_proxy@internal@tbb@@SAXAEAVtask@3@@Z ) -__TBB_SYMBOL( ?internal_set_ref_count@task@tbb@@AEAAXH@Z ) -__TBB_SYMBOL( ?internal_decrement_ref_count@task@tbb@@AEAA_JXZ ) -__TBB_SYMBOL( ?is_owned_by_current_thread@task@tbb@@QEBA_NXZ ) -__TBB_SYMBOL( ?note_affinity@task@tbb@@UEAAXG@Z ) -__TBB_SYMBOL( ?self@task@tbb@@SAAEAV12@XZ ) -__TBB_SYMBOL( ?spawn_and_wait_for_all@task@tbb@@QEAAXAEAVtask_list@2@@Z ) -__TBB_SYMBOL( ?default_num_threads@task_scheduler_init@tbb@@SAHXZ ) -__TBB_SYMBOL( ?initialize@task_scheduler_init@tbb@@QEAAXH_K@Z ) -__TBB_SYMBOL( ?initialize@task_scheduler_init@tbb@@QEAAXH@Z ) -__TBB_SYMBOL( ?terminate@task_scheduler_init@tbb@@QEAAXXZ ) -__TBB_SYMBOL( ?internal_blocking_terminate@task_scheduler_init@tbb@@AEAA_N_N@Z ) -#if __TBB_SCHEDULER_OBSERVER -__TBB_SYMBOL( ?observe@task_scheduler_observer_v3@internal@tbb@@QEAAX_N@Z ) -#endif /* __TBB_SCHEDULER_OBSERVER */ - -/* arena.cpp */ -__TBB_SYMBOL( ?internal_max_concurrency@task_arena_base@internal@interface7@tbb@@KAHPEBVtask_arena@34@@Z ) -__TBB_SYMBOL( ?internal_current_slot@task_arena_base@internal@interface7@tbb@@KAHXZ ) -__TBB_SYMBOL( ?internal_initialize@task_arena_base@internal@interface7@tbb@@IEAAXXZ ) -__TBB_SYMBOL( ?internal_terminate@task_arena_base@internal@interface7@tbb@@IEAAXXZ ) -__TBB_SYMBOL( ?internal_attach@task_arena_base@internal@interface7@tbb@@IEAAXXZ ) -__TBB_SYMBOL( ?internal_enqueue@task_arena_base@internal@interface7@tbb@@IEBAXAEAVtask@4@_J@Z ) -__TBB_SYMBOL( ?internal_execute@task_arena_base@internal@interface7@tbb@@IEBAXAEAVdelegate_base@234@@Z ) -__TBB_SYMBOL( ?internal_wait@task_arena_base@internal@interface7@tbb@@IEBAXXZ ) -#if __TBB_TASK_ISOLATION -__TBB_SYMBOL( ?isolate_within_arena@internal@interface7@tbb@@YAXAEAVdelegate_base@123@_J@Z ) -#endif /* __TBB_TASK_ISOLATION */ -#if !TBB_NO_LEGACY -// task_v2.cpp -__TBB_SYMBOL( ?destroy@task@tbb@@QEAAXAEAV12@@Z ) -#endif - -// Exception handling in task scheduler -#if __TBB_TASK_GROUP_CONTEXT -__TBB_SYMBOL( ?allocate@allocate_root_with_context_proxy@internal@tbb@@QEBAAEAVtask@3@_K@Z ) -__TBB_SYMBOL( ?free@allocate_root_with_context_proxy@internal@tbb@@QEBAXAEAVtask@3@@Z ) -__TBB_SYMBOL( ?change_group@task@tbb@@QEAAXAEAVtask_group_context@2@@Z ) -__TBB_SYMBOL( ?is_group_execution_cancelled@task_group_context@tbb@@QEBA_NXZ ) -__TBB_SYMBOL( ?cancel_group_execution@task_group_context@tbb@@QEAA_NXZ ) -__TBB_SYMBOL( ?reset@task_group_context@tbb@@QEAAXXZ ) -__TBB_SYMBOL( ?capture_fp_settings@task_group_context@tbb@@QEAAXXZ ) -__TBB_SYMBOL( ?init@task_group_context@tbb@@IEAAXXZ ) -__TBB_SYMBOL( ?register_pending_exception@task_group_context@tbb@@QEAAXXZ ) -__TBB_SYMBOL( ??1task_group_context@tbb@@QEAA@XZ ) -#if __TBB_TASK_PRIORITY -__TBB_SYMBOL( ?set_priority@task_group_context@tbb@@QEAAXW4priority_t@2@@Z ) -__TBB_SYMBOL( ?priority@task_group_context@tbb@@QEBA?AW4priority_t@2@XZ ) -#endif /* __TBB_TASK_PRIORITY */ -__TBB_SYMBOL( ?name@captured_exception@tbb@@UEBAPEBDXZ ) -__TBB_SYMBOL( ?what@captured_exception@tbb@@UEBAPEBDXZ ) -__TBB_SYMBOL( ??1captured_exception@tbb@@UEAA@XZ ) -__TBB_SYMBOL( ?move@captured_exception@tbb@@UEAAPEAV12@XZ ) -__TBB_SYMBOL( ?destroy@captured_exception@tbb@@UEAAXXZ ) -__TBB_SYMBOL( ?set@captured_exception@tbb@@QEAAXPEBD0@Z ) -__TBB_SYMBOL( ?clear@captured_exception@tbb@@QEAAXXZ ) -#endif /* __TBB_TASK_GROUP_CONTEXT */ - -// Symbols for exceptions thrown from TBB -__TBB_SYMBOL( ?throw_bad_last_alloc_exception_v4@internal@tbb@@YAXXZ ) -__TBB_SYMBOL( ?throw_exception_v4@internal@tbb@@YAXW4exception_id@12@@Z ) -__TBB_SYMBOL( ?what@bad_last_alloc@tbb@@UEBAPEBDXZ ) -__TBB_SYMBOL( ?what@missing_wait@tbb@@UEBAPEBDXZ ) -__TBB_SYMBOL( ?what@invalid_multiple_scheduling@tbb@@UEBAPEBDXZ ) -__TBB_SYMBOL( ?what@improper_lock@tbb@@UEBAPEBDXZ ) -__TBB_SYMBOL( ?what@user_abort@tbb@@UEBAPEBDXZ ) - -// tbb_misc.cpp -__TBB_SYMBOL( ?assertion_failure@tbb@@YAXPEBDH00@Z ) -__TBB_SYMBOL( ?get_initial_auto_partitioner_divisor@internal@tbb@@YA_KXZ ) -__TBB_SYMBOL( ?handle_perror@internal@tbb@@YAXHPEBD@Z ) -__TBB_SYMBOL( ?set_assertion_handler@tbb@@YAP6AXPEBDH00@ZP6AX0H00@Z@Z ) -__TBB_SYMBOL( ?runtime_warning@internal@tbb@@YAXPEBDZZ ) -__TBB_SYMBOL( TBB_runtime_interface_version ) - -// tbb_main.cpp -__TBB_SYMBOL( ?itt_load_pointer_with_acquire_v3@internal@tbb@@YAPEAXPEBX@Z ) -__TBB_SYMBOL( ?itt_store_pointer_with_release_v3@internal@tbb@@YAXPEAX0@Z ) -__TBB_SYMBOL( ?call_itt_notify_v5@internal@tbb@@YAXHPEAX@Z ) -__TBB_SYMBOL( ?itt_load_pointer_v3@internal@tbb@@YAPEAXPEBX@Z ) -__TBB_SYMBOL( ?itt_set_sync_name_v3@internal@tbb@@YAXPEAXPEB_W@Z ) -#if __TBB_ITT_STRUCTURE_API -__TBB_SYMBOL( ?itt_make_task_group_v7@internal@tbb@@YAXW4itt_domain_enum@12@PEAX_K12W4string_index@12@@Z ) -__TBB_SYMBOL( ?itt_metadata_str_add_v7@internal@tbb@@YAXW4itt_domain_enum@12@PEAX_KW4string_index@12@PEBD@Z ) -__TBB_SYMBOL( ?itt_relation_add_v7@internal@tbb@@YAXW4itt_domain_enum@12@PEAX_KW4itt_relation@12@12@Z ) -__TBB_SYMBOL( ?itt_task_begin_v7@internal@tbb@@YAXW4itt_domain_enum@12@PEAX_K12W4string_index@12@@Z ) -__TBB_SYMBOL( ?itt_task_end_v7@internal@tbb@@YAXW4itt_domain_enum@12@@Z ) -__TBB_SYMBOL( ?itt_region_begin_v9@internal@tbb@@YAXW4itt_domain_enum@12@PEAX_K12W4string_index@12@@Z ) -__TBB_SYMBOL( ?itt_region_end_v9@internal@tbb@@YAXW4itt_domain_enum@12@PEAX_K@Z ) -#endif - -// pipeline.cpp -__TBB_SYMBOL( ??_7pipeline@tbb@@6B@ ) -__TBB_SYMBOL( ??0pipeline@tbb@@QEAA@XZ ) -__TBB_SYMBOL( ??1filter@tbb@@UEAA@XZ ) -__TBB_SYMBOL( ??1pipeline@tbb@@UEAA@XZ ) -__TBB_SYMBOL( ?add_filter@pipeline@tbb@@QEAAXAEAVfilter@2@@Z ) -__TBB_SYMBOL( ?clear@pipeline@tbb@@QEAAXXZ ) -__TBB_SYMBOL( ?inject_token@pipeline@tbb@@AEAAXAEAVtask@2@@Z ) -__TBB_SYMBOL( ?run@pipeline@tbb@@QEAAX_K@Z ) -#if __TBB_TASK_GROUP_CONTEXT -__TBB_SYMBOL( ?run@pipeline@tbb@@QEAAX_KAEAVtask_group_context@2@@Z ) -#endif -__TBB_SYMBOL( ?process_item@thread_bound_filter@tbb@@QEAA?AW4result_type@12@XZ ) -__TBB_SYMBOL( ?try_process_item@thread_bound_filter@tbb@@QEAA?AW4result_type@12@XZ ) -__TBB_SYMBOL( ?set_end_of_input@filter@tbb@@IEAAXXZ ) - -// queuing_rw_mutex.cpp -__TBB_SYMBOL( ?internal_construct@queuing_rw_mutex@tbb@@QEAAXXZ ) -__TBB_SYMBOL( ?acquire@scoped_lock@queuing_rw_mutex@tbb@@QEAAXAEAV23@_N@Z ) -__TBB_SYMBOL( ?downgrade_to_reader@scoped_lock@queuing_rw_mutex@tbb@@QEAA_NXZ ) -__TBB_SYMBOL( ?release@scoped_lock@queuing_rw_mutex@tbb@@QEAAXXZ ) -__TBB_SYMBOL( ?upgrade_to_writer@scoped_lock@queuing_rw_mutex@tbb@@QEAA_NXZ ) -__TBB_SYMBOL( ?try_acquire@scoped_lock@queuing_rw_mutex@tbb@@QEAA_NAEAV23@_N@Z ) - -// reader_writer_lock.cpp -__TBB_SYMBOL( ?try_lock_read@reader_writer_lock@interface5@tbb@@QEAA_NXZ ) -__TBB_SYMBOL( ?try_lock@reader_writer_lock@interface5@tbb@@QEAA_NXZ ) -__TBB_SYMBOL( ?unlock@reader_writer_lock@interface5@tbb@@QEAAXXZ ) -__TBB_SYMBOL( ?lock_read@reader_writer_lock@interface5@tbb@@QEAAXXZ ) -__TBB_SYMBOL( ?lock@reader_writer_lock@interface5@tbb@@QEAAXXZ ) -__TBB_SYMBOL( ?internal_construct@reader_writer_lock@interface5@tbb@@AEAAXXZ ) -__TBB_SYMBOL( ?internal_destroy@reader_writer_lock@interface5@tbb@@AEAAXXZ ) -__TBB_SYMBOL( ?internal_construct@scoped_lock@reader_writer_lock@interface5@tbb@@AEAAXAEAV234@@Z ) -__TBB_SYMBOL( ?internal_destroy@scoped_lock@reader_writer_lock@interface5@tbb@@AEAAXXZ ) -__TBB_SYMBOL( ?internal_construct@scoped_lock_read@reader_writer_lock@interface5@tbb@@AEAAXAEAV234@@Z ) -__TBB_SYMBOL( ?internal_destroy@scoped_lock_read@reader_writer_lock@interface5@tbb@@AEAAXXZ ) - -#if !TBB_NO_LEGACY -// spin_rw_mutex.cpp v2 -__TBB_SYMBOL( ?internal_itt_releasing@spin_rw_mutex@tbb@@CAXPEAV12@@Z ) -__TBB_SYMBOL( ?internal_acquire_writer@spin_rw_mutex@tbb@@CA_NPEAV12@@Z ) -__TBB_SYMBOL( ?internal_acquire_reader@spin_rw_mutex@tbb@@CAXPEAV12@@Z ) -__TBB_SYMBOL( ?internal_downgrade@spin_rw_mutex@tbb@@CAXPEAV12@@Z ) -__TBB_SYMBOL( ?internal_upgrade@spin_rw_mutex@tbb@@CA_NPEAV12@@Z ) -__TBB_SYMBOL( ?internal_release_reader@spin_rw_mutex@tbb@@CAXPEAV12@@Z ) -__TBB_SYMBOL( ?internal_release_writer@spin_rw_mutex@tbb@@CAXPEAV12@@Z ) -__TBB_SYMBOL( ?internal_try_acquire_writer@spin_rw_mutex@tbb@@CA_NPEAV12@@Z ) -__TBB_SYMBOL( ?internal_try_acquire_reader@spin_rw_mutex@tbb@@CA_NPEAV12@@Z ) -#endif - -// spin_rw_mutex v3 -__TBB_SYMBOL( ?internal_construct@spin_rw_mutex_v3@tbb@@AEAAXXZ ) -__TBB_SYMBOL( ?internal_upgrade@spin_rw_mutex_v3@tbb@@AEAA_NXZ ) -__TBB_SYMBOL( ?internal_downgrade@spin_rw_mutex_v3@tbb@@AEAAXXZ ) -__TBB_SYMBOL( ?internal_acquire_reader@spin_rw_mutex_v3@tbb@@AEAAXXZ ) -__TBB_SYMBOL( ?internal_acquire_writer@spin_rw_mutex_v3@tbb@@AEAA_NXZ ) -__TBB_SYMBOL( ?internal_release_reader@spin_rw_mutex_v3@tbb@@AEAAXXZ ) -__TBB_SYMBOL( ?internal_release_writer@spin_rw_mutex_v3@tbb@@AEAAXXZ ) -__TBB_SYMBOL( ?internal_try_acquire_reader@spin_rw_mutex_v3@tbb@@AEAA_NXZ ) -__TBB_SYMBOL( ?internal_try_acquire_writer@spin_rw_mutex_v3@tbb@@AEAA_NXZ ) - -// x86_rtm_rw_mutex.cpp -__TBB_SYMBOL( ?internal_acquire_writer@x86_rtm_rw_mutex@internal@interface8@tbb@@AEAAXAEAVscoped_lock@1234@_N@Z ) -__TBB_SYMBOL( ?internal_acquire_reader@x86_rtm_rw_mutex@internal@interface8@tbb@@AEAAXAEAVscoped_lock@1234@_N@Z ) -__TBB_SYMBOL( ?internal_upgrade@x86_rtm_rw_mutex@internal@interface8@tbb@@AEAA_NAEAVscoped_lock@1234@@Z ) -__TBB_SYMBOL( ?internal_downgrade@x86_rtm_rw_mutex@internal@interface8@tbb@@AEAA_NAEAVscoped_lock@1234@@Z ) -__TBB_SYMBOL( ?internal_try_acquire_writer@x86_rtm_rw_mutex@internal@interface8@tbb@@AEAA_NAEAVscoped_lock@1234@@Z ) -__TBB_SYMBOL( ?internal_release@x86_rtm_rw_mutex@internal@interface8@tbb@@AEAAXAEAVscoped_lock@1234@@Z ) -__TBB_SYMBOL( ?internal_construct@x86_rtm_rw_mutex@internal@interface8@tbb@@AEAAXXZ ) - -// spin_mutex.cpp -__TBB_SYMBOL( ?internal_construct@spin_mutex@tbb@@QEAAXXZ ) -__TBB_SYMBOL( ?internal_acquire@scoped_lock@spin_mutex@tbb@@AEAAXAEAV23@@Z ) -__TBB_SYMBOL( ?internal_release@scoped_lock@spin_mutex@tbb@@AEAAXXZ ) -__TBB_SYMBOL( ?internal_try_acquire@scoped_lock@spin_mutex@tbb@@AEAA_NAEAV23@@Z ) - -// mutex.cpp -__TBB_SYMBOL( ?internal_acquire@scoped_lock@mutex@tbb@@AEAAXAEAV23@@Z ) -__TBB_SYMBOL( ?internal_release@scoped_lock@mutex@tbb@@AEAAXXZ ) -__TBB_SYMBOL( ?internal_try_acquire@scoped_lock@mutex@tbb@@AEAA_NAEAV23@@Z ) -__TBB_SYMBOL( ?internal_construct@mutex@tbb@@AEAAXXZ ) -__TBB_SYMBOL( ?internal_destroy@mutex@tbb@@AEAAXXZ ) - -// recursive_mutex.cpp -__TBB_SYMBOL( ?internal_construct@recursive_mutex@tbb@@AEAAXXZ ) -__TBB_SYMBOL( ?internal_destroy@recursive_mutex@tbb@@AEAAXXZ ) -__TBB_SYMBOL( ?internal_acquire@scoped_lock@recursive_mutex@tbb@@AEAAXAEAV23@@Z ) -__TBB_SYMBOL( ?internal_try_acquire@scoped_lock@recursive_mutex@tbb@@AEAA_NAEAV23@@Z ) -__TBB_SYMBOL( ?internal_release@scoped_lock@recursive_mutex@tbb@@AEAAXXZ ) - -// queuing_mutex.cpp -__TBB_SYMBOL( ?internal_construct@queuing_mutex@tbb@@QEAAXXZ ) -__TBB_SYMBOL( ?acquire@scoped_lock@queuing_mutex@tbb@@QEAAXAEAV23@@Z ) -__TBB_SYMBOL( ?release@scoped_lock@queuing_mutex@tbb@@QEAAXXZ ) -__TBB_SYMBOL( ?try_acquire@scoped_lock@queuing_mutex@tbb@@QEAA_NAEAV23@@Z ) - -//critical_section.cpp -__TBB_SYMBOL( ?internal_construct@critical_section_v4@internal@tbb@@QEAAXXZ ) - -#if !TBB_NO_LEGACY -// concurrent_hash_map.cpp -__TBB_SYMBOL( ?internal_grow_predicate@hash_map_segment_base@internal@tbb@@QEBA_NXZ ) - -// concurrent_queue.cpp v2 -__TBB_SYMBOL( ??0concurrent_queue_base@internal@tbb@@IEAA@_K@Z ) -__TBB_SYMBOL( ??0concurrent_queue_iterator_base@internal@tbb@@IEAA@AEBVconcurrent_queue_base@12@@Z ) -__TBB_SYMBOL( ??1concurrent_queue_base@internal@tbb@@MEAA@XZ ) -__TBB_SYMBOL( ??1concurrent_queue_iterator_base@internal@tbb@@IEAA@XZ ) -__TBB_SYMBOL( ?advance@concurrent_queue_iterator_base@internal@tbb@@IEAAXXZ ) -__TBB_SYMBOL( ?assign@concurrent_queue_iterator_base@internal@tbb@@IEAAXAEBV123@@Z ) -__TBB_SYMBOL( ?internal_pop@concurrent_queue_base@internal@tbb@@IEAAXPEAX@Z ) -__TBB_SYMBOL( ?internal_pop_if_present@concurrent_queue_base@internal@tbb@@IEAA_NPEAX@Z ) -__TBB_SYMBOL( ?internal_push@concurrent_queue_base@internal@tbb@@IEAAXPEBX@Z ) -__TBB_SYMBOL( ?internal_push_if_not_full@concurrent_queue_base@internal@tbb@@IEAA_NPEBX@Z ) -__TBB_SYMBOL( ?internal_set_capacity@concurrent_queue_base@internal@tbb@@IEAAX_J_K@Z ) -__TBB_SYMBOL( ?internal_size@concurrent_queue_base@internal@tbb@@IEBA_JXZ ) -#endif - -// concurrent_queue v3 -__TBB_SYMBOL( ??0concurrent_queue_iterator_base_v3@internal@tbb@@IEAA@AEBVconcurrent_queue_base_v3@12@@Z ) -__TBB_SYMBOL( ??0concurrent_queue_iterator_base_v3@internal@tbb@@IEAA@AEBVconcurrent_queue_base_v3@12@_K@Z ) -__TBB_SYMBOL( ??1concurrent_queue_iterator_base_v3@internal@tbb@@IEAA@XZ ) -__TBB_SYMBOL( ?assign@concurrent_queue_iterator_base_v3@internal@tbb@@IEAAXAEBV123@@Z ) -__TBB_SYMBOL( ?advance@concurrent_queue_iterator_base_v3@internal@tbb@@IEAAXXZ ) -__TBB_SYMBOL( ??0concurrent_queue_base_v3@internal@tbb@@IEAA@_K@Z ) -__TBB_SYMBOL( ??1concurrent_queue_base_v3@internal@tbb@@MEAA@XZ ) -__TBB_SYMBOL( ?internal_push@concurrent_queue_base_v3@internal@tbb@@IEAAXPEBX@Z ) -__TBB_SYMBOL( ?internal_push_move@concurrent_queue_base_v8@internal@tbb@@IEAAXPEBX@Z ) -__TBB_SYMBOL( ?internal_push_if_not_full@concurrent_queue_base_v3@internal@tbb@@IEAA_NPEBX@Z ) -__TBB_SYMBOL( ?internal_push_move_if_not_full@concurrent_queue_base_v8@internal@tbb@@IEAA_NPEBX@Z ) -__TBB_SYMBOL( ?internal_pop@concurrent_queue_base_v3@internal@tbb@@IEAAXPEAX@Z ) -__TBB_SYMBOL( ?internal_pop_if_present@concurrent_queue_base_v3@internal@tbb@@IEAA_NPEAX@Z ) -__TBB_SYMBOL( ?internal_abort@concurrent_queue_base_v3@internal@tbb@@IEAAXXZ ) -__TBB_SYMBOL( ?internal_size@concurrent_queue_base_v3@internal@tbb@@IEBA_JXZ ) -__TBB_SYMBOL( ?internal_empty@concurrent_queue_base_v3@internal@tbb@@IEBA_NXZ ) -__TBB_SYMBOL( ?internal_finish_clear@concurrent_queue_base_v3@internal@tbb@@IEAAXXZ ) -__TBB_SYMBOL( ?internal_set_capacity@concurrent_queue_base_v3@internal@tbb@@IEAAX_J_K@Z ) -__TBB_SYMBOL( ?internal_throw_exception@concurrent_queue_base_v3@internal@tbb@@IEBAXXZ ) -__TBB_SYMBOL( ?assign@concurrent_queue_base_v3@internal@tbb@@IEAAXAEBV123@@Z ) -__TBB_SYMBOL( ?move_content@concurrent_queue_base_v8@internal@tbb@@IEAAXAEAV123@@Z ) - -#if !TBB_NO_LEGACY -// concurrent_vector.cpp v2 -__TBB_SYMBOL( ?internal_assign@concurrent_vector_base@internal@tbb@@IEAAXAEBV123@_KP6AXPEAX1@ZP6AX2PEBX1@Z5@Z ) -__TBB_SYMBOL( ?internal_capacity@concurrent_vector_base@internal@tbb@@IEBA_KXZ ) -__TBB_SYMBOL( ?internal_clear@concurrent_vector_base@internal@tbb@@IEAAXP6AXPEAX_K@Z_N@Z ) -__TBB_SYMBOL( ?internal_copy@concurrent_vector_base@internal@tbb@@IEAAXAEBV123@_KP6AXPEAXPEBX1@Z@Z ) -__TBB_SYMBOL( ?internal_grow_by@concurrent_vector_base@internal@tbb@@IEAA_K_K0P6AXPEAX0@Z@Z ) -__TBB_SYMBOL( ?internal_grow_to_at_least@concurrent_vector_base@internal@tbb@@IEAAX_K0P6AXPEAX0@Z@Z ) -__TBB_SYMBOL( ?internal_push_back@concurrent_vector_base@internal@tbb@@IEAAPEAX_KAEA_K@Z ) -__TBB_SYMBOL( ?internal_reserve@concurrent_vector_base@internal@tbb@@IEAAX_K00@Z ) -#endif - -// concurrent_vector v3 -__TBB_SYMBOL( ??1concurrent_vector_base_v3@internal@tbb@@IEAA@XZ ) -__TBB_SYMBOL( ?internal_assign@concurrent_vector_base_v3@internal@tbb@@IEAAXAEBV123@_KP6AXPEAX1@ZP6AX2PEBX1@Z5@Z ) -__TBB_SYMBOL( ?internal_capacity@concurrent_vector_base_v3@internal@tbb@@IEBA_KXZ ) -__TBB_SYMBOL( ?internal_clear@concurrent_vector_base_v3@internal@tbb@@IEAA_KP6AXPEAX_K@Z@Z ) -__TBB_SYMBOL( ?internal_copy@concurrent_vector_base_v3@internal@tbb@@IEAAXAEBV123@_KP6AXPEAXPEBX1@Z@Z ) -__TBB_SYMBOL( ?internal_grow_by@concurrent_vector_base_v3@internal@tbb@@IEAA_K_K0P6AXPEAXPEBX0@Z2@Z ) -__TBB_SYMBOL( ?internal_grow_to_at_least@concurrent_vector_base_v3@internal@tbb@@IEAAX_K0P6AXPEAXPEBX0@Z2@Z ) -__TBB_SYMBOL( ?internal_push_back@concurrent_vector_base_v3@internal@tbb@@IEAAPEAX_KAEA_K@Z ) -__TBB_SYMBOL( ?internal_reserve@concurrent_vector_base_v3@internal@tbb@@IEAAX_K00@Z ) -__TBB_SYMBOL( ?internal_compact@concurrent_vector_base_v3@internal@tbb@@IEAAPEAX_KPEAXP6AX10@ZP6AX1PEBX0@Z@Z ) -__TBB_SYMBOL( ?internal_swap@concurrent_vector_base_v3@internal@tbb@@IEAAXAEAV123@@Z ) -__TBB_SYMBOL( ?internal_throw_exception@concurrent_vector_base_v3@internal@tbb@@IEBAX_K@Z ) -__TBB_SYMBOL( ?internal_resize@concurrent_vector_base_v3@internal@tbb@@IEAAX_K00PEBXP6AXPEAX0@ZP6AX210@Z@Z ) -__TBB_SYMBOL( ?internal_grow_to_at_least_with_result@concurrent_vector_base_v3@internal@tbb@@IEAA_K_K0P6AXPEAXPEBX0@Z2@Z ) - -// tbb_thread -__TBB_SYMBOL( ?allocate_closure_v3@internal@tbb@@YAPEAX_K@Z ) -__TBB_SYMBOL( ?detach@tbb_thread_v3@internal@tbb@@QEAAXXZ ) -__TBB_SYMBOL( ?free_closure_v3@internal@tbb@@YAXPEAX@Z ) -__TBB_SYMBOL( ?hardware_concurrency@tbb_thread_v3@internal@tbb@@SAIXZ ) -__TBB_SYMBOL( ?internal_start@tbb_thread_v3@internal@tbb@@AEAAXP6AIPEAX@Z0@Z ) -__TBB_SYMBOL( ?join@tbb_thread_v3@internal@tbb@@QEAAXXZ ) -__TBB_SYMBOL( ?move_v3@internal@tbb@@YAXAEAVtbb_thread_v3@12@0@Z ) -__TBB_SYMBOL( ?thread_get_id_v3@internal@tbb@@YA?AVid@tbb_thread_v3@12@XZ ) -__TBB_SYMBOL( ?thread_sleep_v3@internal@tbb@@YAXAEBVinterval_t@tick_count@2@@Z ) -__TBB_SYMBOL( ?thread_yield_v3@internal@tbb@@YAXXZ ) - -// condition_variable -__TBB_SYMBOL( ?internal_initialize_condition_variable@internal@interface5@tbb@@YAXAEATcondvar_impl_t@123@@Z ) -__TBB_SYMBOL( ?internal_condition_variable_wait@internal@interface5@tbb@@YA_NAEATcondvar_impl_t@123@PEAVmutex@3@PEBVinterval_t@tick_count@3@@Z ) -__TBB_SYMBOL( ?internal_condition_variable_notify_one@internal@interface5@tbb@@YAXAEATcondvar_impl_t@123@@Z ) -__TBB_SYMBOL( ?internal_condition_variable_notify_all@internal@interface5@tbb@@YAXAEATcondvar_impl_t@123@@Z ) -__TBB_SYMBOL( ?internal_destroy_condition_variable@internal@interface5@tbb@@YAXAEATcondvar_impl_t@123@@Z ) - -// global parameter -__TBB_SYMBOL( ?active_value@global_control@interface9@tbb@@CA_KH@Z ) -__TBB_SYMBOL( ?internal_create@global_control@interface9@tbb@@AEAAXXZ ) -__TBB_SYMBOL( ?internal_destroy@global_control@interface9@tbb@@AEAAXXZ ) - -#undef __TBB_SYMBOL diff --git a/src/tbb/src/tbb/winrt-tbb-export.lst b/src/tbb/src/tbb/winrt-tbb-export.lst deleted file mode 100644 index da551170b..000000000 --- a/src/tbb/src/tbb/winrt-tbb-export.lst +++ /dev/null @@ -1,300 +0,0 @@ -; Copyright (c) 2005-2017 Intel Corporation -; -; Licensed under the Apache License, Version 2.0 (the "License"); -; you may not use this file except in compliance with the License. -; You may obtain a copy of the License at -; -; http://www.apache.org/licenses/LICENSE-2.0 -; -; Unless required by applicable law or agreed to in writing, software -; distributed under the License is distributed on an "AS IS" BASIS, -; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -; See the License for the specific language governing permissions and -; limitations under the License. -; -; -; -; - -#include "tbb/tbb_config.h" - -// cache_aligned_allocator.cpp -__TBB_SYMBOL( ?NFS_Allocate@internal@tbb@@YAPAXIIPAX@Z ) -__TBB_SYMBOL( ?NFS_GetLineSize@internal@tbb@@YAIXZ ) -__TBB_SYMBOL( ?NFS_Free@internal@tbb@@YAXPAX@Z ) -__TBB_SYMBOL( ?allocate_via_handler_v3@internal@tbb@@YAPAXI@Z ) -__TBB_SYMBOL( ?deallocate_via_handler_v3@internal@tbb@@YAXPAX@Z ) -__TBB_SYMBOL( ?is_malloc_used_v3@internal@tbb@@YA_NXZ ) - -// task.cpp v3 -__TBB_SYMBOL( ?allocate@allocate_additional_child_of_proxy@internal@tbb@@QBAAAVtask@3@I@Z ) -__TBB_SYMBOL( ?allocate@allocate_child_proxy@internal@tbb@@QBAAAVtask@3@I@Z ) -__TBB_SYMBOL( ?allocate@allocate_continuation_proxy@internal@tbb@@QBAAAVtask@3@I@Z ) -__TBB_SYMBOL( ?allocate@allocate_root_proxy@internal@tbb@@SAAAVtask@3@I@Z ) -__TBB_SYMBOL( ?destroy@task_base@internal@interface5@tbb@@SAXAAVtask@4@@Z ) -__TBB_SYMBOL( ?free@allocate_additional_child_of_proxy@internal@tbb@@QBAXAAVtask@3@@Z ) -__TBB_SYMBOL( ?free@allocate_child_proxy@internal@tbb@@QBAXAAVtask@3@@Z ) -__TBB_SYMBOL( ?free@allocate_continuation_proxy@internal@tbb@@QBAXAAVtask@3@@Z ) -__TBB_SYMBOL( ?free@allocate_root_proxy@internal@tbb@@SAXAAVtask@3@@Z ) -__TBB_SYMBOL( ?internal_set_ref_count@task@tbb@@AAAXH@Z ) -__TBB_SYMBOL( ?internal_decrement_ref_count@task@tbb@@AAAHXZ ) -__TBB_SYMBOL( ?is_owned_by_current_thread@task@tbb@@QBA_NXZ ) -__TBB_SYMBOL( ?note_affinity@task@tbb@@UAAXG@Z ) -__TBB_SYMBOL( ?resize@affinity_partitioner_base_v3@internal@tbb@@AAAXI@Z ) -__TBB_SYMBOL( ?self@task@tbb@@SAAAV12@XZ ) -__TBB_SYMBOL( ?spawn_and_wait_for_all@task@tbb@@QAAXAAVtask_list@2@@Z ) -__TBB_SYMBOL( ?default_num_threads@task_scheduler_init@tbb@@SAHXZ ) -__TBB_SYMBOL( ?initialize@task_scheduler_init@tbb@@QAAXHI@Z ) -__TBB_SYMBOL( ?initialize@task_scheduler_init@tbb@@QAAXH@Z ) -__TBB_SYMBOL( ?terminate@task_scheduler_init@tbb@@QAAXXZ ) -#if __TBB_SCHEDULER_OBSERVER -__TBB_SYMBOL( ?observe@task_scheduler_observer_v3@internal@tbb@@QAAX_N@Z ) -#endif /* __TBB_SCHEDULER_OBSERVER */ - -/* arena.cpp */ -__TBB_SYMBOL( ?internal_max_concurrency@task_arena_base@internal@interface7@tbb@@KAHPBVtask_arena@34@@Z ) -__TBB_SYMBOL( ?internal_current_slot@task_arena_base@internal@interface7@tbb@@KAHXZ ) -__TBB_SYMBOL( ?internal_initialize@task_arena_base@internal@interface7@tbb@@IAAXXZ ) -__TBB_SYMBOL( ?internal_terminate@task_arena_base@internal@interface7@tbb@@IAAXXZ ) -__TBB_SYMBOL( ?internal_attach@task_arena_base@internal@interface7@tbb@@IAAXXZ ) -__TBB_SYMBOL( ?internal_enqueue@task_arena_base@internal@interface7@tbb@@IBAXAAVtask@4@H@Z ) -__TBB_SYMBOL( ?internal_execute@task_arena_base@internal@interface7@tbb@@IBAXAAVdelegate_base@234@@Z ) -__TBB_SYMBOL( ?internal_wait@task_arena_base@internal@interface7@tbb@@IBAXXZ ) -#if __TBB_TASK_ISOLATION -__TBB_SYMBOL( ?isolate_within_arena@internal@interface7@tbb@@YAXAAVdelegate_base@123@H@Z ) -#endif /* __TBB_TASK_ISOLATION */ - -#if !TBB_NO_LEGACY -// task_v2.cpp -__TBB_SYMBOL( ?destroy@task@tbb@@QAAXAAV12@@Z ) -#endif - -// exception handling support -#if __TBB_TASK_GROUP_CONTEXT -__TBB_SYMBOL( ?allocate@allocate_root_with_context_proxy@internal@tbb@@QBAAAVtask@3@I@Z ) -__TBB_SYMBOL( ?free@allocate_root_with_context_proxy@internal@tbb@@QBAXAAVtask@3@@Z ) -__TBB_SYMBOL( ?change_group@task@tbb@@QAAXAAVtask_group_context@2@@Z ) -__TBB_SYMBOL( ?is_group_execution_cancelled@task_group_context@tbb@@QBA_NXZ ) -__TBB_SYMBOL( ?cancel_group_execution@task_group_context@tbb@@QAA_NXZ ) -__TBB_SYMBOL( ?reset@task_group_context@tbb@@QAAXXZ ) -__TBB_SYMBOL( ?capture_fp_settings@task_group_context@tbb@@QAAXXZ ) -__TBB_SYMBOL( ?init@task_group_context@tbb@@IAAXXZ ) -__TBB_SYMBOL( ?register_pending_exception@task_group_context@tbb@@QAAXXZ ) -__TBB_SYMBOL( ??1task_group_context@tbb@@QAA@XZ ) -#if __TBB_TASK_PRIORITY -__TBB_SYMBOL( ?set_priority@task_group_context@tbb@@QAAXW4priority_t@2@@Z ) -__TBB_SYMBOL( ?priority@task_group_context@tbb@@QBA?AW4priority_t@2@XZ ) -#endif /* __TBB_TASK_PRIORITY */ -__TBB_SYMBOL( ?name@captured_exception@tbb@@UBAPBDXZ ) -__TBB_SYMBOL( ?what@captured_exception@tbb@@UBAPBDXZ ) -__TBB_SYMBOL( ??1captured_exception@tbb@@UAA@XZ ) -__TBB_SYMBOL( ?move@captured_exception@tbb@@UAAPAV12@XZ ) -__TBB_SYMBOL( ?destroy@captured_exception@tbb@@UAAXXZ ) -__TBB_SYMBOL( ?set@captured_exception@tbb@@QAAXPBD0@Z ) -__TBB_SYMBOL( ?clear@captured_exception@tbb@@QAAXXZ ) -#endif /* __TBB_TASK_GROUP_CONTEXT */ - -// Symbols for exceptions thrown from TBB -__TBB_SYMBOL( ?throw_bad_last_alloc_exception_v4@internal@tbb@@YAXXZ ) -__TBB_SYMBOL( ?throw_exception_v4@internal@tbb@@YAXW4exception_id@12@@Z ) -__TBB_SYMBOL( ?what@bad_last_alloc@tbb@@UBAPBDXZ ) -__TBB_SYMBOL( ?what@missing_wait@tbb@@UBAPBDXZ ) -__TBB_SYMBOL( ?what@invalid_multiple_scheduling@tbb@@UBAPBDXZ ) -__TBB_SYMBOL( ?what@improper_lock@tbb@@UBAPBDXZ ) -__TBB_SYMBOL( ?what@user_abort@tbb@@UBAPBDXZ ) - -// tbb_misc.cpp -__TBB_SYMBOL( ?assertion_failure@tbb@@YAXPBDH00@Z ) -__TBB_SYMBOL( ?get_initial_auto_partitioner_divisor@internal@tbb@@YAIXZ ) -__TBB_SYMBOL( ?handle_perror@internal@tbb@@YAXHPBD@Z ) -__TBB_SYMBOL( ?set_assertion_handler@tbb@@YAP6AXPBDH00@ZP6AX0H00@Z@Z ) -__TBB_SYMBOL( ?runtime_warning@internal@tbb@@YAXPBDZZ ) -__TBB_SYMBOL( TBB_runtime_interface_version ) - -// tbb_main.cpp -__TBB_SYMBOL( ?itt_load_pointer_with_acquire_v3@internal@tbb@@YAPAXPBX@Z ) -__TBB_SYMBOL( ?itt_store_pointer_with_release_v3@internal@tbb@@YAXPAX0@Z ) -__TBB_SYMBOL( ?call_itt_notify_v5@internal@tbb@@YAXHPAX@Z ) -__TBB_SYMBOL( ?itt_set_sync_name_v3@internal@tbb@@YAXPAXPB_W@Z ) -__TBB_SYMBOL( ?itt_load_pointer_v3@internal@tbb@@YAPAXPBX@Z ) - -// pipeline.cpp -__TBB_SYMBOL( ??0pipeline@tbb@@QAA@XZ ) -__TBB_SYMBOL( ??1filter@tbb@@UAA@XZ ) -__TBB_SYMBOL( ??1pipeline@tbb@@UAA@XZ ) -__TBB_SYMBOL( ??_7pipeline@tbb@@6B@ ) -__TBB_SYMBOL( ?add_filter@pipeline@tbb@@QAAXAAVfilter@2@@Z ) -__TBB_SYMBOL( ?clear@pipeline@tbb@@QAAXXZ ) -__TBB_SYMBOL( ?inject_token@pipeline@tbb@@AAAXAAVtask@2@@Z ) -__TBB_SYMBOL( ?run@pipeline@tbb@@QAAXI@Z ) -#if __TBB_TASK_GROUP_CONTEXT -__TBB_SYMBOL( ?run@pipeline@tbb@@QAAXIAAVtask_group_context@2@@Z ) -#endif -__TBB_SYMBOL( ?process_item@thread_bound_filter@tbb@@QAA?AW4result_type@12@XZ ) -__TBB_SYMBOL( ?try_process_item@thread_bound_filter@tbb@@QAA?AW4result_type@12@XZ ) -__TBB_SYMBOL( ?set_end_of_input@filter@tbb@@IAAXXZ ) - -// queuing_rw_mutex.cpp -__TBB_SYMBOL( ?internal_construct@queuing_rw_mutex@tbb@@QAAXXZ ) -__TBB_SYMBOL( ?acquire@scoped_lock@queuing_rw_mutex@tbb@@QAAXAAV23@_N@Z ) -__TBB_SYMBOL( ?downgrade_to_reader@scoped_lock@queuing_rw_mutex@tbb@@QAA_NXZ ) -__TBB_SYMBOL( ?release@scoped_lock@queuing_rw_mutex@tbb@@QAAXXZ ) -__TBB_SYMBOL( ?upgrade_to_writer@scoped_lock@queuing_rw_mutex@tbb@@QAA_NXZ ) -__TBB_SYMBOL( ?try_acquire@scoped_lock@queuing_rw_mutex@tbb@@QAA_NAAV23@_N@Z ) - -// reader_writer_lock.cpp -__TBB_SYMBOL( ?try_lock_read@reader_writer_lock@interface5@tbb@@QAA_NXZ ) -__TBB_SYMBOL( ?try_lock@reader_writer_lock@interface5@tbb@@QAA_NXZ ) -__TBB_SYMBOL( ?unlock@reader_writer_lock@interface5@tbb@@QAAXXZ ) -__TBB_SYMBOL( ?lock_read@reader_writer_lock@interface5@tbb@@QAAXXZ ) -__TBB_SYMBOL( ?lock@reader_writer_lock@interface5@tbb@@QAAXXZ ) -__TBB_SYMBOL( ?internal_construct@reader_writer_lock@interface5@tbb@@AAAXXZ ) -__TBB_SYMBOL( ?internal_destroy@reader_writer_lock@interface5@tbb@@AAAXXZ ) -__TBB_SYMBOL( ?internal_construct@scoped_lock@reader_writer_lock@interface5@tbb@@AAAXAAV234@@Z ) -__TBB_SYMBOL( ?internal_destroy@scoped_lock@reader_writer_lock@interface5@tbb@@AAAXXZ ) -__TBB_SYMBOL( ?internal_construct@scoped_lock_read@reader_writer_lock@interface5@tbb@@AAAXAAV234@@Z ) -__TBB_SYMBOL( ?internal_destroy@scoped_lock_read@reader_writer_lock@interface5@tbb@@AAAXXZ ) - -#if !TBB_NO_LEGACY -// spin_rw_mutex.cpp v2 -__TBB_SYMBOL( ?internal_acquire_reader@spin_rw_mutex@tbb@@CAXPAV12@@Z ) -__TBB_SYMBOL( ?internal_acquire_writer@spin_rw_mutex@tbb@@CA_NPAV12@@Z ) -__TBB_SYMBOL( ?internal_downgrade@spin_rw_mutex@tbb@@CAXPAV12@@Z ) -__TBB_SYMBOL( ?internal_itt_releasing@spin_rw_mutex@tbb@@CAXPAV12@@Z ) -__TBB_SYMBOL( ?internal_release_reader@spin_rw_mutex@tbb@@CAXPAV12@@Z ) -__TBB_SYMBOL( ?internal_release_writer@spin_rw_mutex@tbb@@CAXPAV12@@Z ) -__TBB_SYMBOL( ?internal_upgrade@spin_rw_mutex@tbb@@CA_NPAV12@@Z ) -__TBB_SYMBOL( ?internal_try_acquire_writer@spin_rw_mutex@tbb@@CA_NPAV12@@Z ) -__TBB_SYMBOL( ?internal_try_acquire_reader@spin_rw_mutex@tbb@@CA_NPAV12@@Z ) -#endif - -// spin_rw_mutex v3 -__TBB_SYMBOL( ?internal_construct@spin_rw_mutex_v3@tbb@@AAAXXZ ) -__TBB_SYMBOL( ?internal_upgrade@spin_rw_mutex_v3@tbb@@AAA_NXZ ) -__TBB_SYMBOL( ?internal_downgrade@spin_rw_mutex_v3@tbb@@AAAXXZ ) -__TBB_SYMBOL( ?internal_acquire_reader@spin_rw_mutex_v3@tbb@@AAAXXZ ) -__TBB_SYMBOL( ?internal_acquire_writer@spin_rw_mutex_v3@tbb@@AAA_NXZ ) -__TBB_SYMBOL( ?internal_release_reader@spin_rw_mutex_v3@tbb@@AAAXXZ ) -__TBB_SYMBOL( ?internal_release_writer@spin_rw_mutex_v3@tbb@@AAAXXZ ) -__TBB_SYMBOL( ?internal_try_acquire_reader@spin_rw_mutex_v3@tbb@@AAA_NXZ ) -__TBB_SYMBOL( ?internal_try_acquire_writer@spin_rw_mutex_v3@tbb@@AAA_NXZ ) - -// spin_mutex.cpp -__TBB_SYMBOL( ?internal_construct@spin_mutex@tbb@@QAAXXZ ) -__TBB_SYMBOL( ?internal_acquire@scoped_lock@spin_mutex@tbb@@AAAXAAV23@@Z ) -__TBB_SYMBOL( ?internal_release@scoped_lock@spin_mutex@tbb@@AAAXXZ ) -__TBB_SYMBOL( ?internal_try_acquire@scoped_lock@spin_mutex@tbb@@AAA_NAAV23@@Z ) - -// mutex.cpp -__TBB_SYMBOL( ?internal_acquire@scoped_lock@mutex@tbb@@AAAXAAV23@@Z ) -__TBB_SYMBOL( ?internal_release@scoped_lock@mutex@tbb@@AAAXXZ ) -__TBB_SYMBOL( ?internal_try_acquire@scoped_lock@mutex@tbb@@AAA_NAAV23@@Z ) -__TBB_SYMBOL( ?internal_construct@mutex@tbb@@AAAXXZ ) -__TBB_SYMBOL( ?internal_destroy@mutex@tbb@@AAAXXZ ) - -// recursive_mutex.cpp -__TBB_SYMBOL( ?internal_acquire@scoped_lock@recursive_mutex@tbb@@AAAXAAV23@@Z ) -__TBB_SYMBOL( ?internal_release@scoped_lock@recursive_mutex@tbb@@AAAXXZ ) -__TBB_SYMBOL( ?internal_try_acquire@scoped_lock@recursive_mutex@tbb@@AAA_NAAV23@@Z ) -__TBB_SYMBOL( ?internal_construct@recursive_mutex@tbb@@AAAXXZ ) -__TBB_SYMBOL( ?internal_destroy@recursive_mutex@tbb@@AAAXXZ ) - -// queuing_mutex.cpp -__TBB_SYMBOL( ?internal_construct@queuing_mutex@tbb@@QAAXXZ ) -__TBB_SYMBOL( ?acquire@scoped_lock@queuing_mutex@tbb@@QAAXAAV23@@Z ) -__TBB_SYMBOL( ?release@scoped_lock@queuing_mutex@tbb@@QAAXXZ ) -__TBB_SYMBOL( ?try_acquire@scoped_lock@queuing_mutex@tbb@@QAA_NAAV23@@Z ) - -// critical_section.cpp -__TBB_SYMBOL( ?internal_construct@critical_section_v4@internal@tbb@@QAAXXZ ) - -#if !TBB_NO_LEGACY -// concurrent_hash_map.cpp -__TBB_SYMBOL( ?internal_grow_predicate@hash_map_segment_base@internal@tbb@@QBA_NXZ ) - -// concurrent_queue.cpp v2 -__TBB_SYMBOL( ?advance@concurrent_queue_iterator_base@internal@tbb@@IAAXXZ ) -__TBB_SYMBOL( ?assign@concurrent_queue_iterator_base@internal@tbb@@IAAXABV123@@Z ) -__TBB_SYMBOL( ?internal_size@concurrent_queue_base@internal@tbb@@IBAHXZ ) -__TBB_SYMBOL( ??0concurrent_queue_base@internal@tbb@@IAA@I@Z ) -__TBB_SYMBOL( ??0concurrent_queue_iterator_base@internal@tbb@@IAA@ABVconcurrent_queue_base@12@@Z ) -__TBB_SYMBOL( ??1concurrent_queue_base@internal@tbb@@MAA@XZ ) -__TBB_SYMBOL( ??1concurrent_queue_iterator_base@internal@tbb@@IAA@XZ ) -__TBB_SYMBOL( ?internal_pop@concurrent_queue_base@internal@tbb@@IAAXPAX@Z ) -__TBB_SYMBOL( ?internal_pop_if_present@concurrent_queue_base@internal@tbb@@IAA_NPAX@Z ) -__TBB_SYMBOL( ?internal_push@concurrent_queue_base@internal@tbb@@IAAXPBX@Z ) -__TBB_SYMBOL( ?internal_push_if_not_full@concurrent_queue_base@internal@tbb@@IAA_NPBX@Z ) -__TBB_SYMBOL( ?internal_set_capacity@concurrent_queue_base@internal@tbb@@IAAXHI@Z ) -#endif - -// concurrent_queue v3 -__TBB_SYMBOL( ??1concurrent_queue_iterator_base_v3@internal@tbb@@IAA@XZ ) -__TBB_SYMBOL( ??0concurrent_queue_iterator_base_v3@internal@tbb@@IAA@ABVconcurrent_queue_base_v3@12@@Z ) -__TBB_SYMBOL( ??0concurrent_queue_iterator_base_v3@internal@tbb@@IAA@ABVconcurrent_queue_base_v3@12@I@Z ) -__TBB_SYMBOL( ?advance@concurrent_queue_iterator_base_v3@internal@tbb@@IAAXXZ ) -__TBB_SYMBOL( ?assign@concurrent_queue_iterator_base_v3@internal@tbb@@IAAXABV123@@Z ) -__TBB_SYMBOL( ??0concurrent_queue_base_v3@internal@tbb@@IAA@I@Z ) -__TBB_SYMBOL( ??1concurrent_queue_base_v3@internal@tbb@@MAA@XZ ) -__TBB_SYMBOL( ?internal_pop@concurrent_queue_base_v3@internal@tbb@@IAAXPAX@Z ) -__TBB_SYMBOL( ?internal_pop_if_present@concurrent_queue_base_v3@internal@tbb@@IAA_NPAX@Z ) -__TBB_SYMBOL( ?internal_abort@concurrent_queue_base_v3@internal@tbb@@IAAXXZ ) -__TBB_SYMBOL( ?internal_push@concurrent_queue_base_v3@internal@tbb@@IAAXPBX@Z ) -__TBB_SYMBOL( ?internal_push_move@concurrent_queue_base_v8@internal@tbb@@IAAXPBX@Z ) -__TBB_SYMBOL( ?internal_push_if_not_full@concurrent_queue_base_v3@internal@tbb@@IAA_NPBX@Z ) -__TBB_SYMBOL( ?internal_push_move_if_not_full@concurrent_queue_base_v8@internal@tbb@@IAA_NPBX@Z ) -__TBB_SYMBOL( ?internal_size@concurrent_queue_base_v3@internal@tbb@@IBAHXZ ) -__TBB_SYMBOL( ?internal_empty@concurrent_queue_base_v3@internal@tbb@@IBA_NXZ ) -__TBB_SYMBOL( ?internal_set_capacity@concurrent_queue_base_v3@internal@tbb@@IAAXHI@Z ) -__TBB_SYMBOL( ?internal_finish_clear@concurrent_queue_base_v3@internal@tbb@@IAAXXZ ) -__TBB_SYMBOL( ?internal_throw_exception@concurrent_queue_base_v3@internal@tbb@@IBAXXZ ) -__TBB_SYMBOL( ?assign@concurrent_queue_base_v3@internal@tbb@@IAAXABV123@@Z ) -__TBB_SYMBOL( ?move_content@concurrent_queue_base_v8@internal@tbb@@IAAXAAV123@@Z ) - -#if !TBB_NO_LEGACY -// concurrent_vector.cpp v2 -__TBB_SYMBOL( ?internal_assign@concurrent_vector_base@internal@tbb@@IAAXABV123@IP6AXPAXI@ZP6AX1PBXI@Z4@Z ) -__TBB_SYMBOL( ?internal_capacity@concurrent_vector_base@internal@tbb@@IBAIXZ ) -__TBB_SYMBOL( ?internal_clear@concurrent_vector_base@internal@tbb@@IAAXP6AXPAXI@Z_N@Z ) -__TBB_SYMBOL( ?internal_copy@concurrent_vector_base@internal@tbb@@IAAXABV123@IP6AXPAXPBXI@Z@Z ) -__TBB_SYMBOL( ?internal_grow_by@concurrent_vector_base@internal@tbb@@IAAIIIP6AXPAXI@Z@Z ) -__TBB_SYMBOL( ?internal_grow_to_at_least@concurrent_vector_base@internal@tbb@@IAAXIIP6AXPAXI@Z@Z ) -__TBB_SYMBOL( ?internal_push_back@concurrent_vector_base@internal@tbb@@IAAPAXIAAI@Z ) -__TBB_SYMBOL( ?internal_reserve@concurrent_vector_base@internal@tbb@@IAAXIII@Z ) -#endif - -// concurrent_vector v3 -__TBB_SYMBOL( ??1concurrent_vector_base_v3@internal@tbb@@IAA@XZ ) -__TBB_SYMBOL( ?internal_assign@concurrent_vector_base_v3@internal@tbb@@IAAXABV123@IP6AXPAXI@ZP6AX1PBXI@Z4@Z ) -__TBB_SYMBOL( ?internal_capacity@concurrent_vector_base_v3@internal@tbb@@IBAIXZ ) -__TBB_SYMBOL( ?internal_clear@concurrent_vector_base_v3@internal@tbb@@IAAIP6AXPAXI@Z@Z ) -__TBB_SYMBOL( ?internal_copy@concurrent_vector_base_v3@internal@tbb@@IAAXABV123@IP6AXPAXPBXI@Z@Z ) -__TBB_SYMBOL( ?internal_grow_by@concurrent_vector_base_v3@internal@tbb@@IAAIIIP6AXPAXPBXI@Z1@Z ) -__TBB_SYMBOL( ?internal_grow_to_at_least@concurrent_vector_base_v3@internal@tbb@@IAAXIIP6AXPAXPBXI@Z1@Z ) -__TBB_SYMBOL( ?internal_push_back@concurrent_vector_base_v3@internal@tbb@@IAAPAXIAAI@Z ) -__TBB_SYMBOL( ?internal_reserve@concurrent_vector_base_v3@internal@tbb@@IAAXIII@Z ) -__TBB_SYMBOL( ?internal_compact@concurrent_vector_base_v3@internal@tbb@@IAAPAXIPAXP6AX0I@ZP6AX0PBXI@Z@Z ) -__TBB_SYMBOL( ?internal_swap@concurrent_vector_base_v3@internal@tbb@@IAAXAAV123@@Z ) -__TBB_SYMBOL( ?internal_throw_exception@concurrent_vector_base_v3@internal@tbb@@IBAXI@Z ) -__TBB_SYMBOL( ?internal_resize@concurrent_vector_base_v3@internal@tbb@@IAAXIIIPBXP6AXPAXI@ZP6AX10I@Z@Z ) -__TBB_SYMBOL( ?internal_grow_to_at_least_with_result@concurrent_vector_base_v3@internal@tbb@@IAAIIIP6AXPAXPBXI@Z1@Z ) - -// tbb_thread -__TBB_SYMBOL( ?join@tbb_thread_v3@internal@tbb@@QAAXXZ ) -__TBB_SYMBOL( ?detach@tbb_thread_v3@internal@tbb@@QAAXXZ ) -__TBB_SYMBOL( ?internal_start@tbb_thread_v3@internal@tbb@@AAAXP6AIPAX@Z0@Z ) -__TBB_SYMBOL( ?allocate_closure_v3@internal@tbb@@YAPAXI@Z ) -__TBB_SYMBOL( ?free_closure_v3@internal@tbb@@YAXPAX@Z ) -__TBB_SYMBOL( ?hardware_concurrency@tbb_thread_v3@internal@tbb@@SAIXZ ) -__TBB_SYMBOL( ?thread_yield_v3@internal@tbb@@YAXXZ ) -__TBB_SYMBOL( ?thread_sleep_v3@internal@tbb@@YAXABVinterval_t@tick_count@2@@Z ) -__TBB_SYMBOL( ?move_v3@internal@tbb@@YAXAAVtbb_thread_v3@12@0@Z ) -__TBB_SYMBOL( ?thread_get_id_v3@internal@tbb@@YA?AVid@tbb_thread_v3@12@XZ ) - -// condition_variable -__TBB_SYMBOL( ?internal_initialize_condition_variable@internal@interface5@tbb@@YAXAATcondvar_impl_t@123@@Z ) -__TBB_SYMBOL( ?internal_condition_variable_wait@internal@interface5@tbb@@YA_NAATcondvar_impl_t@123@PAVmutex@3@PBVinterval_t@tick_count@3@@Z ) -__TBB_SYMBOL( ?internal_condition_variable_notify_one@internal@interface5@tbb@@YAXAATcondvar_impl_t@123@@Z ) -__TBB_SYMBOL( ?internal_condition_variable_notify_all@internal@interface5@tbb@@YAXAATcondvar_impl_t@123@@Z ) -__TBB_SYMBOL( ?internal_destroy_condition_variable@internal@interface5@tbb@@YAXAATcondvar_impl_t@123@@Z ) - -#undef __TBB_SYMBOL diff --git a/src/tbb/src/tbb/x86_rtm_rw_mutex.cpp b/src/tbb/src/tbb/x86_rtm_rw_mutex.cpp deleted file mode 100644 index 7f2027fab..000000000 --- a/src/tbb/src/tbb/x86_rtm_rw_mutex.cpp +++ /dev/null @@ -1,277 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#include "tbb/tbb_config.h" -#if __TBB_TSX_AVAILABLE -#include "tbb/spin_rw_mutex.h" -#include "tbb/tbb_machine.h" -#include "itt_notify.h" -#include "governor.h" -#include "tbb/atomic.h" - -// __TBB_RW_MUTEX_DELAY_TEST shifts the point where flags aborting speculation are -// added to the read-set of the operation. If 1, will add the test just before -// the transaction is ended; this technique is called lazy subscription. -// CAUTION: due to proven issues of lazy subscription, use of __TBB_RW_MUTEX_DELAY_TEST is discouraged! -#ifndef __TBB_RW_MUTEX_DELAY_TEST - #define __TBB_RW_MUTEX_DELAY_TEST 0 -#endif - -#if defined(_MSC_VER) && defined(_Wp64) - // Workaround for overzealous compiler warnings in /Wp64 mode - #pragma warning (disable: 4244) -#endif - -namespace tbb { - -namespace interface8 { -namespace internal { - -// abort code for mutexes that detect a conflict with another thread. -// value is hexadecimal -enum { - speculation_transaction_aborted = 0x01, - speculation_can_retry = 0x02, - speculation_memadd_conflict = 0x04, - speculation_buffer_overflow = 0x08, - speculation_breakpoint_hit = 0x10, - speculation_nested_abort = 0x20, - speculation_xabort_mask = 0xFF000000, - speculation_xabort_shift = 24, - speculation_retry = speculation_transaction_aborted - | speculation_can_retry - | speculation_memadd_conflict -}; - -// maximum number of times to retry -// TODO: experiment on retry values. -static const int retry_threshold_read = 10; -static const int retry_threshold_write = 10; - -//! Release speculative mutex -void x86_rtm_rw_mutex::internal_release(x86_rtm_rw_mutex::scoped_lock& s) { - switch(s.transaction_state) { - case RTM_transacting_writer: - case RTM_transacting_reader: - { - __TBB_ASSERT(__TBB_machine_is_in_transaction(), "transaction_state && not speculating"); -#if __TBB_RW_MUTEX_DELAY_TEST - if(s.transaction_state == RTM_transacting_reader) { - if(this->w_flag) __TBB_machine_transaction_conflict_abort(); - } else { - if(this->state) __TBB_machine_transaction_conflict_abort(); - } -#endif - __TBB_machine_end_transaction(); - s.my_scoped_lock.internal_set_mutex(NULL); - } - break; - case RTM_real_reader: - __TBB_ASSERT(!this->w_flag, "w_flag set but read lock acquired"); - s.my_scoped_lock.release(); - break; - case RTM_real_writer: - __TBB_ASSERT(this->w_flag, "w_flag unset but write lock acquired"); - this->w_flag = false; - s.my_scoped_lock.release(); - break; - case RTM_not_in_mutex: - __TBB_ASSERT(false, "RTM_not_in_mutex, but in release"); - default: - __TBB_ASSERT(false, "invalid transaction_state"); - } - s.transaction_state = RTM_not_in_mutex; -} - -//! Acquire write lock on the given mutex. -void x86_rtm_rw_mutex::internal_acquire_writer(x86_rtm_rw_mutex::scoped_lock& s, bool only_speculate) -{ - __TBB_ASSERT(s.transaction_state == RTM_not_in_mutex, "scoped_lock already in transaction"); - if(tbb::internal::governor::speculation_enabled()) { - int num_retries = 0; - unsigned int abort_code; - do { - tbb::internal::atomic_backoff backoff; - if(this->state) { - if(only_speculate) return; - do { - backoff.pause(); // test the spin_rw_mutex (real readers or writers) - } while(this->state); - } - // _xbegin returns -1 on success or the abort code, so capture it - if(( abort_code = __TBB_machine_begin_transaction()) == ~(unsigned int)(0) ) - { - // started speculation -#if !__TBB_RW_MUTEX_DELAY_TEST - if(this->state) { // add spin_rw_mutex to read-set. - // reader or writer grabbed the lock, so abort. - __TBB_machine_transaction_conflict_abort(); - } -#endif - s.transaction_state = RTM_transacting_writer; - s.my_scoped_lock.internal_set_mutex(this); // need mutex for release() - return; // successfully started speculation - } - ++num_retries; - } while( (abort_code & speculation_retry) != 0 && (num_retries < retry_threshold_write) ); - } - - if(only_speculate) return; // should apply a real try_lock... - s.my_scoped_lock.acquire(*this, true); // kill transactional writers - __TBB_ASSERT(!w_flag, "After acquire for write, w_flag already true"); - w_flag = true; // kill transactional readers - s.transaction_state = RTM_real_writer; - return; -} - -//! Acquire read lock on given mutex. -// only_speculate : true if we are doing a try_acquire. If true and we fail to speculate, don't -// really acquire the lock, return and do a try_acquire on the contained spin_rw_mutex. If -// the lock is already held by a writer, just return. -void x86_rtm_rw_mutex::internal_acquire_reader(x86_rtm_rw_mutex::scoped_lock& s, bool only_speculate) { - __TBB_ASSERT(s.transaction_state == RTM_not_in_mutex, "scoped_lock already in transaction"); - if(tbb::internal::governor::speculation_enabled()) { - int num_retries = 0; - unsigned int abort_code; - do { - tbb::internal::atomic_backoff backoff; - // if in try_acquire, and lock is held as writer, don't attempt to speculate. - if(w_flag) { - if(only_speculate) return; - do { - backoff.pause(); // test the spin_rw_mutex (real readers or writers) - } while(w_flag); - } - // _xbegin returns -1 on success or the abort code, so capture it - if((abort_code = __TBB_machine_begin_transaction()) == ~(unsigned int)(0) ) - { - // started speculation -#if !__TBB_RW_MUTEX_DELAY_TEST - if(w_flag) { // add w_flag to read-set. - __TBB_machine_transaction_conflict_abort(); // writer grabbed the lock, so abort. - } -#endif - s.transaction_state = RTM_transacting_reader; - s.my_scoped_lock.internal_set_mutex(this); // need mutex for release() - return; // successfully started speculation - } - // fallback path - // retry only if there is any hope of getting into a transaction soon - // Retry in the following cases (from Section 8.3.5 of Intel(R) - // Architecture Instruction Set Extensions Programming Reference): - // 1. abort caused by XABORT instruction (bit 0 of EAX register is set) - // 2. the transaction may succeed on a retry (bit 1 of EAX register is set) - // 3. if another logical processor conflicted with a memory address - // that was part of the transaction that aborted (bit 2 of EAX register is set) - // That is, retry if (abort_code & 0x7) is non-zero - ++num_retries; - } while( (abort_code & speculation_retry) != 0 && (num_retries < retry_threshold_read) ); - } - - if(only_speculate) return; - s.my_scoped_lock.acquire( *this, false ); - s.transaction_state = RTM_real_reader; -} - -//! Upgrade reader to become a writer. -/** Returns whether the upgrade happened without releasing and re-acquiring the lock */ -bool x86_rtm_rw_mutex::internal_upgrade(x86_rtm_rw_mutex::scoped_lock& s) -{ - switch(s.transaction_state) { - case RTM_real_reader: { - s.transaction_state = RTM_real_writer; - bool no_release = s.my_scoped_lock.upgrade_to_writer(); - __TBB_ASSERT(!w_flag, "After upgrade_to_writer, w_flag already true"); - w_flag = true; - return no_release; - } - case RTM_transacting_reader: -#if !__TBB_RW_MUTEX_DELAY_TEST - if(this->state) { // add spin_rw_mutex to read-set. - // Real reader or writer holds the lock; so commit the read and re-acquire for write. - internal_release(s); - internal_acquire_writer(s); - return false; - } else -#endif - { - s.transaction_state = RTM_transacting_writer; - return true; - } - default: - __TBB_ASSERT(false, "Invalid state for upgrade"); - return false; - } -} - -//! Downgrade writer to a reader. -bool x86_rtm_rw_mutex::internal_downgrade(x86_rtm_rw_mutex::scoped_lock& s) { - switch(s.transaction_state) { - case RTM_real_writer: - s.transaction_state = RTM_real_reader; - __TBB_ASSERT(w_flag, "Before downgrade_to_reader w_flag not true"); - w_flag = false; - return s.my_scoped_lock.downgrade_to_reader(); - case RTM_transacting_writer: -#if __TBB_RW_MUTEX_DELAY_TEST - if(this->state) { // a reader or writer has acquired mutex for real. - __TBB_machine_transaction_conflict_abort(); - } -#endif - s.transaction_state = RTM_transacting_reader; - return true; - default: - __TBB_ASSERT(false, "Invalid state for downgrade"); - return false; - } -} - -//! Try to acquire write lock on the given mutex. -// There may be reader(s) which acquired the spin_rw_mutex, as well as possibly -// transactional reader(s). If this is the case, the acquire will fail, and assigning -// w_flag will kill the transactors. So we only assign w_flag if we have successfully -// acquired the lock. -bool x86_rtm_rw_mutex::internal_try_acquire_writer(x86_rtm_rw_mutex::scoped_lock& s) -{ - internal_acquire_writer(s, /*only_speculate=*/true); - if(s.transaction_state == RTM_transacting_writer) { - return true; - } - __TBB_ASSERT(s.transaction_state == RTM_not_in_mutex, "Trying to acquire writer which is already allocated"); - // transacting write acquire failed. try_acquire the real mutex - bool result = s.my_scoped_lock.try_acquire(*this, true); - if(result) { - // only shoot down readers if we're not transacting ourselves - __TBB_ASSERT(!w_flag, "After try_acquire_writer, w_flag already true"); - w_flag = true; - s.transaction_state = RTM_real_writer; - } - return result; -} - -void x86_rtm_rw_mutex::internal_construct() { - ITT_SYNC_CREATE(this, _T("tbb::x86_rtm_rw_mutex"), _T("")); -} - -} // namespace internal -} // namespace interface8 -} // namespace tbb - -#endif /* __TBB_TSX_AVAILABLE */ diff --git a/src/tbb/src/tbbbind/CMakeLists.txt b/src/tbb/src/tbbbind/CMakeLists.txt new file mode 100644 index 000000000..b5e9261ab --- /dev/null +++ b/src/tbb/src/tbbbind/CMakeLists.txt @@ -0,0 +1,103 @@ +# Copyright (c) 2020-2023 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set(CMAKE_SKIP_BUILD_RPATH TRUE) + +function(tbbbind_build TBBBIND_NAME REQUIRED_HWLOC_TARGET) + if (NOT TARGET ${REQUIRED_HWLOC_TARGET}) + message(STATUS "HWLOC target ${REQUIRED_HWLOC_TARGET} doesn't exist." + " The ${TBBBIND_NAME} target cannot be created") + return() + endif() + add_library(${TBBBIND_NAME} tbb_bind.cpp) + + if (WIN32) + target_sources(${TBBBIND_NAME} PRIVATE tbb_bind.rc) + endif() + + add_library(TBB::${TBBBIND_NAME} ALIAS ${TBBBIND_NAME}) + + target_compile_definitions(${TBBBIND_NAME} + PUBLIC + $<$<CONFIG:DEBUG>:TBB_USE_DEBUG> + PRIVATE + __TBBBIND_BUILD) + target_include_directories(${TBBBIND_NAME} + PUBLIC + $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../../include> + $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}> + ${HWLOC_INCLUDE_DIRS} # pkg-config defined + ) + + target_compile_options(${TBBBIND_NAME} + PRIVATE + ${TBB_CXX_STD_FLAG} # TODO: consider making it PUBLIC. + ${TBB_MMD_FLAG} + ${TBB_DSE_FLAG} + ${TBB_WARNING_LEVEL} + ${TBB_LIB_COMPILE_FLAGS} + ${TBB_COMMON_COMPILE_FLAGS} + ) + + # Avoid use of target_link_libraries here as it changes /DEF option to \DEF on Windows. + set_target_properties(${TBBBIND_NAME} PROPERTIES + DEFINE_SYMBOL "" + ) + + tbb_handle_ipo(${TBBBIND_NAME}) + + if (TBB_DEF_FILE_PREFIX) # If there's no prefix, assume we're using export directives + set_target_properties(${TBBBIND_NAME} PROPERTIES + LINK_FLAGS "${TBB_LINK_DEF_FILE_FLAG}\"${CMAKE_CURRENT_SOURCE_DIR}/def/${TBB_DEF_FILE_PREFIX}-tbbbind.def\"" + LINK_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/def/${TBB_DEF_FILE_PREFIX}-tbbbind.def" + ) + endif() + + # Prefer using target_link_options instead of target_link_libraries to specify link options because + # target_link_libraries may incorrectly handle some options (on Windows, for example). + if (COMMAND target_link_options) + target_link_options(${TBBBIND_NAME} + PRIVATE + ${TBB_LIB_LINK_FLAGS} + ${TBB_COMMON_LINK_FLAGS} + ) + else() + target_link_libraries(${TBBBIND_NAME} + PRIVATE + ${TBB_LIB_LINK_FLAGS} + ${TBB_COMMON_LINK_FLAGS} + ) + endif() + + target_link_libraries(${TBBBIND_NAME} + PUBLIC + ${REQUIRED_HWLOC_TARGET} + PRIVATE + ${TBB_LIB_LINK_LIBS} + ${TBB_COMMON_LINK_LIBS} + ) + + tbb_install_target(${TBBBIND_NAME}) + +endfunction() + +if (NOT DEFINED HWLOC_TARGET_EXPLICITLY_DEFINED AND TARGET PkgConfig::HWLOC) + message(STATUS "The ${TBBBIND_LIBRARY_NAME} target will be configured using the HWLOC ${HWLOC_VERSION}") + tbbbind_build(${TBBBIND_LIBRARY_NAME} PkgConfig::HWLOC) +else() + tbbbind_build(tbbbind HWLOC::hwloc_1_11) + tbbbind_build(tbbbind_2_0 HWLOC::hwloc_2 ) + tbbbind_build(tbbbind_2_5 HWLOC::hwloc_2_5 ) +endif() + diff --git a/src/tbb/src/tbb/win64-gcc-tbb-export.def b/src/tbb/src/tbbbind/def/lin32-tbbbind.def similarity index 64% rename from src/tbb/src/tbb/win64-gcc-tbb-export.def rename to src/tbb/src/tbbbind/def/lin32-tbbbind.def index 6159cf85f..221dc8467 100644 --- a/src/tbb/src/tbb/win64-gcc-tbb-export.def +++ b/src/tbb/src/tbbbind/def/lin32-tbbbind.def @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2019-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,33 +12,15 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ { global: - -#define __TBB_SYMBOL( sym ) sym; -#include "win64-gcc-tbb-export.lst" - -local: - -/* TBB symbols */ -*3tbb*; -*__TBB*; - -/* Intel Compiler (libirc) symbols */ -__intel_*; -_intel_*; -get_msg_buf; -get_text_buf; -message_catalog; -print_buf; -irc__get_msg; -irc__print; - +__TBB_internal_initialize_system_topology; +__TBB_internal_apply_affinity; +__TBB_internal_restore_affinity; +__TBB_internal_allocate_binding_handler; +__TBB_internal_deallocate_binding_handler; +__TBB_internal_get_default_concurrency; +__TBB_internal_destroy_system_topology; }; - diff --git a/src/tbb/src/tbb/lin64-tbb-export.def b/src/tbb/src/tbbbind/def/lin64-tbbbind.def similarity index 63% rename from src/tbb/src/tbb/lin64-tbb-export.def rename to src/tbb/src/tbbbind/def/lin64-tbbbind.def index 1f3cac3ae..221dc8467 100644 --- a/src/tbb/src/tbb/lin64-tbb-export.def +++ b/src/tbb/src/tbbbind/def/lin64-tbbbind.def @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2019-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,35 +12,15 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ { global: - -#define __TBB_SYMBOL( sym ) sym; -#include "lin64-tbb-export.lst" - -local: - -/* TBB symbols */ -*3tbb*; -*__TBB*; - -/* ITT symbols */ -__itt_*; - -/* Intel Compiler (libirc) symbols */ -__intel_*; -_intel_*; -get_msg_buf; -get_text_buf; -message_catalog; -print_buf; -irc__get_msg; -irc__print; - +__TBB_internal_initialize_system_topology; +__TBB_internal_apply_affinity; +__TBB_internal_restore_affinity; +__TBB_internal_allocate_binding_handler; +__TBB_internal_deallocate_binding_handler; +__TBB_internal_get_default_concurrency; +__TBB_internal_destroy_system_topology; }; diff --git a/src/tbb/src/tbbbind/def/mac64-tbbbind.def b/src/tbb/src/tbbbind/def/mac64-tbbbind.def new file mode 100755 index 000000000..be72bcf9a --- /dev/null +++ b/src/tbb/src/tbbbind/def/mac64-tbbbind.def @@ -0,0 +1,18 @@ +# Copyright (c) 2023 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +___TBB_internal_initialize_system_topology +___TBB_internal_get_default_concurrency +___TBB_internal_destroy_system_topology + diff --git a/src/tbb/src/rml/server/win64-rml-export.def b/src/tbb/src/tbbbind/def/win32-tbbbind.def similarity index 64% rename from src/tbb/src/rml/server/win64-rml-export.def rename to src/tbb/src/tbbbind/def/win32-tbbbind.def index 2a550d0b5..be4f714ed 100644 --- a/src/tbb/src/rml/server/win64-rml-export.def +++ b/src/tbb/src/tbbbind/def/win32-tbbbind.def @@ -1,4 +1,4 @@ -; Copyright (c) 2005-2017 Intel Corporation +; Copyright (c) 2019-2021 Intel Corporation ; ; Licensed under the Apache License, Version 2.0 (the "License"); ; you may not use this file except in compliance with the License. @@ -11,17 +11,13 @@ ; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ; See the License for the specific language governing permissions and ; limitations under the License. -; -; -; -; EXPORTS -__RML_open_factory -__RML_close_factory -__TBB_make_rml_server -__KMP_make_rml_server -__TBB_call_with_my_server_info -__KMP_call_with_my_server_info - +__TBB_internal_initialize_system_topology +__TBB_internal_apply_affinity +__TBB_internal_restore_affinity +__TBB_internal_allocate_binding_handler +__TBB_internal_deallocate_binding_handler +__TBB_internal_get_default_concurrency +__TBB_internal_destroy_system_topology diff --git a/src/tbb/src/tbb/win64-tbb-export.def b/src/tbb/src/tbbbind/def/win64-tbbbind.def similarity index 64% rename from src/tbb/src/tbb/win64-tbb-export.def rename to src/tbb/src/tbbbind/def/win64-tbbbind.def index d3f3d1a57..be4f714ed 100644 --- a/src/tbb/src/tbb/win64-tbb-export.def +++ b/src/tbb/src/tbbbind/def/win64-tbbbind.def @@ -1,4 +1,4 @@ -; Copyright (c) 2005-2017 Intel Corporation +; Copyright (c) 2019-2021 Intel Corporation ; ; Licensed under the Apache License, Version 2.0 (the "License"); ; you may not use this file except in compliance with the License. @@ -11,16 +11,13 @@ ; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ; See the License for the specific language governing permissions and ; limitations under the License. -; -; -; -; - -; This file is organized with a section for each .cpp file. -; Each of these sections is in alphabetical order. EXPORTS -#define __TBB_SYMBOL( sym ) sym -#include "win64-tbb-export.lst" - +__TBB_internal_initialize_system_topology +__TBB_internal_apply_affinity +__TBB_internal_restore_affinity +__TBB_internal_allocate_binding_handler +__TBB_internal_deallocate_binding_handler +__TBB_internal_get_default_concurrency +__TBB_internal_destroy_system_topology diff --git a/src/tbb/src/tbbbind/tbb_bind.cpp b/src/tbb/src/tbbbind/tbb_bind.cpp new file mode 100644 index 000000000..a15a37ac9 --- /dev/null +++ b/src/tbb/src/tbbbind/tbb_bind.cpp @@ -0,0 +1,541 @@ +/* + Copyright (c) 2019-2024 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include <vector> +#include <mutex> + +#include "../tbb/assert_impl.h" // Out-of-line TBB assertion handling routines are instantiated here. +#include "oneapi/tbb/detail/_assert.h" +#include "oneapi/tbb/detail/_config.h" + +#if _MSC_VER && !__INTEL_COMPILER && !__clang__ +// #pragma warning( push ) +// #pragma warning( disable : 4100 ) +#elif _MSC_VER && __clang__ +// #pragma GCC diagnostic push +// #pragma GCC diagnostic ignored "-Wunused-parameter" +#endif +#include <hwloc.h> +#if _MSC_VER && !__INTEL_COMPILER && !__clang__ +// #pragma warning( pop ) +#elif _MSC_VER && __clang__ +// #pragma GCC diagnostic pop +#endif + +#define __TBBBIND_HWLOC_HYBRID_CPUS_INTERFACES_PRESENT (HWLOC_API_VERSION >= 0x20400) +#define __TBBBIND_HWLOC_TOPOLOGY_FLAG_RESTRICT_TO_CPUBINDING_PRESENT (HWLOC_API_VERSION >= 0x20500) + +// Most of hwloc calls returns negative exit code on error. +// This macro tracks error codes that are returned from the hwloc interfaces. +#define assertion_hwloc_wrapper(command, ...) \ + __TBB_ASSERT_EX( (command(__VA_ARGS__)) >= 0, "Error occurred during call to hwloc API."); + +namespace tbb { +namespace detail { +namespace r1 { + +//------------------------------------------------------------------------ +// Information about the machine's hardware TBB is happen to work on +//------------------------------------------------------------------------ +class system_topology { + friend class binding_handler; + + // Common topology members + hwloc_topology_t topology{nullptr}; + hwloc_cpuset_t process_cpu_affinity_mask{nullptr}; + hwloc_nodeset_t process_node_affinity_mask{nullptr}; + std::size_t number_of_processors_groups{1}; + + // NUMA API related topology members + std::vector<hwloc_cpuset_t> numa_affinity_masks_list{}; + std::vector<int> numa_indexes_list{}; + int numa_nodes_count{0}; + + // Hybrid CPUs API related topology members + std::vector<hwloc_cpuset_t> core_types_affinity_masks_list{}; + std::vector<int> core_types_indexes_list{}; + + enum init_stages { uninitialized, + started, + topology_allocated, + topology_loaded, + topology_parsed } initialization_state; + + // Binding threads that locate in another Windows Processor groups + // is allowed only if machine topology contains several Windows Processors groups + // and process affinity mask wasn't limited manually (affinity mask cannot violates + // processors group boundaries). + bool intergroup_binding_allowed(std::size_t groups_num) { return groups_num > 1; } + +private: + void topology_initialization(std::size_t groups_num) { + initialization_state = started; + + // Parse topology + if ( hwloc_topology_init( &topology ) == 0 ) { + initialization_state = topology_allocated; +#if __TBBBIND_HWLOC_TOPOLOGY_FLAG_RESTRICT_TO_CPUBINDING_PRESENT + unsigned long flags = 0; + if (groups_num > 1) { + // HWLOC x86 backend might interfere with process affinity mask on + // Windows systems with multiple processor groups. + flags = HWLOC_TOPOLOGY_FLAG_DONT_CHANGE_BINDING; + } else { + flags = HWLOC_TOPOLOGY_FLAG_IS_THISSYSTEM | HWLOC_TOPOLOGY_FLAG_RESTRICT_TO_CPUBINDING; + } + if (hwloc_topology_set_flags(topology, flags) != 0) { + return; + } +#endif + if ( hwloc_topology_load( topology ) == 0 ) { + initialization_state = topology_loaded; + } + } + if ( initialization_state != topology_loaded ) + return; + +#if __TBB_CPUBIND_PRESENT + // Getting process affinity mask + if ( intergroup_binding_allowed(groups_num) ) { + process_cpu_affinity_mask = hwloc_bitmap_dup(hwloc_topology_get_complete_cpuset (topology)); + process_node_affinity_mask = hwloc_bitmap_dup(hwloc_topology_get_complete_nodeset(topology)); + } else { + process_cpu_affinity_mask = hwloc_bitmap_alloc(); + process_node_affinity_mask = hwloc_bitmap_alloc(); + + assertion_hwloc_wrapper(hwloc_get_cpubind, topology, process_cpu_affinity_mask, 0); + hwloc_cpuset_to_nodeset(topology, process_cpu_affinity_mask, process_node_affinity_mask); + } +#else + process_cpu_affinity_mask = hwloc_bitmap_dup(hwloc_topology_get_complete_cpuset (topology)); + process_node_affinity_mask = hwloc_bitmap_dup(hwloc_topology_get_complete_nodeset(topology)); +#endif + + number_of_processors_groups = groups_num; + } + + void numa_topology_parsing() { + // Fill parameters with stubs if topology parsing is broken. + if ( initialization_state != topology_loaded ) { + numa_nodes_count = 1; + numa_indexes_list.push_back(-1); + return; + } + + // If system contains no NUMA nodes, HWLOC 1.11 returns an infinitely filled bitmap. + // hwloc_bitmap_weight() returns negative value for such bitmaps, so we use this check + // to change way of topology initialization. + numa_nodes_count = hwloc_bitmap_weight(process_node_affinity_mask); + if (numa_nodes_count <= 0) { + // numa_nodes_count may be empty if the process affinity mask is empty too (invalid case) + // or if some internal HWLOC error occurred. + // So we place -1 as index in this case. + numa_indexes_list.push_back(numa_nodes_count == 0 ? -1 : 0); + numa_nodes_count = 1; + + numa_affinity_masks_list.push_back(hwloc_bitmap_dup(process_cpu_affinity_mask)); + } else { + // Get NUMA logical indexes list + unsigned counter = 0; + int i = 0; + int max_numa_index = -1; + numa_indexes_list.resize(numa_nodes_count); + hwloc_obj_t node_buffer; + hwloc_bitmap_foreach_begin(i, process_node_affinity_mask) { + node_buffer = hwloc_get_numanode_obj_by_os_index(topology, i); + numa_indexes_list[counter] = static_cast<int>(node_buffer->logical_index); + + if ( numa_indexes_list[counter] > max_numa_index ) { + max_numa_index = numa_indexes_list[counter]; + } + + counter++; + } hwloc_bitmap_foreach_end(); + __TBB_ASSERT(max_numa_index >= 0, "Maximal NUMA index must not be negative"); + + // Fill concurrency and affinity masks lists + numa_affinity_masks_list.resize(max_numa_index + 1); + int index = 0; + hwloc_bitmap_foreach_begin(i, process_node_affinity_mask) { + node_buffer = hwloc_get_numanode_obj_by_os_index(topology, i); + index = static_cast<int>(node_buffer->logical_index); + + hwloc_cpuset_t& current_mask = numa_affinity_masks_list[index]; + current_mask = hwloc_bitmap_dup(node_buffer->cpuset); + + hwloc_bitmap_and(current_mask, current_mask, process_cpu_affinity_mask); + __TBB_ASSERT(!hwloc_bitmap_iszero(current_mask), "hwloc detected unavailable NUMA node"); + } hwloc_bitmap_foreach_end(); + } + } + + void core_types_topology_parsing() { + // Fill parameters with stubs if topology parsing is broken. + if ( initialization_state != topology_loaded ) { + core_types_indexes_list.push_back(-1); + return; + } +#if __TBBBIND_HWLOC_HYBRID_CPUS_INTERFACES_PRESENT + __TBB_ASSERT(hwloc_get_api_version() >= 0x20400, "Hybrid CPUs support interfaces required HWLOC >= 2.4"); + // Parsing the hybrid CPU topology + int core_types_number = hwloc_cpukinds_get_nr(topology, 0); + bool core_types_parsing_broken = core_types_number <= 0; + if (!core_types_parsing_broken) { + core_types_affinity_masks_list.resize(core_types_number); + int efficiency{-1}; + + for (int core_type = 0; core_type < core_types_number; ++core_type) { + hwloc_cpuset_t& current_mask = core_types_affinity_masks_list[core_type]; + current_mask = hwloc_bitmap_alloc(); + + if (!hwloc_cpukinds_get_info(topology, core_type, current_mask, &efficiency, nullptr, nullptr, 0) + && efficiency >= 0 + ) { + hwloc_bitmap_and(current_mask, current_mask, process_cpu_affinity_mask); + + if (hwloc_bitmap_weight(current_mask) > 0) { + core_types_indexes_list.push_back(core_type); + } + __TBB_ASSERT(hwloc_bitmap_weight(current_mask) >= 0, "Infinivitely filled core type mask"); + } else { + core_types_parsing_broken = true; + break; + } + } + } +#else /*!__TBBBIND_HWLOC_HYBRID_CPUS_INTERFACES_PRESENT*/ + bool core_types_parsing_broken{true}; +#endif /*__TBBBIND_HWLOC_HYBRID_CPUS_INTERFACES_PRESENT*/ + + if (core_types_parsing_broken) { + for (auto& core_type_mask : core_types_affinity_masks_list) { + hwloc_bitmap_free(core_type_mask); + } + core_types_affinity_masks_list.resize(1); + core_types_indexes_list.resize(1); + + core_types_affinity_masks_list[0] = hwloc_bitmap_dup(process_cpu_affinity_mask); + core_types_indexes_list[0] = -1; + } + } + + void enforce_hwloc_2_5_runtime_linkage() { + // Without the call of this function HWLOC 2.4 can be successfully loaded during the tbbbind_2_5 loading. + // It is possible since tbbbind_2_5 don't use any new entry points that were introduced in HWLOC 2.5 + // But tbbbind_2_5 compiles with HWLOC 2.5 header, therefore such situation requires binary forward compatibility + // which are not guaranteed by the HWLOC library. To enforce linkage tbbbind_2_5 only with HWLOC >= 2.5 version + // this function calls the interface that is available in the HWLOC 2.5 only. +#if HWLOC_API_VERSION >= 0x20500 + auto some_core = hwloc_get_next_obj_by_type(topology, HWLOC_OBJ_CORE, nullptr); + hwloc_get_obj_with_same_locality(topology, some_core, HWLOC_OBJ_CORE, nullptr, nullptr, 0); +#endif + } + + + void initialize( std::size_t groups_num ) { + if ( initialization_state != uninitialized ) + return; + + topology_initialization(groups_num); + numa_topology_parsing(); + core_types_topology_parsing(); + + enforce_hwloc_2_5_runtime_linkage(); + + if (initialization_state == topology_loaded) + initialization_state = topology_parsed; + } + + static system_topology* instance_ptr; +public: + typedef hwloc_cpuset_t affinity_mask; + typedef hwloc_const_cpuset_t const_affinity_mask; + + bool is_topology_parsed() { return initialization_state == topology_parsed; } + + static void construct( std::size_t groups_num ) { + if (instance_ptr == nullptr) { + instance_ptr = new system_topology(); + instance_ptr->initialize(groups_num); + } + } + + static system_topology& instance() { + __TBB_ASSERT(instance_ptr != nullptr, "Getting instance of non-constructed topology"); + return *instance_ptr; + } + + static void destroy() { + __TBB_ASSERT(instance_ptr != nullptr, "Destroying non-constructed topology"); + delete instance_ptr; + } + + ~system_topology() { + if ( is_topology_parsed() ) { + for (auto& numa_node_mask : numa_affinity_masks_list) { + hwloc_bitmap_free(numa_node_mask); + } + + for (auto& core_type_mask : core_types_affinity_masks_list) { + hwloc_bitmap_free(core_type_mask); + } + + hwloc_bitmap_free(process_node_affinity_mask); + hwloc_bitmap_free(process_cpu_affinity_mask); + } + + if ( initialization_state >= topology_allocated ) { + hwloc_topology_destroy(topology); + } + + initialization_state = uninitialized; + } + + void fill_topology_information( + int& _numa_nodes_count, int*& _numa_indexes_list, + int& _core_types_count, int*& _core_types_indexes_list + ) { + __TBB_ASSERT(is_topology_parsed(), "Trying to get access to uninitialized system_topology"); + _numa_nodes_count = numa_nodes_count; + _numa_indexes_list = numa_indexes_list.data(); + + _core_types_count = (int)core_types_indexes_list.size(); + _core_types_indexes_list = core_types_indexes_list.data(); + } + + void fill_constraints_affinity_mask(affinity_mask input_mask, int numa_node_index, int core_type_index, int max_threads_per_core) { + __TBB_ASSERT(is_topology_parsed(), "Trying to get access to uninitialized system_topology"); + __TBB_ASSERT(numa_node_index < (int)numa_affinity_masks_list.size(), "Wrong NUMA node id"); + __TBB_ASSERT(core_type_index < (int)core_types_affinity_masks_list.size(), "Wrong core type id"); + __TBB_ASSERT(max_threads_per_core == -1 || max_threads_per_core > 0, "Wrong max_threads_per_core"); + + hwloc_cpuset_t constraints_mask = hwloc_bitmap_alloc(); + hwloc_cpuset_t core_mask = hwloc_bitmap_alloc(); + + hwloc_bitmap_copy(constraints_mask, process_cpu_affinity_mask); + if (numa_node_index >= 0) { + hwloc_bitmap_and(constraints_mask, constraints_mask, numa_affinity_masks_list[numa_node_index]); + } + if (core_type_index >= 0) { + hwloc_bitmap_and(constraints_mask, constraints_mask, core_types_affinity_masks_list[core_type_index]); + } + if (max_threads_per_core > 0) { + // clear input mask + hwloc_bitmap_zero(input_mask); + + hwloc_obj_t current_core = nullptr; + while ((current_core = hwloc_get_next_obj_by_type(topology, HWLOC_OBJ_CORE, current_core)) != nullptr) { + hwloc_bitmap_and(core_mask, constraints_mask, current_core->cpuset); + + // fit the core mask to required bits number + int current_threads_per_core = 0; + for (int id = hwloc_bitmap_first(core_mask); id != -1; id = hwloc_bitmap_next(core_mask, id)) { + if (++current_threads_per_core > max_threads_per_core) { + hwloc_bitmap_clr(core_mask, id); + } + } + + hwloc_bitmap_or(input_mask, input_mask, core_mask); + } + } else { + hwloc_bitmap_copy(input_mask, constraints_mask); + } + + hwloc_bitmap_free(core_mask); + hwloc_bitmap_free(constraints_mask); + } + + void fit_num_threads_per_core(affinity_mask result_mask, affinity_mask current_mask, affinity_mask constraints_mask) { + hwloc_bitmap_zero(result_mask); + hwloc_obj_t current_core = nullptr; + while ((current_core = hwloc_get_next_obj_by_type(topology, HWLOC_OBJ_CORE, current_core)) != nullptr) { + if (hwloc_bitmap_intersects(current_mask, current_core->cpuset)) { + hwloc_bitmap_or(result_mask, result_mask, current_core->cpuset); + } + } + hwloc_bitmap_and(result_mask, result_mask, constraints_mask); + } + + int get_default_concurrency(int numa_node_index, int core_type_index, int max_threads_per_core) { + __TBB_ASSERT(is_topology_parsed(), "Trying to get access to uninitialized system_topology"); + + hwloc_cpuset_t constraints_mask = hwloc_bitmap_alloc(); + fill_constraints_affinity_mask(constraints_mask, numa_node_index, core_type_index, max_threads_per_core); + + int default_concurrency = hwloc_bitmap_weight(constraints_mask); + hwloc_bitmap_free(constraints_mask); + return default_concurrency; + } + + affinity_mask allocate_process_affinity_mask() { + __TBB_ASSERT(is_topology_parsed(), "Trying to get access to uninitialized system_topology"); + return hwloc_bitmap_dup(process_cpu_affinity_mask); + } + + void free_affinity_mask( affinity_mask mask_to_free ) { + hwloc_bitmap_free(mask_to_free); // If bitmap is nullptr, no operation is performed. + } + + void store_current_affinity_mask( affinity_mask current_mask ) { + assertion_hwloc_wrapper(hwloc_get_cpubind, topology, current_mask, HWLOC_CPUBIND_THREAD); + + hwloc_bitmap_and(current_mask, current_mask, process_cpu_affinity_mask); + __TBB_ASSERT(!hwloc_bitmap_iszero(current_mask), + "Current affinity mask must intersects with process affinity mask"); + } + + void set_affinity_mask( const_affinity_mask mask ) { + if (hwloc_bitmap_weight(mask) > 0) { + assertion_hwloc_wrapper(hwloc_set_cpubind, topology, mask, HWLOC_CPUBIND_THREAD); + } + } +}; + +system_topology* system_topology::instance_ptr{nullptr}; + +class binding_handler { + // Following vector saves thread affinity mask on scheduler entry to return it to this thread + // on scheduler exit. + typedef std::vector<system_topology::affinity_mask> affinity_masks_container; + affinity_masks_container affinity_backup; + system_topology::affinity_mask handler_affinity_mask; + +#ifdef _WIN32 + affinity_masks_container affinity_buffer; + int my_numa_node_id; + int my_core_type_id; + int my_max_threads_per_core; +#endif + +public: + binding_handler( std::size_t size, int numa_node_id, int core_type_id, int max_threads_per_core ) + : affinity_backup(size) +#ifdef _WIN32 + , affinity_buffer(size) + , my_numa_node_id(numa_node_id) + , my_core_type_id(core_type_id) + , my_max_threads_per_core(max_threads_per_core) +#endif + { + for (std::size_t i = 0; i < size; ++i) { + affinity_backup[i] = system_topology::instance().allocate_process_affinity_mask(); +#ifdef _WIN32 + affinity_buffer[i] = system_topology::instance().allocate_process_affinity_mask(); +#endif + } + handler_affinity_mask = system_topology::instance().allocate_process_affinity_mask(); + system_topology::instance().fill_constraints_affinity_mask + (handler_affinity_mask, numa_node_id, core_type_id, max_threads_per_core); + } + + ~binding_handler() { + for (std::size_t i = 0; i < affinity_backup.size(); ++i) { + system_topology::instance().free_affinity_mask(affinity_backup[i]); +#ifdef _WIN32 + system_topology::instance().free_affinity_mask(affinity_buffer[i]); +#endif + } + system_topology::instance().free_affinity_mask(handler_affinity_mask); + } + + void apply_affinity( unsigned slot_num ) { + auto& topology = system_topology::instance(); + __TBB_ASSERT(slot_num < affinity_backup.size(), + "The slot number is greater than the number of slots in the arena"); + __TBB_ASSERT(topology.is_topology_parsed(), + "Trying to get access to uninitialized system_topology"); + + topology.store_current_affinity_mask(affinity_backup[slot_num]); + +#ifdef _WIN32 + // TBBBind supports only systems where NUMA nodes and core types do not cross the border + // between several processor groups. So if a certain NUMA node or core type constraint + // specified, then the constraints affinity mask will not cross the processor groups' border. + + // But if we have constraint based only on the max_threads_per_core setting, then the + // constraints affinity mask does may cross the border between several processor groups + // on machines with more then 64 hardware threads. That is why we need to use the special + // function, which regulates the number of threads in the current threads mask. + if (topology.number_of_processors_groups > 1 && my_max_threads_per_core != -1 && + (my_numa_node_id == -1 || topology.numa_indexes_list.size() == 1) && + (my_core_type_id == -1 || topology.core_types_indexes_list.size() == 1) + ) { + topology.fit_num_threads_per_core(affinity_buffer[slot_num], affinity_backup[slot_num], handler_affinity_mask); + topology.set_affinity_mask(affinity_buffer[slot_num]); + return; + } +#endif + topology.set_affinity_mask(handler_affinity_mask); + } + + void restore_previous_affinity_mask( unsigned slot_num ) { + auto& topology = system_topology::instance(); + __TBB_ASSERT(topology.is_topology_parsed(), + "Trying to get access to uninitialized system_topology"); + topology.set_affinity_mask(affinity_backup[slot_num]); + }; + +}; + +extern "C" { // exported to TBB interfaces + +TBBBIND_EXPORT void __TBB_internal_initialize_system_topology( + std::size_t groups_num, + int& numa_nodes_count, int*& numa_indexes_list, + int& core_types_count, int*& core_types_indexes_list +) { + system_topology::construct(groups_num); + system_topology::instance().fill_topology_information( + numa_nodes_count, numa_indexes_list, + core_types_count, core_types_indexes_list + ); +} + +TBBBIND_EXPORT binding_handler* __TBB_internal_allocate_binding_handler(int number_of_slots, int numa_id, int core_type_id, int max_threads_per_core) { + __TBB_ASSERT(number_of_slots > 0, "Trying to create numa handler for 0 threads."); + return new binding_handler(number_of_slots, numa_id, core_type_id, max_threads_per_core); +} + +TBBBIND_EXPORT void __TBB_internal_deallocate_binding_handler(binding_handler* handler_ptr) { + __TBB_ASSERT(handler_ptr != nullptr, "Trying to deallocate nullptr pointer."); + delete handler_ptr; +} + +TBBBIND_EXPORT void __TBB_internal_apply_affinity(binding_handler* handler_ptr, int slot_num) { + __TBB_ASSERT(handler_ptr != nullptr, "Trying to get access to uninitialized metadata."); + handler_ptr->apply_affinity(slot_num); +} + +TBBBIND_EXPORT void __TBB_internal_restore_affinity(binding_handler* handler_ptr, int slot_num) { + __TBB_ASSERT(handler_ptr != nullptr, "Trying to get access to uninitialized metadata."); + handler_ptr->restore_previous_affinity_mask(slot_num); +} + +TBBBIND_EXPORT int __TBB_internal_get_default_concurrency(int numa_id, int core_type_id, int max_threads_per_core) { + return system_topology::instance().get_default_concurrency(numa_id, core_type_id, max_threads_per_core); +} + +void __TBB_internal_destroy_system_topology() { + return system_topology::destroy(); +} + +} // extern "C" + +} // namespace r1 +} // namespace detail +} // namespace tbb + +#undef assertion_hwloc_wrapper diff --git a/src/tbb/src/rml/server/irml.rc b/src/tbb/src/tbbbind/tbb_bind.rc similarity index 51% rename from src/tbb/src/rml/server/irml.rc rename to src/tbb/src/tbbbind/tbb_bind.rc index 73a84f83e..2d2b806e2 100644 --- a/src/tbb/src/rml/server/irml.rc +++ b/src/tbb/src/tbbbind/tbb_bind.rc @@ -1,4 +1,4 @@ -// Copyright (c) 2005-2017 Intel Corporation +// Copyright (c) 2005-2024 Intel Corporation // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -11,55 +11,28 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -// -// -// -// - -// Microsoft Visual C++ generated resource script. -// -#ifdef APSTUDIO_INVOKED -#ifndef APSTUDIO_READONLY_SYMBOLS -#define _APS_NO_MFC 1 -#define _APS_NEXT_RESOURCE_VALUE 102 -#define _APS_NEXT_COMMAND_VALUE 40001 -#define _APS_NEXT_CONTROL_VALUE 1001 -#define _APS_NEXT_SYMED_VALUE 101 -#endif -#endif -#define APSTUDIO_READONLY_SYMBOLS ///////////////////////////////////////////////////////////////////////////// // -// Generated from the TEXTINCLUDE 2 resource. +// Includes // #include <winresrc.h> -#define ENDL "\r\n" -#include "tbb/tbb_version.h" - -///////////////////////////////////////////////////////////////////////////// -#undef APSTUDIO_READONLY_SYMBOLS +#include "../../include/oneapi/tbb/version.h" ///////////////////////////////////////////////////////////////////////////// // Neutral resources -#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_NEU) #ifdef _WIN32 LANGUAGE LANG_NEUTRAL, SUBLANG_NEUTRAL #pragma code_page(1252) #endif //_WIN32 -///////////////////////////////////////////////////////////////////////////// -// manifest integration -#ifdef TBB_MANIFEST -#include "winuser.h" -2 RT_MANIFEST tbbmanifest.exe.manifest -#endif - ///////////////////////////////////////////////////////////////////////////// // // Version // +#define TBB_VERNUMBERS TBB_VERSION_MAJOR,TBB_VERSION_MINOR,TBB_VERSION_PATCH +#define TBB_VERSION TBB_VERSION_STRING VS_VERSION_INFO VERSIONINFO FILEVERSION TBB_VERNUMBERS @@ -79,16 +52,16 @@ BEGIN BLOCK "000004b0" BEGIN VALUE "CompanyName", "Intel Corporation\0" - VALUE "FileDescription", "Intel(R) Threading Building Blocks resource manager library\0" + VALUE "FileDescription", "oneAPI Threading Building Blocks (oneTBB) library\0" VALUE "FileVersion", TBB_VERSION "\0" - VALUE "LegalCopyright", "Copyright 2005-2017 Intel Corporation. All Rights Reserved.\0" + VALUE "LegalCopyright", "Copyright 2005-2024 Intel Corporation. All Rights Reserved.\0" VALUE "LegalTrademarks", "\0" #ifndef TBB_USE_DEBUG - VALUE "OriginalFilename", "irml.dll\0" + VALUE "OriginalFilename", "tbbbind.dll\0" #else - VALUE "OriginalFilename", "irml_debug.dll\0" + VALUE "OriginalFilename", "tbbbind_debug.dll\0" #endif - VALUE "ProductName", "Intel(R) Threading Building Blocks for Windows\0" + VALUE "ProductName", "oneAPI Threading Building Blocks (oneTBB)\0" VALUE "ProductVersion", TBB_VERSION "\0" VALUE "PrivateBuild", "\0" VALUE "SpecialBuild", "\0" @@ -99,18 +72,3 @@ BEGIN VALUE "Translation", 0x0, 1200 END END - -#endif // Neutral resources -///////////////////////////////////////////////////////////////////////////// - - -#ifndef APSTUDIO_INVOKED -///////////////////////////////////////////////////////////////////////////// -// -// Generated from the TEXTINCLUDE 3 resource. -// - - -///////////////////////////////////////////////////////////////////////////// -#endif // not APSTUDIO_INVOKED - diff --git a/src/tbb/src/tbbmalloc/CMakeLists.txt b/src/tbb/src/tbbmalloc/CMakeLists.txt new file mode 100644 index 000000000..26c1891d7 --- /dev/null +++ b/src/tbb/src/tbbmalloc/CMakeLists.txt @@ -0,0 +1,121 @@ +# Copyright (c) 2020-2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_library(tbbmalloc + backend.cpp + backref.cpp + frontend.cpp + large_objects.cpp + tbbmalloc.cpp + ../tbb/itt_notify.cpp) + +if (WIN32) + target_sources(tbbmalloc PRIVATE tbbmalloc.rc) +endif() + +add_library(TBB::tbbmalloc ALIAS tbbmalloc) + +target_compile_definitions(tbbmalloc + PUBLIC + $<$<CONFIG:DEBUG>:TBB_USE_DEBUG> + PRIVATE + __TBBMALLOC_BUILD + $<$<NOT:$<BOOL:${BUILD_SHARED_LIBS}>>:__TBB_DYNAMIC_LOAD_ENABLED=0> + $<$<NOT:$<BOOL:${BUILD_SHARED_LIBS}>>:__TBB_SOURCE_DIRECTLY_INCLUDED=1>) + +if (NOT ("${CMAKE_SYSTEM_PROCESSOR}" MATCHES "(armv7-a|aarch64|mips|arm64|riscv)" OR + "${CMAKE_OSX_ARCHITECTURES}" MATCHES "arm64" OR + WINDOWS_STORE OR + TBB_WINDOWS_DRIVER OR + TBB_SANITIZE MATCHES "thread")) + target_compile_definitions(tbbmalloc PRIVATE __TBB_USE_ITT_NOTIFY) +endif() + +target_include_directories(tbbmalloc + PUBLIC + $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../../include> + $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>) + +# TODO: fix warnings +if (MSVC) + # signed unsigned mismatch, declaration hides class member + set(TBB_WARNING_SUPPRESS ${TBB_WARNING_SUPPRESS} /wd4267 /wd4244 /wd4245 /wd4458) +endif() + +target_compile_options(tbbmalloc + PRIVATE + ${TBB_CXX_STD_FLAG} # TODO: consider making it PUBLIC. + ${TBB_MMD_FLAG} + ${TBB_DSE_FLAG} + ${TBB_WARNING_LEVEL} + ${TBB_WARNING_SUPPRESS} + ${TBB_LIB_COMPILE_FLAGS} + ${TBBMALLOC_LIB_COMPILE_FLAGS} + ${TBB_COMMON_COMPILE_FLAGS} +) + +enable_language(C) + +# Avoid use of target_link_libraries here as it changes /DEF option to \DEF on Windows. +set_target_properties(tbbmalloc PROPERTIES + DEFINE_SYMBOL "" + LINKER_LANGUAGE C +) + +tbb_handle_ipo(tbbmalloc) + +if (TBB_DEF_FILE_PREFIX) # If there's no prefix, assume we're using export directives + set_target_properties(tbbmalloc PROPERTIES + LINK_FLAGS "${TBB_LINK_DEF_FILE_FLAG}\"${CMAKE_CURRENT_SOURCE_DIR}/def/${TBB_DEF_FILE_PREFIX}-tbbmalloc.def\"" + LINK_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/def/${TBB_DEF_FILE_PREFIX}-tbbmalloc.def" + ) +endif() + +set(CMAKE_CXX_IMPLICIT_LINK_LIBRARIES "") + +# Prefer using target_link_options instead of target_link_libraries to specify link options because +# target_link_libraries may incorrectly handle some options (on Windows, for example). +if (COMMAND target_link_options) + target_link_options(tbbmalloc + PRIVATE + ${TBB_LIB_LINK_FLAGS} + ${TBB_COMMON_LINK_FLAGS} + ) +else() + target_link_libraries(tbbmalloc + PRIVATE + ${TBB_LIB_LINK_FLAGS} + ${TBB_COMMON_LINK_FLAGS} + ) +endif() + +target_link_libraries(tbbmalloc + PRIVATE + Threads::Threads + ${TBB_LIB_LINK_LIBS} + ${TBB_COMMON_LINK_LIBS} +) + +if(TBB_BUILD_APPLE_FRAMEWORKS) + set_target_properties(tbbmalloc PROPERTIES + FRAMEWORK TRUE + FRAMEWORK_VERSION ${TBBMALLOC_BINARY_VERSION}.${TBB_BINARY_MINOR_VERSION} + XCODE_ATTRIBUTE_PRODUCT_BUNDLE_IDENTIFIER com.intel.tbbmalloc + MACOSX_FRAMEWORK_IDENTIFIER com.intel.tbbmalloc + MACOSX_FRAMEWORK_BUNDLE_VERSION ${TBBMALLOC_BINARY_VERSION}.${TBB_BINARY_MINOR_VERSION} + MACOSX_FRAMEWORK_SHORT_VERSION_STRING ${TBBMALLOC_BINARY_VERSION} + ) +endif() + +tbb_install_target(tbbmalloc) diff --git a/src/tbb/src/tbbmalloc/Customize.h b/src/tbb/src/tbbmalloc/Customize.h index 4fdf06896..fdb616432 100644 --- a/src/tbb/src/tbbmalloc/Customize.h +++ b/src/tbb/src/tbbmalloc/Customize.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2023 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,144 +12,65 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef _TBB_malloc_Customize_H_ #define _TBB_malloc_Customize_H_ // customizing MALLOC_ASSERT macro -#include "tbb/tbb_stddef.h" #define MALLOC_ASSERT(assertion, message) __TBB_ASSERT(assertion, message) +#define MALLOC_ASSERT_EX(assertion, message) __TBB_ASSERT_EX(assertion, message) #ifndef MALLOC_DEBUG #define MALLOC_DEBUG TBB_USE_DEBUG #endif -#include "tbb/tbb_machine.h" +#include "oneapi/tbb/detail/_utils.h" +#include "oneapi/tbb/detail/_assert.h" + +#include "Synchronize.h" -#if DO_ITT_NOTIFY -#include "tbb/itt_notify.h" +#if __TBB_USE_ITT_NOTIFY +#include "../tbb/itt_notify.h" #define MALLOC_ITT_SYNC_PREPARE(pointer) ITT_NOTIFY(sync_prepare, (pointer)) #define MALLOC_ITT_SYNC_ACQUIRED(pointer) ITT_NOTIFY(sync_acquired, (pointer)) #define MALLOC_ITT_SYNC_RELEASING(pointer) ITT_NOTIFY(sync_releasing, (pointer)) #define MALLOC_ITT_SYNC_CANCEL(pointer) ITT_NOTIFY(sync_cancel, (pointer)) #define MALLOC_ITT_FINI_ITTLIB() ITT_FINI_ITTLIB() +#define MALLOC_ITT_RELEASE_RESOURCES() ITT_RELEASE_RESOURCES() #else #define MALLOC_ITT_SYNC_PREPARE(pointer) ((void)0) #define MALLOC_ITT_SYNC_ACQUIRED(pointer) ((void)0) #define MALLOC_ITT_SYNC_RELEASING(pointer) ((void)0) #define MALLOC_ITT_SYNC_CANCEL(pointer) ((void)0) #define MALLOC_ITT_FINI_ITTLIB() ((void)0) +#define MALLOC_ITT_RELEASE_RESOURCES() ((void)0) #endif -//! Stripped down version of spin_mutex. -/** Instances of MallocMutex must be declared in memory that is zero-initialized. - There are no constructors. This is a feature that lets it be - used in situations where the mutex might be used while file-scope constructors - are running. - - There are no methods "acquire" or "release". The scoped_lock must be used - in a strict block-scoped locking pattern. Omitting these methods permitted - further simplification. */ -class MallocMutex : tbb::internal::no_copy { - __TBB_atomic_flag flag; - -public: - class scoped_lock : tbb::internal::no_copy { - MallocMutex& mutex; - bool taken; - public: - scoped_lock( MallocMutex& m ) : mutex(m), taken(true) { __TBB_LockByte(m.flag); } - scoped_lock( MallocMutex& m, bool block, bool *locked ) : mutex(m), taken(false) { - if (block) { - __TBB_LockByte(m.flag); - taken = true; - } else { - taken = __TBB_TryLockByte(m.flag); - } - if (locked) *locked = taken; - } - ~scoped_lock() { - if (taken) __TBB_UnlockByte(mutex.flag); - } - }; - friend class scoped_lock; -}; - -// TODO: use signed/unsigned in atomics more consistently -inline intptr_t AtomicIncrement( volatile intptr_t& counter ) { - return __TBB_FetchAndAddW( &counter, 1 )+1; -} - -inline uintptr_t AtomicAdd( volatile intptr_t& counter, intptr_t value ) { - return __TBB_FetchAndAddW( &counter, value ); -} - -inline intptr_t AtomicCompareExchange( volatile intptr_t& location, intptr_t new_value, intptr_t comparand) { - return __TBB_CompareAndSwapW( &location, new_value, comparand ); -} - -inline uintptr_t AtomicFetchStore(volatile void* location, uintptr_t value) { - return __TBB_FetchAndStoreW(location, value); -} - -inline void AtomicOr(volatile void *operand, uintptr_t addend) { - __TBB_AtomicOR(operand, addend); -} - -inline void AtomicAnd(volatile void *operand, uintptr_t addend) { - __TBB_AtomicAND(operand, addend); -} - -inline intptr_t FencedLoad( const volatile intptr_t &location ) { - return __TBB_load_with_acquire(location); -} - -inline void FencedStore( volatile intptr_t &location, intptr_t value ) { - __TBB_store_with_release(location, value); -} - -inline void SpinWaitWhileEq(const volatile intptr_t &location, const intptr_t value) { - tbb::internal::spin_wait_while_eq(location, value); -} - -class AtomicBackoff { - tbb::internal::atomic_backoff backoff; -public: - AtomicBackoff() {} - void pause() { backoff.pause(); } -}; - -inline void SpinWaitUntilEq(const volatile intptr_t &location, const intptr_t value) { - tbb::internal::spin_wait_until_eq(location, value); -} - inline intptr_t BitScanRev(uintptr_t x) { - return !x? -1 : __TBB_Log2(x); + return x == 0 ? -1 : static_cast<intptr_t>(tbb::detail::log2(x)); } template<typename T> static inline bool isAligned(T* arg, uintptr_t alignment) { - return tbb::internal::is_aligned(arg,alignment); + return tbb::detail::is_aligned(arg,alignment); } static inline bool isPowerOfTwo(uintptr_t arg) { - return tbb::internal::is_power_of_two(arg); + return tbb::detail::is_power_of_two(arg); } static inline bool isPowerOfTwoAtLeast(uintptr_t arg, uintptr_t power2) { - return arg && tbb::internal::is_power_of_two_at_least(arg,power2); + return arg && tbb::detail::is_power_of_two_at_least(arg,power2); } -#define MALLOC_STATIC_ASSERT(condition,msg) __TBB_STATIC_ASSERT(condition,msg) +inline void do_yield() { + tbb::detail::yield(); +} #define USE_DEFAULT_MEMORY_MAPPING 1 // To support malloc replacement -#include "proxy.h" +#include "../tbbmalloc_proxy/proxy.h" #if MALLOC_UNIXLIKE_OVERLOAD_ENABLED #define malloc_proxy __TBB_malloc_proxy @@ -170,62 +91,49 @@ namespace internal { // Need these to work regardless of tools support. namespace tbb { - namespace internal { - - enum notify_type {prepare=0, cancel, acquired, releasing}; - -#if TBB_USE_THREADING_TOOLS - inline void call_itt_notify(notify_type t, void *ptr) { - switch ( t ) { - case prepare: - MALLOC_ITT_SYNC_PREPARE( ptr ); - break; - case cancel: - MALLOC_ITT_SYNC_CANCEL( ptr ); - break; - case acquired: - MALLOC_ITT_SYNC_ACQUIRED( ptr ); - break; - case releasing: - MALLOC_ITT_SYNC_RELEASING( ptr ); - break; - } +namespace detail { +namespace d1 { + + enum notify_type {prepare=0, cancel, acquired, releasing}; + +#if TBB_USE_PROFILING_TOOLS + inline void call_itt_notify(notify_type t, void *ptr) { + // unreferenced formal parameter warning + detail::suppress_unused_warning(ptr); + switch ( t ) { + case prepare: + MALLOC_ITT_SYNC_PREPARE( ptr ); + break; + case cancel: + MALLOC_ITT_SYNC_CANCEL( ptr ); + break; + case acquired: + MALLOC_ITT_SYNC_ACQUIRED( ptr ); + break; + case releasing: + MALLOC_ITT_SYNC_RELEASING( ptr ); + break; } + } #else - inline void call_itt_notify(notify_type /*t*/, void * /*ptr*/) {} -#endif // TBB_USE_THREADING_TOOLS - - template <typename T> - inline void itt_store_word_with_release(T& dst, T src) { -#if TBB_USE_THREADING_TOOLS - call_itt_notify(releasing, &dst); -#endif // TBB_USE_THREADING_TOOLS - FencedStore(*(intptr_t*)&dst, src); - } + inline void call_itt_notify(notify_type /*t*/, void * /*ptr*/) {} +#endif // TBB_USE_PROFILING_TOOLS - template <typename T> - inline T itt_load_word_with_acquire(T& src) { - T result = FencedLoad(*(intptr_t*)&src); -#if TBB_USE_THREADING_TOOLS - call_itt_notify(acquired, &src); -#endif // TBB_USE_THREADING_TOOLS - return result; - - } - } // namespace internal +} // namespace d1 +} // namespace detail } // namespace tbb -#include "tbb/internal/_aggregator_impl.h" +#include "oneapi/tbb/detail/_aggregator.h" template <typename OperationType> struct MallocAggregator { - typedef tbb::internal::aggregator_generic<OperationType> type; + typedef tbb::detail::d1::aggregator_generic<OperationType> type; }; //! aggregated_operation base class template <typename Derived> struct MallocAggregatedOperation { - typedef tbb::internal::aggregated_operation<Derived> type; + typedef tbb::detail::d1::aggregated_operation<Derived> type; }; #endif /* _TBB_malloc_Customize_H_ */ diff --git a/src/tbb/src/tbbmalloc/MapMemory.h b/src/tbb/src/tbbmalloc/MapMemory.h index 552917a43..63b4bc375 100644 --- a/src/tbb/src/tbbmalloc/MapMemory.h +++ b/src/tbb/src/tbbmalloc/MapMemory.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2022 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef _itt_shared_malloc_MapMemory_H @@ -23,16 +19,7 @@ #include <stdlib.h> -void *ErrnoPreservingMalloc(size_t bytes) -{ - int prevErrno = errno; - void *ret = malloc( bytes ); - if (!ret) - errno = prevErrno; - return ret; -} - -#if __linux__ || __APPLE__ || __sun || __FreeBSD__ +#if __unix__ || __APPLE__ || __sun || __FreeBSD__ #if __sun && !defined(_XPG4_2) // To have void* as mmap's 1st argument @@ -41,7 +28,7 @@ void *ErrnoPreservingMalloc(size_t bytes) #endif #include <sys/mman.h> -#if __linux__ +#if __unix__ /* __TBB_MAP_HUGETLB is MAP_HUGETLB from system header linux/mman.h. The header is not included here, as on some Linux flavors inclusion of linux/mman.h leads to compilation error, @@ -57,20 +44,108 @@ void *ErrnoPreservingMalloc(size_t bytes) #undef XPG4_WAS_DEFINED #endif -#define MEMORY_MAPPING_USES_MALLOC 0 -void* MapMemory (size_t bytes, bool hugePages) -{ - void* result = 0; - int prevErrno = errno; +inline void* mmap_impl(size_t map_size, void* map_hint = nullptr, int map_flags = 0) { #ifndef MAP_ANONYMOUS // macOS* defines MAP_ANON, which is deprecated in Linux*. #define MAP_ANONYMOUS MAP_ANON #endif /* MAP_ANONYMOUS */ - int addFlags = hugePages? __TBB_MAP_HUGETLB : 0; - result = mmap(NULL, bytes, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|addFlags, -1, 0); - if (result==MAP_FAILED) + return mmap(map_hint, map_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | map_flags, -1, 0); +} + +inline void* mmapTHP(size_t bytes) { + // Initializes in zero-initialized data section + static void* hint; + + // Optimistically try to use a last huge page aligned region end + // as a hint for mmap. + hint = hint ? (void*)((uintptr_t)hint - bytes) : hint; + void* result = mmap_impl(bytes, hint); + + // Something went wrong + if (result == MAP_FAILED) { + hint = nullptr; + return MAP_FAILED; + } + + // Otherwise, fall back to the slow path - map oversized region + // and trim excess parts. + if (!isAligned(result, HUGE_PAGE_SIZE)) { + // Undo previous try + munmap(result, bytes); + + // Map oversized on huge page size region + result = mmap_impl(bytes + HUGE_PAGE_SIZE); + + // Something went wrong + if (result == MAP_FAILED) { + hint = nullptr; + return MAP_FAILED; + } + + // Misalignment offset + uintptr_t offset = 0; + + if (!isAligned(result, HUGE_PAGE_SIZE)) { + // Trim excess head of a region if it is no aligned + offset = HUGE_PAGE_SIZE - ((uintptr_t)result & (HUGE_PAGE_SIZE - 1)); + munmap(result, offset); + + // New region beginning + result = (void*)((uintptr_t)result + offset); + } + + // Trim excess tail of a region + munmap((void*)((uintptr_t)result + bytes), HUGE_PAGE_SIZE - offset); + } + + // Assume, that mmap virtual addresses grow down by default + // So, set a hint as a result of a last successful allocation + // and then use it minus requested size as a new mapping point. + // TODO: Atomic store is meant here, fence not needed, but + // currently we don't have such function. + hint = result; + + MALLOC_ASSERT(isAligned(result, HUGE_PAGE_SIZE), "Mapped address is not aligned on huge page size."); + + return result; +} + +#define MEMORY_MAPPING_USES_MALLOC 0 +void* MapMemory (size_t bytes, PageType pageType) +{ + void* result = nullptr; + int prevErrno = errno; + + switch (pageType) { + case REGULAR: + { + result = mmap_impl(bytes); + break; + } + case PREALLOCATED_HUGE_PAGE: + { + MALLOC_ASSERT((bytes % HUGE_PAGE_SIZE) == 0, "Mapping size should be divisible by huge page size"); + result = mmap_impl(bytes, nullptr, __TBB_MAP_HUGETLB); + break; + } + case TRANSPARENT_HUGE_PAGE: + { + MALLOC_ASSERT((bytes % HUGE_PAGE_SIZE) == 0, "Mapping size should be divisible by huge page size"); + result = mmapTHP(bytes); + break; + } + default: + { + MALLOC_ASSERT(false, "Unknown page type"); + } + } + + if (result == MAP_FAILED) { errno = prevErrno; - return result==MAP_FAILED? 0: result; + return nullptr; + } + + return result; } int UnmapMemory(void *area, size_t bytes) @@ -86,10 +161,10 @@ int UnmapMemory(void *area, size_t bytes) #include <windows.h> #define MEMORY_MAPPING_USES_MALLOC 0 -void* MapMemory (size_t bytes, bool) +void* MapMemory (size_t bytes, PageType) { /* Is VirtualAlloc thread safe? */ - return VirtualAlloc(NULL, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); + return VirtualAlloc(nullptr, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); } int UnmapMemory(void *area, size_t /*bytes*/) @@ -100,8 +175,17 @@ int UnmapMemory(void *area, size_t /*bytes*/) #else +void *ErrnoPreservingMalloc(size_t bytes) +{ + int prevErrno = errno; + void *ret = malloc( bytes ); + if (!ret) + errno = prevErrno; + return ret; +} + #define MEMORY_MAPPING_USES_MALLOC 1 -void* MapMemory (size_t bytes, bool) +void* MapMemory (size_t bytes, PageType) { return ErrnoPreservingMalloc( bytes ); } diff --git a/src/tbb/src/tbbmalloc/Statistics.h b/src/tbb/src/tbbmalloc/Statistics.h index 392442e90..a8767a819 100644 --- a/src/tbb/src/tbbmalloc/Statistics.h +++ b/src/tbb/src/tbbmalloc/Statistics.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2022 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #define MAX_THREADS 1024 @@ -67,7 +63,7 @@ static inline int STAT_increment(int thread, int bin, int ctr) static inline void initStatisticsCollection() { #if defined(MALLOCENV_COLLECT_STATISTICS) - if (NULL != getenv(MALLOCENV_COLLECT_STATISTICS)) + if (nullptr != getenv(MALLOCENV_COLLECT_STATISTICS)) reportAllocationStatistics = true; #endif } diff --git a/src/tbb/src/tbbmalloc/Synchronize.h b/src/tbb/src/tbbmalloc/Synchronize.h new file mode 100644 index 000000000..b25d7e245 --- /dev/null +++ b/src/tbb/src/tbbmalloc/Synchronize.h @@ -0,0 +1,97 @@ +/* + Copyright (c) 2005-2023 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_malloc_Synchronize_H_ +#define __TBB_malloc_Synchronize_H_ + +#include "oneapi/tbb/detail/_utils.h" + +#include <atomic> + +//! Stripped down version of spin_mutex. +/** Instances of MallocMutex must be declared in memory that is zero-initialized. + There are no constructors. This is a feature that lets it be + used in situations where the mutex might be used while file-scope constructors + are running. + + There are no methods "acquire" or "release". The scoped_lock must be used + in a strict block-scoped locking pattern. Omitting these methods permitted + further simplification. */ +class MallocMutex : tbb::detail::no_copy { + std::atomic_flag m_flag = ATOMIC_FLAG_INIT; + + void lock() { + tbb::detail::atomic_backoff backoff; + while (m_flag.test_and_set()) backoff.pause(); + } + bool try_lock() { + return !m_flag.test_and_set(); + } + void unlock() { + m_flag.clear(std::memory_order_release); + } + +public: + class scoped_lock : tbb::detail::no_copy { + MallocMutex& m_mutex; + bool m_taken; + + public: + scoped_lock(MallocMutex& m) : m_mutex(m), m_taken(true) { + m.lock(); + } + scoped_lock(MallocMutex& m, bool block, bool *locked) : m_mutex(m), m_taken(false) { + if (block) { + m.lock(); + m_taken = true; + } else { + m_taken = m.try_lock(); + } + if (locked) *locked = m_taken; + } + + scoped_lock(scoped_lock& other) = delete; + scoped_lock& operator=(scoped_lock&) = delete; + + ~scoped_lock() { + if (m_taken) { + m_mutex.unlock(); + } + } + }; + friend class scoped_lock; +}; + +inline void SpinWaitWhileEq(const std::atomic<intptr_t>& location, const intptr_t value) { + tbb::detail::spin_wait_while_eq(location, value); +} + +#if USE_PTHREAD && __TBB_SOURCE_DIRECTLY_INCLUDED + +inline void SpinWaitUntilEq(const std::atomic<intptr_t>& location, const intptr_t value) { + tbb::detail::spin_wait_until_eq(location, value); +} + +#endif + +class AtomicBackoff { + tbb::detail::atomic_backoff backoff; +public: + AtomicBackoff() {} + void pause() { backoff.pause(); } +}; + +#endif /* __TBB_malloc_Synchronize_H_ */ diff --git a/src/tbb/src/tbbmalloc/TypeDefinitions.h b/src/tbb/src/tbbmalloc/TypeDefinitions.h index 9b514ca4a..bfadf61d6 100644 --- a/src/tbb/src/tbbmalloc/TypeDefinitions.h +++ b/src/tbb/src/tbbmalloc/TypeDefinitions.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2023 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef _itt_shared_malloc_TypeDefinitions_H_ @@ -29,7 +25,7 @@ # define __ARCH_ipf 1 # elif defined(_M_IX86)||defined(__i386__) // the latter for MinGW support # define __ARCH_x86_32 1 -# elif defined(_M_ARM) +# elif defined(_M_ARM) || defined(_M_ARM64) || defined(__aarch64__) // the latter for MinGW support # define __ARCH_other 1 # else # error Unknown processor architecture for Windows diff --git a/src/tbb/src/tbbmalloc/backend.cpp b/src/tbb/src/tbbmalloc/backend.cpp index 88212dd68..87531f816 100644 --- a/src/tbb/src/tbbmalloc/backend.cpp +++ b/src/tbb/src/tbbmalloc/backend.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2023 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include <string.h> /* for memset */ @@ -43,118 +39,81 @@ namespace internal { /* assume MapMemory and UnmapMemory are customized */ #endif -void* getRawMemory (size_t size, bool hugePages) { - return MapMemory(size, hugePages); +void* getRawMemory (size_t size, PageType pageType) { + return MapMemory(size, pageType); } int freeRawMemory (void *object, size_t size) { return UnmapMemory(object, size); } -void HugePagesStatus::registerAllocation(bool gotPage) -{ - if (gotPage) { - if (!wasObserved) - FencedStore(wasObserved, 1); - } else - FencedStore(enabled, 0); - // reports huge page status only once - if (needActualStatusPrint - && AtomicCompareExchange(needActualStatusPrint, 0, 1)) - doPrintStatus(gotPage, "available"); -} - -void HugePagesStatus::registerReleasing(void* addr, size_t size) -{ - // We: 1) got huge page at least once, - // 2) something that looks like a huge page is been released, - // and 3) user requested huge pages, - // so a huge page might be available at next allocation. - // TODO: keep page status in regions and use exact check here - if (FencedLoad(wasObserved) && size>=pageSize && isAligned(addr, pageSize)) - FencedStore(enabled, requestedMode.get()); -} - -void HugePagesStatus::printStatus() { - doPrintStatus(requestedMode.get(), "requested"); - if (requestedMode.get()) { // report actual status iff requested - if (pageSize) - FencedStore(needActualStatusPrint, 1); - else - doPrintStatus(/*state=*/false, "available"); - } -} - -void HugePagesStatus::doPrintStatus(bool state, const char *stateName) -{ - // Under macOS* fprintf/snprintf acquires an internal lock, so when - // 1st allocation is done under the lock, we got a deadlock. - // Do not use fprintf etc during initialization. - fputs("TBBmalloc: huge pages\t", stderr); - if (!state) - fputs("not ", stderr); - fputs(stateName, stderr); - fputs("\n", stderr); -} - #if CHECK_ALLOCATION_RANGE void Backend::UsedAddressRange::registerAlloc(uintptr_t left, uintptr_t right) { MallocMutex::scoped_lock lock(mutex); - if (left < leftBound) - leftBound = left; - if (right > rightBound) - rightBound = right; - MALLOC_ASSERT(leftBound, ASSERT_TEXT); - MALLOC_ASSERT(leftBound < rightBound, ASSERT_TEXT); - MALLOC_ASSERT(leftBound <= left && right <= rightBound, ASSERT_TEXT); + if (left < leftBound.load(std::memory_order_relaxed)) + leftBound.store(left, std::memory_order_relaxed); + if (right > rightBound.load(std::memory_order_relaxed)) + rightBound.store(right, std::memory_order_relaxed); + MALLOC_ASSERT(leftBound.load(std::memory_order_relaxed), ASSERT_TEXT); + MALLOC_ASSERT(leftBound.load(std::memory_order_relaxed) < rightBound.load(std::memory_order_relaxed), ASSERT_TEXT); + MALLOC_ASSERT(leftBound.load(std::memory_order_relaxed) <= left && right <= rightBound.load(std::memory_order_relaxed), ASSERT_TEXT); } void Backend::UsedAddressRange::registerFree(uintptr_t left, uintptr_t right) { MallocMutex::scoped_lock lock(mutex); - if (leftBound == left) { - if (rightBound == right) { - leftBound = ADDRESS_UPPER_BOUND; - rightBound = 0; + if (leftBound.load(std::memory_order_relaxed) == left) { + if (rightBound.load(std::memory_order_relaxed) == right) { + leftBound.store(ADDRESS_UPPER_BOUND, std::memory_order_relaxed); + rightBound.store(0, std::memory_order_relaxed); } else - leftBound = right; - } else if (rightBound == right) - rightBound = left; - MALLOC_ASSERT((!rightBound && leftBound == ADDRESS_UPPER_BOUND) - || leftBound < rightBound, ASSERT_TEXT); + leftBound.store(right, std::memory_order_relaxed); + } else if (rightBound.load(std::memory_order_relaxed) == right) + rightBound.store(left, std::memory_order_relaxed); + MALLOC_ASSERT((!rightBound.load(std::memory_order_relaxed) && leftBound.load(std::memory_order_relaxed) == ADDRESS_UPPER_BOUND) + || leftBound.load(std::memory_order_relaxed) < rightBound.load(std::memory_order_relaxed), ASSERT_TEXT); } #endif // CHECK_ALLOCATION_RANGE +// Initialized in frontend inside defaultMemPool +extern HugePagesStatus hugePages; + void *Backend::allocRawMem(size_t &size) { - void *res = NULL; - size_t allocSize; + void *res = nullptr; + size_t allocSize = 0; if (extMemPool->userPool()) { - if (extMemPool->fixedPool && bootsrapMemDone==FencedLoad(bootsrapMemStatus)) - return NULL; - MALLOC_ASSERT(bootsrapMemStatus!=bootsrapMemNotDone, + if (extMemPool->fixedPool && bootsrapMemDone == bootsrapMemStatus.load(std::memory_order_acquire)) + return nullptr; + MALLOC_ASSERT(bootsrapMemStatus != bootsrapMemNotDone, "Backend::allocRawMem() called prematurely?"); // TODO: support for raw mem not aligned at sizeof(uintptr_t) // memory from fixed pool is asked once and only once allocSize = alignUpGeneric(size, extMemPool->granularity); res = (*extMemPool->rawAlloc)(extMemPool->poolId, allocSize); } else { - // check if alignment to huge page size is recommended - size_t hugePageSize = hugePages.recommendedGranularity(); - allocSize = alignUpGeneric(size, hugePageSize? hugePageSize : extMemPool->granularity); - // try to get them at 1st allocation and still use, if successful - // if 1st try is unsuccessful, no more trying - if (FencedLoad(hugePages.enabled)) { - MALLOC_ASSERT(hugePageSize, "Inconsistent state of HugePagesStatus"); - res = getRawMemory(allocSize, /*hugePages=*/true); - hugePages.registerAllocation(res); + // Align allocation on page size + size_t pageSize = hugePages.isEnabled ? hugePages.getGranularity() : extMemPool->granularity; + MALLOC_ASSERT(pageSize, "Page size cannot be zero."); + allocSize = alignUpGeneric(size, pageSize); + + // If user requested huge pages and they are available, try to use preallocated ones firstly. + // If there are none, lets check transparent huge pages support and use them instead. + if (hugePages.isEnabled) { + if (hugePages.isHPAvailable) { + res = getRawMemory(allocSize, PREALLOCATED_HUGE_PAGE); + } + if (!res && hugePages.isTHPAvailable) { + res = getRawMemory(allocSize, TRANSPARENT_HUGE_PAGE); + } } - if (!res) - res = getRawMemory(allocSize, /*hugePages=*/false); + if (!res) { + res = getRawMemory(allocSize, REGULAR); + } } if (res) { @@ -166,7 +125,7 @@ void *Backend::allocRawMem(size_t &size) volatile size_t curTotalSize = totalMemSize; // to read global value once MALLOC_ASSERT(curTotalSize+size > curTotalSize, "Overflow allocation size."); #endif - AtomicAdd((intptr_t&)totalMemSize, size); + totalMemSize.fetch_add(size); } return res; @@ -179,13 +138,12 @@ bool Backend::freeRawMem(void *object, size_t size) volatile size_t curTotalSize = totalMemSize; // to read global value once MALLOC_ASSERT(curTotalSize-size < curTotalSize, "Negative allocation size."); #endif - AtomicAdd((intptr_t&)totalMemSize, -size); + totalMemSize.fetch_sub(size); if (extMemPool->userPool()) { MALLOC_ASSERT(!extMemPool->fixedPool, "No free for fixed-size pools."); fail = (*extMemPool->rawFree)(extMemPool->poolId, object, size); } else { usedAddrRange.registerFree((uintptr_t)object, (uintptr_t)object + size); - hugePages.registerReleasing(object, size); fail = freeRawMemory(object, size); } // TODO: use result in all freeRawMem() callers @@ -196,8 +154,8 @@ bool Backend::freeRawMem(void *object, size_t size) // Protected object size. After successful locking returns size of locked block, // and releasing requires setting block size. -class GuardedSize : tbb::internal::no_copy { - uintptr_t value; +class GuardedSize : tbb::detail::no_copy { + std::atomic<uintptr_t> value; public: enum State { LOCKED, @@ -208,31 +166,30 @@ class GuardedSize : tbb::internal::no_copy { MAX_SPEC_VAL = LAST_REGION_BLOCK }; - void initLocked() { value = LOCKED; } + void initLocked() { value.store(LOCKED, std::memory_order_release); } // TBB_REVAMP_TODO: was relaxed void makeCoalscing() { - MALLOC_ASSERT(value == LOCKED, ASSERT_TEXT); - value = COAL_BLOCK; + MALLOC_ASSERT(value.load(std::memory_order_relaxed) == LOCKED, ASSERT_TEXT); + value.store(COAL_BLOCK, std::memory_order_release); // TBB_REVAMP_TODO: was relaxed } size_t tryLock(State state) { - size_t szVal, sz; MALLOC_ASSERT(state <= MAX_LOCKED_VAL, ASSERT_TEXT); + size_t sz = value.load(std::memory_order_acquire); for (;;) { - sz = FencedLoad((intptr_t&)value); - if (sz <= MAX_LOCKED_VAL) + if (sz <= MAX_LOCKED_VAL) { break; - szVal = AtomicCompareExchange((intptr_t&)value, state, sz); - - if (szVal==sz) + } + if (value.compare_exchange_strong(sz, state)) { break; + } } return sz; } void unlock(size_t size) { - MALLOC_ASSERT(value <= MAX_LOCKED_VAL, "The lock is not locked"); + MALLOC_ASSERT(value.load(std::memory_order_relaxed) <= MAX_LOCKED_VAL, "The lock is not locked"); MALLOC_ASSERT(size > MAX_LOCKED_VAL, ASSERT_TEXT); - FencedStore((intptr_t&)value, size); + value.store(size, std::memory_order_release); } - bool isLastRegionBlock() const { return value==LAST_REGION_BLOCK; } + bool isLastRegionBlock() const { return value.load(std::memory_order_relaxed) == LAST_REGION_BLOCK; } friend void Backend::IndexedBins::verify(); }; @@ -263,7 +220,7 @@ class FreeBlock : BlockMutexes { // valid only when block is in processing, i.e. one is not free and not size_t sizeTmp; // used outside of backend int myBin; // bin that is owner of the block - bool aligned; + bool slabAligned; bool blockInBin; // this block in myBin already FreeBlock *rightNeig(size_t sz) const { @@ -300,12 +257,12 @@ class FreeBlock : BlockMutexes { myL.makeCoalscing(); rightNeig(blockSz)->leftL.makeCoalscing(); sizeTmp = blockSz; - nextToFree = NULL; + nextToFree = nullptr; } void markUsed() { myL.initLocked(); rightNeig(sizeTmp)->leftL.initLocked(); - nextToFree = NULL; + nextToFree = nullptr; } static void markBlocks(FreeBlock *fBlock, int num, size_t size) { for (int i=1; i<num; i++) { @@ -340,12 +297,13 @@ inline bool BackendSync::waitTillBlockReleased(intptr_t startModifiedCnt) }; ITT_Guard ittGuard(&inFlyBlocks); #endif - for (intptr_t myBinsInFlyBlocks = FencedLoad(inFlyBlocks), - myCoalescQInFlyBlocks = backend->blocksInCoalescing(); ; - backoff.pause()) { - MALLOC_ASSERT(myBinsInFlyBlocks>=0 && myCoalescQInFlyBlocks>=0, NULL); - intptr_t currBinsInFlyBlocks = FencedLoad(inFlyBlocks), - currCoalescQInFlyBlocks = backend->blocksInCoalescing(); + intptr_t myBinsInFlyBlocks = inFlyBlocks.load(std::memory_order_acquire); + intptr_t myCoalescQInFlyBlocks = backend->blocksInCoalescing(); + while (true) { + MALLOC_ASSERT(myBinsInFlyBlocks>=0 && myCoalescQInFlyBlocks>=0, nullptr); + + intptr_t currBinsInFlyBlocks = inFlyBlocks.load(std::memory_order_acquire); + intptr_t currCoalescQInFlyBlocks = backend->blocksInCoalescing(); WhiteboxTestingYield(); // Stop waiting iff: @@ -361,11 +319,20 @@ inline bool BackendSync::waitTillBlockReleased(intptr_t startModifiedCnt) if (currCoalescQInFlyBlocks > 0 && backend->scanCoalescQ(/*forceCoalescQDrop=*/false)) break; // 4) when there are no blocks - if (!currBinsInFlyBlocks && !currCoalescQInFlyBlocks) + if (!currBinsInFlyBlocks && !currCoalescQInFlyBlocks) { // re-scan make sense only if bins were modified since scanned + auto pool = backend->extMemPool; + if (pool->hardCachesCleanupInProgress.load(std::memory_order_acquire) || + pool->softCachesCleanupInProgress.load(std::memory_order_acquire)) { + backoff.pause(); + continue; + } + return startModifiedCnt != getNumOfMods(); + } myBinsInFlyBlocks = currBinsInFlyBlocks; myCoalescQInFlyBlocks = currCoalescQInFlyBlocks; + backoff.pause(); } return true; } @@ -375,34 +342,30 @@ void CoalRequestQ::putBlock(FreeBlock *fBlock) MALLOC_ASSERT(fBlock->sizeTmp >= FreeBlock::minBlockSize, ASSERT_TEXT); fBlock->markUsed(); // the block is in the queue, do not forget that it's here - AtomicIncrement(inFlyBlocks); + inFlyBlocks++; + FreeBlock *myBlToFree = blocksToFree.load(std::memory_order_acquire); for (;;) { - FreeBlock *myBlToFree = (FreeBlock*)FencedLoad((intptr_t&)blocksToFree); - fBlock->nextToFree = myBlToFree; - if (myBlToFree == - (FreeBlock*)AtomicCompareExchange((intptr_t&)blocksToFree, - (intptr_t)fBlock, - (intptr_t)myBlToFree)) + if (blocksToFree.compare_exchange_strong(myBlToFree, fBlock)) { return; + } } } FreeBlock *CoalRequestQ::getAll() { for (;;) { - FreeBlock *myBlToFree = (FreeBlock*)FencedLoad((intptr_t&)blocksToFree); - - if (!myBlToFree) - return NULL; - else { - if (myBlToFree == - (FreeBlock*)AtomicCompareExchange((intptr_t&)blocksToFree, - 0, (intptr_t)myBlToFree)) + FreeBlock *myBlToFree = blocksToFree.load(std::memory_order_acquire); + + if (!myBlToFree) { + return nullptr; + } else { + if (blocksToFree.compare_exchange_strong(myBlToFree, nullptr)) { return myBlToFree; - else + } else { continue; + } } } } @@ -410,7 +373,8 @@ FreeBlock *CoalRequestQ::getAll() inline void CoalRequestQ::blockWasProcessed() { bkndSync->binsModified(); - int prev = AtomicAdd(inFlyBlocks, -1); + int prev = inFlyBlocks.fetch_sub(1); + tbb::detail::suppress_unused_warning(prev); MALLOC_ASSERT(prev > 0, ASSERT_TEXT); } @@ -419,23 +383,22 @@ inline void CoalRequestQ::blockWasProcessed() // split the block without removing it. // If the free space should go to other bin(s), remove the block. // alignedBin is true, if all blocks in the bin have slab-aligned right side. -FreeBlock *Backend::IndexedBins::getFromBin(int binIdx, BackendSync *sync, - size_t size, bool needAlignedRes, bool alignedBin, bool wait, - int *binLocked) +FreeBlock *Backend::IndexedBins::getFromBin(int binIdx, BackendSync *sync, size_t size, + bool needAlignedRes, bool alignedBin, bool wait, int *binLocked) { Bin *b = &freeBins[binIdx]; try_next: - FreeBlock *fBlock = NULL; - if (b->head) { - bool locked; + FreeBlock *fBlock = nullptr; + if (!b->empty()) { + bool locked = false; MallocMutex::scoped_lock scopedLock(b->tLock, wait, &locked); if (!locked) { if (binLocked) (*binLocked)++; - return NULL; + return nullptr; } - for (FreeBlock *curr = b->head; curr; curr = curr->next) { + for (FreeBlock *curr = b->head.load(std::memory_order_relaxed); curr; curr = curr->next) { size_t szBlock = curr->tryLockBlock(); if (!szBlock) { // block is locked, re-do bin lock, as there is no place to spin @@ -443,50 +406,36 @@ FreeBlock *Backend::IndexedBins::getFromBin(int binIdx, BackendSync *sync, goto try_next; } + // GENERAL CASE if (alignedBin || !needAlignedRes) { size_t splitSz = szBlock - size; - // If we got a block as split result, - // it must have a room for control structures. - if (szBlock >= size && (splitSz >= FreeBlock::minBlockSize || - !splitSz)) + // If we got a block as split result, it must have a room for control structures. + if (szBlock >= size && (splitSz >= FreeBlock::minBlockSize || !splitSz)) fBlock = curr; } else { + // SPECIAL CASE, to get aligned block from unaligned bin we have to cut the middle of a block + // and return remaining left and right part. Possible only in fixed pool scenario, assert for this + // is set inside splitBlock() function. + void *newB = alignUp(curr, slabSize); uintptr_t rightNew = (uintptr_t)newB + size; uintptr_t rightCurr = (uintptr_t)curr + szBlock; - // appropriate size, and left and right split results - // are either big enough or non-existent + // Check if the block size is sufficient, + // and also left and right split results are either big enough or non-existent if (rightNew <= rightCurr - && (newB==curr || - (uintptr_t)newB-(uintptr_t)curr >= FreeBlock::minBlockSize) - && (rightNew==rightCurr || - rightCurr - rightNew >= FreeBlock::minBlockSize)) + && (newB == curr || ((uintptr_t)newB - (uintptr_t)curr) >= FreeBlock::minBlockSize) + && (rightNew == rightCurr || (rightCurr - rightNew) >= FreeBlock::minBlockSize)) fBlock = curr; } + if (fBlock) { - // consume must be called before result of removing from a bin - // is visible externally. + // consume must be called before result of removing from a bin is visible externally. sync->blockConsumed(); - if (alignedBin && needAlignedRes && - Backend::sizeToBin(szBlock-size) == Backend::sizeToBin(szBlock)) { - // free remainder of fBlock stay in same bin, - // so no need to remove it from the bin - // TODO: add more "still here" cases - FreeBlock *newFBlock = fBlock; - // return block from right side of fBlock - fBlock = (FreeBlock*)((uintptr_t)newFBlock + szBlock - size); - MALLOC_ASSERT(isAligned(fBlock, slabSize), "Invalid free block"); - fBlock->initHeader(); - fBlock->setLeftFree(szBlock - size); - newFBlock->setMeFree(szBlock - size); - - fBlock->sizeTmp = size; - } else { - b->removeBlock(fBlock); - if (freeBins[binIdx].empty()) - bitMask.set(binIdx, false); - fBlock->sizeTmp = szBlock; - } + // TODO: think about cases when block stays in the same bin + b->removeBlock(fBlock); + if (freeBins[binIdx].empty()) + bitMask.set(binIdx, false); + fBlock->sizeTmp = szBlock; break; } else { // block size is not valid, search for next block in the bin curr->setMeFree(szBlock); @@ -500,14 +449,14 @@ FreeBlock *Backend::IndexedBins::getFromBin(int binIdx, BackendSync *sync, bool Backend::IndexedBins::tryReleaseRegions(int binIdx, Backend *backend) { Bin *b = &freeBins[binIdx]; - FreeBlock *fBlockList = NULL; + FreeBlock *fBlockList = nullptr; // got all blocks from the bin and re-do coalesce on them // to release single-block regions try_next: - if (b->head) { + if (!b->empty()) { MallocMutex::scoped_lock binLock(b->tLock); - for (FreeBlock *curr = b->head; curr; ) { + for (FreeBlock *curr = b->head.load(std::memory_order_relaxed); curr; ) { size_t szBlock = curr->tryLockBlock(); if (!szBlock) goto try_next; @@ -527,10 +476,10 @@ bool Backend::IndexedBins::tryReleaseRegions(int binIdx, Backend *backend) void Backend::Bin::removeBlock(FreeBlock *fBlock) { - MALLOC_ASSERT(fBlock->next||fBlock->prev||fBlock==head, + MALLOC_ASSERT(fBlock->next||fBlock->prev||fBlock== head.load(std::memory_order_relaxed), "Detected that a block is not in the bin."); - if (head == fBlock) - head = fBlock->next; + if (head.load(std::memory_order_relaxed) == fBlock) + head.store(fBlock->next, std::memory_order_relaxed); if (tail == fBlock) tail = fBlock->prev; if (fBlock->prev) @@ -539,13 +488,11 @@ void Backend::Bin::removeBlock(FreeBlock *fBlock) fBlock->next->prev = fBlock->prev; } -void Backend::IndexedBins::addBlock(int binIdx, FreeBlock *fBlock, size_t blockSz, bool addToTail) +void Backend::IndexedBins::addBlock(int binIdx, FreeBlock *fBlock, size_t /* blockSz */, bool addToTail) { Bin *b = &freeBins[binIdx]; - fBlock->myBin = binIdx; - fBlock->aligned = toAlignedBin(fBlock, blockSz); - fBlock->next = fBlock->prev = NULL; + fBlock->next = fBlock->prev = nullptr; { MallocMutex::scoped_lock scopedLock(b->tLock); if (addToTail) { @@ -553,11 +500,11 @@ void Backend::IndexedBins::addBlock(int binIdx, FreeBlock *fBlock, size_t blockS b->tail = fBlock; if (fBlock->prev) fBlock->prev->next = fBlock; - if (!b->head) - b->head = fBlock; + if (!b->head.load(std::memory_order_relaxed)) + b->head.store(fBlock, std::memory_order_relaxed); } else { - fBlock->next = b->head; - b->head = fBlock; + fBlock->next = b->head.load(std::memory_order_relaxed); + b->head.store(fBlock, std::memory_order_relaxed); if (fBlock->next) fBlock->next->prev = fBlock; if (!b->tail) @@ -569,13 +516,11 @@ void Backend::IndexedBins::addBlock(int binIdx, FreeBlock *fBlock, size_t blockS bool Backend::IndexedBins::tryAddBlock(int binIdx, FreeBlock *fBlock, bool addToTail) { - bool locked; + bool locked = false; Bin *b = &freeBins[binIdx]; - fBlock->myBin = binIdx; - fBlock->aligned = toAlignedBin(fBlock, fBlock->sizeTmp); if (addToTail) { - fBlock->next = NULL; + fBlock->next = nullptr; { MallocMutex::scoped_lock scopedLock(b->tLock, /*wait=*/false, &locked); if (!locked) @@ -584,17 +529,17 @@ bool Backend::IndexedBins::tryAddBlock(int binIdx, FreeBlock *fBlock, bool addTo b->tail = fBlock; if (fBlock->prev) fBlock->prev->next = fBlock; - if (!b->head) - b->head = fBlock; + if (!b->head.load(std::memory_order_relaxed)) + b->head.store(fBlock, std::memory_order_relaxed); } } else { - fBlock->prev = NULL; + fBlock->prev = nullptr; { MallocMutex::scoped_lock scopedLock(b->tLock, /*wait=*/false, &locked); if (!locked) return false; - fBlock->next = b->head; - b->head = fBlock; + fBlock->next = b->head.load(std::memory_order_relaxed); + b->head.store(fBlock, std::memory_order_relaxed); if (fBlock->next) fBlock->next->prev = fBlock; if (!b->tail) @@ -607,7 +552,7 @@ bool Backend::IndexedBins::tryAddBlock(int binIdx, FreeBlock *fBlock, bool addTo void Backend::IndexedBins::reset() { - for (int i=0; i<Backend::freeBinsNum; i++) + for (unsigned i=0; i<Backend::freeBinsNum; i++) freeBins[i].reset(); bitMask.reset(); } @@ -625,77 +570,65 @@ bool ExtMemoryPool::regionsAreReleaseable() const return !keepAllMemory && !delayRegsReleasing; } -FreeBlock *Backend::splitUnalignedBlock(FreeBlock *fBlock, int num, size_t size, - bool needAlignedBlock) +FreeBlock *Backend::splitBlock(FreeBlock *fBlock, int num, size_t size, bool blockIsAligned, bool needAlignedBlock) { - const size_t totalSize = num*size; - if (needAlignedBlock) { - size_t fBlockSz = fBlock->sizeTmp; - uintptr_t fBlockEnd = (uintptr_t)fBlock + fBlockSz; - FreeBlock *newB = alignUp(fBlock, slabSize); - FreeBlock *rightPart = (FreeBlock*)((uintptr_t)newB + totalSize); - - // Space to use is in the middle, - // ... return free right part + const size_t totalSize = num * size; + + // SPECIAL CASE, for unaligned block we have to cut the middle of a block + // and return remaining left and right part. Possible only in a fixed pool scenario. + if (needAlignedBlock && !blockIsAligned) { + MALLOC_ASSERT(extMemPool->fixedPool, + "Aligned block request from unaligned bin possible only in fixed pool scenario."); + + // Space to use is in the middle + FreeBlock *newBlock = alignUp(fBlock, slabSize); + FreeBlock *rightPart = (FreeBlock*)((uintptr_t)newBlock + totalSize); + uintptr_t fBlockEnd = (uintptr_t)fBlock + fBlock->sizeTmp; + + // Return free right part if ((uintptr_t)rightPart != fBlockEnd) { rightPart->initHeader(); // to prevent coalescing rightPart with fBlock - coalescAndPut(rightPart, fBlockEnd - (uintptr_t)rightPart); + size_t rightSize = fBlockEnd - (uintptr_t)rightPart; + coalescAndPut(rightPart, rightSize, toAlignedBin(rightPart, rightSize)); } - // ... and free left part - if (newB != fBlock) { - newB->initHeader(); // to prevent coalescing fBlock with newB - coalescAndPut(fBlock, (uintptr_t)newB - (uintptr_t)fBlock); + // And free left part + if (newBlock != fBlock) { + newBlock->initHeader(); // to prevent coalescing fBlock with newB + size_t leftSize = (uintptr_t)newBlock - (uintptr_t)fBlock; + coalescAndPut(fBlock, leftSize, toAlignedBin(fBlock, leftSize)); } - - fBlock = newB; - MALLOC_ASSERT(isAligned(fBlock, slabSize), ASSERT_TEXT); - } else { - if (size_t splitSz = fBlock->sizeTmp - totalSize) { - // split block and return free right part - FreeBlock *splitB = (FreeBlock*)((uintptr_t)fBlock + totalSize); - splitB->initHeader(); - coalescAndPut(splitB, splitSz); - } - } - FreeBlock::markBlocks(fBlock, num, size); - return fBlock; -} - -FreeBlock *Backend::splitAlignedBlock(FreeBlock *fBlock, int num, size_t size, - bool needAlignedBlock) -{ - if (fBlock->sizeTmp != num*size) { // i.e., need to split the block - FreeBlock *newAlgnd; - size_t newSz; - + fBlock = newBlock; + } else if (size_t splitSize = fBlock->sizeTmp - totalSize) { // need to split the block + // GENERAL CASE, cut the left or right part of the block + FreeBlock *splitBlock = nullptr; if (needAlignedBlock) { - newAlgnd = fBlock; - fBlock = (FreeBlock*)((uintptr_t)newAlgnd + newAlgnd->sizeTmp - - num*size); - MALLOC_ASSERT(isAligned(fBlock, slabSize), "Invalid free block"); + // For slab aligned blocks cut the right side of the block + // and return it to a requester, original block returns to backend + splitBlock = fBlock; + fBlock = (FreeBlock*)((uintptr_t)splitBlock + splitSize); fBlock->initHeader(); - newSz = newAlgnd->sizeTmp - num*size; } else { - newAlgnd = (FreeBlock*)((uintptr_t)fBlock + num*size); - newSz = fBlock->sizeTmp - num*size; - newAlgnd->initHeader(); + // For large object blocks cut original block and put free right part to backend + splitBlock = (FreeBlock*)((uintptr_t)fBlock + totalSize); + splitBlock->initHeader(); } - coalescAndPut(newAlgnd, newSz); + // Mark free block as it`s parent only when the requested type (needAlignedBlock) + // and returned from Bins/OS block (isAligned) are equal (XOR operation used) + bool markAligned = (blockIsAligned ^ needAlignedBlock) ? toAlignedBin(splitBlock, splitSize) : blockIsAligned; + coalescAndPut(splitBlock, splitSize, markAligned); } - MALLOC_ASSERT(!needAlignedBlock || isAligned(fBlock, slabSize), - "Expect to get aligned block, if one was requested."); + MALLOC_ASSERT(!needAlignedBlock || isAligned(fBlock, slabSize), "Expect to get aligned block, if one was requested."); FreeBlock::markBlocks(fBlock, num, size); return fBlock; } -inline size_t Backend::getMaxBinnedSize() const +size_t Backend::getMaxBinnedSize() const { - return hugePages.wasObserved && !inUserPool()? + return hugePages.isEnabled && !inUserPool() ? maxBinned_HugePage : maxBinned_SmallPage; } -inline bool Backend::MaxRequestComparator::operator()(size_t oldMaxReq, - size_t requestSize) const +inline bool Backend::MaxRequestComparator::operator()(size_t oldMaxReq, size_t requestSize) const { return requestSize > oldMaxReq && requestSize < backend->getMaxBinnedSize(); } @@ -705,21 +638,23 @@ FreeBlock *Backend::releaseMemInCaches(intptr_t startModifiedCnt, int *lockedBinsThreshold, int numOfLockedBins) { // something released from caches - if (extMemPool->hardCachesCleanup() - // ..or can use blocks that are in processing now - || bkndSync.waitTillBlockReleased(startModifiedCnt)) + if (extMemPool->hardCachesCleanup(false)) + return (FreeBlock*)VALID_BLOCK_IN_BIN; + + if (bkndSync.waitTillBlockReleased(startModifiedCnt)) return (FreeBlock*)VALID_BLOCK_IN_BIN; + // OS can't give us more memory, but we have some in locked bins if (*lockedBinsThreshold && numOfLockedBins) { *lockedBinsThreshold = 0; return (FreeBlock*)VALID_BLOCK_IN_BIN; } - return NULL; // nothing found, give up + return nullptr; // nothing found, give up } FreeBlock *Backend::askMemFromOS(size_t blockSize, intptr_t startModifiedCnt, int *lockedBinsThreshold, int numOfLockedBins, - bool *splittableRet) + bool *splittableRet, bool needSlabRegion) { FreeBlock *block; // The block sizes can be divided into 3 groups: @@ -768,20 +703,21 @@ FreeBlock *Backend::askMemFromOS(size_t blockSize, intptr_t startModifiedCnt, // This must be done carefully, because blocks in bins can be released // in releaseCachesToLimit(). const unsigned NUM_OF_REG = 3; - block = addNewRegion(regSz_sizeBased, MEMREG_FLEXIBLE_SIZE, /*addToBin=*/false); + MemRegionType regType = needSlabRegion ? MEMREG_SLAB_BLOCKS : MEMREG_LARGE_BLOCKS; + block = addNewRegion(regSz_sizeBased, regType, /*addToBin=*/false); if (block) for (unsigned idx=0; idx<NUM_OF_REG; idx++) - if (! addNewRegion(regSz_sizeBased, MEMREG_FLEXIBLE_SIZE, /*addToBin=*/true)) + if (! addNewRegion(regSz_sizeBased, regType, /*addToBin=*/true)) break; } else { - block = addNewRegion(regSz_sizeBased, MEMREG_SEVERAL_BLOCKS, /*addToBin=*/false); + block = addNewRegion(regSz_sizeBased, MEMREG_LARGE_BLOCKS, /*addToBin=*/false); } memExtendingSema.signal(); // no regions found, try to clean cache if (!block || block == (FreeBlock*)VALID_BLOCK_IN_BIN) return releaseMemInCaches(startModifiedCnt, lockedBinsThreshold, numOfLockedBins); - // Since a region can hold more than one block it can be splitted. + // Since a region can hold more than one block it can be split. *splittableRet = true; } // after asking memory from OS, release caches if we above the memory limits @@ -792,44 +728,50 @@ FreeBlock *Backend::askMemFromOS(size_t blockSize, intptr_t startModifiedCnt, void Backend::releaseCachesToLimit() { - if (!memSoftLimit || totalMemSize <= memSoftLimit) + if (!memSoftLimit.load(std::memory_order_relaxed) + || totalMemSize.load(std::memory_order_relaxed) <= memSoftLimit.load(std::memory_order_relaxed)) { return; + } size_t locTotalMemSize, locMemSoftLimit; scanCoalescQ(/*forceCoalescQDrop=*/false); if (extMemPool->softCachesCleanup() && - (locTotalMemSize = FencedLoad((intptr_t&)totalMemSize)) <= - (locMemSoftLimit = FencedLoad((intptr_t&)memSoftLimit))) + (locTotalMemSize = totalMemSize.load(std::memory_order_acquire)) <= + (locMemSoftLimit = memSoftLimit.load(std::memory_order_acquire))) return; // clean global large-object cache, if this is not enough, clean local caches // do this in several tries, because backend fragmentation can prevent // region from releasing for (int cleanLocal = 0; cleanLocal<2; cleanLocal++) - while (cleanLocal? - extMemPool->allLocalCaches.cleanup(extMemPool, /*cleanOnlyUnused=*/true) - : extMemPool->loc.decreasingCleanup()) - if ((locTotalMemSize = FencedLoad((intptr_t&)totalMemSize)) <= - (locMemSoftLimit = FencedLoad((intptr_t&)memSoftLimit))) + while (cleanLocal ? + extMemPool->allLocalCaches.cleanup(/*cleanOnlyUnused=*/true) : + extMemPool->loc.decreasingCleanup()) + if ((locTotalMemSize = totalMemSize.load(std::memory_order_acquire)) <= + (locMemSoftLimit = memSoftLimit.load(std::memory_order_acquire))) return; // last chance to match memSoftLimit - extMemPool->hardCachesCleanup(); + extMemPool->hardCachesCleanup(true); } -FreeBlock *Backend::IndexedBins:: - findBlock(int nativeBin, BackendSync *sync, size_t size, - bool resSlabAligned, bool alignedBin, int *numOfLockedBins) +int Backend::IndexedBins::getMinNonemptyBin(unsigned startBin) const { - for (int i=getMinNonemptyBin(nativeBin); i<freeBinsNum; i=getMinNonemptyBin(i+1)) - if (FreeBlock *block = getFromBin(i, sync, size, resSlabAligned, alignedBin, - /*wait=*/false, numOfLockedBins)) + int p = bitMask.getMinTrue(startBin); + return p == -1 ? Backend::freeBinsNum : p; +} + +FreeBlock *Backend::IndexedBins::findBlock(int nativeBin, BackendSync *sync, size_t size, + bool needAlignedBlock, bool alignedBin, int *numOfLockedBins) +{ + for (int i=getMinNonemptyBin(nativeBin); i<(int)freeBinsNum; i=getMinNonemptyBin(i+1)) + if (FreeBlock *block = getFromBin(i, sync, size, needAlignedBlock, alignedBin, /*wait=*/false, numOfLockedBins)) return block; - return NULL; + return nullptr; } void Backend::requestBootstrapMem() { - if (bootsrapMemDone == FencedLoad(bootsrapMemStatus)) + if (bootsrapMemDone == bootsrapMemStatus.load(std::memory_order_acquire)) return; MallocMutex::scoped_lock lock( bootsrapMemStatusMutex ); if (bootsrapMemDone == bootsrapMemStatus) @@ -837,8 +779,8 @@ void Backend::requestBootstrapMem() MALLOC_ASSERT(bootsrapMemNotDone == bootsrapMemStatus, ASSERT_TEXT); bootsrapMemStatus = bootsrapMemInitializing; // request some rather big region during bootstrap in advance - // ok to get NULL here, as later we re-do a request with more modest size - addNewRegion(2*1024*1024, MEMREG_FLEXIBLE_SIZE, /*addToBin=*/true); + // ok to get nullptr here, as later we re-do a request with more modest size + addNewRegion(2*1024*1024, MEMREG_SLAB_BLOCKS, /*addToBin=*/true); bootsrapMemStatus = bootsrapMemDone; } @@ -846,7 +788,7 @@ void Backend::requestBootstrapMem() // needAlignedRes is true if result must be slab-aligned FreeBlock *Backend::genericGetBlock(int num, size_t size, bool needAlignedBlock) { - FreeBlock *block = NULL; + FreeBlock *block = nullptr; const size_t totalReqSize = num*size; // no splitting after requesting new region, asks exact size const int nativeBin = sizeToBin(totalReqSize); @@ -865,56 +807,52 @@ FreeBlock *Backend::genericGetBlock(int num, size_t size, bool needAlignedBlock) for (;;) { const intptr_t startModifiedCnt = bkndSync.getNumOfMods(); int numOfLockedBins; - + intptr_t cleanCnt; do { + cleanCnt = backendCleanCnt.load(std::memory_order_acquire); numOfLockedBins = 0; - - // TODO: try different bin search order if (needAlignedBlock) { - block = freeAlignedBins.findBlock(nativeBin, &bkndSync, num*size, - /*needAlignedBlock=*/true, /*alignedBin=*/true, - &numOfLockedBins); - if (!block) - block = freeLargeBins.findBlock(nativeBin, &bkndSync, num*size, - /*needAlignedBlock=*/true, /*alignedBin=*/false, - &numOfLockedBins); + block = freeSlabAlignedBins.findBlock(nativeBin, &bkndSync, num*size, needAlignedBlock, + /*alignedBin=*/true, &numOfLockedBins); + if (!block && extMemPool->fixedPool) + block = freeLargeBlockBins.findBlock(nativeBin, &bkndSync, num*size, needAlignedBlock, + /*alignedBin=*/false, &numOfLockedBins); } else { - block = freeLargeBins.findBlock(nativeBin, &bkndSync, num*size, - /*needAlignedBlock=*/false, /*alignedBin=*/false, - &numOfLockedBins); - if (!block) - block = freeAlignedBins.findBlock(nativeBin, &bkndSync, num*size, - /*needAlignedBlock=*/false, /*alignedBin=*/true, - &numOfLockedBins); + block = freeLargeBlockBins.findBlock(nativeBin, &bkndSync, num*size, needAlignedBlock, + /*alignedBin=*/false, &numOfLockedBins); + if (!block && extMemPool->fixedPool) + block = freeSlabAlignedBins.findBlock(nativeBin, &bkndSync, num*size, needAlignedBlock, + /*alignedBin=*/true, &numOfLockedBins); } - } while (!block && numOfLockedBins>lockedBinsThreshold); + } while (!block && (numOfLockedBins>lockedBinsThreshold || cleanCnt % 2 == 1 || + cleanCnt != backendCleanCnt.load(std::memory_order_acquire))); if (block) break; - if (!(scanCoalescQ(/*forceCoalescQDrop=*/true) - | extMemPool->softCachesCleanup())) { + bool retScanCoalescQ = scanCoalescQ(/*forceCoalescQDrop=*/true); + bool retSoftCachesCleanup = extMemPool->softCachesCleanup(); + if (!(retScanCoalescQ || retSoftCachesCleanup)) { // bins are not updated, // only remaining possibility is to ask for more memory - block = - askMemFromOS(totalReqSize, startModifiedCnt, &lockedBinsThreshold, - numOfLockedBins, &splittable); + block = askMemFromOS(totalReqSize, startModifiedCnt, &lockedBinsThreshold, + numOfLockedBins, &splittable, needAlignedBlock); if (!block) - return NULL; + return nullptr; if (block != (FreeBlock*)VALID_BLOCK_IN_BIN) { // size can be increased in askMemFromOS, that's why >= MALLOC_ASSERT(block->sizeTmp >= size, ASSERT_TEXT); break; } // valid block somewhere in bins, let's find it - block = NULL; + block = nullptr; } } MALLOC_ASSERT(block, ASSERT_TEXT); - if (splittable) - block = toAlignedBin(block, block->sizeTmp)? - splitAlignedBlock(block, num, size, needAlignedBlock) : - splitUnalignedBlock(block, num, size, needAlignedBlock); + if (splittable) { + // At this point we have to be sure that slabAligned attribute describes the right block state + block = splitBlock(block, num, size, block->slabAligned, needAlignedBlock); + } // matched blockConsumed() from startUseBlock() bkndSync.blockReleased(); @@ -933,12 +871,22 @@ LargeMemoryBlock *Backend::getLargeBlock(size_t size) return lmb; } +BlockI *Backend::getSlabBlock(int num) { + BlockI *b = (BlockI*)genericGetBlock(num, slabSize, /*slabAligned=*/true); + MALLOC_ASSERT(isAligned(b, slabSize), ASSERT_TEXT); + return b; +} + +void Backend::putSlabBlock(BlockI *block) { + genericPutBlock((FreeBlock *)block, slabSize, /*slabAligned=*/true); +} + void *Backend::getBackRefSpace(size_t size, bool *rawMemUsed) { // This block is released only at shutdown, so it can prevent // a entire region releasing when it's received from the backend, // so prefer getRawMemory using. - if (void *ret = getRawMemory(size, /*hugePages=*/false)) { + if (void *ret = getRawMemory(size, REGULAR)) { *rawMemUsed = true; return ret; } @@ -957,24 +905,24 @@ void Backend::putBackRefSpace(void *b, size_t size, bool rawMemUsed) void Backend::removeBlockFromBin(FreeBlock *fBlock) { if (fBlock->myBin != Backend::NO_BIN) { - if (fBlock->aligned) - freeAlignedBins.lockRemoveBlock(fBlock->myBin, fBlock); + if (fBlock->slabAligned) + freeSlabAlignedBins.lockRemoveBlock(fBlock->myBin, fBlock); else - freeLargeBins.lockRemoveBlock(fBlock->myBin, fBlock); + freeLargeBlockBins.lockRemoveBlock(fBlock->myBin, fBlock); } } -void Backend::genericPutBlock(FreeBlock *fBlock, size_t blockSz) +void Backend::genericPutBlock(FreeBlock *fBlock, size_t blockSz, bool slabAligned) { bkndSync.blockConsumed(); - coalescAndPut(fBlock, blockSz); + coalescAndPut(fBlock, blockSz, slabAligned); bkndSync.blockReleased(); } void AllLargeBlocksList::add(LargeMemoryBlock *lmb) { MallocMutex::scoped_lock scoped_cs(largeObjLock); - lmb->gPrev = NULL; + lmb->gPrev = nullptr; lmb->gNext = loHead; if (lmb->gNext) lmb->gNext->gPrev = lmb; @@ -996,7 +944,7 @@ void Backend::putLargeBlock(LargeMemoryBlock *lmb) { if (extMemPool->userPool()) extMemPool->lmbList.remove(lmb); - genericPutBlock((FreeBlock *)lmb, lmb->unalignedSize); + genericPutBlock((FreeBlock *)lmb, lmb->unalignedSize, false); } void Backend::returnLargeObject(LargeMemoryBlock *lmb) @@ -1014,7 +962,7 @@ void *Backend::remap(void *ptr, size_t oldSize, size_t newSize, size_t alignment // during remap, can't guarantee alignment more strict than current or // more strict than page alignment || !isAligned(ptr, alignment) || alignment>extMemPool->granularity) - return NULL; + return nullptr; const LargeMemoryBlock* lmbOld = ((LargeObjectHdr *)ptr - 1)->memoryBlock; const size_t oldUnalignedSize = lmbOld->unalignedSize; FreeBlock *oldFBlock = (FreeBlock *)lmbOld; @@ -1022,34 +970,39 @@ void *Backend::remap(void *ptr, size_t oldSize, size_t newSize, size_t alignment // in every region only one block can have LAST_REGION_BLOCK on right, // so don't need no synchronization if (!right->isLastRegionBlock()) - return NULL; + return nullptr; MemRegion *oldRegion = static_cast<LastFreeBlock*>(right)->memRegion; MALLOC_ASSERT( oldRegion < ptr, ASSERT_TEXT ); const size_t oldRegionSize = oldRegion->allocSz; if (oldRegion->type != MEMREG_ONE_BLOCK) - return NULL; // we are not single in the region + return nullptr; // we are not single in the region const size_t userOffset = (uintptr_t)ptr - (uintptr_t)oldRegion; + const size_t alignedSize = LargeObjectCache::alignToBin(newSize + userOffset); const size_t requestSize = - alignUp(userOffset + newSize + sizeof(LastFreeBlock), extMemPool->granularity); - if (requestSize < newSize) // is wrapped around? - return NULL; + alignUp(sizeof(MemRegion) + alignedSize + sizeof(LastFreeBlock), extMemPool->granularity); + if (requestSize < alignedSize) // is wrapped around? + return nullptr; regionList.remove(oldRegion); + // The deallocation should be registered in address range before mremap to + // prevent a race condition with allocation on another thread. + // (OS can reuse the memory and registerAlloc will be missed on another thread) + usedAddrRange.registerFree((uintptr_t)oldRegion, (uintptr_t)oldRegion + oldRegionSize); + void *ret = mremap(oldRegion, oldRegion->allocSz, requestSize, MREMAP_MAYMOVE); if (MAP_FAILED == ret) { // can't remap, revert and leave regionList.add(oldRegion); - return NULL; + usedAddrRange.registerAlloc((uintptr_t)oldRegion, (uintptr_t)oldRegion + oldRegionSize); + return nullptr; } MemRegion *region = (MemRegion*)ret; MALLOC_ASSERT(region->type == MEMREG_ONE_BLOCK, ASSERT_TEXT); region->allocSz = requestSize; + region->blockSz = alignedSize; FreeBlock *fBlock = (FreeBlock *)alignUp((uintptr_t)region + sizeof(MemRegion), largeObjectAlignment); - // put LastFreeBlock at the very end of region - const uintptr_t fBlockEnd = (uintptr_t)region + requestSize - sizeof(LastFreeBlock); - region->blockSz = fBlockEnd - (uintptr_t)fBlock; regionList.add(region); startUseBlock(region, fBlock, /*addToBin=*/false); @@ -1058,7 +1011,7 @@ void *Backend::remap(void *ptr, size_t oldSize, size_t newSize, size_t alignment // TODO: get rid of useless pair blockConsumed()/blockReleased() bkndSync.blockReleased(); - // object must start at same offest from region's start + // object must start at same offset from region's start void *object = (void*)((uintptr_t)region + userOffset); MALLOC_ASSERT(isAligned(object, alignment), ASSERT_TEXT); LargeObjectHdr *header = (LargeObjectHdr*)object - 1; @@ -1072,9 +1025,8 @@ void *Backend::remap(void *ptr, size_t oldSize, size_t newSize, size_t alignment MALLOC_ASSERT((uintptr_t)lmb + lmb->unalignedSize >= (uintptr_t)object + lmb->objectSize, "An object must fit to the block."); - usedAddrRange.registerFree((uintptr_t)oldRegion, (uintptr_t)oldRegion + oldRegionSize); usedAddrRange.registerAlloc((uintptr_t)region, (uintptr_t)region + requestSize); - AtomicAdd((intptr_t&)totalMemSize, region->allocSz - oldRegionSize); + totalMemSize.fetch_add(region->allocSz - oldRegionSize); return object; } @@ -1091,7 +1043,7 @@ FreeBlock *Backend::doCoalesc(FreeBlock *fBlock, MemRegion **mRegion) { FreeBlock *resBlock = fBlock; size_t resSize = fBlock->sizeTmp; - MemRegion *memRegion = NULL; + MemRegion *memRegion = nullptr; fBlock->markCoalescing(resSize); resBlock->blockInBin = false; @@ -1101,14 +1053,14 @@ FreeBlock *Backend::doCoalesc(FreeBlock *fBlock, MemRegion **mRegion) if (leftSz != GuardedSize::LOCKED) { if (leftSz == GuardedSize::COAL_BLOCK) { coalescQ.putBlock(fBlock); - return NULL; + return nullptr; } else { FreeBlock *left = fBlock->leftNeig(leftSz); size_t lSz = left->trySetMeUsed(GuardedSize::COAL_BLOCK); if (lSz <= GuardedSize::MAX_LOCKED_VAL) { fBlock->setLeftFree(leftSz); // rollback coalescQ.putBlock(fBlock); - return NULL; + return nullptr; } else { MALLOC_ASSERT(lSz == leftSz, "Invalid header"); left->blockInBin = true; @@ -1132,7 +1084,7 @@ FreeBlock *Backend::doCoalesc(FreeBlock *fBlock, MemRegion **mRegion) removeBlockFromBin(resBlock); } coalescQ.putBlock(resBlock); - return NULL; + return nullptr; } else { size_t rSz = right->rightNeig(rightSz)-> trySetLeftUsed(GuardedSize::COAL_BLOCK); @@ -1143,7 +1095,7 @@ FreeBlock *Backend::doCoalesc(FreeBlock *fBlock, MemRegion **mRegion) removeBlockFromBin(resBlock); } coalescQ.putBlock(resBlock); - return NULL; + return nullptr; } else { MALLOC_ASSERT(rSz == rightSz, "Invalid header"); removeBlockFromBin(right); @@ -1168,13 +1120,12 @@ FreeBlock *Backend::doCoalesc(FreeBlock *fBlock, MemRegion **mRegion) MALLOC_ASSERT((uintptr_t)memRegion < (uintptr_t)resBlock, ASSERT_TEXT); *mRegion = memRegion; } else - *mRegion = NULL; + *mRegion = nullptr; resBlock->sizeTmp = resSize; return resBlock; } -bool Backend::coalescAndPutList(FreeBlock *list, bool forceCoalescQDrop, - bool reportBlocksProcessed) +bool Backend::coalescAndPutList(FreeBlock *list, bool forceCoalescQDrop, bool reportBlocksProcessed) { bool regionReleased = false; @@ -1204,12 +1155,12 @@ bool Backend::coalescAndPutList(FreeBlock *list, bool forceCoalescQDrop, } size_t currSz = toRet->sizeTmp; int bin = sizeToBin(currSz); - bool toAligned = toAlignedBin(toRet, currSz); + bool toAligned = extMemPool->fixedPool ? toAlignedBin(toRet, currSz) : toRet->slabAligned; bool needAddToBin = true; if (toRet->blockInBin) { // Does it stay in same bin? - if (toRet->myBin == bin && toRet->aligned == toAligned) + if (toRet->myBin == bin && toRet->slabAligned == toAligned) needAddToBin = false; else { toRet->blockInBin = false; @@ -1219,14 +1170,15 @@ bool Backend::coalescAndPutList(FreeBlock *list, bool forceCoalescQDrop, // Does not stay in same bin, or bin-less; add it if (needAddToBin) { - toRet->prev = toRet->next = toRet->nextToFree = NULL; + toRet->prev = toRet->next = toRet->nextToFree = nullptr; toRet->myBin = NO_BIN; + toRet->slabAligned = toAligned; // If the block is too small to fit in any bin, keep it bin-less. // It's not a leak because the block later can be coalesced. if (currSz >= minBinnedSize) { toRet->sizeTmp = currSz; - IndexedBins *target = toAligned? &freeAlignedBins : &freeLargeBins; + IndexedBins *target = toRet->slabAligned ? &freeSlabAlignedBins : &freeLargeBlockBins; if (forceCoalescQDrop) { target->addBlock(bin, toRet, toRet->sizeTmp, addToTail); } else if (!target->tryAddBlock(bin, toRet, addToTail)) { @@ -1249,10 +1201,11 @@ bool Backend::coalescAndPutList(FreeBlock *list, bool forceCoalescQDrop, // Coalesce fBlock and add it back to a bin; // processing delayed coalescing requests. -void Backend::coalescAndPut(FreeBlock *fBlock, size_t blockSz) +void Backend::coalescAndPut(FreeBlock *fBlock, size_t blockSz, bool slabAligned) { fBlock->sizeTmp = blockSz; - fBlock->nextToFree = NULL; + fBlock->nextToFree = nullptr; + fBlock->slabAligned = slabAligned; coalescAndPutList(fBlock, /*forceCoalescQDrop=*/false, /*reportBlocksProcessed=*/false); } @@ -1266,7 +1219,7 @@ bool Backend::scanCoalescQ(bool forceCoalescQDrop) // matches blockConsumed() from CoalRequestQ::putBlock() coalescAndPutList(currCoalescList, forceCoalescQDrop, /*reportBlocksProcessed=*/true); - // returns status of coalescQ.getAll(), as an indication of possibe changes in backend + // returns status of coalescQ.getAll(), as an indication of possible changes in backend // TODO: coalescAndPutList() may report is some new free blocks became available or not return currCoalescList; } @@ -1278,34 +1231,32 @@ FreeBlock *Backend::findBlockInRegion(MemRegion *region, size_t exactBlockSize) uintptr_t fBlockEnd, lastFreeBlock = (uintptr_t)region + region->allocSz - sizeof(LastFreeBlock); - MALLOC_STATIC_ASSERT(sizeof(LastFreeBlock) % sizeof(uintptr_t) == 0, + static_assert(sizeof(LastFreeBlock) % sizeof(uintptr_t) == 0, "Atomic applied on LastFreeBlock, and we put it at the end of region, that" " is uintptr_t-aligned, so no unaligned atomic operations are possible."); // right bound is slab-aligned, keep LastFreeBlock after it - if (region->type==MEMREG_FLEXIBLE_SIZE) { - fBlock = (FreeBlock *)alignUp((uintptr_t)region + sizeof(MemRegion), - sizeof(uintptr_t)); + if (region->type == MEMREG_SLAB_BLOCKS) { + fBlock = (FreeBlock *)alignUp((uintptr_t)region + sizeof(MemRegion), sizeof(uintptr_t)); fBlockEnd = alignDown(lastFreeBlock, slabSize); } else { - fBlock = (FreeBlock *)alignUp((uintptr_t)region + sizeof(MemRegion), - largeObjectAlignment); + fBlock = (FreeBlock *)alignUp((uintptr_t)region + sizeof(MemRegion), largeObjectAlignment); fBlockEnd = (uintptr_t)fBlock + exactBlockSize; MALLOC_ASSERT(fBlockEnd <= lastFreeBlock, ASSERT_TEXT); } if (fBlockEnd <= (uintptr_t)fBlock) - return NULL; // allocSz is too small + return nullptr; // allocSz is too small blockSz = fBlockEnd - (uintptr_t)fBlock; // TODO: extend getSlabBlock to support degradation, i.e. getting less blocks // then requested, and then relax this check // (now all or nothing is implemented, check according to this) if (blockSz < numOfSlabAllocOnMiss*slabSize) - return NULL; + return nullptr; region->blockSz = blockSz; return fBlock; } -// startUseBlock adds free block to a bin, the block can be used and +// startUseBlock may add the free block to a bin, the block can be used and // even released after this, so the region must be added to regionList already void Backend::startUseBlock(MemRegion *region, FreeBlock *fBlock, bool addToBin) { @@ -1315,7 +1266,7 @@ void Backend::startUseBlock(MemRegion *region, FreeBlock *fBlock, bool addToBin) LastFreeBlock *lastBl = static_cast<LastFreeBlock*>(fBlock->rightNeig(blockSz)); // to not get unaligned atomics during LastFreeBlock access - MALLOC_ASSERT(isAligned(lastBl, sizeof(uintptr_t)), NULL); + MALLOC_ASSERT(isAligned(lastBl, sizeof(uintptr_t)), nullptr); lastBl->initHeader(); lastBl->setMeFree(GuardedSize::LAST_REGION_BLOCK); lastBl->setLeftFree(blockSz); @@ -1326,23 +1277,26 @@ void Backend::startUseBlock(MemRegion *region, FreeBlock *fBlock, bool addToBin) unsigned targetBin = sizeToBin(blockSz); // during adding advance regions, register bin for a largest block in region advRegBins.registerBin(targetBin); - if (region->type!=MEMREG_ONE_BLOCK && toAlignedBin(fBlock, blockSz)) { - freeAlignedBins.addBlock(targetBin, fBlock, blockSz, /*addToTail=*/false); + if (region->type == MEMREG_SLAB_BLOCKS) { + fBlock->slabAligned = true; + freeSlabAlignedBins.addBlock(targetBin, fBlock, blockSz, /*addToTail=*/false); } else { - freeLargeBins.addBlock(targetBin, fBlock, blockSz, /*addToTail=*/false); + fBlock->slabAligned = false; + freeLargeBlockBins.addBlock(targetBin, fBlock, blockSz, /*addToTail=*/false); } } else { // to match with blockReleased() in genericGetBlock bkndSync.blockConsumed(); + // Understand our alignment for correct splitBlock operation + fBlock->slabAligned = region->type == MEMREG_SLAB_BLOCKS ? true : false; fBlock->sizeTmp = fBlock->tryLockBlock(); - MALLOC_ASSERT(fBlock->sizeTmp >= FreeBlock::minBlockSize, - "Locking must be successful"); + MALLOC_ASSERT(fBlock->sizeTmp >= FreeBlock::minBlockSize, "Locking must be successful"); } } void MemRegionList::add(MemRegion *r) { - r->prev = NULL; + r->prev = nullptr; MallocMutex::scoped_lock lock(regionListLock); r->next = head; head = r; @@ -1376,13 +1330,12 @@ int MemRegionList::reportStat(FILE *f) FreeBlock *Backend::addNewRegion(size_t size, MemRegionType memRegType, bool addToBin) { - MALLOC_STATIC_ASSERT(sizeof(BlockMutexes) <= sizeof(BlockI), - "Header must be not overwritten in used blocks"); + static_assert(sizeof(BlockMutexes) <= sizeof(BlockI), "Header must be not overwritten in used blocks"); MALLOC_ASSERT(FreeBlock::minBlockSize > GuardedSize::MAX_SPEC_VAL, "Block length must not conflict with special values of GuardedSize"); - // If the region is not "flexible size" we should reserve some space for + // If the region is not "for slabs" we should reserve some space for // a region header, the worst case alignment and the last block mark. - const size_t requestSize = memRegType == MEMREG_FLEXIBLE_SIZE ? size : + const size_t requestSize = memRegType == MEMREG_SLAB_BLOCKS ? size : size + sizeof(MemRegion) + largeObjectAlignment + FreeBlock::minBlockSize + sizeof(LastFreeBlock); @@ -1390,12 +1343,12 @@ FreeBlock *Backend::addNewRegion(size_t size, MemRegionType memRegType, bool add MemRegion *region = (MemRegion*)allocRawMem(rawSize); if (!region) { MALLOC_ASSERT(rawSize==requestSize, "getRawMem has not allocated memory but changed the allocated size."); - return NULL; + return nullptr; } if (rawSize < sizeof(MemRegion)) { if (!extMemPool->fixedPool) freeRawMem(region, rawSize); - return NULL; + return nullptr; } region->type = memRegType; @@ -1404,7 +1357,7 @@ FreeBlock *Backend::addNewRegion(size_t size, MemRegionType memRegType, bool add if (!fBlock) { if (!extMemPool->fixedPool) freeRawMem(region, rawSize); - return NULL; + return nullptr; } regionList.add(region); startUseBlock(region, fBlock, addToBin); @@ -1426,8 +1379,8 @@ void Backend::reset() // no active threads are allowed in backend while reset() called verify(); - freeLargeBins.reset(); - freeAlignedBins.reset(); + freeLargeBlockBins.reset(); + freeSlabAlignedBins.reset(); advRegBins.reset(); for (MemRegion *curr = regionList.head; curr; curr = curr->next) { @@ -1443,8 +1396,8 @@ bool Backend::destroy() // no active threads are allowed in backend while destroy() called verify(); if (!inUserPool()) { - freeLargeBins.reset(); - freeAlignedBins.reset(); + freeLargeBlockBins.reset(); + freeSlabAlignedBins.reset(); } while (regionList.head) { MemRegion *helper = regionList.head->next; @@ -1457,25 +1410,29 @@ bool Backend::destroy() bool Backend::clean() { scanCoalescQ(/*forceCoalescQDrop=*/false); - + // Backend::clean is always called under synchronization so only one thread can + // enter to this method at once. + // backendCleanCnt%2== 1 means that clean operation is in progress + backendCleanCnt.fetch_add(1, std::memory_order_acq_rel); bool res = false; // We can have several blocks occupying a whole region, // because such regions are added in advance (see askMemFromOS() and reset()), // and never used. Release them all. for (int i = advRegBins.getMinUsedBin(0); i != -1; i = advRegBins.getMinUsedBin(i+1)) { - if (i == freeAlignedBins.getMinNonemptyBin(i)) - res |= freeAlignedBins.tryReleaseRegions(i, this); - if (i == freeLargeBins.getMinNonemptyBin(i)) - res |= freeLargeBins.tryReleaseRegions(i, this); + if (i == freeSlabAlignedBins.getMinNonemptyBin(i)) + res |= freeSlabAlignedBins.tryReleaseRegions(i, this); + if (i == freeLargeBlockBins.getMinNonemptyBin(i)) + res |= freeLargeBlockBins.tryReleaseRegions(i, this); } - + backendCleanCnt.fetch_add(1, std::memory_order_acq_rel); return res; } void Backend::IndexedBins::verify() { - for (int i=0; i<freeBinsNum; i++) { - for (FreeBlock *fb = freeBins[i].head; fb; fb=fb->next) { +#if MALLOC_DEBUG + for (int i=0; i<(int)freeBinsNum; i++) { + for (FreeBlock *fb = freeBins[i].head.load(std::memory_order_relaxed); fb; fb=fb->next) { uintptr_t mySz = fb->myL.value; MALLOC_ASSERT(mySz>GuardedSize::MAX_SPEC_VAL, ASSERT_TEXT); FreeBlock *right = (FreeBlock*)((uintptr_t)fb + mySz); @@ -1485,6 +1442,7 @@ void Backend::IndexedBins::verify() MALLOC_ASSERT(fb->leftL.value<=GuardedSize::MAX_SPEC_VAL, ASSERT_TEXT); } } +#endif } // For correct operation, it must be called when no other threads @@ -1493,10 +1451,10 @@ void Backend::verify() { #if MALLOC_DEBUG scanCoalescQ(/*forceCoalescQDrop=*/false); - - freeLargeBins.verify(); - freeAlignedBins.verify(); #endif // MALLOC_DEBUG + + freeLargeBlockBins.verify(); + freeSlabAlignedBins.verify(); } #if __TBB_MALLOC_BACKEND_STAT @@ -1518,6 +1476,7 @@ size_t Backend::Bin::reportFreeBlocks(FILE *f) for (FreeBlock *fb = head; fb; fb = fb->next) { size_t sz = fb->tryLockBlock(); fb->setMeFree(sz); + fb->rightNeig(sz)->setLeftFree(sz); fprintf(f, " [%p;%p]", fb, (void*)((uintptr_t)fb+sz)); totalSz += sz; } @@ -1544,9 +1503,9 @@ void Backend::reportStat(FILE *f) int regNum = regionList.reportStat(f); fprintf(f, "\n%d regions, %lu KB in all regions\n free bins:\nlarge bins: ", regNum, totalMemSize/1024); - freeLargeBins.reportStat(f); + freeLargeBlockBins.reportStat(f); fprintf(f, "\naligned bins: "); - freeAlignedBins.reportStat(f); + freeSlabAlignedBins.reportStat(f); fprintf(f, "\n"); } #endif // __TBB_MALLOC_BACKEND_STAT diff --git a/src/tbb/src/tbbmalloc/backend.h b/src/tbb/src/tbbmalloc/backend.h new file mode 100644 index 000000000..cbf62c0b0 --- /dev/null +++ b/src/tbb/src/tbbmalloc/backend.h @@ -0,0 +1,392 @@ +/* + Copyright (c) 2005-2023 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_tbbmalloc_internal_H + #error tbbmalloc_internal.h must be included at this point +#endif + +#ifndef __TBB_backend_H +#define __TBB_backend_H + +// Included from namespace rml::internal + +// global state of blocks currently in processing +class BackendSync { + // Class instances should reside in zero-initialized memory! + // The number of blocks currently removed from a bin and not returned back + std::atomic<intptr_t> inFlyBlocks; // to another + std::atomic<intptr_t> binsModifications; // incremented on every bin modification + Backend *backend; +public: + void init(Backend *b) { backend = b; } + void blockConsumed() { inFlyBlocks++; } + void binsModified() { binsModifications++; } + void blockReleased() { +#if __TBB_MALLOC_BACKEND_STAT + MALLOC_ITT_SYNC_RELEASING(&inFlyBlocks); +#endif + binsModifications++; + intptr_t prev = inFlyBlocks.fetch_sub(1); + MALLOC_ASSERT(prev > 0, ASSERT_TEXT); + suppress_unused_warning(prev); + } + intptr_t getNumOfMods() const { return binsModifications.load(std::memory_order_acquire); } + // return true if need re-do the blocks search + inline bool waitTillBlockReleased(intptr_t startModifiedCnt); +}; + +class CoalRequestQ { // queue of free blocks that coalescing was delayed +private: + std::atomic<FreeBlock*> blocksToFree; + BackendSync *bkndSync; + // counted blocks in blocksToFree and that are leaved blocksToFree + // and still in active coalescing + std::atomic<intptr_t> inFlyBlocks; +public: + void init(BackendSync *bSync) { bkndSync = bSync; } + FreeBlock *getAll(); // return current list of blocks and make queue empty + void putBlock(FreeBlock *fBlock); + inline void blockWasProcessed(); + intptr_t blocksInFly() const { return inFlyBlocks.load(std::memory_order_acquire); } +}; + +class MemExtendingSema { + std::atomic<intptr_t> active; +public: + bool wait() { + bool rescanBins = false; + // up to 3 threads can add more memory from OS simultaneously, + // rest of threads have to wait + intptr_t prevCnt = active.load(std::memory_order_acquire); + for (;;) { + if (prevCnt < 3) { + if (active.compare_exchange_strong(prevCnt, prevCnt + 1)) { + break; + } + } else { + SpinWaitWhileEq(active, prevCnt); + rescanBins = true; + break; + } + } + return rescanBins; + } + void signal() { active.fetch_sub(1); } +}; + +enum MemRegionType { + // The region holds only slabs + MEMREG_SLAB_BLOCKS = 0, + // The region can hold several large object blocks + MEMREG_LARGE_BLOCKS, + // The region holds only one block with a requested size + MEMREG_ONE_BLOCK +}; + +class MemRegionList { + MallocMutex regionListLock; +public: + MemRegion *head; + void add(MemRegion *r); + void remove(MemRegion *r); + int reportStat(FILE *f); +}; + +class Backend { +private: +/* Blocks in range [minBinnedSize; getMaxBinnedSize()] are kept in bins, + one region can contains several blocks. Larger blocks are allocated directly + and one region always contains one block. +*/ + enum { + minBinnedSize = 8*1024UL, + /* If huge pages are available, maxBinned_HugePage used. + If not, maxBinned_SmallPage is the threshold. + TODO: use pool's granularity for upper bound setting.*/ + maxBinned_SmallPage = 1024*1024UL, + // TODO: support other page sizes + maxBinned_HugePage = 4*1024*1024UL + }; + enum { + VALID_BLOCK_IN_BIN = 1 // valid block added to bin, not returned as result + }; +public: + // Backend bins step is the same as CacheStep for large object cache + static const size_t freeBinsStep = LargeObjectCache::LargeBSProps::CacheStep; + static const unsigned freeBinsNum = (maxBinned_HugePage-minBinnedSize)/freeBinsStep + 1; + + // if previous access missed per-thread slabs pool, + // allocate numOfSlabAllocOnMiss blocks in advance + static const int numOfSlabAllocOnMiss = 2; + + enum { + NO_BIN = -1, + // special bin for blocks >= maxBinned_HugePage, blocks go to this bin + // when pool is created with keepAllMemory policy + // TODO: currently this bin is scanned using "1st fit", as it accumulates + // blocks of different sizes, "best fit" is preferred in terms of fragmentation + HUGE_BIN = freeBinsNum-1 + }; + + // Bin keeps 2-linked list of free blocks. It must be 2-linked + // because during coalescing a block it's removed from a middle of the list. + struct Bin { + std::atomic<FreeBlock*> head; + FreeBlock* tail; + MallocMutex tLock; + + void removeBlock(FreeBlock *fBlock); + void reset() { + head.store(nullptr, std::memory_order_relaxed); + tail = nullptr; + } + bool empty() const { return !head.load(std::memory_order_relaxed); } + + size_t countFreeBlocks(); + size_t reportFreeBlocks(FILE *f); + void reportStat(FILE *f); + }; + + typedef BitMaskMin<Backend::freeBinsNum> BitMaskBins; + + // array of bins supplemented with bitmask for fast finding of non-empty bins + class IndexedBins { + BitMaskBins bitMask; + Bin freeBins[Backend::freeBinsNum]; + FreeBlock *getFromBin(int binIdx, BackendSync *sync, size_t size, + bool needAlignedBlock, bool alignedBin, bool wait, int *resLocked); + public: + FreeBlock *findBlock(int nativeBin, BackendSync *sync, size_t size, + bool needAlignedBlock, bool alignedBin,int *numOfLockedBins); + bool tryReleaseRegions(int binIdx, Backend *backend); + void lockRemoveBlock(int binIdx, FreeBlock *fBlock); + void addBlock(int binIdx, FreeBlock *fBlock, size_t blockSz, bool addToTail); + bool tryAddBlock(int binIdx, FreeBlock *fBlock, bool addToTail); + int getMinNonemptyBin(unsigned startBin) const; + void verify(); + void reset(); + void reportStat(FILE *f); + }; + +private: + class AdvRegionsBins { + BitMaskBins bins; + public: + void registerBin(int regBin) { bins.set(regBin, 1); } + int getMinUsedBin(int start) const { return bins.getMinTrue(start); } + void reset() { bins.reset(); } + }; + // auxiliary class to atomic maximum request finding + class MaxRequestComparator { + const Backend *backend; + public: + MaxRequestComparator(const Backend *be) : backend(be) {} + inline bool operator()(size_t oldMaxReq, size_t requestSize) const; + }; + +#if CHECK_ALLOCATION_RANGE + // Keep min and max of all addresses requested from OS, + // use it for checking memory possibly allocated by replaced allocators + // and for debugging purposes. Valid only for default memory pool. + class UsedAddressRange { + static const uintptr_t ADDRESS_UPPER_BOUND = UINTPTR_MAX; + + std::atomic<uintptr_t> leftBound, + rightBound; + MallocMutex mutex; + public: + // rightBound is zero-initialized + void init() { leftBound.store(ADDRESS_UPPER_BOUND, std::memory_order_relaxed); } + void registerAlloc(uintptr_t left, uintptr_t right); + void registerFree(uintptr_t left, uintptr_t right); + // as only left and right bounds are kept, we can return true + // for pointer not allocated by us, if more than single region + // was requested from OS + bool inRange(void *ptr) const { + const uintptr_t p = (uintptr_t)ptr; + return leftBound.load(std::memory_order_relaxed)<=p && + p<=rightBound.load(std::memory_order_relaxed); + } + }; +#else + class UsedAddressRange { + public: + void init() { } + void registerAlloc(uintptr_t, uintptr_t) {} + void registerFree(uintptr_t, uintptr_t) {} + bool inRange(void *) const { return true; } + }; +#endif + + ExtMemoryPool *extMemPool; + // used for release every region on pool destroying + MemRegionList regionList; + + CoalRequestQ coalescQ; // queue of coalescing requests + BackendSync bkndSync; + // semaphore protecting adding more more memory from OS + MemExtendingSema memExtendingSema; + //size_t totalMemSize, + // memSoftLimit; + std::atomic<size_t> totalMemSize; + std::atomic<size_t> memSoftLimit; + UsedAddressRange usedAddrRange; + // to keep 1st allocation large than requested, keep bootstrapping status + enum { + bootsrapMemNotDone = 0, + bootsrapMemInitializing, + bootsrapMemDone + }; + std::atomic<intptr_t> bootsrapMemStatus; + MallocMutex bootsrapMemStatusMutex; + + // Using of maximal observed requested size allows decrease + // memory consumption for small requests and decrease fragmentation + // for workloads when small and large allocation requests are mixed. + // TODO: decrease, not only increase it + std::atomic<size_t> maxRequestedSize; + + // register bins related to advance regions + AdvRegionsBins advRegBins; + // Storage for split FreeBlocks + IndexedBins freeLargeBlockBins, + freeSlabAlignedBins; + + std::atomic<intptr_t> backendCleanCnt; + // Our friends + friend class BackendSync; + + /******************************** Backend methods ******************************/ + + /*--------------------------- Coalescing functions ----------------------------*/ + void coalescAndPut(FreeBlock *fBlock, size_t blockSz, bool slabAligned); + bool coalescAndPutList(FreeBlock *head, bool forceCoalescQDrop, bool reportBlocksProcessed); + + // Main coalescing operation + FreeBlock *doCoalesc(FreeBlock *fBlock, MemRegion **memRegion); + + // Queue for conflicted blocks during coalescing + bool scanCoalescQ(bool forceCoalescQDrop); + intptr_t blocksInCoalescing() const { return coalescQ.blocksInFly(); } + + /*--------------------- FreeBlock backend accessors ---------------------------*/ + FreeBlock *genericGetBlock(int num, size_t size, bool slabAligned); + void genericPutBlock(FreeBlock *fBlock, size_t blockSz, bool slabAligned); + + // Split the block and return remaining parts to backend if possible + FreeBlock *splitBlock(FreeBlock *fBlock, int num, size_t size, bool isAligned, bool needAlignedBlock); + + void removeBlockFromBin(FreeBlock *fBlock); + + // TODO: combine with returnLargeObject + void putLargeBlock(LargeMemoryBlock *lmb); + + /*------------------- Starting point for OS allocation ------------------------*/ + void requestBootstrapMem(); + FreeBlock *askMemFromOS(size_t totalReqSize, intptr_t startModifiedCnt, + int *lockedBinsThreshold, int numOfLockedBins, + bool *splittable, bool needSlabRegion); + + /*---------------------- Memory regions allocation ----------------------------*/ + FreeBlock *addNewRegion(size_t size, MemRegionType type, bool addToBin); + void releaseRegion(MemRegion *region); + + // TODO: combine in one initMemoryRegion function + FreeBlock *findBlockInRegion(MemRegion *region, size_t exactBlockSize); + void startUseBlock(MemRegion *region, FreeBlock *fBlock, bool addToBin); + + /*------------------------- Raw memory accessors ------------------------------*/ + void *allocRawMem(size_t &size); + bool freeRawMem(void *object, size_t size); + + /*------------------------------ Cleanup functions ----------------------------*/ + // Clean all memory from all caches (extMemPool hard cleanup) + FreeBlock *releaseMemInCaches(intptr_t startModifiedCnt, int *lockedBinsThreshold, int numOfLockedBins); + // Soft heap limit (regular cleanup, then maybe hard cleanup) + void releaseCachesToLimit(); + + /*---------------------------------- Utility ----------------------------------*/ + // TODO: move inside IndexedBins class + static int sizeToBin(size_t size) { + if (size >= maxBinned_HugePage) + return HUGE_BIN; + else if (size < minBinnedSize) + return NO_BIN; + + int bin = (size - minBinnedSize)/freeBinsStep; + + MALLOC_ASSERT(bin < HUGE_BIN, "Invalid size."); + return bin; + } + static bool toAlignedBin(FreeBlock *block, size_t size) { + return isAligned((char*)block + size, slabSize) && size >= slabSize; + } + +public: + /*--------------------- Init, reset, destroy, verify -------------------------*/ + void init(ExtMemoryPool *extMemoryPool); + bool destroy(); + + void verify(); + void reset(); + bool clean(); // clean on caches cleanup + + /*------------------------- Slab block request --------------------------------*/ + BlockI *getSlabBlock(int num); + void putSlabBlock(BlockI *block); + + /*-------------------------- Large object request -----------------------------*/ + LargeMemoryBlock *getLargeBlock(size_t size); + // TODO: make consistent with getLargeBlock + void returnLargeObject(LargeMemoryBlock *lmb); + + /*-------------------------- Backreference memory request ----------------------*/ + void *getBackRefSpace(size_t size, bool *rawMemUsed); + void putBackRefSpace(void *b, size_t size, bool rawMemUsed); + + /*----------------------------- Remap object ----------------------------------*/ + void *remap(void *ptr, size_t oldSize, size_t newSize, size_t alignment); + + /*---------------------------- Validation -------------------------------------*/ + bool inUserPool() const; + bool ptrCanBeValid(void *ptr) const { return usedAddrRange.inRange(ptr); } + + /*-------------------------- Configuration API --------------------------------*/ + // Soft heap limit + void setRecommendedMaxSize(size_t softLimit) { + memSoftLimit = softLimit; + releaseCachesToLimit(); + } + + /*------------------------------- Info ----------------------------------------*/ + size_t getMaxBinnedSize() const; + + /*-------------------------- Testing, statistics ------------------------------*/ +#if __TBB_MALLOC_WHITEBOX_TEST + size_t getTotalMemSize() const { return totalMemSize.load(std::memory_order_relaxed); } +#endif +#if __TBB_MALLOC_BACKEND_STAT + void reportStat(FILE *f); +private: + static size_t binToSize(int bin) { + MALLOC_ASSERT(bin <= HUGE_BIN, "Invalid bin."); + + return bin*freeBinsStep + minBinnedSize; + } +#endif +}; + +#endif // __TBB_backend_H diff --git a/src/tbb/src/tbbmalloc/backref.cpp b/src/tbb/src/tbbmalloc/backref.cpp index 3343e5b7f..b0ea83069 100644 --- a/src/tbb/src/tbbmalloc/backref.cpp +++ b/src/tbb/src/tbbmalloc/backref.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2023 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include "tbbmalloc_internal.h" @@ -35,52 +31,52 @@ struct BackRefBlock : public BlockI { FreeObject *freeList; // list of all blocks that were allocated from raw mem (i.e., not from backend) BackRefBlock *nextRawMemBlock; - int allocatedCount; // the number of objects allocated - BackRefIdx::master_t myNum; // the index in the master + std::atomic<int> allocatedCount; // the number of objects allocated + BackRefIdx::main_t myNum; // the index in the main MallocMutex blockMutex; // true if this block has been added to the listForUse chain, - // modifications protected by masterMutex - bool addedToForUse; + // modifications protected by mainMutex + std::atomic<bool> addedToForUse; BackRefBlock(const BackRefBlock *blockToUse, intptr_t num) : - nextForUse(NULL), bumpPtr((FreeObject*)((uintptr_t)blockToUse + slabSize - sizeof(void*))), - freeList(NULL), nextRawMemBlock(NULL), allocatedCount(0), myNum(num), + nextForUse(nullptr), bumpPtr((FreeObject*)((uintptr_t)blockToUse + slabSize - sizeof(void*))), + freeList(nullptr), nextRawMemBlock(nullptr), allocatedCount(0), myNum(num), addedToForUse(false) { - memset(&blockMutex, 0, sizeof(MallocMutex)); + memset(static_cast<void*>(&blockMutex), 0, sizeof(MallocMutex)); - MALLOC_ASSERT(!(num >> CHAR_BIT*sizeof(BackRefIdx::master_t)), - "index in BackRefMaster must fit to BackRefIdx::master"); + MALLOC_ASSERT(!(num >> CHAR_BIT*sizeof(BackRefIdx::main_t)), + "index in BackRefMain must fit to BackRefIdx::main"); } // clean all but header - void zeroSet() { memset(this+1, 0, BackRefBlock::bytes-sizeof(BackRefBlock)); } + void zeroSet() { memset(static_cast<void*>(this+1), 0, BackRefBlock::bytes-sizeof(BackRefBlock)); } static const int bytes = slabSize; }; // max number of backreference pointers in slab block static const int BR_MAX_CNT = (BackRefBlock::bytes-sizeof(BackRefBlock))/sizeof(void*); -struct BackRefMaster { +struct BackRefMain { /* On 64-bit systems a slab block can hold up to ~2K back pointers to slab blocks - * or large objects, so it can address at least 32MB. The master array of 256KB + * or large objects, so it can address at least 32MB. The main array of 256KB * holds 32K pointers to such blocks, addressing ~1 TB. * On 32-bit systems there is ~4K back pointers in a slab block, so ~64MB can be addressed. - * The master array of 8KB holds 2K pointers to leaves, so ~128 GB can addressed. + * The main array of 8KB holds 2K pointers to leaves, so ~128 GB can addressed. */ static const size_t bytes = sizeof(uintptr_t)>4? 256*1024 : 8*1024; static const int dataSz; -/* space is reserved for master table and 4 leaves +/* space is reserved for main table and 4 leaves taking into account VirtualAlloc allocation granularity */ static const int leaves = 4; - static const size_t masterSize = BackRefMaster::bytes+leaves*BackRefBlock::bytes; + static const size_t mainSize = BackRefMain::bytes+leaves*BackRefBlock::bytes; // The size of memory request for a few more leaf blocks; // selected to match VirtualAlloc granularity static const size_t blockSpaceSize = 64*1024; Backend *backend; - BackRefBlock *active; // if defined, use it for allocations - BackRefBlock *listForUse; // the chain of data blocks with free items + std::atomic<BackRefBlock*> active; // if defined, use it for allocations + std::atomic<BackRefBlock*> listForUse; // the chain of data blocks with free items BackRefBlock *allRawMemBlocks; - intptr_t lastUsed; // index of the last used block + std::atomic <intptr_t> lastUsed; // index of the last used block bool rawMemUsed; MallocMutex requestNewSpaceMutex; BackRefBlock *backRefBl[1]; // the real size of the array is dataSz @@ -91,161 +87,175 @@ struct BackRefMaster { bool requestNewSpace(); }; -const int BackRefMaster::dataSz - = 1+(BackRefMaster::bytes-sizeof(BackRefMaster))/sizeof(BackRefBlock*); +const int BackRefMain::dataSz + = 1+(BackRefMain::bytes-sizeof(BackRefMain))/sizeof(BackRefBlock*); -static MallocMutex masterMutex; -static BackRefMaster *backRefMaster; +static MallocMutex mainMutex; +static std::atomic<BackRefMain*> backRefMain; -bool initBackRefMaster(Backend *backend) +bool initBackRefMain(Backend *backend) { bool rawMemUsed; - BackRefMaster *master = - (BackRefMaster*)backend->getBackRefSpace(BackRefMaster::masterSize, + BackRefMain *main = + (BackRefMain*)backend->getBackRefSpace(BackRefMain::mainSize, &rawMemUsed); - if (! master) + if (! main) return false; - master->backend = backend; - master->listForUse = master->allRawMemBlocks = NULL; - master->rawMemUsed = rawMemUsed; - master->lastUsed = -1; - memset(&master->requestNewSpaceMutex, 0, sizeof(MallocMutex)); - for (int i=0; i<BackRefMaster::leaves; i++) { - BackRefBlock *bl = (BackRefBlock*)((uintptr_t)master + BackRefMaster::bytes + i*BackRefBlock::bytes); + main->backend = backend; + main->listForUse.store(nullptr, std::memory_order_relaxed); + main->allRawMemBlocks = nullptr; + main->rawMemUsed = rawMemUsed; + main->lastUsed = -1; + memset(static_cast<void*>(&main->requestNewSpaceMutex), 0, sizeof(MallocMutex)); + for (int i=0; i<BackRefMain::leaves; i++) { + BackRefBlock *bl = (BackRefBlock*)((uintptr_t)main + BackRefMain::bytes + i*BackRefBlock::bytes); bl->zeroSet(); - master->initEmptyBackRefBlock(bl); + main->initEmptyBackRefBlock(bl); if (i) - master->addToForUseList(bl); + main->addToForUseList(bl); else // active leaf is not needed in listForUse - master->active = bl; + main->active.store(bl, std::memory_order_relaxed); } - // backRefMaster is read in getBackRef, so publish it in consistent state - FencedStore((intptr_t&)backRefMaster, (intptr_t)master); + // backRefMain is read in getBackRef, so publish it in consistent state + backRefMain.store(main, std::memory_order_release); return true; } -void destroyBackRefMaster(Backend *backend) +#if __TBB_SOURCE_DIRECTLY_INCLUDED +void destroyBackRefMain(Backend *backend) { - if (backRefMaster) { // Is initBackRefMaster() called? - for (BackRefBlock *curr=backRefMaster->allRawMemBlocks; curr; ) { + if (backRefMain.load(std::memory_order_acquire)) { // Is initBackRefMain() called? + for (BackRefBlock *curr = backRefMain.load(std::memory_order_relaxed)->allRawMemBlocks; curr; ) { BackRefBlock *next = curr->nextRawMemBlock; // allRawMemBlocks list is only for raw mem blocks - backend->putBackRefSpace(curr, BackRefMaster::blockSpaceSize, + backend->putBackRefSpace(curr, BackRefMain::blockSpaceSize, /*rawMemUsed=*/true); curr = next; } - backend->putBackRefSpace(backRefMaster, BackRefMaster::masterSize, - backRefMaster->rawMemUsed); + backend->putBackRefSpace(backRefMain.load(std::memory_order_relaxed), BackRefMain::mainSize, + backRefMain.load(std::memory_order_relaxed)->rawMemUsed); } } +#endif -void BackRefMaster::addToForUseList(BackRefBlock *bl) +void BackRefMain::addToForUseList(BackRefBlock *bl) { - bl->nextForUse = listForUse; - listForUse = bl; - bl->addedToForUse = true; + bl->nextForUse = listForUse.load(std::memory_order_relaxed); + listForUse.store(bl, std::memory_order_relaxed); + bl->addedToForUse.store(true, std::memory_order_relaxed); } -void BackRefMaster::initEmptyBackRefBlock(BackRefBlock *newBl) +void BackRefMain::initEmptyBackRefBlock(BackRefBlock *newBl) { intptr_t nextLU = lastUsed+1; new (newBl) BackRefBlock(newBl, nextLU); - MALLOC_ASSERT(nextLU < dataSz, NULL); + MALLOC_ASSERT(nextLU < dataSz, nullptr); backRefBl[nextLU] = newBl; // lastUsed is read in getBackRef, and access to backRefBl[lastUsed] // is possible only after checking backref against current lastUsed - FencedStore(lastUsed, nextLU); + lastUsed.store(nextLU, std::memory_order_release); } -bool BackRefMaster::requestNewSpace() +bool BackRefMain::requestNewSpace() { - bool rawMemUsed; - MALLOC_STATIC_ASSERT(!(blockSpaceSize % BackRefBlock::bytes), + bool isRawMemUsed; + static_assert(!(blockSpaceSize % BackRefBlock::bytes), "Must request space for whole number of blocks."); - if (backRefMaster->dataSz <= lastUsed + 1) // no space in master + if (BackRefMain::dataSz <= lastUsed + 1) // no space in main return false; // only one thread at a time may add blocks MallocMutex::scoped_lock newSpaceLock(requestNewSpaceMutex); - if (listForUse) // double check that only one block is available + if (listForUse.load(std::memory_order_relaxed)) // double check that only one block is available return true; - BackRefBlock *newBl = - (BackRefBlock*)backend->getBackRefSpace(blockSpaceSize, &rawMemUsed); + BackRefBlock *newBl = (BackRefBlock*)backend->getBackRefSpace(blockSpaceSize, &isRawMemUsed); if (!newBl) return false; - // touch a page for the 1st time without taking masterMutex ... + // touch a page for the 1st time without taking mainMutex ... for (BackRefBlock *bl = newBl; (uintptr_t)bl < (uintptr_t)newBl + blockSpaceSize; - bl = (BackRefBlock*)((uintptr_t)bl + BackRefBlock::bytes)) + bl = (BackRefBlock*)((uintptr_t)bl + BackRefBlock::bytes)) { bl->zeroSet(); + } - MallocMutex::scoped_lock lock(masterMutex); // ... and share under lock + MallocMutex::scoped_lock lock(mainMutex); // ... and share under lock - const size_t numOfUnusedIdxs = backRefMaster->dataSz - lastUsed - 1; - if (numOfUnusedIdxs <= 0) { // no space in master under lock, roll back - backend->putBackRefSpace(newBl, blockSpaceSize, rawMemUsed); + const size_t numOfUnusedIdxs = BackRefMain::dataSz - lastUsed - 1; + if (numOfUnusedIdxs <= 0) { // no space in main under lock, roll back + backend->putBackRefSpace(newBl, blockSpaceSize, isRawMemUsed); return false; } - // It's possible that only part of newBl is used, due to lack of indices in master. + // It's possible that only part of newBl is used, due to lack of indices in main. // This is OK as such underutilization is possible only once for backreferneces table. int blocksToUse = min(numOfUnusedIdxs, blockSpaceSize / BackRefBlock::bytes); // use the first block in the batch to maintain the list of "raw" memory // to be released at shutdown - if (rawMemUsed) { - newBl->nextRawMemBlock = backRefMaster->allRawMemBlocks; - backRefMaster->allRawMemBlocks = newBl; + if (isRawMemUsed) { + newBl->nextRawMemBlock = backRefMain.load(std::memory_order_relaxed)->allRawMemBlocks; + backRefMain.load(std::memory_order_relaxed)->allRawMemBlocks = newBl; } - for (BackRefBlock *bl = newBl; blocksToUse>0; - bl = (BackRefBlock*)((uintptr_t)bl + BackRefBlock::bytes), blocksToUse--) { + for (BackRefBlock *bl = newBl; blocksToUse>0; bl = (BackRefBlock*)((uintptr_t)bl + BackRefBlock::bytes), blocksToUse--) { initEmptyBackRefBlock(bl); - if (active->allocatedCount == BR_MAX_CNT) - active = bl; // active leaf is not needed in listForUse - else + if (active.load(std::memory_order_relaxed)->allocatedCount.load(std::memory_order_relaxed) == BR_MAX_CNT) { + active.store(bl, std::memory_order_release); // active leaf is not needed in listForUse + } else { addToForUseList(bl); + } } return true; } -BackRefBlock *BackRefMaster::findFreeBlock() +BackRefBlock *BackRefMain::findFreeBlock() { - if (active->allocatedCount < BR_MAX_CNT) - return active; - - if (listForUse) { // use released list - MallocMutex::scoped_lock lock(masterMutex); - - if (active->allocatedCount == BR_MAX_CNT && listForUse) { - active = listForUse; - listForUse = listForUse->nextForUse; - MALLOC_ASSERT(active->addedToForUse, ASSERT_TEXT); - active->addedToForUse = false; + BackRefBlock* active_block = active.load(std::memory_order_acquire); + MALLOC_ASSERT(active_block, ASSERT_TEXT); + + if (active_block->allocatedCount.load(std::memory_order_relaxed) < BR_MAX_CNT) + return active_block; + + if (listForUse.load(std::memory_order_relaxed)) { // use released list + MallocMutex::scoped_lock lock(mainMutex); + + if (active_block->allocatedCount.load(std::memory_order_relaxed) == BR_MAX_CNT) { + active_block = listForUse.load(std::memory_order_relaxed); + if (active_block) { + active.store(active_block, std::memory_order_release); + listForUse.store(active_block->nextForUse, std::memory_order_relaxed); + MALLOC_ASSERT(active_block->addedToForUse.load(std::memory_order_relaxed), ASSERT_TEXT); + active_block->addedToForUse.store(false, std::memory_order_relaxed); + } } } else // allocate new data node if (!requestNewSpace()) - return NULL; - return active; + return nullptr; + return active.load(std::memory_order_acquire); // reread because of requestNewSpace } void *getBackRef(BackRefIdx backRefIdx) { - // !backRefMaster means no initialization done, so it can't be valid memory + // !backRefMain means no initialization done, so it can't be valid memory // see addEmptyBackRefBlock for fences around lastUsed - if (!FencedLoad((intptr_t&)backRefMaster) - || backRefIdx.getMaster() > FencedLoad(backRefMaster->lastUsed) + if (!(backRefMain.load(std::memory_order_acquire)) + || backRefIdx.getMain() > (backRefMain.load(std::memory_order_relaxed)->lastUsed.load(std::memory_order_acquire)) || backRefIdx.getOffset() >= BR_MAX_CNT) - return NULL; - return *(void**)((uintptr_t)backRefMaster->backRefBl[backRefIdx.getMaster()] - + sizeof(BackRefBlock)+backRefIdx.getOffset()*sizeof(void*)); + { + return nullptr; + } + std::atomic<void*>& backRefEntry = *(std::atomic<void*>*)( + (uintptr_t)backRefMain.load(std::memory_order_relaxed)->backRefBl[backRefIdx.getMain()] + + sizeof(BackRefBlock) + backRefIdx.getOffset() * sizeof(std::atomic<void*>) + ); + return backRefEntry.load(std::memory_order_relaxed); } void setBackRef(BackRefIdx backRefIdx, void *newPtr) { - MALLOC_ASSERT(backRefIdx.getMaster()<=backRefMaster->lastUsed && backRefIdx.getOffset()<BR_MAX_CNT, - ASSERT_TEXT); - *(void**)((uintptr_t)backRefMaster->backRefBl[backRefIdx.getMaster()] - + sizeof(BackRefBlock) + backRefIdx.getOffset()*sizeof(void*)) = newPtr; + MALLOC_ASSERT(backRefIdx.getMain()<=backRefMain.load(std::memory_order_relaxed)->lastUsed.load(std::memory_order_relaxed) + && backRefIdx.getOffset()<BR_MAX_CNT, ASSERT_TEXT); + ((std::atomic<void*>*)((uintptr_t)backRefMain.load(std::memory_order_relaxed)->backRefBl[backRefIdx.getMain()] + + sizeof(BackRefBlock) + backRefIdx.getOffset() * sizeof(void*)))->store(newPtr, std::memory_order_relaxed); } BackRefIdx BackRefIdx::newBackRef(bool largeObj) @@ -256,11 +266,11 @@ BackRefIdx BackRefIdx::newBackRef(bool largeObj) bool lastBlockFirstUsed = false; do { - MALLOC_ASSERT(backRefMaster, ASSERT_TEXT); - blockToUse = backRefMaster->findFreeBlock(); + MALLOC_ASSERT(backRefMain.load(std::memory_order_relaxed), ASSERT_TEXT); + blockToUse = backRefMain.load(std::memory_order_relaxed)->findFreeBlock(); if (!blockToUse) return BackRefIdx(); - toUse = NULL; + toUse = nullptr; { // the block is locked to find a reference MallocMutex::scoped_lock lock(blockToUse->blockMutex); @@ -271,30 +281,32 @@ BackRefIdx BackRefIdx::newBackRef(bool largeObj) ((uintptr_t)blockToUse->freeList>=(uintptr_t)blockToUse && (uintptr_t)blockToUse->freeList < (uintptr_t)blockToUse + slabSize), ASSERT_TEXT); - } else if (blockToUse->allocatedCount < BR_MAX_CNT) { + } else if (blockToUse->allocatedCount.load(std::memory_order_relaxed) < BR_MAX_CNT) { toUse = (void**)blockToUse->bumpPtr; blockToUse->bumpPtr = (FreeObject*)((uintptr_t)blockToUse->bumpPtr - sizeof(void*)); - if (blockToUse->allocatedCount == BR_MAX_CNT-1) { + if (blockToUse->allocatedCount.load(std::memory_order_relaxed) == BR_MAX_CNT-1) { MALLOC_ASSERT((uintptr_t)blockToUse->bumpPtr < (uintptr_t)blockToUse+sizeof(BackRefBlock), ASSERT_TEXT); - blockToUse->bumpPtr = NULL; + blockToUse->bumpPtr = nullptr; } } if (toUse) { - if (!blockToUse->allocatedCount && !backRefMaster->listForUse) + if (!blockToUse->allocatedCount.load(std::memory_order_relaxed) && + !backRefMain.load(std::memory_order_relaxed)->listForUse.load(std::memory_order_relaxed)) { lastBlockFirstUsed = true; - blockToUse->allocatedCount++; + } + blockToUse->allocatedCount.store(blockToUse->allocatedCount.load(std::memory_order_relaxed) + 1, std::memory_order_relaxed); } } // end of lock scope } while (!toUse); // The first thread that uses the last block requests new space in advance; // possible failures are ignored. if (lastBlockFirstUsed) - backRefMaster->requestNewSpace(); + backRefMain.load(std::memory_order_relaxed)->requestNewSpace(); - res.master = blockToUse->myNum; + res.main = blockToUse->myNum; uintptr_t offset = ((uintptr_t)toUse - ((uintptr_t)blockToUse + sizeof(BackRefBlock)))/sizeof(void*); // Is offset too big? @@ -308,30 +320,34 @@ BackRefIdx BackRefIdx::newBackRef(bool largeObj) void removeBackRef(BackRefIdx backRefIdx) { MALLOC_ASSERT(!backRefIdx.isInvalid(), ASSERT_TEXT); - MALLOC_ASSERT(backRefIdx.getMaster()<=backRefMaster->lastUsed + MALLOC_ASSERT(backRefIdx.getMain()<=backRefMain.load(std::memory_order_relaxed)->lastUsed.load(std::memory_order_relaxed) && backRefIdx.getOffset()<BR_MAX_CNT, ASSERT_TEXT); - BackRefBlock *currBlock = backRefMaster->backRefBl[backRefIdx.getMaster()]; - FreeObject *freeObj = (FreeObject*)((uintptr_t)currBlock + sizeof(BackRefBlock) - + backRefIdx.getOffset()*sizeof(void*)); - MALLOC_ASSERT(((uintptr_t)freeObj>(uintptr_t)currBlock && - (uintptr_t)freeObj<(uintptr_t)currBlock + slabSize), ASSERT_TEXT); + BackRefBlock *currBlock = backRefMain.load(std::memory_order_relaxed)->backRefBl[backRefIdx.getMain()]; + std::atomic<void*>& backRefEntry = *(std::atomic<void*>*)((uintptr_t)currBlock + sizeof(BackRefBlock) + + backRefIdx.getOffset()*sizeof(std::atomic<void*>)); + MALLOC_ASSERT(((uintptr_t)&backRefEntry >(uintptr_t)currBlock && + (uintptr_t)&backRefEntry <(uintptr_t)currBlock + slabSize), ASSERT_TEXT); { MallocMutex::scoped_lock lock(currBlock->blockMutex); - freeObj->next = currBlock->freeList; - MALLOC_ASSERT(!freeObj->next || - ((uintptr_t)freeObj->next > (uintptr_t)currBlock - && (uintptr_t)freeObj->next < - (uintptr_t)currBlock + slabSize), ASSERT_TEXT); - currBlock->freeList = freeObj; - currBlock->allocatedCount--; + backRefEntry.store(currBlock->freeList, std::memory_order_relaxed); +#if MALLOC_DEBUG + uintptr_t backRefEntryValue = (uintptr_t)backRefEntry.load(std::memory_order_relaxed); + MALLOC_ASSERT(!backRefEntryValue || + (backRefEntryValue > (uintptr_t)currBlock + && backRefEntryValue < (uintptr_t)currBlock + slabSize), ASSERT_TEXT); +#endif + currBlock->freeList = (FreeObject*)&backRefEntry; + currBlock->allocatedCount.store(currBlock->allocatedCount.load(std::memory_order_relaxed)-1, std::memory_order_relaxed); } // TODO: do we need double-check here? - if (!currBlock->addedToForUse && currBlock!=backRefMaster->active) { - MallocMutex::scoped_lock lock(masterMutex); + if (!currBlock->addedToForUse.load(std::memory_order_relaxed) && + currBlock!=backRefMain.load(std::memory_order_relaxed)->active.load(std::memory_order_relaxed)) { + MallocMutex::scoped_lock lock(mainMutex); - if (!currBlock->addedToForUse && currBlock!=backRefMaster->active) - backRefMaster->addToForUseList(currBlock); + if (!currBlock->addedToForUse.load(std::memory_order_relaxed) && + currBlock!=backRefMain.load(std::memory_order_relaxed)->active.load(std::memory_order_relaxed)) + backRefMain.load(std::memory_order_relaxed)->addToForUseList(currBlock); } } diff --git a/src/tbb/src/tbbmalloc/lin32-tbbmalloc-export.def b/src/tbb/src/tbbmalloc/def/lin32-tbbmalloc.def similarity index 95% rename from src/tbb/src/tbbmalloc/lin32-tbbmalloc-export.def rename to src/tbb/src/tbbmalloc/def/lin32-tbbmalloc.def index 33eff0f1c..e096de62f 100644 --- a/src/tbb/src/tbbmalloc/lin32-tbbmalloc-export.def +++ b/src/tbb/src/tbbmalloc/def/lin32-tbbmalloc.def @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ { @@ -49,6 +45,7 @@ _ZN3rml12pool_reallocEPNS_10MemoryPoolEPvj; _ZN3rml20pool_aligned_reallocEPNS_10MemoryPoolEPvjj; _ZN3rml19pool_aligned_mallocEPNS_10MemoryPoolEjj; _ZN3rml13pool_identifyEPv; +_ZN3rml10pool_msizeEPNS_10MemoryPoolEPv; local: diff --git a/src/tbb/src/tbbmalloc/lin64-tbbmalloc-export.def b/src/tbb/src/tbbmalloc/def/lin64-tbbmalloc.def similarity index 95% rename from src/tbb/src/tbbmalloc/lin64-tbbmalloc-export.def rename to src/tbb/src/tbbmalloc/def/lin64-tbbmalloc.def index a57c933b6..96abb2bff 100644 --- a/src/tbb/src/tbbmalloc/lin64-tbbmalloc-export.def +++ b/src/tbb/src/tbbmalloc/def/lin64-tbbmalloc.def @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ { @@ -49,6 +45,7 @@ _ZN3rml12pool_reallocEPNS_10MemoryPoolEPvm; _ZN3rml20pool_aligned_reallocEPNS_10MemoryPoolEPvmm; _ZN3rml19pool_aligned_mallocEPNS_10MemoryPoolEmm; _ZN3rml13pool_identifyEPv; +_ZN3rml10pool_msizeEPNS_10MemoryPoolEPv; local: diff --git a/src/tbb/src/tbbmalloc/mac32-tbbmalloc-export.def b/src/tbb/src/tbbmalloc/def/mac64-tbbmalloc.def similarity index 58% rename from src/tbb/src/tbbmalloc/mac32-tbbmalloc-export.def rename to src/tbb/src/tbbmalloc/def/mac64-tbbmalloc.def index 71cc10795..71253e0aa 100644 --- a/src/tbb/src/tbbmalloc/mac32-tbbmalloc-export.def +++ b/src/tbb/src/tbbmalloc/def/mac64-tbbmalloc.def @@ -1,22 +1,16 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ +# Copyright (c) 2005-2021 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. _scalable_calloc _scalable_free @@ -46,3 +40,5 @@ __ZN3rml12pool_reallocEPNS_10MemoryPoolEPvm __ZN3rml20pool_aligned_reallocEPNS_10MemoryPoolEPvmm __ZN3rml19pool_aligned_mallocEPNS_10MemoryPoolEmm __ZN3rml13pool_identifyEPv +__ZN3rml10pool_msizeEPNS_10MemoryPoolEPv + diff --git a/src/tbb/src/tbbmalloc/win32-tbbmalloc-export.def b/src/tbb/src/tbbmalloc/def/win32-tbbmalloc.def similarity index 93% rename from src/tbb/src/tbbmalloc/win32-tbbmalloc-export.def rename to src/tbb/src/tbbmalloc/def/win32-tbbmalloc.def index f02c208be..bfad79f4e 100644 --- a/src/tbb/src/tbbmalloc/win32-tbbmalloc-export.def +++ b/src/tbb/src/tbbmalloc/def/win32-tbbmalloc.def @@ -1,4 +1,4 @@ -; Copyright (c) 2005-2017 Intel Corporation +; Copyright (c) 2005-2021 Intel Corporation ; ; Licensed under the Apache License, Version 2.0 (the "License"); ; you may not use this file except in compliance with the License. @@ -11,10 +11,6 @@ ; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ; See the License for the specific language governing permissions and ; limitations under the License. -; -; -; -; EXPORTS @@ -35,6 +31,8 @@ __TBB_malloc_safer_realloc __TBB_malloc_safer_msize __TBB_malloc_safer_aligned_msize __TBB_malloc_safer_aligned_realloc + +; memory pool stuff ?pool_create@rml@@YAPAVMemoryPool@1@HPBUMemPoolPolicy@1@@Z ?pool_create_v1@rml@@YA?AW4MemPoolError@1@HPBUMemPoolPolicy@1@PAPAVMemoryPool@1@@Z ?pool_destroy@rml@@YA_NPAVMemoryPool@1@@Z @@ -45,3 +43,5 @@ __TBB_malloc_safer_aligned_realloc ?pool_aligned_realloc@rml@@YAPAXPAVMemoryPool@1@PAXII@Z ?pool_aligned_malloc@rml@@YAPAXPAVMemoryPool@1@II@Z ?pool_identify@rml@@YAPAVMemoryPool@1@PAX@Z +?pool_msize@rml@@YAIPAVMemoryPool@1@PAX@Z + diff --git a/src/tbb/src/tbbmalloc/win64-tbbmalloc-export.def b/src/tbb/src/tbbmalloc/def/win64-tbbmalloc.def similarity index 94% rename from src/tbb/src/tbbmalloc/win64-tbbmalloc-export.def rename to src/tbb/src/tbbmalloc/def/win64-tbbmalloc.def index 0e335a497..ba6ae2d2e 100644 --- a/src/tbb/src/tbbmalloc/win64-tbbmalloc-export.def +++ b/src/tbb/src/tbbmalloc/def/win64-tbbmalloc.def @@ -1,4 +1,4 @@ -; Copyright (c) 2005-2017 Intel Corporation +; Copyright (c) 2005-2021 Intel Corporation ; ; Licensed under the Apache License, Version 2.0 (the "License"); ; you may not use this file except in compliance with the License. @@ -11,10 +11,6 @@ ; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ; See the License for the specific language governing permissions and ; limitations under the License. -; -; -; -; EXPORTS @@ -35,6 +31,7 @@ __TBB_malloc_safer_realloc __TBB_malloc_safer_msize __TBB_malloc_safer_aligned_msize __TBB_malloc_safer_aligned_realloc + ; memory pool stuff ?pool_create@rml@@YAPEAVMemoryPool@1@_JPEBUMemPoolPolicy@1@@Z ?pool_create_v1@rml@@YA?AW4MemPoolError@1@_JPEBUMemPoolPolicy@1@PEAPEAVMemoryPool@1@@Z @@ -46,3 +43,5 @@ __TBB_malloc_safer_aligned_realloc ?pool_aligned_realloc@rml@@YAPEAXPEAVMemoryPool@1@PEAX_K2@Z ?pool_aligned_malloc@rml@@YAPEAXPEAVMemoryPool@1@_K1@Z ?pool_identify@rml@@YAPEAVMemoryPool@1@PEAX@Z +?pool_msize@rml@@YA_KPEAVMemoryPool@1@PEAX@Z + diff --git a/src/tbb/src/tbbmalloc/frontend.cpp b/src/tbb/src/tbbmalloc/frontend.cpp index 9aa9a1358..c9aaf4633 100644 --- a/src/tbb/src/tbbmalloc/frontend.cpp +++ b/src/tbb/src/tbbmalloc/frontend.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2024 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,19 +12,15 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ - #include "tbbmalloc_internal.h" #include <errno.h> #include <new> /* for placement new */ #include <string.h> /* for memset */ -#include "../tbb/tbb_version.h" +#include "oneapi/tbb/version.h" +#include "../tbb/environment.h" #include "../tbb/itt_notify.h" // for __TBB_load_ittnotify() #if USE_PTHREAD @@ -32,7 +28,6 @@ #define TlsGetValue_func pthread_getspecific #define GetMyTID() pthread_self() #include <sched.h> - inline void do_yield() {sched_yield();} extern "C" { static void mallocThreadShutdownNotification(void*); } #if __sun || __SUNPRO_CC #define __asm__ asm @@ -44,22 +39,18 @@ #include<thread> #define TlsSetValue_func FlsSetValue #define TlsGetValue_func FlsGetValue - #define TlsAlloc() FlsAlloc(NULL) + #define TlsAlloc() FlsAlloc(nullptr) #define TLS_ALLOC_FAILURE FLS_OUT_OF_INDEXES #define TlsFree FlsFree - inline void do_yield() {std::this_thread::yield();} #else #define TlsSetValue_func TlsSetValue #define TlsGetValue_func TlsGetValue #define TLS_ALLOC_FAILURE TLS_OUT_OF_INDEXES - inline void do_yield() {SwitchToThread();} #endif #else #error Must define USE_PTHREAD or USE_WINTHREAD - #endif - #define FREELIST_NONBLOCKING 1 namespace rml { @@ -73,12 +64,6 @@ class MemoryPool; inline bool isMallocInitialized(); -bool RecursiveMallocCallProtector::noRecursion() { - MALLOC_ASSERT(isMallocInitialized(), - "Recursion status can be checked only when initialization was done."); - return !mallocRecursionDetected; -} - #endif // MALLOC_CHECK_RECURSION /** Support for handling the special UNUSABLE pointer state **/ @@ -123,7 +108,7 @@ class BootStrapBlocks { #if USE_INTERNAL_TID class ThreadId { static tls_key_t Tid_key; - static intptr_t ThreadCount; + std::atomic<intptr_t> ThreadCount; unsigned int id; @@ -133,23 +118,27 @@ class ThreadId { RecursiveMallocCallProtector scoped; // Thread-local value is zero -> first call from this thread, // need to initialize with next ID value (IDs start from 1) - result = AtomicIncrement(ThreadCount); // returned new value! + result = ++ThreadCount; // returned new value! TlsSetValue_func( Tid_key, reinterpret_cast<void*>(result) ); } return result; } public: - static void init() { + static bool init() { #if USE_WINTHREAD Tid_key = TlsAlloc(); + if (Tid_key == TLS_ALLOC_FAILURE) + return false; #else - int status = pthread_key_create( &Tid_key, NULL ); + int status = pthread_key_create( &Tid_key, nullptr ); if ( status ) { - fprintf (stderr, "The memory manager cannot create tls key during initialization; exiting \n"); - exit(1); + fprintf (stderr, "The memory manager cannot create tls key during initialization\n"); + return false; } #endif /* USE_WINTHREAD */ + return true; } +#if __TBB_SOURCE_DIRECTLY_INCLUDED static void destroy() { if( Tid_key ) { #if USE_WINTHREAD @@ -157,13 +146,12 @@ class ThreadId { #else int status = pthread_key_delete( Tid_key ); #endif /* USE_WINTHREAD */ - if ( status ) { - fprintf (stderr, "The memory manager cannot delete tls key; exiting \n"); - exit(1); - } + if ( status ) + fprintf (stderr, "The memory manager cannot delete tls key\n"); Tid_key = 0; } } +#endif ThreadId() : id(ThreadId::tlsNumber()) {} bool isCurrentThreadId() const { return id == ThreadId::tlsNumber(); } @@ -172,7 +160,7 @@ class ThreadId { friend unsigned int getThreadId() { return ThreadId::tlsNumber(); } #endif #if COLLECT_STATISTICS - static unsigned getMaxThreadId() { return ThreadCount; } + static unsigned getMaxThreadId() { return ThreadCount.load(std::memory_order_relaxed); } friend int STAT_increment(ThreadId tid, int bin, int ctr); #endif @@ -192,19 +180,28 @@ int STAT_increment(ThreadId tid, int bin, int ctr) class ThreadId { #if USE_PTHREAD - pthread_t tid; + std::atomic<pthread_t> tid; #else - DWORD tid; + std::atomic<DWORD> tid; #endif public: ThreadId() : tid(GetMyTID()) {} + ThreadId(ThreadId &other) = delete; + ~ThreadId() = default; + #if USE_PTHREAD - bool isCurrentThreadId() const { return pthread_equal(pthread_self(), tid); } + bool isCurrentThreadId() const { return pthread_equal(pthread_self(), tid.load(std::memory_order_relaxed)); } #else - bool isCurrentThreadId() const { return GetCurrentThreadId() == tid; } + bool isCurrentThreadId() const { return GetCurrentThreadId() == tid.load(std::memory_order_relaxed); } #endif - static void init() {} + ThreadId& operator=(const ThreadId& other) { + tid.store(other.tid.load(std::memory_order_relaxed), std::memory_order_relaxed); + return *this; + } + static bool init() { return true; } +#if __TBB_SOURCE_DIRECTLY_INCLUDED static void destroy() {} +#endif }; #endif // USE_INTERNAL_TID @@ -266,7 +263,7 @@ class MemoryPool { // and set reasonable low granularity static const size_t defaultGranularity = estimatedCacheLineSize; - MemoryPool(); // deny + MemoryPool() = delete; // deny public: static MallocMutex memPoolListLock; @@ -277,14 +274,15 @@ class MemoryPool { ExtMemoryPool extMemPool; BootStrapBlocks bootStrapBlocks; - bool init(intptr_t poolId, const MemPoolPolicy* memPoolPolicy); static void initDefaultPool(); + + bool init(intptr_t poolId, const MemPoolPolicy* memPoolPolicy); bool reset(); bool destroy(); - void processThreadShutdown(TLSData *tlsData); + void onThreadShutdown(TLSData *tlsData); inline TLSData *getTLS(bool create); - void clearTLS() { extMemPool.tlsPointerKey.setThreadMallocTLS(NULL); } + void clearTLS() { extMemPool.tlsPointerKey.setThreadMallocTLS(nullptr); } Block *getEmptyBlock(size_t size); void returnEmptyBlock(Block *block, bool poolTheBlock); @@ -320,8 +318,8 @@ struct Padding : PaddingImpl<N/sizeof(size_t)> {}; // fields and fields commonly accessed by not owner threads. class GlobalBlockFields : public BlockI { protected: - FreeObject *publicFreeList; - Block *nextPrivatizable; + std::atomic<FreeObject*> publicFreeList; + std::atomic<Block*> nextPrivatizable; MemoryPool *poolPtr; }; @@ -333,8 +331,8 @@ class LocalBlockFields : public GlobalBlockFields, Padding<blockHeaderAlignment FreeObject *freeList; /* Pointer to local data for the owner thread. Used for fast finding tls when releasing object from a block that current thread owned. - NULL for orphaned blocks. */ - TLSData *tlsPtr; + nullptr for orphaned blocks. */ + std::atomic<TLSData*> tlsPtr; ThreadId ownerTid; /* the ID of the thread that owns or last owned the block */ BackRefIdx backRefIdx; uint16_t allocatedCount; /* Number of objects allocated (obviously by the owning thread) */ @@ -355,17 +353,26 @@ class LocalBlockFields : public GlobalBlockFields, Padding<blockHeaderAlignment class Block : public LocalBlockFields, Padding<2*blockHeaderAlignment - sizeof(LocalBlockFields)> { public: - bool empty() const { return allocatedCount==0 && !isSolidPtr(publicFreeList); } + bool empty() const { + if (allocatedCount > 0) return false; + MALLOC_ASSERT(!isSolidPtr(publicFreeList.load(std::memory_order_relaxed)), ASSERT_TEXT); + return true; + } inline FreeObject* allocate(); inline FreeObject *allocateFromFreeList(); - inline bool emptyEnoughToUse(); + + inline bool adjustFullness(); + void adjustPositionInBin(Bin* bin = nullptr); +#if MALLOC_DEBUG bool freeListNonNull() { return freeList; } +#endif void freePublicObject(FreeObject *objectToFree); inline void freeOwnObject(void *object); void reset(); - void privatizePublicFreeList( bool cleanup = false ); + void privatizePublicFreeList( bool reset = true ); void restoreBumpPtr(); void privatizeOrphaned(TLSData *tls, unsigned index); + bool readyToShare(); void shareOrphaned(intptr_t binTag, unsigned index); unsigned int getSize() const { MALLOC_ASSERT(isStartupAllocObject() || objectSize<minLargeObjectSize, @@ -383,8 +390,10 @@ class Block : public LocalBlockFields, // try to check this before this dereference as for invalid objects // this may be unreadable MALLOC_ASSERT(isAligned(object, sizeof(size_t)), "Try to free invalid small object"); +#if !__TBB_USE_THREAD_SANITIZER // releasing to free slab MALLOC_ASSERT(allocatedCount>0, msg); +#endif // must not point to slab's header MALLOC_ASSERT((uintptr_t)object - (uintptr_t)this >= sizeof(Block), msg); if (startupAllocObjSizeMark == objectSize) // startup block @@ -392,15 +401,17 @@ class Block : public LocalBlockFields, else { // non-startup objects are 8 Byte aligned MALLOC_ASSERT(isAligned(object, 8), "Try to free invalid small object"); + FreeObject *toFree = findObjectToFree(object); +#if !__TBB_USE_THREAD_SANITIZER MALLOC_ASSERT(allocatedCount <= (slabSize-sizeof(Block))/objectSize && (!bumpPtr || object>bumpPtr), msg); - FreeObject *toFree = findObjectToFree(object); // check against head of freeList, as this is mostly // expected after double free MALLOC_ASSERT(toFree != freeList, msg); +#endif // check against head of publicFreeList, to detect double free // involving foreign thread - MALLOC_ASSERT(toFree != publicFreeList, msg); + MALLOC_ASSERT(toFree != publicFreeList.load(std::memory_order_relaxed), msg); } #else suppress_unused_warning(object); @@ -414,19 +425,21 @@ class Block : public LocalBlockFields, void cleanBlockHeader(); private: - static const float emptyEnoughRatio; /* "Reactivate" a block if this share of its objects is free. */ + static const float emptyEnoughRatio; /* Threshold on free space needed to "reactivate" a block */ inline FreeObject *allocateFromBumpPtr(); inline FreeObject *findAllocatedObject(const void *address) const; +#if MALLOC_DEBUG inline bool isProperlyPlaced(const void *object) const; +#endif inline void markOwned(TLSData *tls) { - MALLOC_ASSERT(!tlsPtr, ASSERT_TEXT); + MALLOC_ASSERT(!tlsPtr.load(std::memory_order_relaxed), ASSERT_TEXT); ownerTid = ThreadId(); /* save the ID of the current thread */ - tlsPtr = tls; + tlsPtr.store(tls, std::memory_order_relaxed); } inline void markOrphaned() { - MALLOC_ASSERT(tlsPtr, ASSERT_TEXT); - tlsPtr = NULL; + MALLOC_ASSERT(tlsPtr.load(std::memory_order_relaxed), ASSERT_TEXT); + tlsPtr.store(nullptr, std::memory_order_relaxed); } friend class Bin; @@ -436,33 +449,38 @@ class Block : public LocalBlockFields, const float Block::emptyEnoughRatio = 1.0 / 4.0; -MALLOC_STATIC_ASSERT(sizeof(Block) <= 2*estimatedCacheLineSize, +static_assert(sizeof(Block) <= 2*estimatedCacheLineSize, "The class Block does not fit into 2 cache lines on this platform. " "Defining USE_INTERNAL_TID may help to fix it."); class Bin { - Block *activeBlk; - Block *mailbox; +private: +public: + Block *activeBlk; + std::atomic<Block*> mailbox; MallocMutex mailLock; public: inline Block* getActiveBlock() const { return activeBlk; } - void resetActiveBlock() { activeBlk = 0; } - bool activeBlockUnused() const { return activeBlk && !activeBlk->allocatedCount; } + void resetActiveBlock() { activeBlk = nullptr; } inline void setActiveBlock(Block *block); inline Block* setPreviousBlockActive(); - Block* getPublicFreeListBlock(); + Block* getPrivatizedFreeListBlock(); void moveBlockToFront(Block *block); - void processLessUsedBlock(MemoryPool *memPool, Block *block); + bool cleanPublicFreeLists(); + void processEmptyBlock(Block *block, bool poolTheBlock); + void addPublicFreeListBlock(Block* block); void outofTLSBin(Block* block); void verifyTLSBin(size_t size) const; void pushTLSBin(Block* block); +#if MALLOC_DEBUG void verifyInitState() const { - MALLOC_ASSERT( activeBlk == 0, ASSERT_TEXT ); - MALLOC_ASSERT( mailbox == 0, ASSERT_TEXT ); + MALLOC_ASSERT( !activeBlk, ASSERT_TEXT ); + MALLOC_ASSERT( !mailbox.load(std::memory_order_relaxed), ASSERT_TEXT ); } +#endif friend void Block::freePublicObject (FreeObject *objectToFree); }; @@ -518,16 +536,16 @@ const uint32_t minLargeObjectSize = fittingSize5 + 1; * threads memory that are likely in local cache(s) of our CPU. */ class FreeBlockPool { - Block *head; +private: + std::atomic<Block*> head; int size; Backend *backend; - bool lastAccessMiss; public: static const int POOL_HIGH_MARK = 32; static const int POOL_LOW_MARK = 8; class ResOfGet { - ResOfGet(); + ResOfGet() = delete; public: Block* block; bool lastAccMiss; @@ -543,10 +561,11 @@ class FreeBlockPool { template<int LOW_MARK, int HIGH_MARK> class LocalLOCImpl { +private: static const size_t MAX_TOTAL_SIZE = 4*1024*1024; // TODO: can single-linked list be faster here? - LargeMemoryBlock *head, - *tail; // need it when do releasing on overflow + LargeMemoryBlock *tail; // need it when do releasing on overflow + std::atomic<LargeMemoryBlock*> head; size_t totalSize; int numOfBlocks; public: @@ -554,7 +573,7 @@ class LocalLOCImpl { LargeMemoryBlock *get(size_t size); bool externalCleanup(ExtMemoryPool *extMemPool); #if __TBB_MALLOC_WHITEBOX_TEST - LocalLOCImpl() : head(NULL), tail(NULL), totalSize(0), numOfBlocks(0) {} + LocalLOCImpl() : tail(nullptr), head(nullptr), totalSize(0), numOfBlocks(0) {} static size_t getMaxSize() { return MAX_TOTAL_SIZE; } static const int LOC_HIGH_MARK = HIGH_MARK; #else @@ -572,20 +591,25 @@ class TLSData : public TLSRemote { LocalLOC lloc; unsigned currCacheIdx; private: - bool unused; + std::atomic<bool> unused; public: - TLSData(MemoryPool *mPool, Backend *bknd) : memPool(mPool), freeSlabBlocks(bknd) {} + TLSData(MemoryPool *mPool, Backend *bknd) : memPool(mPool), freeSlabBlocks(bknd), currCacheIdx(0) {} MemoryPool *getMemPool() const { return memPool; } Bin* getAllocationBin(size_t size); - void release(MemoryPool *mPool); - bool externalCleanup(ExtMemoryPool *mPool, bool cleanOnlyUnused) { - if (!unused && cleanOnlyUnused) return false; + void release(); + bool externalCleanup(bool cleanOnlyUnused, bool cleanBins) { + if (!unused.load(std::memory_order_relaxed) && cleanOnlyUnused) return false; + // Heavy operation in terms of synchronization complexity, + // should be called only for the current thread + bool released = cleanBins ? cleanupBlockBins() : false; // both cleanups to be called, and the order is not important - return lloc.externalCleanup(mPool) | freeSlabBlocks.externalCleanup(); + bool lloc_cleaned = lloc.externalCleanup(&memPool->extMemPool); + bool free_slab_blocks_cleaned = freeSlabBlocks.externalCleanup(); + return released || lloc_cleaned || free_slab_blocks_cleaned; } - bool cleanUnusedActiveBlocks(Backend *backend, bool userPool); - void markUsed() { unused = false; } // called by owner when TLS touched - void markUnused() { unused = true; } // can be called by not owner thread + bool cleanupBlockBins(); + void markUsed() { unused.store(false, std::memory_order_relaxed); } // called by owner when TLS touched + void markUnused() { unused.store(true, std::memory_order_relaxed); } // can be called by not owner thread }; TLSData *TLSKey::createTLS(MemoryPool *memPool, Backend *backend) @@ -593,7 +617,7 @@ TLSData *TLSKey::createTLS(MemoryPool *memPool, Backend *backend) MALLOC_ASSERT( sizeof(TLSData) >= sizeof(Bin) * numBlockBins + sizeof(FreeBlockPool), ASSERT_TEXT ); TLSData* tls = (TLSData*) memPool->bootStrapBlocks.allocate(memPool, sizeof(TLSData)); if ( !tls ) - return NULL; + return nullptr; new(tls) TLSData(memPool, backend); /* the block contains zeroes after bootStrapMalloc, so bins are initialized */ #if MALLOC_DEBUG @@ -605,38 +629,38 @@ TLSData *TLSKey::createTLS(MemoryPool *memPool, Backend *backend) return tls; } -bool TLSData::cleanUnusedActiveBlocks(Backend *backend, bool userPool) +bool TLSData::cleanupBlockBins() { bool released = false; - // active blocks can be not used, so return them to backend - for (uint32_t i=0; i<numBlockBinLimit; i++) - if (bin[i].activeBlockUnused()) { - Block *block = bin[i].getActiveBlock(); + for (uint32_t i = 0; i < numBlockBinLimit; i++) { + released |= bin[i].cleanPublicFreeLists(); + // After cleaning public free lists, only the active block might be empty. + // Do not use processEmptyBlock because it will just restore bumpPtr. + Block *block = bin[i].getActiveBlock(); + if (block && block->empty()) { bin[i].outofTLSBin(block); - // slab blocks in user's pools do not have valid backRefIdx - if (!userPool) - removeBackRef(*(block->getBackRefIdx())); - backend->putSlabBlock(block); - + memPool->returnEmptyBlock(block, /*poolTheBlock=*/false); released = true; } + } return released; } bool ExtMemoryPool::releaseAllLocalCaches() { - bool released = allLocalCaches.cleanup(this, /*cleanOnlyUnused=*/false); + // Iterate all registered TLS data and clean LLOC and Slab pools + bool released = allLocalCaches.cleanup(/*cleanOnlyUnused=*/false); + // Bins privatization is done only for the current thread if (TLSData *tlsData = tlsPointerKey.getThreadMallocTLS()) - // released only for current thread for now - released |= tlsData->cleanUnusedActiveBlocks(&backend, userPool()); + released |= tlsData->cleanupBlockBins(); return released; } void AllLocalCaches::registerThread(TLSRemote *tls) { - tls->prev = NULL; + tls->prev = nullptr; MallocMutex::scoped_lock lock(listLock); MALLOC_ASSERT(head!=tls, ASSERT_TEXT); tls->next = head; @@ -659,22 +683,20 @@ void AllLocalCaches::unregisterThread(TLSRemote *tls) MALLOC_ASSERT(!tls->next || tls->next->next!=tls->next, ASSERT_TEXT); } -bool AllLocalCaches::cleanup(ExtMemoryPool *extPool, bool cleanOnlyUnused) +bool AllLocalCaches::cleanup(bool cleanOnlyUnused) { - bool total = false; + bool released = false; { MallocMutex::scoped_lock lock(listLock); - for (TLSRemote *curr=head; curr; curr=curr->next) - total |= static_cast<TLSData*>(curr)-> - externalCleanup(extPool, cleanOnlyUnused); + released |= static_cast<TLSData*>(curr)->externalCleanup(cleanOnlyUnused, /*cleanBins=*/false); } - return total; + return released; } void AllLocalCaches::markUnused() { - bool locked; + bool locked = false; MallocMutex::scoped_lock lock(listLock, /*block=*/false, &locked); if (!locked) // not wait for marking if someone doing something with it return; @@ -685,8 +707,8 @@ void AllLocalCaches::markUnused() #if MALLOC_CHECK_RECURSION MallocMutex RecursiveMallocCallProtector::rmc_mutex; -pthread_t RecursiveMallocCallProtector::owner_thread; -void *RecursiveMallocCallProtector::autoObjPtr; +std::atomic<pthread_t> RecursiveMallocCallProtector::owner_thread; +std::atomic<void*> RecursiveMallocCallProtector::autoObjPtr; bool RecursiveMallocCallProtector::mallocRecursionDetected; #if __FreeBSD__ bool RecursiveMallocCallProtector::canUsePthread; @@ -703,7 +725,14 @@ enum MemoryOrigin { unknownMem // can be allocated by system allocator or TBB allocator }; -template<MemoryOrigin> bool isLargeObject(void *object); +template<MemoryOrigin> +#if __TBB_USE_THREAD_SANITIZER +// We have a real race when accessing the large object header for +// non large objects (e.g. small or foreign objects). +// Therefore, we need to hide this access from the thread sanitizer +__attribute__((no_sanitize("thread"))) +#endif +bool isLargeObject(void *object); static void *internalMalloc(size_t size); static void internalFree(void *object); static void *internalPoolMalloc(MemoryPool* mPool, size_t size); @@ -733,7 +762,7 @@ ALWAYSINLINE( bool isMallocInitialized() ); /* * Given a number return the highest non-zero bit in it. It is intended to work with 32-bit values only. - * Moreover, on IPF, for sake of simplicity and performance, it is narrowed to only serve for 64 to 1023. + * Moreover, on some platforms, for sake of simplicity and performance, it is narrowed to only serve for 64 to 1023. * This is enough for current algorithm of distribution of sizes among bins. * __TBB_Log2 is not used here to minimize dependencies on TBB specific sources. */ @@ -747,7 +776,7 @@ static inline unsigned int highestBitPos(unsigned int n) unsigned int pos; #if __ARCH_x86_32||__ARCH_x86_64 -# if __linux__||__APPLE__||__FreeBSD__||__NetBSD__||__sun||__MINGW32__ +# if __unix__||__APPLE__||__MINGW32__ __asm__ ("bsr %1,%0" : "=r"(pos) : "r"(n)); # elif (_WIN32 && (!_WIN64 || __INTEL_COMPILER)) __asm @@ -774,28 +803,27 @@ static inline unsigned int highestBitPos(unsigned int n) return pos; } -template<bool Is32Bit> unsigned int getSmallObjectIndex(unsigned int size) { - return (size-1)>>3; -} -template<> -unsigned int getSmallObjectIndex</*Is32Bit=*/false>(unsigned int size) -{ - // For 64-bit malloc, 16 byte alignment is needed except for bin 0. unsigned int result = (size-1)>>3; - if (result) result |= 1; // 0,1,3,5,7; bins 2,4,6 are not aligned to 16 bytes + constexpr bool is_64bit = (8 == sizeof(void*)); + if (is_64bit) { + // For 64-bit malloc, 16 byte alignment is needed except for bin 0. + if (result) result |= 1; // 0,1,3,5,7; bins 2,4,6 are not aligned to 16 bytes + } return result; } + /* * Depending on indexRequest, for a given size return either the index into the bin * for objects of this size, or the actual size of objects in this bin. + * TODO: Change return type to unsigned short. */ template<bool indexRequest> static unsigned int getIndexOrObjectSize (unsigned int size) { if (size <= maxSmallObjectSize) { // selection from 8/16/24/32/40/48/56/64 - unsigned int index = getSmallObjectIndex</*Is32Bit=*/(sizeof(size_t)<=4)>( size ); + unsigned int index = getSmallObjectIndex( size ); /* Bin 0 is for 8 bytes, bin 1 is for 16, and so forth */ return indexRequest ? index : (index+1)<<3; } @@ -859,19 +887,18 @@ void *BootStrapBlocks::allocate(MemoryPool *memPool, size_t size) } else { if (!bootStrapBlock) { bootStrapBlock = memPool->getEmptyBlock(size); - if (!bootStrapBlock) return NULL; + if (!bootStrapBlock) return nullptr; } result = bootStrapBlock->bumpPtr; bootStrapBlock->bumpPtr = (FreeObject *)((uintptr_t)bootStrapBlock->bumpPtr - bootStrapBlock->objectSize); if ((uintptr_t)bootStrapBlock->bumpPtr < (uintptr_t)bootStrapBlock+sizeof(Block)) { - bootStrapBlock->bumpPtr = NULL; + bootStrapBlock->bumpPtr = nullptr; bootStrapBlock->next = bootStrapBlockUsed; bootStrapBlockUsed = bootStrapBlock; - bootStrapBlock = NULL; + bootStrapBlock = nullptr; } } } // Unlock with release - memset (result, 0, size); return (void*)result; } @@ -888,8 +915,8 @@ void BootStrapBlocks::free(void* ptr) void BootStrapBlocks::reset() { - bootStrapBlock = bootStrapBlockUsed = NULL; - bootStrapObjectList = NULL; + bootStrapBlock = bootStrapBlockUsed = nullptr; + bootStrapObjectList = nullptr; } #if !(FREELIST_NONBLOCKING) @@ -903,27 +930,27 @@ static MallocMutex publicFreeListLock; // lock for changes of publicFreeList * On Linux, it seems to be necessary to avoid linking with C++ libraries. * * By usage convention there is no race on the initialization. */ -LifoList::LifoList( ) : top(NULL) +LifoList::LifoList( ) : top(nullptr) { // MallocMutex assumes zero initialization - memset(&lock, 0, sizeof(MallocMutex)); + memset(static_cast<void*>(&lock), 0, sizeof(MallocMutex)); } void LifoList::push(Block *block) { MallocMutex::scoped_lock scoped_cs(lock); - block->next = top; - top = block; + block->next = top.load(std::memory_order_relaxed); + top.store(block, std::memory_order_relaxed); } Block *LifoList::pop() { - Block *block=NULL; - if (top) { + Block* block = nullptr; + if (top.load(std::memory_order_relaxed)) { MallocMutex::scoped_lock scoped_cs(lock); - if (top) { - block = top; - top = block->next; + block = top.load(std::memory_order_relaxed); + if (block) { + top.store(block->next, std::memory_order_relaxed); } } return block; @@ -931,11 +958,11 @@ Block *LifoList::pop() Block *LifoList::grab() { - Block *block = NULL; - if (top) { + Block *block = nullptr; + if (top.load(std::memory_order_relaxed)) { MallocMutex::scoped_lock scoped_cs(lock); - block = top; - top = NULL; + block = top.load(std::memory_order_relaxed); + top.store(nullptr, std::memory_order_relaxed); } return block; } @@ -944,7 +971,7 @@ Block *LifoList::grab() template<bool poolDestroy> void AllLargeBlocksList::releaseAll(Backend *backend) { LargeMemoryBlock *next, *lmb = loHead; - loHead = NULL; + loHead = nullptr; for (; lmb; lmb = next) { next = lmb->gNext; @@ -955,7 +982,7 @@ template<bool poolDestroy> void AllLargeBlocksList::releaseAll(Backend *backend) } else { // clean g(Next|Prev) to prevent removing lmb // from AllLargeBlocksList inside returnLargeObject - lmb->gNext = lmb->gPrev = NULL; + lmb->gNext = lmb->gPrev = nullptr; backend->returnLargeObject(lmb); } } @@ -980,10 +1007,10 @@ inline Bin* TLSData::getAllocationBin(size_t size) /* Return an empty uninitialized block in a non-blocking fashion. */ Block *MemoryPool::getEmptyBlock(size_t size) { - TLSData* tls = extMemPool.tlsPointerKey.getThreadMallocTLS(); + TLSData* tls = getTLS(/*create=*/false); // try to use per-thread cache, if TLS available FreeBlockPool::ResOfGet resOfGet = tls? - tls->freeSlabBlocks.getBlock() : FreeBlockPool::ResOfGet(NULL, false); + tls->freeSlabBlocks.getBlock() : FreeBlockPool::ResOfGet(nullptr, false); Block *result = resOfGet.block; if (!result) { // not found in local cache, asks backend for slabs @@ -991,7 +1018,7 @@ Block *MemoryPool::getEmptyBlock(size_t size) BackRefIdx backRefIdx[Backend::numOfSlabAllocOnMiss]; result = static_cast<Block*>(extMemPool.backend.getSlabBlock(num)); - if (!result) return NULL; + if (!result) return nullptr; if (!extMemPool.userPool()) for (int i=0; i<num; i++) { @@ -1003,7 +1030,7 @@ Block *MemoryPool::getEmptyBlock(size_t size) Block *b = result; for (int j=0; j<num; b=(Block*)((uintptr_t)b+slabSize), j++) extMemPool.backend.putSlabBlock(b); - return NULL; + return nullptr; } } // resources were allocated, register blocks @@ -1016,7 +1043,7 @@ Block *MemoryPool::getEmptyBlock(size_t size) setBackRef(backRefIdx[i], b); b->backRefIdx = backRefIdx[i]; } - b->tlsPtr = tls; + b->tlsPtr.store(tls, std::memory_order_relaxed); b->poolPtr = this; // all but first one go to per-thread pool if (i > 0) { @@ -1035,9 +1062,8 @@ void MemoryPool::returnEmptyBlock(Block *block, bool poolTheBlock) { block->reset(); if (poolTheBlock) { - extMemPool.tlsPointerKey.getThreadMallocTLS()->freeSlabBlocks.returnBlock(block); - } - else { + getTLS(/*create=*/false)->freeSlabBlocks.returnBlock(block); + } else { // slab blocks in user's pools do not have valid backRefIdx if (!extMemPool.userPool()) removeBackRef(*(block->getBackRefIdx())); @@ -1056,11 +1082,11 @@ bool ExtMemoryPool::init(intptr_t poolId, rawAllocType rawAlloc, this->keepAllMemory = keepAllMemory; this->fixedPool = fixedPool; this->delayRegsReleasing = false; - if (! initTLS()) + if (!initTLS()) return false; loc.init(this); backend.init(this); - MALLOC_ASSERT(isPoolValid(), NULL); + MALLOC_ASSERT(isPoolValid(), nullptr); return true; } @@ -1123,7 +1149,7 @@ bool MemoryPool::destroy() extMemPool.lmbList.releaseAll</*poolDestroy=*/true>(&extMemPool.backend); else { // only one non-userPool() is supported now - MALLOC_ASSERT(this==defaultMemPool, NULL); + MALLOC_ASSERT(this==defaultMemPool, nullptr); // There and below in extMemPool.destroy(), do not restore initial state // for user pool, because it's just about to be released. But for system // pool restoring, we do not want to do zeroing of it on subsequent reload. @@ -1133,11 +1159,13 @@ bool MemoryPool::destroy() return extMemPool.destroy(); } -void MemoryPool::processThreadShutdown(TLSData *tlsData) +void MemoryPool::onThreadShutdown(TLSData *tlsData) { - tlsData->release(this); - bootStrapBlocks.free(tlsData); - clearTLS(); + if (tlsData) { // might be called for "empty" TLS + tlsData->release(); + bootStrapBlocks.free(tlsData); + clearTLS(); + } } #if MALLOC_DEBUG @@ -1186,8 +1214,8 @@ void Bin::pushTLSBin(Block* block) MALLOC_ASSERT( block->isOwnedByCurrentThread(), ASSERT_TEXT ); MALLOC_ASSERT( block->objectSize != 0, ASSERT_TEXT ); - MALLOC_ASSERT( block->next == NULL, ASSERT_TEXT ); - MALLOC_ASSERT( block->previous == NULL, ASSERT_TEXT ); + MALLOC_ASSERT( block->next == nullptr, ASSERT_TEXT ); + MALLOC_ASSERT( block->previous == nullptr, ASSERT_TEXT ); MALLOC_ASSERT( this, ASSERT_TEXT ); verifyTLSBin(size); @@ -1221,7 +1249,7 @@ void Bin::outofTLSBin(Block* block) if (block == activeBlk) { activeBlk = block->previous? block->previous : block->next; } - /* Delink the block */ + /* Unlink the block */ if (block->previous) { MALLOC_ASSERT( block->previous->next == block, ASSERT_TEXT ); block->previous->next = block->next; @@ -1230,116 +1258,157 @@ void Bin::outofTLSBin(Block* block) MALLOC_ASSERT( block->next->previous == block, ASSERT_TEXT ); block->next->previous = block->previous; } - block->next = NULL; - block->previous = NULL; + block->next = nullptr; + block->previous = nullptr; verifyTLSBin(size); } -Block* Bin::getPublicFreeListBlock() +Block* Bin::getPrivatizedFreeListBlock() { Block* block; MALLOC_ASSERT( this, ASSERT_TEXT ); // if this method is called, active block usage must be unsuccessful - MALLOC_ASSERT( !activeBlk && !mailbox || activeBlk && activeBlk->isFull, ASSERT_TEXT ); + MALLOC_ASSERT( (!activeBlk && !mailbox.load(std::memory_order_relaxed)) || (activeBlk && activeBlk->isFull), ASSERT_TEXT ); // the counter should be changed STAT_increment(getThreadId(), ThreadCommonCounters, lockPublicFreeList); - if (!FencedLoad((intptr_t&)mailbox)) // hotpath is empty mailbox - return NULL; + if (!mailbox.load(std::memory_order_acquire)) // hotpath is empty mailbox + return nullptr; else { // mailbox is not empty, take lock and inspect it MallocMutex::scoped_lock scoped_cs(mailLock); - block = mailbox; + block = mailbox.load(std::memory_order_relaxed); if( block ) { MALLOC_ASSERT( block->isOwnedByCurrentThread(), ASSERT_TEXT ); - MALLOC_ASSERT( !isNotForUse(block->nextPrivatizable), ASSERT_TEXT ); - mailbox = block->nextPrivatizable; - block->nextPrivatizable = (Block*) this; + MALLOC_ASSERT( !isNotForUse(block->nextPrivatizable.load(std::memory_order_relaxed)), ASSERT_TEXT ); + mailbox.store(block->nextPrivatizable.load(std::memory_order_relaxed), std::memory_order_relaxed); + block->nextPrivatizable.store((Block*)this, std::memory_order_relaxed); } } if( block ) { - MALLOC_ASSERT( isSolidPtr(block->publicFreeList), ASSERT_TEXT ); + MALLOC_ASSERT( isSolidPtr(block->publicFreeList.load(std::memory_order_relaxed)), ASSERT_TEXT ); block->privatizePublicFreeList(); + block->adjustPositionInBin(this); } return block; } -bool Block::emptyEnoughToUse() +void Bin::addPublicFreeListBlock(Block* block) { - const float threshold = (slabSize - sizeof(Block)) * (1-emptyEnoughRatio); + MallocMutex::scoped_lock scoped_cs(mailLock); + block->nextPrivatizable.store(mailbox.load(std::memory_order_relaxed), std::memory_order_relaxed); + mailbox.store(block, std::memory_order_relaxed); +} +// Process publicly freed objects in all blocks and return empty blocks +// to the backend in order to reduce overall footprint. +bool Bin::cleanPublicFreeLists() +{ + Block* block; + if (!mailbox.load(std::memory_order_acquire)) + return false; + else { + // Grab all the blocks in the mailbox + MallocMutex::scoped_lock scoped_cs(mailLock); + block = mailbox.load(std::memory_order_relaxed); + mailbox.store(nullptr, std::memory_order_relaxed); + } + bool released = false; + while (block) { + MALLOC_ASSERT( block->isOwnedByCurrentThread(), ASSERT_TEXT ); + Block* tmp = block->nextPrivatizable.load(std::memory_order_relaxed); + block->nextPrivatizable.store((Block*)this, std::memory_order_relaxed); + block->privatizePublicFreeList(); + if (block->empty()) { + processEmptyBlock(block, /*poolTheBlock=*/false); + released = true; + } else + block->adjustPositionInBin(this); + block = tmp; + } + return released; +} + +bool Block::adjustFullness() +{ if (bumpPtr) { /* If we are still using a bump ptr for this block it is empty enough to use. */ STAT_increment(getThreadId(), getIndex(objectSize), examineEmptyEnough); isFull = false; - return 1; - } - - /* allocatedCount shows how many objects in the block are in use; however it still counts - blocks freed by other threads; so prior call to privatizePublicFreeList() is recommended */ - isFull = (allocatedCount*objectSize > threshold)? true: false; + } else { + const float threshold = (slabSize - sizeof(Block)) * (1 - emptyEnoughRatio); + /* allocatedCount shows how many objects in the block are in use; however it still counts + * blocks freed by other threads; so prior call to privatizePublicFreeList() is recommended */ + isFull = (allocatedCount*objectSize > threshold) ? true : false; #if COLLECT_STATISTICS - if (isFull) - STAT_increment(getThreadId(), getIndex(objectSize), examineNotEmpty); - else - STAT_increment(getThreadId(), getIndex(objectSize), examineEmptyEnough); + if (isFull) + STAT_increment(getThreadId(), getIndex(objectSize), examineNotEmpty); + else + STAT_increment(getThreadId(), getIndex(objectSize), examineEmptyEnough); #endif - return !isFull; + } + return isFull; +} + +// This method resides in class Block, and not in class Bin, in order to avoid +// calling getAllocationBin on a reasonably hot path in Block::freeOwnObject +void Block::adjustPositionInBin(Bin* bin/*=nullptr*/) +{ + // If the block were full, but became empty enough to use, + // move it to the front of the list + if (isFull && !adjustFullness()) { + if (!bin) + bin = tlsPtr.load(std::memory_order_relaxed)->getAllocationBin(objectSize); + bin->moveBlockToFront(this); + } } /* Restore the bump pointer for an empty block that is planned to use */ void Block::restoreBumpPtr() { MALLOC_ASSERT( allocatedCount == 0, ASSERT_TEXT ); - MALLOC_ASSERT( publicFreeList == NULL, ASSERT_TEXT ); + MALLOC_ASSERT( !isSolidPtr(publicFreeList.load(std::memory_order_relaxed)), ASSERT_TEXT ); STAT_increment(getThreadId(), getIndex(objectSize), freeRestoreBumpPtr); bumpPtr = (FreeObject *)((uintptr_t)this + slabSize - objectSize); - freeList = NULL; - isFull = 0; + freeList = nullptr; + isFull = false; } void Block::freeOwnObject(void *object) { - tlsPtr->markUsed(); + tlsPtr.load(std::memory_order_relaxed)->markUsed(); allocatedCount--; MALLOC_ASSERT( allocatedCount < (slabSize-sizeof(Block))/objectSize, ASSERT_TEXT ); #if COLLECT_STATISTICS // Note that getAllocationBin is not called on the hottest path with statistics off. - if (tlsPtr->getAllocationBin(objectSize)->getActiveBlock() != this) + if (tlsPtr.load(std::memory_order_relaxed)->getAllocationBin(objectSize)->getActiveBlock() != this) STAT_increment(getThreadId(), getIndex(objectSize), freeToInactiveBlock); else STAT_increment(getThreadId(), getIndex(objectSize), freeToActiveBlock); #endif if (empty()) { - // The bump pointer is about to be restored for the block, - // no need to find objectToFree here (this is costly). - - // if the last object of a slab is freed, the slab cannot be marked full + // If the last object of a slab is freed, the slab cannot be marked full MALLOC_ASSERT(!isFull, ASSERT_TEXT); - tlsPtr->getAllocationBin(objectSize)->processLessUsedBlock(poolPtr, this); - } else { + tlsPtr.load(std::memory_order_relaxed)->getAllocationBin(objectSize)->processEmptyBlock(this, /*poolTheBlock=*/true); + } else { // hot path FreeObject *objectToFree = findObjectToFree(object); objectToFree->next = freeList; freeList = objectToFree; - - if (isFull && emptyEnoughToUse()) - tlsPtr->getAllocationBin(objectSize)->moveBlockToFront(this); + adjustPositionInBin(); } } void Block::freePublicObject (FreeObject *objectToFree) { - FreeObject *localPublicFreeList; + FreeObject* localPublicFreeList{}; MALLOC_ITT_SYNC_RELEASING(&publicFreeList); #if FREELIST_NONBLOCKING - FreeObject *temp = publicFreeList; + // TBB_REVAMP_TODO: make it non atomic in non-blocking scenario + localPublicFreeList = publicFreeList.load(std::memory_order_relaxed); do { - localPublicFreeList = objectToFree->next = temp; - temp = (FreeObject*)AtomicCompareExchange( - (intptr_t&)publicFreeList, - (intptr_t)objectToFree, (intptr_t)localPublicFreeList ); + objectToFree->next = localPublicFreeList; // no backoff necessary because trying to make change, not waiting for a change - } while( temp != localPublicFreeList ); + } while( !publicFreeList.compare_exchange_strong(localPublicFreeList, objectToFree) ); #else STAT_increment(getThreadId(), ThreadCommonCounters, lockPublicFreeList); { @@ -1349,41 +1418,47 @@ void Block::freePublicObject (FreeObject *objectToFree) } #endif - if( localPublicFreeList==NULL ) { + if( localPublicFreeList==nullptr ) { // if the block is abandoned, its nextPrivatizable pointer should be UNUSABLE // otherwise, it should point to the bin the block belongs to. // reading nextPrivatizable is thread-safe below, because: - // 1) the executing thread atomically got publicFreeList==NULL and changed it to non-NULL; - // 2) only owning thread can change it back to NULL, + // 1) the executing thread atomically got publicFreeList==nullptr and changed it to non-nullptr; + // 2) only owning thread can change it back to nullptr, // 3) but it can not be done until the block is put to the mailbox // So the executing thread is now the only one that can change nextPrivatizable - if( !isNotForUse(nextPrivatizable) ) { - MALLOC_ASSERT( nextPrivatizable!=NULL, ASSERT_TEXT ); - Bin* theBin = (Bin*) nextPrivatizable; - MallocMutex::scoped_lock scoped_cs(theBin->mailLock); - nextPrivatizable = theBin->mailbox; - theBin->mailbox = this; + Block* next = nextPrivatizable.load(std::memory_order_acquire); + if( !isNotForUse(next) ) { + MALLOC_ASSERT( next!=nullptr, ASSERT_TEXT ); + Bin* theBin = (Bin*) next; +#if MALLOC_DEBUG && TBB_REVAMP_TODO + // FIXME: The thread that returns the block is not the block's owner. + // The below assertion compares 'theBin' against the caller's local bin, thus, it always fails. + // Need to find a way to get the correct remote bin for comparison. + { // check that nextPrivatizable points to the bin the block belongs to + uint32_t index = getIndex( objectSize ); + TLSData* tls = getThreadMallocTLS(); + MALLOC_ASSERT( theBin==tls->bin+index, ASSERT_TEXT ); + } +#endif // MALLOC_DEBUG + theBin->addPublicFreeListBlock(this); } } STAT_increment(getThreadId(), ThreadCommonCounters, freeToOtherThread); - STAT_increment(ownerTid, getIndex(objectSize), freeByOtherThread); + STAT_increment(ownerTid.load(std::memory_order_relaxed), getIndex(objectSize), freeByOtherThread); } -void Block::privatizePublicFreeList( bool cleanup ) +// Make objects freed by other threads available for use again +void Block::privatizePublicFreeList( bool reset ) { - FreeObject *temp, *localPublicFreeList; - const intptr_t endMarker = cleanup? UNUSABLE : 0; + FreeObject *localPublicFreeList; + // If reset is false, publicFreeList should not be zeroed but set to UNUSABLE + // to properly synchronize with other threads freeing objects to this slab. + const intptr_t endMarker = reset ? 0 : UNUSABLE; - // During cleanup of orphaned blocks, the calling thread is not registered as the owner - MALLOC_ASSERT( cleanup || isOwnedByCurrentThread(), ASSERT_TEXT ); + // Only the owner thread may reset the pointer to nullptr + MALLOC_ASSERT( isOwnedByCurrentThread() || !reset, ASSERT_TEXT ); #if FREELIST_NONBLOCKING - temp = publicFreeList; - do { - localPublicFreeList = temp; - temp = (FreeObject*)AtomicCompareExchange( (intptr_t&)publicFreeList, - endMarker, (intptr_t)localPublicFreeList); - // no backoff necessary because trying to make change, not waiting for a change - } while( temp != localPublicFreeList ); + localPublicFreeList = publicFreeList.exchange((FreeObject*)endMarker); #else STAT_increment(getThreadId(), ThreadCommonCounters, lockPublicFreeList); { @@ -1391,18 +1466,18 @@ void Block::privatizePublicFreeList( bool cleanup ) localPublicFreeList = publicFreeList; publicFreeList = endMarker; } - temp = localPublicFreeList; #endif MALLOC_ITT_SYNC_ACQUIRED(&publicFreeList); + MALLOC_ASSERT( !(reset && isNotForUse(publicFreeList)), ASSERT_TEXT ); - // publicFreeList must have been UNUSABLE (possible for orphaned blocks) or valid, but not NULL - MALLOC_ASSERT( localPublicFreeList!=NULL, ASSERT_TEXT ); - MALLOC_ASSERT( localPublicFreeList==temp, ASSERT_TEXT ); - if( isSolidPtr(temp) ) { + // publicFreeList must have been UNUSABLE or valid, but not nullptr + MALLOC_ASSERT( localPublicFreeList!=nullptr, ASSERT_TEXT ); + if( isSolidPtr(localPublicFreeList) ) { MALLOC_ASSERT( allocatedCount <= (slabSize-sizeof(Block))/objectSize, ASSERT_TEXT ); /* other threads did not change the counter freeing our blocks */ allocatedCount--; - while( isSolidPtr(temp->next) ){ // the list will end with either NULL or UNUSABLE + FreeObject *temp = localPublicFreeList; + while( isSolidPtr(temp->next) ){ // the list will end with either nullptr or UNUSABLE temp = temp->next; allocatedCount--; MALLOC_ASSERT( allocatedCount < (slabSize-sizeof(Block))/objectSize, ASSERT_TEXT ); @@ -1418,78 +1493,85 @@ void Block::privatizeOrphaned(TLSData *tls, unsigned index) { Bin* bin = tls->bin + index; STAT_increment(getThreadId(), index, allocBlockPublic); - next = NULL; - previous = NULL; - MALLOC_ASSERT( publicFreeList!=NULL, ASSERT_TEXT ); + next = nullptr; + previous = nullptr; + MALLOC_ASSERT( publicFreeList.load(std::memory_order_relaxed) != nullptr, ASSERT_TEXT ); /* There is not a race here since no other thread owns this block */ markOwned(tls); // It is safe to change nextPrivatizable, as publicFreeList is not null - MALLOC_ASSERT( isNotForUse(nextPrivatizable), ASSERT_TEXT ); - nextPrivatizable = (Block*)bin; + MALLOC_ASSERT( isNotForUse(nextPrivatizable.load(std::memory_order_relaxed)), ASSERT_TEXT ); + nextPrivatizable.store((Block*)bin, std::memory_order_relaxed); // the next call is required to change publicFreeList to 0 privatizePublicFreeList(); - if( allocatedCount ) { - emptyEnoughToUse(); // check its fullness and set result->isFull - } else { + if( empty() ) { restoreBumpPtr(); + } else { + adjustFullness(); // check the block fullness and set isFull + } + MALLOC_ASSERT( !isNotForUse(publicFreeList.load(std::memory_order_relaxed)), ASSERT_TEXT ); +} + + +bool Block::readyToShare() +{ + FreeObject* oldVal = nullptr; +#if FREELIST_NONBLOCKING + publicFreeList.compare_exchange_strong(oldVal, (FreeObject*)UNUSABLE); +#else + STAT_increment(getThreadId(), ThreadCommonCounters, lockPublicFreeList); + { + MallocMutex::scoped_lock scoped_cs(publicFreeListLock); + if ( (oldVal=publicFreeList)==nullptr ) + publicFreeList = reinterpret_cast<FreeObject *>(UNUSABLE); } - MALLOC_ASSERT( !isNotForUse(publicFreeList), ASSERT_TEXT ); +#endif + return oldVal==nullptr; } void Block::shareOrphaned(intptr_t binTag, unsigned index) { MALLOC_ASSERT( binTag, ASSERT_TEXT ); + // unreferenced formal parameter warning + tbb::detail::suppress_unused_warning(index); STAT_increment(getThreadId(), index, freeBlockPublic); markOrphaned(); - // need to set publicFreeList to non-zero, so other threads - // will not change nextPrivatizable and it can be zeroed. - if ((intptr_t)nextPrivatizable==binTag) { - void* oldval; -#if FREELIST_NONBLOCKING - oldval = (void*)AtomicCompareExchange((intptr_t&)publicFreeList, UNUSABLE, 0); -#else - STAT_increment(getThreadId(), ThreadCommonCounters, lockPublicFreeList); - { - MallocMutex::scoped_lock scoped_cs(publicFreeListLock); - if ( (oldval=publicFreeList)==NULL ) - (intptr_t&)(publicFreeList) = UNUSABLE; - } -#endif - if ( oldval!=NULL ) { + if ((intptr_t)nextPrivatizable.load(std::memory_order_relaxed) == binTag) { + // First check passed: the block is not in mailbox yet. + // Need to set publicFreeList to non-zero, so other threads + // will not change nextPrivatizable and it can be zeroed. + if ( !readyToShare() ) { // another thread freed an object; we need to wait until it finishes. // There is no need for exponential backoff, as the wait here is not for a lock; // but need to yield, so the thread we wait has a chance to run. // TODO: add a pause to also be friendly to hyperthreads int count = 256; - while( (intptr_t)const_cast<Block* volatile &>(nextPrivatizable)==binTag ) { + while ((intptr_t)nextPrivatizable.load(std::memory_order_relaxed) == binTag) { if (--count==0) { do_yield(); count = 256; } } } - } else { - MALLOC_ASSERT( isSolidPtr(publicFreeList), ASSERT_TEXT ); } - MALLOC_ASSERT( publicFreeList!=NULL, ASSERT_TEXT ); + MALLOC_ASSERT( publicFreeList.load(std::memory_order_relaxed) !=nullptr, ASSERT_TEXT ); // now it is safe to change our data - previous = NULL; + previous = nullptr; // it is caller responsibility to ensure that the list of blocks // formed by nextPrivatizable pointers is kept consistent if required. // if only called from thread shutdown code, it does not matter. - (intptr_t&)(nextPrivatizable) = UNUSABLE; + nextPrivatizable.store((Block*)UNUSABLE, std::memory_order_relaxed); } void Block::cleanBlockHeader() { - next = NULL; - previous = NULL; - freeList = NULL; + next = nullptr; + previous = nullptr; + freeList = nullptr; allocatedCount = 0; - isFull = 0; - tlsPtr = NULL; + isFull = false; + tlsPtr.store(nullptr, std::memory_order_relaxed); - publicFreeList = NULL; + publicFreeList.store(nullptr, std::memory_order_relaxed); } void Block::initEmptyBlock(TLSData *tls, size_t size) @@ -1500,16 +1582,17 @@ void Block::initEmptyBlock(TLSData *tls, size_t size) unsigned int objSz = getObjectSize(size); cleanBlockHeader(); + MALLOC_ASSERT(objSz <= USHRT_MAX, "objSz must not be less 2^16-1"); objectSize = objSz; markOwned(tls); // bump pointer should be prepared for first allocation - thus mode it down to objectSize bumpPtr = (FreeObject *)((uintptr_t)this + slabSize - objectSize); // each block should have the address where the head of the list of "privatizable" blocks is kept - // the only exception is a block for boot strap which is initialized when TLS is yet NULL - nextPrivatizable = tls? (Block*)(tls->bin + index) : NULL; + // the only exception is a block for boot strap which is initialized when TLS is yet nullptr + nextPrivatizable.store( tls? (Block*)(tls->bin + index) : nullptr, std::memory_order_relaxed); TRACEF(( "[ScalableMalloc trace] Empty block %p is initialized, owner is %ld, objectSize is %d, bumpPtr is %p\n", - this, tlsPtr ? getThreadId() : -1, objectSize, bumpPtr )); + this, tlsPtr.load(std::memory_order_relaxed) ? getThreadId() : -1, objectSize, bumpPtr )); } Block *OrphanedBlocks::get(TLSData *tls, unsigned int size) @@ -1540,20 +1623,20 @@ void OrphanedBlocks::reset() bool OrphanedBlocks::cleanup(Backend* backend) { - bool result = false; + bool released = false; for (uint32_t i=0; i<numBlockBinLimit; i++) { Block* block = bins[i].grab(); MALLOC_ITT_SYNC_ACQUIRED(bins+i); while (block) { Block* next = block->next; - block->privatizePublicFreeList( /*cleanup=*/true ); + block->privatizePublicFreeList( /*reset=*/false ); // do not set publicFreeList to nullptr if (block->empty()) { block->reset(); // slab blocks in user's pools do not have valid backRefIdx if (!backend->inUserPool()) removeBackRef(*(block->getBackRefIdx())); backend->putSlabBlock(block); - result = true; + released = true; } else { MALLOC_ITT_SYNC_RELEASING(bins+i); bins[i].push(block); @@ -1561,32 +1644,33 @@ bool OrphanedBlocks::cleanup(Backend* backend) block = next; } } - return result; + return released; } FreeBlockPool::ResOfGet FreeBlockPool::getBlock() { - Block *b = (Block*)AtomicFetchStore(&head, 0); + Block *b = head.exchange(nullptr); + bool lastAccessMiss; if (b) { size--; Block *newHead = b->next; lastAccessMiss = false; - FencedStore((intptr_t&)head, (intptr_t)newHead); - } else + head.store(newHead, std::memory_order_release); + } else { lastAccessMiss = true; - + } return ResOfGet(b, lastAccessMiss); } void FreeBlockPool::returnBlock(Block *block) { MALLOC_ASSERT( size <= POOL_HIGH_MARK, ASSERT_TEXT ); - Block *localHead = (Block*)AtomicFetchStore(&head, 0); + Block *localHead = head.exchange(nullptr); - if (!localHead) + if (!localHead) { size = 0; // head was stolen by externalClean, correct size accordingly - else if (size == POOL_HIGH_MARK) { + } else if (size == POOL_HIGH_MARK) { // release cold blocks and add hot one, // so keep POOL_LOW_MARK-1 blocks and add new block to head Block *headToFree = localHead, *helper; @@ -1594,7 +1678,7 @@ void FreeBlockPool::returnBlock(Block *block) headToFree = headToFree->next; Block *last = headToFree; headToFree = headToFree->next; - last->next = NULL; + last->next = nullptr; size = POOL_LOW_MARK-1; for (Block *currBl = headToFree; currBl; currBl = helper) { helper = currBl->next; @@ -1606,23 +1690,23 @@ void FreeBlockPool::returnBlock(Block *block) } size++; block->next = localHead; - FencedStore((intptr_t&)head, (intptr_t)block); + head.store(block, std::memory_order_release); } bool FreeBlockPool::externalCleanup() { Block *helper; - bool nonEmpty = false; + bool released = false; - for (Block *currBl=(Block*)AtomicFetchStore(&head, 0); currBl; currBl=helper) { + for (Block *currBl=head.exchange(nullptr); currBl; currBl=helper) { helper = currBl->next; // slab blocks in user's pools do not have valid backRefIdx if (!backend->inUserPool()) removeBackRef(currBl->backRefIdx); backend->putSlabBlock(currBl); - nonEmpty = true; + released = true; } - return nonEmpty; + return released; } /* Prepare the block for returning to FreeBlockPool */ @@ -1630,13 +1714,13 @@ void Block::reset() { // it is caller's responsibility to ensure no data is lost before calling this MALLOC_ASSERT( allocatedCount==0, ASSERT_TEXT ); - MALLOC_ASSERT( !isSolidPtr(publicFreeList), ASSERT_TEXT ); + MALLOC_ASSERT( !isSolidPtr(publicFreeList.load(std::memory_order_relaxed)), ASSERT_TEXT ); if (!isStartupAllocObject()) STAT_increment(getThreadId(), getIndex(objectSize), freeBlockBack); cleanBlockHeader(); - nextPrivatizable = NULL; + nextPrivatizable.store(nullptr, std::memory_order_relaxed); objectSize = 0; // for an empty block, bump pointer should point right after the end of the block @@ -1656,14 +1740,14 @@ inline Block* Bin::setPreviousBlockActive() MALLOC_ASSERT( activeBlk, ASSERT_TEXT ); Block* temp = activeBlk->previous; if( temp ) { - MALLOC_ASSERT( temp->isFull == 0, ASSERT_TEXT ); + MALLOC_ASSERT( !(temp->isFull), ASSERT_TEXT ); activeBlk = temp; } return temp; } inline bool Block::isOwnedByCurrentThread() const { - return tlsPtr && ownerTid.isCurrentThreadId(); + return tlsPtr.load(std::memory_order_relaxed) && ownerTid.isCurrentThreadId(); } FreeObject *Block::findObjectToFree(const void *object) const @@ -1691,23 +1775,25 @@ FreeObject *Block::findObjectToFree(const void *object) const return objectToFree; } -void TLSData::release(MemoryPool *mPool) +void TLSData::release() { - mPool->extMemPool.allLocalCaches.unregisterThread(this); - externalCleanup(&mPool->extMemPool, /*cleanOnlyUnused=*/false); + memPool->extMemPool.allLocalCaches.unregisterThread(this); + externalCleanup(/*cleanOnlyUnused=*/false, /*cleanBins=*/false); for (unsigned index = 0; index < numBlockBins; index++) { Block *activeBlk = bin[index].getActiveBlock(); if (!activeBlk) continue; Block *threadlessBlock = activeBlk->previous; + bool syncOnMailbox = false; while (threadlessBlock) { Block *threadBlock = threadlessBlock->previous; if (threadlessBlock->empty()) { /* we destroy the thread, so not use its block pool */ - mPool->returnEmptyBlock(threadlessBlock, /*poolTheBlock=*/false); + memPool->returnEmptyBlock(threadlessBlock, /*poolTheBlock=*/false); } else { - mPool->extMemPool.orphanedBlocks.put(intptr_t(bin+index), threadlessBlock); + memPool->extMemPool.orphanedBlocks.put(intptr_t(bin+index), threadlessBlock); + syncOnMailbox = true; } threadlessBlock = threadBlock; } @@ -1716,13 +1802,21 @@ void TLSData::release(MemoryPool *mPool) Block *threadBlock = threadlessBlock->next; if (threadlessBlock->empty()) { /* we destroy the thread, so not use its block pool */ - mPool->returnEmptyBlock(threadlessBlock, /*poolTheBlock=*/false); + memPool->returnEmptyBlock(threadlessBlock, /*poolTheBlock=*/false); } else { - mPool->extMemPool.orphanedBlocks.put(intptr_t(bin+index), threadlessBlock); + memPool->extMemPool.orphanedBlocks.put(intptr_t(bin+index), threadlessBlock); + syncOnMailbox = true; } threadlessBlock = threadBlock; } bin[index].resetActiveBlock(); + + if (syncOnMailbox) { + // Although, we synchronized on nextPrivatizable inside a block, we still need to + // synchronize on the bin lifetime because the thread releasing an object into the public + // free list is touching the bin (mailbox and mailLock) + MallocMutex::scoped_lock scoped_cs(bin[index].mailLock); + } } } @@ -1758,11 +1852,11 @@ static StartupBlock *firstStartupBlock; StartupBlock *StartupBlock::getBlock() { BackRefIdx backRefIdx = BackRefIdx::newBackRef(/*largeObj=*/false); - if (backRefIdx.isInvalid()) return NULL; + if (backRefIdx.isInvalid()) return nullptr; StartupBlock *block = static_cast<StartupBlock*>( defaultMemPool->extMemPool.backend.getSlabBlock(1)); - if (!block) return NULL; + if (!block) return nullptr; block->cleanBlockHeader(); setBackRef(backRefIdx, block); @@ -1776,43 +1870,31 @@ StartupBlock *StartupBlock::getBlock() FreeObject *StartupBlock::allocate(size_t size) { FreeObject *result; - StartupBlock *newBlock = NULL; - bool newBlockUnused = false; + StartupBlock *newBlock = nullptr; /* Objects must be aligned on their natural bounds, and objects bigger than word on word's bound. */ size = alignUp(size, sizeof(size_t)); // We need size of an object to implement msize. size_t reqSize = size + sizeof(size_t); - // speculatively allocates newBlock to try avoid allocation while holding lock - /* TODO: The function is called when malloc nested call is detected, - so simultaneous usage from different threads seems unlikely. - If pre-allocation is found useless, the code might be simplified. */ - if (!firstStartupBlock || firstStartupBlock->availableSize() < reqSize) { - newBlock = StartupBlock::getBlock(); - if (!newBlock) return NULL; - } { MallocMutex::scoped_lock scoped_cs(startupMallocLock); // Re-check whether we need a new block (conditions might have changed) if (!firstStartupBlock || firstStartupBlock->availableSize() < reqSize) { if (!newBlock) { newBlock = StartupBlock::getBlock(); - if (!newBlock) return NULL; + if (!newBlock) return nullptr; } newBlock->next = (Block*)firstStartupBlock; if (firstStartupBlock) firstStartupBlock->previous = (Block*)newBlock; firstStartupBlock = newBlock; - } else - newBlockUnused = true; + } result = firstStartupBlock->bumpPtr; firstStartupBlock->allocatedCount++; firstStartupBlock->bumpPtr = (FreeObject *)((uintptr_t)firstStartupBlock->bumpPtr + reqSize); } - if (newBlock && newBlockUnused) - defaultMemPool->returnEmptyBlock(newBlock, /*poolTheBlock=*/false); // keep object size at the negative offset *((size_t*)result) = size; @@ -1821,7 +1903,7 @@ FreeObject *StartupBlock::allocate(size_t size) void StartupBlock::free(void *ptr) { - Block* blockToRelease = NULL; + Block* blockToRelease = nullptr; { MallocMutex::scoped_lock scoped_cs(startupMallocLock); @@ -1848,7 +1930,7 @@ void StartupBlock::free(void *ptr) } } if (blockToRelease) { - blockToRelease->previous = blockToRelease->next = NULL; + blockToRelease->previous = blockToRelease->next = nullptr; defaultMemPool->returnEmptyBlock(blockToRelease, /*poolTheBlock=*/false); } } @@ -1866,62 +1948,14 @@ void StartupBlock::free(void *ptr) * In theory, we only need values 0 and 2. But value 1 is nonetheless * useful for detecting errors in the double-check pattern. */ -static intptr_t mallocInitialized; // implicitly initialized to 0 +static std::atomic<intptr_t> mallocInitialized{0}; // implicitly initialized to 0 static MallocMutex initMutex; /** The leading "\0" is here so that applying "strings" to the binary delivers a clean result. */ static char VersionString[] = "\0" TBBMALLOC_VERSION_STRINGS; -#if __TBB_WIN8UI_SUPPORT -bool GetBoolEnvironmentVariable(const char *) { return false; } -#else -bool GetBoolEnvironmentVariable(const char *name) -{ - if (const char* s = getenv(name)) - return strcmp(s,"0") != 0; - return false; -} -#endif - -void AllocControlledMode::initReadEnv(const char *envName, intptr_t defaultVal) -{ - if (!setDone) { -#if !__TBB_WIN8UI_SUPPORT - // TODO: use strtol to get the actual value of the envirable - const char *envVal = getenv(envName); - if (envVal && !strcmp(envVal, "1")) - val = 1; - else -#endif - val = defaultVal; - setDone = true; - } -} - -void MemoryPool::initDefaultPool() -{ - long long unsigned hugePageSize = 0; -#if __linux__ - if (FILE *f = fopen("/proc/meminfo", "r")) { - const int READ_BUF_SIZE = 100; - char buf[READ_BUF_SIZE]; - MALLOC_STATIC_ASSERT(sizeof(hugePageSize) >= 8, - "At least 64 bits required for keeping page size/numbers."); - - while (fgets(buf, READ_BUF_SIZE, f)) { - if (1 == sscanf(buf, "Hugepagesize: %llu kB", &hugePageSize)) { - hugePageSize *= 1024; - break; - } - } - fclose(f); - } -#endif - hugePages.init(hugePageSize); -} - -#if USE_PTHREAD && (__TBB_SOURCE_DIRECTLY_INCLUDED || __TBB_USE_DLOPEN_REENTRANCY_WORKAROUND) +#if USE_PTHREAD && __TBB_SOURCE_DIRECTLY_INCLUDED /* Decrease race interval between dynamic library unloading and pthread key destructor. Protect only Pthreads with supported unloading. */ @@ -1929,27 +1963,28 @@ class ShutdownSync { /* flag is the number of threads in pthread key dtor body (i.e., between threadDtorStart() and threadDtorDone()) or the signal to skip dtor, if flag < 0 */ - intptr_t flag; + std::atomic<intptr_t> flag; static const intptr_t skipDtor = INTPTR_MIN/2; public: - void init() { flag = 0; } -/* Suppose that 2*abs(skipDtor) or more threads never call threadExitStart() - simultaneously, so flag is never becomes negative because of that. */ + void init() { flag.store(0, std::memory_order_release); } +/* Suppose that 2*abs(skipDtor) or more threads never call threadDtorStart() + simultaneously, so flag never becomes negative because of that. */ bool threadDtorStart() { - if (flag < 0) + if (flag.load(std::memory_order_acquire) < 0) return false; - if (AtomicIncrement(flag) <= 0) { // note that new value returned - AtomicAdd(flag, -1); // flag is spoiled by us, restore it + if (++flag <= 0) { // note that new value returned + flag.fetch_sub(1); // flag is spoiled by us, restore it return false; } return true; } void threadDtorDone() { - AtomicAdd(flag, -1); + flag.fetch_sub(1); } void processExit() { - if (AtomicAdd(flag, skipDtor) != 0) + if (flag.fetch_add(skipDtor) != 0) { SpinWaitUntilEq(flag, skipDtor); + } } }; @@ -1963,28 +1998,28 @@ class ShutdownSync { void processExit() { } }; -#endif // USE_PTHREAD && (__TBB_SOURCE_DIRECTLY_INCLUDED || __TBB_USE_DLOPEN_REENTRANCY_WORKAROUND) +#endif // USE_PTHREAD && __TBB_SOURCE_DIRECTLY_INCLUDED static ShutdownSync shutdownSync; inline bool isMallocInitialized() { // Load must have acquire fence; otherwise thread taking "initialized" path // might perform textually later loads *before* mallocInitialized becomes 2. - return 2 == FencedLoad(mallocInitialized); + return 2 == mallocInitialized.load(std::memory_order_acquire); } -bool isMallocInitializedExt() { - return isMallocInitialized(); -} - -/** Caller is responsible for ensuring this routine is called exactly once. */ +/* Caller is responsible for ensuring this routine is called exactly once. */ extern "C" void MallocInitializeITT() { -#if DO_ITT_NOTIFY +#if __TBB_USE_ITT_NOTIFY if (!usedBySrcIncluded) - tbb::internal::__TBB_load_ittnotify(); + tbb::detail::r1::__TBB_load_ittnotify(); #endif } +void MemoryPool::initDefaultPool() { + hugePages.init(); +} + /* * Allocator initialization routine; * it is called lazily on the very first scalable_malloc call. @@ -2004,13 +2039,17 @@ static bool initMemoryManager() // POSIX.1-2001-compliant way to get page size const size_t granularity = sysconf(_SC_PAGESIZE); #endif + if (!defaultMemPool) { + // Do not rely on static constructors and do the assignment in case + // of library static section not initialized at this call yet. + defaultMemPool = (MemoryPool*)defaultMemPool_space; + } bool initOk = defaultMemPool-> - extMemPool.init(0, NULL, NULL, granularity, + extMemPool.init(0, nullptr, nullptr, granularity, /*keepAllMemory=*/false, /*fixedPool=*/false); // TODO: extMemPool.init() to not allocate memory - if (!initOk || !initBackRefMaster(&defaultMemPool->extMemPool.backend)) + if (!initOk || !initBackRefMain(&defaultMemPool->extMemPool.backend) || !ThreadId::init()) return false; - ThreadId::init(); // Create keys for thread id MemoryPool::initDefaultPool(); // init() is required iff initMemoryManager() is called // after mallocProcessShutdownNotification() @@ -2021,18 +2060,22 @@ static bool initMemoryManager() return true; } +static bool GetBoolEnvironmentVariable(const char* name) { + return tbb::detail::r1::GetBoolEnvironmentVariable(name); +} + //! Ensures that initMemoryManager() is called once and only once. /** Does not return until initMemoryManager() has been completed by a thread. There is no need to call this routine if mallocInitialized==2 . */ static bool doInitialization() { MallocMutex::scoped_lock lock( initMutex ); - if (mallocInitialized!=2) { - MALLOC_ASSERT( mallocInitialized==0, ASSERT_TEXT ); - mallocInitialized = 1; + if (mallocInitialized.load(std::memory_order_relaxed)!=2) { + MALLOC_ASSERT( mallocInitialized.load(std::memory_order_relaxed)==0, ASSERT_TEXT ); + mallocInitialized.store(1, std::memory_order_relaxed); RecursiveMallocCallProtector scoped; if (!initMemoryManager()) { - mallocInitialized = 0; // restore and out + mallocInitialized.store(0, std::memory_order_relaxed); // restore and out return false; } #ifdef MALLOC_EXTRA_INITIALIZATION @@ -2041,18 +2084,18 @@ static bool doInitialization() #if MALLOC_CHECK_RECURSION RecursiveMallocCallProtector::detectNaiveOverload(); #endif - MALLOC_ASSERT( mallocInitialized==1, ASSERT_TEXT ); + MALLOC_ASSERT( mallocInitialized.load(std::memory_order_relaxed)==1, ASSERT_TEXT ); // Store must have release fence, otherwise mallocInitialized==2 // might become remotely visible before side effects of // initMemoryManager() become remotely visible. - FencedStore( mallocInitialized, 2 ); + mallocInitialized.store(2, std::memory_order_release); if( GetBoolEnvironmentVariable("TBB_VERSION") ) { fputs(VersionString+1,stderr); hugePages.printStatus(); } } /* It can't be 0 or I would have initialized it */ - MALLOC_ASSERT( mallocInitialized==2, ASSERT_TEXT ); + MALLOC_ASSERT( mallocInitialized.load(std::memory_order_relaxed)==2, ASSERT_TEXT ); return true; } @@ -2065,7 +2108,7 @@ FreeObject *Block::allocateFromFreeList() { FreeObject *result; - if (!freeList) return NULL; + if (!freeList) return nullptr; result = freeList; MALLOC_ASSERT( result, ASSERT_TEXT ); @@ -2084,7 +2127,7 @@ FreeObject *Block::allocateFromBumpPtr() if (result) { bumpPtr = (FreeObject *) ((uintptr_t) bumpPtr - objectSize); if ( (uintptr_t)bumpPtr < (uintptr_t)this+sizeof(Block) ) { - bumpPtr = NULL; + bumpPtr = nullptr; } MALLOC_ASSERT( allocatedCount < (slabSize-sizeof(Block))/objectSize, ASSERT_TEXT ); allocatedCount++; @@ -2110,8 +2153,8 @@ inline FreeObject* Block::allocate() MALLOC_ASSERT( !bumpPtr, ASSERT_TEXT ); /* the block is considered full. */ - isFull = 1; - return NULL; + isFull = true; + return nullptr; } size_t Block::findObjectSize(void *object) const @@ -2139,12 +2182,12 @@ void Bin::moveBlockToFront(Block *block) pushTLSBin(block); } -void Bin::processLessUsedBlock(MemoryPool *memPool, Block *block) +void Bin::processEmptyBlock(Block *block, bool poolTheBlock) { if (block != activeBlk) { - /* We are not actively using this block; return it to the general block pool */ + /* We are not using this block; return it to the pool */ outofTLSBin(block); - memPool->returnEmptyBlock(block, /*poolTheBlock=*/true); + block->getMemPool()->returnEmptyBlock(block, poolTheBlock); } else { /* all objects are free - let's restore the bump pointer */ block->restoreBumpPtr(); @@ -2158,9 +2201,9 @@ bool LocalLOCImpl<LOW_MARK, HIGH_MARK>::put(LargeMemoryBlock *object, ExtMemoryP // not spoil cache with too large object, that can cause its total cleanup if (size > MAX_TOTAL_SIZE) return false; - LargeMemoryBlock *localHead = (LargeMemoryBlock*)AtomicFetchStore(&head, 0); + LargeMemoryBlock *localHead = head.exchange(nullptr); - object->prev = NULL; + object->prev = nullptr; object->next = localHead; if (localHead) localHead->prev = object; @@ -2182,27 +2225,28 @@ bool LocalLOCImpl<LOW_MARK, HIGH_MARK>::put(LargeMemoryBlock *object, ExtMemoryP tail = tail->prev; } LargeMemoryBlock *headToRelease = tail->next; - tail->next = NULL; + tail->next = nullptr; extMemPool->freeLargeObjectList(headToRelease); } - FencedStore((intptr_t&)head, (intptr_t)localHead); + head.store(localHead, std::memory_order_release); return true; } template<int LOW_MARK, int HIGH_MARK> LargeMemoryBlock *LocalLOCImpl<LOW_MARK, HIGH_MARK>::get(size_t size) { - LargeMemoryBlock *localHead, *res=NULL; + LargeMemoryBlock *localHead, *res = nullptr; if (size > MAX_TOTAL_SIZE) - return NULL; + return nullptr; - if (!head || !(localHead = (LargeMemoryBlock*)AtomicFetchStore(&head, 0))) { + // TBB_REVAMP_TODO: review this line + if (!head.load(std::memory_order_acquire) || (localHead = head.exchange(nullptr)) == nullptr) { // do not restore totalSize, numOfBlocks and tail at this point, // as they are used only in put(), where they must be restored - return NULL; + return nullptr; } for (LargeMemoryBlock *curr = localHead; curr; curr=curr->next) { @@ -2221,14 +2265,15 @@ LargeMemoryBlock *LocalLOCImpl<LOW_MARK, HIGH_MARK>::get(size_t size) break; } } - FencedStore((intptr_t&)head, (intptr_t)localHead); + + head.store(localHead, std::memory_order_release); return res; } template<int LOW_MARK, int HIGH_MARK> bool LocalLOCImpl<LOW_MARK, HIGH_MARK>::externalCleanup(ExtMemoryPool *extMemPool) { - if (LargeMemoryBlock *localHead = (LargeMemoryBlock*)AtomicFetchStore(&head, 0)) { + if (LargeMemoryBlock *localHead = head.exchange(nullptr)) { extMemPool->freeLargeObjectList(localHead); return true; } @@ -2237,12 +2282,12 @@ bool LocalLOCImpl<LOW_MARK, HIGH_MARK>::externalCleanup(ExtMemoryPool *extMemPoo void *MemoryPool::getFromLLOCache(TLSData* tls, size_t size, size_t alignment) { - LargeMemoryBlock *lmb = NULL; + LargeMemoryBlock *lmb = nullptr; size_t headersSize = sizeof(LargeMemoryBlock)+sizeof(LargeObjectHdr); size_t allocationSize = LargeObjectCache::alignToBin(size+headersSize+alignment); if (allocationSize < size) // allocationSize is wrapped around after alignToBin - return NULL; + return nullptr; MALLOC_ASSERT(allocationSize >= alignment, "Overflow must be checked before."); if (tls) { @@ -2292,7 +2337,7 @@ void *MemoryPool::getFromLLOCache(TLSData* tls, size_t size, size_t alignment) return alignedArea; } - return NULL; + return nullptr; } void MemoryPool::putToLLOCache(TLSData *tls, void *object) @@ -2327,7 +2372,7 @@ static void *allocateAligned(MemoryPool *memPool, size_t size, size_t alignment) if (!isMallocInitialized()) if (!doInitialization()) - return NULL; + return nullptr; void *result; if (size<=maxSegregatedObjectSize && alignment<=maxSegregatedObjectSize) @@ -2337,7 +2382,7 @@ static void *allocateAligned(MemoryPool *memPool, size_t size, size_t alignment) result = internalPoolMalloc(memPool, size); else if (size+alignment < minLargeObjectSize) { void *unaligned = internalPoolMalloc(memPool, size+alignment); - if (!unaligned) return NULL; + if (!unaligned) return nullptr; result = alignUp(unaligned, alignment); } else goto LargeObjAlloc; @@ -2355,7 +2400,7 @@ static void *allocateAligned(MemoryPool *memPool, size_t size, size_t alignment) } static void *reallocAligned(MemoryPool *memPool, void *ptr, - size_t size, size_t alignment = 0) + size_t newSize, size_t alignment = 0) { void *result; size_t copySize; @@ -2363,42 +2408,58 @@ static void *reallocAligned(MemoryPool *memPool, void *ptr, if (isLargeObject<ourMem>(ptr)) { LargeMemoryBlock* lmb = ((LargeObjectHdr *)ptr - 1)->memoryBlock; copySize = lmb->unalignedSize-((uintptr_t)ptr-(uintptr_t)lmb); - if (size <= copySize && (0==alignment || isAligned(ptr, alignment))) { - lmb->objectSize = size; - return ptr; - } else { - copySize = lmb->objectSize; + + // Apply different strategies if size decreases + if (newSize <= copySize && (0 == alignment || isAligned(ptr, alignment))) { + + // For huge objects (that do not fit in backend cache), keep the same space unless + // the new size is at least twice smaller + bool isMemoryBlockHuge = copySize > memPool->extMemPool.backend.getMaxBinnedSize(); + size_t threshold = isMemoryBlockHuge ? copySize / 2 : 0; + if (newSize > threshold) { + lmb->objectSize = newSize; + return ptr; + } + // TODO: For large objects suitable for the backend cache, + // split out the excessive part and put it to the backend. + } + // Reallocate for real + copySize = lmb->objectSize; #if BACKEND_HAS_MREMAP - if (void *r = memPool->extMemPool.remap(ptr, copySize, size, - alignment<largeObjectAlignment? - largeObjectAlignment : alignment)) - return r; + if (void *r = memPool->extMemPool.remap(ptr, copySize, newSize, + alignment < largeObjectAlignment ? largeObjectAlignment : alignment)) + return r; #endif - result = alignment ? allocateAligned(memPool, size, alignment) : - internalPoolMalloc(memPool, size); - } + result = alignment ? allocateAligned(memPool, newSize, alignment) : + internalPoolMalloc(memPool, newSize); + } else { Block* block = (Block *)alignDown(ptr, slabSize); copySize = block->findObjectSize(ptr); - if (size <= copySize && (0==alignment || isAligned(ptr, alignment))) { + + // TODO: Move object to another bin if size decreases and the current bin is "empty enough". + // Currently, in case of size decreasing, old pointer is returned + if (newSize <= copySize && (0==alignment || isAligned(ptr, alignment))) { return ptr; } else { - result = alignment ? allocateAligned(memPool, size, alignment) : - internalPoolMalloc(memPool, size); + result = alignment ? allocateAligned(memPool, newSize, alignment) : + internalPoolMalloc(memPool, newSize); } } if (result) { - memcpy(result, ptr, copySize<size? copySize: size); + memcpy(result, ptr, copySize < newSize ? copySize : newSize); internalPoolFree(memPool, ptr, 0); } return result; } +#if MALLOC_DEBUG /* A predicate checks if an object is properly placed inside its block */ inline bool Block::isProperlyPlaced(const void *object) const { return 0 == ((uintptr_t)this + slabSize - (uintptr_t)object) % objectSize; } +#endif /* Finds the real object inside the block */ FreeObject *Block::findAllocatedObject(const void *address) const @@ -2422,7 +2483,7 @@ static inline BackRefIdx safer_dereference (const BackRefIdx *ptr) #if _MSC_VER __try { #endif - id = *ptr; + id = dereference(ptr); #if _MSC_VER } __except( GetExceptionCode() == EXCEPTION_ACCESS_VIOLATION? EXCEPTION_EXECUTE_HANDLER : EXCEPTION_CONTINUE_SEARCH ) { @@ -2438,11 +2499,11 @@ bool isLargeObject(void *object) if (!isAligned(object, largeObjectAlignment)) return false; LargeObjectHdr *header = (LargeObjectHdr*)object - 1; - BackRefIdx idx = memOrigin==unknownMem? safer_dereference(&header->backRefIdx) : - header->backRefIdx; + BackRefIdx idx = (memOrigin == unknownMem) ? + safer_dereference(&header->backRefIdx) : dereference(&header->backRefIdx); return idx.isLargeObject() - // in valid LargeObjectHdr memoryBlock is not NULL + // in valid LargeObjectHdr memoryBlock is not nullptr && header->memoryBlock // in valid LargeObjectHdr memoryBlock points somewhere before header // TODO: more strict check @@ -2493,7 +2554,7 @@ static void *internalPoolMalloc(MemoryPool* memPool, size_t size) Bin* bin; Block * mallocBlock; - if (!memPool) return NULL; + if (!memPool) return nullptr; if (!size) size = sizeof(size_t); @@ -2503,7 +2564,7 @@ static void *internalPoolMalloc(MemoryPool* memPool, size_t size) if (size >= minLargeObjectSize) return memPool->getFromLLOCache(tls, size, largeObjectAlignment); - if (!tls) return NULL; + if (!tls) return nullptr; tls->markUsed(); /* @@ -2511,7 +2572,7 @@ static void *internalPoolMalloc(MemoryPool* memPool, size_t size) * It keeps ptr to the active block for allocations of this size */ bin = tls->getAllocationBin(size); - if ( !bin ) return NULL; + if ( !bin ) return nullptr; /* Get a block to try to allocate in. */ for( mallocBlock = bin->getActiveBlock(); mallocBlock; @@ -2524,11 +2585,8 @@ static void *internalPoolMalloc(MemoryPool* memPool, size_t size) /* * else privatize publicly freed objects in some block and allocate from it */ - mallocBlock = bin->getPublicFreeListBlock(); + mallocBlock = bin->getPrivatizedFreeListBlock(); if (mallocBlock) { - if (mallocBlock->emptyEnoughToUse()) { - bin->moveBlockToFront(mallocBlock); - } MALLOC_ASSERT( mallocBlock->freeListNonNull(), ASSERT_TEXT ); if ( FreeObject *result = mallocBlock->allocateFromFreeList() ) return result; @@ -2563,10 +2621,10 @@ static void *internalPoolMalloc(MemoryPool* memPool, size_t size) return internalPoolMalloc(memPool, size); } /* - * else nothing works so return NULL + * else nothing works so return nullptr */ - TRACEF(( "[ScalableMalloc trace] No memory found, returning NULL.\n" )); - return NULL; + TRACEF(( "[ScalableMalloc trace] No memory found, returning nullptr.\n" )); + return nullptr; } // When size==0 (i.e. unknown), detect here whether the object is large. @@ -2598,12 +2656,12 @@ static void *internalMalloc(size_t size) if (RecursiveMallocCallProtector::sameThreadActive()) return size<minLargeObjectSize? StartupBlock::allocate(size) : // nested allocation, so skip tls - (FreeObject*)defaultMemPool->getFromLLOCache(NULL, size, slabSize); + (FreeObject*)defaultMemPool->getFromLLOCache(nullptr, size, slabSize); #endif if (!isMallocInitialized()) if (!doInitialization()) - return NULL; + return nullptr; return internalPoolMalloc(defaultMemPool, size); } @@ -2614,18 +2672,15 @@ static void internalFree(void *object) static size_t internalMsize(void* ptr) { - if (ptr) { - MALLOC_ASSERT(isRecognized(ptr), "Invalid pointer in scalable_msize detected."); - if (isLargeObject<ourMem>(ptr)) { - LargeMemoryBlock* lmb = ((LargeObjectHdr*)ptr - 1)->memoryBlock; - return lmb->objectSize; - } else - return ((Block*)alignDown(ptr, slabSize))->findObjectSize(ptr); + MALLOC_ASSERT(ptr, "Invalid pointer passed to internalMsize"); + if (isLargeObject<ourMem>(ptr)) { + // TODO: return the maximum memory size, that can be written to this object + LargeMemoryBlock* lmb = ((LargeObjectHdr*)ptr - 1)->memoryBlock; + return lmb->objectSize; + } else { + Block *block = (Block*)alignDown(ptr, slabSize); + return block->findObjectSize(ptr); } - errno = EINVAL; - // Unlike _msize, return 0 in case of parameter error. - // Returning size_t(-1) looks more like the way to troubles. - return 0; } } // namespace internal @@ -2634,7 +2689,7 @@ using namespace rml::internal; // legacy entry point saved for compatibility with binaries complied // with pre-6003 versions of TBB -rml::MemoryPool *pool_create(intptr_t pool_id, const MemPoolPolicy *policy) +TBBMALLOC_EXPORT rml::MemoryPool *pool_create(intptr_t pool_id, const MemPoolPolicy *policy) { rml::MemoryPool *pool; MemPoolPolicy pol(policy->pAlloc, policy->pFree, policy->granularity); @@ -2649,29 +2704,31 @@ rml::MemPoolError pool_create_v1(intptr_t pool_id, const MemPoolPolicy *policy, if ( !policy->pAlloc || policy->version<MemPoolPolicy::TBBMALLOC_POOL_VERSION // empty pFree allowed only for fixed pools || !(policy->fixedPool || policy->pFree)) { - *pool = NULL; + *pool = nullptr; return INVALID_POLICY; } if ( policy->version>MemPoolPolicy::TBBMALLOC_POOL_VERSION // future versions are not supported // new flags can be added in place of reserved, but default // behaviour must be supported by this version || policy->reserved ) { - *pool = NULL; + *pool = nullptr; return UNSUPPORTED_POLICY; } if (!isMallocInitialized()) - if (!doInitialization()) + if (!doInitialization()) { + *pool = nullptr; return NO_MEMORY; + } rml::internal::MemoryPool *memPool = (rml::internal::MemoryPool*)internalMalloc((sizeof(rml::internal::MemoryPool))); if (!memPool) { - *pool = NULL; + *pool = nullptr; return NO_MEMORY; } - memset(memPool, 0, sizeof(rml::internal::MemoryPool)); + memset(static_cast<void*>(memPool), 0, sizeof(rml::internal::MemoryPool)); if (!memPool->init(pool_id, policy)) { internalFree(memPool); - *pool = NULL; + *pool = nullptr; return NO_MEMORY; } @@ -2706,7 +2763,7 @@ void *pool_realloc(rml::MemoryPool* mPool, void *object, size_t size) return internalPoolMalloc((rml::internal::MemoryPool*)mPool, size); if (!size) { internalPoolFree((rml::internal::MemoryPool*)mPool, object, 0); - return NULL; + return nullptr; } return reallocAligned((rml::internal::MemoryPool*)mPool, object, size, 0); } @@ -2714,7 +2771,7 @@ void *pool_realloc(rml::MemoryPool* mPool, void *object, size_t size) void *pool_aligned_malloc(rml::MemoryPool* mPool, size_t size, size_t alignment) { if (!isPowerOfTwo(alignment) || 0==size) - return NULL; + return nullptr; return allocateAligned((rml::internal::MemoryPool*)mPool, size, alignment); } @@ -2722,7 +2779,7 @@ void *pool_aligned_malloc(rml::MemoryPool* mPool, size_t size, size_t alignment) void *pool_aligned_realloc(rml::MemoryPool* memPool, void *ptr, size_t size, size_t alignment) { if (!isPowerOfTwo(alignment)) - return NULL; + return nullptr; rml::internal::MemoryPool *mPool = (rml::internal::MemoryPool*)memPool; void *tmp; @@ -2730,7 +2787,7 @@ void *pool_aligned_realloc(rml::MemoryPool* memPool, void *ptr, size_t size, siz tmp = allocateAligned(mPool, size, alignment); else if (!size) { internalPoolFree(mPool, ptr, 0); - return NULL; + return nullptr; } else tmp = reallocAligned(mPool, ptr, size, alignment); @@ -2758,6 +2815,21 @@ rml::MemoryPool *pool_identify(void *object) return (rml::MemoryPool*)pool; } +size_t pool_msize(rml::MemoryPool *mPool, void* object) +{ + if (object) { + // No assert for object recognition, cause objects allocated from non-default + // memory pool do not participate in range checking and do not have valid backreferences for + // small objects. Instead, check that an object belong to the certain memory pool. + MALLOC_ASSERT_EX(mPool == pool_identify(object), "Object does not belong to the specified pool"); + return internalMsize(object); + } + errno = EINVAL; + // Unlike _msize, return 0 in case of parameter error. + // Returning size_t(-1) looks more like the way to troubles. + return 0; +} + } // namespace rml using namespace rml::internal; @@ -2768,50 +2840,74 @@ static unsigned int threadGoingDownCount = 0; /* * When a thread is shutting down this routine should be called to remove all the thread ids - * from the malloc blocks and replace them with a NULL thread id. + * from the malloc blocks and replace them with a nullptr thread id. * * For pthreads, the function is set as a callback in pthread_key_create for TLS bin. - * For non-NULL keys it will be automatically called at thread exit with the key value - * as the argument. + * It will be automatically called at thread exit with the key value as the argument, + * unless that value is nullptr. + * For Windows, it is called from DllMain( DLL_THREAD_DETACH ). + * + * However neither of the above is called for the main process thread, so the routine + * also needs to be called during the process shutdown. * - * for Windows, it should be called directly e.g. from DllMain */ -void mallocThreadShutdownNotification(void* arg) +// TODO: Consider making this function part of class MemoryPool. +void doThreadShutdownNotification(TLSData* tls, bool main_thread) { - // Check whether TLS has been initialized - if (!isMallocInitialized()) return; - TRACEF(( "[ScalableMalloc trace] Thread id %d blocks return start %d\n", getThreadId(), threadGoingDownCount++ )); -#if USE_WINTHREAD - suppress_unused_warning(arg); - MallocMutex::scoped_lock lock(MemoryPool::memPoolListLock); - // The routine is called once per thread, need to walk through all pools on Windows - for (MemoryPool *memPool = defaultMemPool; memPool; memPool = memPool->next) - if (TLSData *tls = memPool->getTLS(/*create=*/false)) - memPool->processThreadShutdown(tls); -#else - if (!shutdownSync.threadDtorStart()) return; - // The routine is called for each memPool, gets memPool from TLSData. - TLSData *tls = (TLSData*)arg; - tls->getMemPool()->processThreadShutdown(tls); - shutdownSync.threadDtorDone(); + +#if USE_PTHREAD + if (tls) { + if (!shutdownSync.threadDtorStart()) return; + tls->getMemPool()->onThreadShutdown(tls); + shutdownSync.threadDtorDone(); + } else #endif + { + suppress_unused_warning(tls); // not used on Windows + // The default pool is safe to use at this point: + // on Linux, only the main thread can go here before destroying defaultMemPool; + // on Windows, shutdown is synchronized via loader lock and isMallocInitialized(). + // See also __TBB_mallocProcessShutdownNotification() + defaultMemPool->onThreadShutdown(defaultMemPool->getTLS(/*create=*/false)); + // Take lock to walk through other pools; but waiting might be dangerous at this point + // (e.g. on Windows the main thread might deadlock) + bool locked = false; + MallocMutex::scoped_lock lock(MemoryPool::memPoolListLock, /*wait=*/!main_thread, &locked); + if (locked) { // the list is safe to process + for (MemoryPool *memPool = defaultMemPool->next; memPool; memPool = memPool->next) + memPool->onThreadShutdown(memPool->getTLS(/*create=*/false)); + } + } TRACEF(( "[ScalableMalloc trace] Thread id %d blocks return end\n", getThreadId() )); } -#if USE_WINTHREAD +#if USE_PTHREAD +void mallocThreadShutdownNotification(void* arg) +{ + // The routine is called for each pool (as TLS dtor) on each thread, except for the main thread + if (!isMallocInitialized()) return; + doThreadShutdownNotification((TLSData*)arg, false); +} +#else extern "C" void __TBB_mallocThreadShutdownNotification() { - mallocThreadShutdownNotification(NULL); + // The routine is called once per thread on Windows + if (!isMallocInitialized()) return; + doThreadShutdownNotification(nullptr, false); } #endif -extern "C" void __TBB_mallocProcessShutdownNotification() +extern "C" void __TBB_mallocProcessShutdownNotification(bool windows_process_dying) { if (!isMallocInitialized()) return; + // Don't clean allocator internals if the entire process is exiting + if (!windows_process_dying) { + doThreadShutdownNotification(nullptr, /*main_thread=*/true); + } #if __TBB_MALLOC_LOCACHE_STAT printf("cache hit ratio %f, size hit %f\n", 1.*cacheHits/mallocCalls, 1.*memHitKB/memAllocKB); @@ -2824,21 +2920,11 @@ extern "C" void __TBB_mallocProcessShutdownNotification() on thread termination when then the tbbmalloc code can be already unloaded. */ defaultMemPool->destroy(); - destroyBackRefMaster(&defaultMemPool->extMemPool.backend); + destroyBackRefMain(&defaultMemPool->extMemPool.backend); ThreadId::destroy(); // Delete key for thread id hugePages.reset(); // new total malloc initialization is possible after this point - FencedStore(mallocInitialized, 0); -#elif __TBB_USE_DLOPEN_REENTRANCY_WORKAROUND -/* In most cases we prevent unloading tbbmalloc, and don't clean up memory - on process shutdown. When impossible to prevent, library unload results - in shutdown notification, and it makes sense to release unused memory - at that point (we can't release all memory because it's possible that - it will be accessed after this point). - TODO: better support systems where we can't prevent unloading by removing - pthread destructors and releasing caches. - */ - defaultMemPool->extMemPool.hardCachesCleanup(); + mallocInitialized.store(0, std::memory_order_release); #endif // __TBB_SOURCE_DIRECTLY_INCLUDED #if COLLECT_STATISTICS @@ -2846,8 +2932,10 @@ extern "C" void __TBB_mallocProcessShutdownNotification() for( int i=1; i<=nThreads && i<MAX_THREADS; ++i ) STAT_print(i); #endif - if (!usedBySrcIncluded) + if (!usedBySrcIncluded) { MALLOC_ITT_FINI_ITTLIB(); + MALLOC_ITT_RELEASE_RESOURCES(); + } } extern "C" void * scalable_malloc(size_t size) @@ -2857,12 +2945,14 @@ extern "C" void * scalable_malloc(size_t size) return ptr; } -extern "C" void scalable_free (void *object) { +extern "C" void scalable_free(void *object) +{ internalFree(object); } #if MALLOC_ZONE_OVERLOAD_ENABLED -extern "C" void __TBB_malloc_free_definite_size(void *object, size_t size) { +extern "C" void __TBB_malloc_free_definite_size(void *object, size_t size) +{ internalPoolFree(defaultMemPool, object, size); } #endif @@ -2871,13 +2961,13 @@ extern "C" void __TBB_malloc_free_definite_size(void *object, size_t size) { * A variant that provides additional memory safety, by checking whether the given address * was obtained with this allocator, and if not redirecting to the provided alternative call. */ -extern "C" void __TBB_malloc_safer_free(void *object, void (*original_free)(void*)) +extern "C" TBBMALLOC_EXPORT void __TBB_malloc_safer_free(void *object, void (*original_free)(void*)) { if (!object) return; // tbbmalloc can allocate object only when tbbmalloc has been initialized - if (FencedLoad(mallocInitialized) && defaultMemPool->extMemPool.backend.ptrCanBeValid(object)) { + if (mallocInitialized.load(std::memory_order_acquire) && defaultMemPool->extMemPool.backend.ptrCanBeValid(object)) { if (isLargeObject<unknownMem>(object)) { // must check 1st for large object, because small object check touches 4 pages on left, // and it can be inaccessible @@ -2903,7 +2993,7 @@ extern "C" void __TBB_malloc_safer_free(void *object, void (*original_free)(void * "realloc changes the size of the object pointed to by p to size. The contents will * be unchanged up to the minimum of the old and the new sizes. If the new size is larger, * the new space is uninitialized. realloc returns a pointer to the new space, or - * NULL if the request cannot be satisfied, in which case *p is unchanged." + * nullptr if the request cannot be satisfied, in which case *p is unchanged." * */ extern "C" void* scalable_realloc(void* ptr, size_t size) @@ -2914,7 +3004,7 @@ extern "C" void* scalable_realloc(void* ptr, size_t size) tmp = internalMalloc(size); else if (!size) { internalFree(ptr); - return NULL; + return nullptr; } else tmp = reallocAligned(defaultMemPool, ptr, size, 0); @@ -2926,16 +3016,16 @@ extern "C" void* scalable_realloc(void* ptr, size_t size) * A variant that provides additional memory safety, by checking whether the given address * was obtained with this allocator, and if not redirecting to the provided alternative call. */ -extern "C" void* __TBB_malloc_safer_realloc(void* ptr, size_t sz, void* original_realloc) +extern "C" TBBMALLOC_EXPORT void* __TBB_malloc_safer_realloc(void* ptr, size_t sz, void* original_realloc) { void *tmp; // TODO: fix warnings about uninitialized use of tmp if (!ptr) { tmp = internalMalloc(sz); - } else if (FencedLoad(mallocInitialized) && isRecognized(ptr)) { + } else if (mallocInitialized.load(std::memory_order_acquire) && isRecognized(ptr)) { if (!sz) { internalFree(ptr); - return NULL; + return nullptr; } else { tmp = reallocAligned(defaultMemPool, ptr, sz, 0); } @@ -2953,7 +3043,7 @@ extern "C" void* __TBB_malloc_safer_realloc(void* ptr, size_t sz, void* original } } } else - tmp = NULL; + tmp = nullptr; } #else else if (original_realloc) { @@ -2963,7 +3053,7 @@ extern "C" void* __TBB_malloc_safer_realloc(void* ptr, size_t sz, void* original tmp = original_realloc_ptr(ptr,sz); } #endif - else tmp = NULL; + else tmp = nullptr; if (!tmp) errno = ENOMEM; return tmp; @@ -2976,7 +3066,7 @@ extern "C" void* __TBB_malloc_safer_realloc(void* ptr, size_t sz, void* original /* * From K&R * calloc returns a pointer to space for an array of nobj objects, - * each of size size, or NULL if the request cannot be satisfied. + * each of size size, or nullptr if the request cannot be satisfied. * The space is initialized to zero bytes. * */ @@ -2991,7 +3081,7 @@ extern "C" void * scalable_calloc(size_t nobj, size_t size) if (nobj>=mult_not_overflow || size>=mult_not_overflow) // 1) heuristic check if (nobj && arraySize / nobj != size) { // 2) exact check errno = ENOMEM; - return NULL; + return nullptr; } void* result = internalMalloc(arraySize); if (result) @@ -3020,7 +3110,7 @@ extern "C" void * scalable_aligned_malloc(size_t size, size_t alignment) { if (!isPowerOfTwo(alignment) || 0==size) { errno = EINVAL; - return NULL; + return nullptr; } void *tmp = allocateAligned(defaultMemPool, size, alignment); if (!tmp) errno = ENOMEM; @@ -3031,7 +3121,7 @@ extern "C" void * scalable_aligned_realloc(void *ptr, size_t size, size_t alignm { if (!isPowerOfTwo(alignment)) { errno = EINVAL; - return NULL; + return nullptr; } void *tmp; @@ -3039,7 +3129,7 @@ extern "C" void * scalable_aligned_realloc(void *ptr, size_t size, size_t alignm tmp = allocateAligned(defaultMemPool, size, alignment); else if (!size) { scalable_free(ptr); - return NULL; + return nullptr; } else tmp = reallocAligned(defaultMemPool, ptr, size, alignment); @@ -3047,21 +3137,21 @@ extern "C" void * scalable_aligned_realloc(void *ptr, size_t size, size_t alignm return tmp; } -extern "C" void * __TBB_malloc_safer_aligned_realloc(void *ptr, size_t size, size_t alignment, void* orig_function) +extern "C" TBBMALLOC_EXPORT void * __TBB_malloc_safer_aligned_realloc(void *ptr, size_t size, size_t alignment, void* orig_function) { /* corner cases left out of reallocAligned to not deal with errno there */ if (!isPowerOfTwo(alignment)) { errno = EINVAL; - return NULL; + return nullptr; } - void *tmp = NULL; + void *tmp = nullptr; if (!ptr) { tmp = allocateAligned(defaultMemPool, size, alignment); - } else if (FencedLoad(mallocInitialized) && isRecognized(ptr)) { + } else if (mallocInitialized.load(std::memory_order_acquire) && isRecognized(ptr)) { if (!size) { internalFree(ptr); - return NULL; + return nullptr; } else { tmp = reallocAligned(defaultMemPool, ptr, size, alignment); } @@ -3087,7 +3177,7 @@ extern "C" void * __TBB_malloc_safer_aligned_realloc(void *ptr, size_t size, siz if ( original_ptrs->aligned_free ){ original_ptrs->aligned_free( ptr ); } - return NULL; + return nullptr; } } #else @@ -3113,23 +3203,30 @@ extern "C" void scalable_aligned_free(void *ptr) */ extern "C" size_t scalable_msize(void* ptr) { - return internalMsize(ptr); + if (ptr) { + MALLOC_ASSERT(isRecognized(ptr), "Invalid pointer in scalable_msize detected."); + return internalMsize(ptr); + } + errno = EINVAL; + // Unlike _msize, return 0 in case of parameter error. + // Returning size_t(-1) looks more like the way to troubles. + return 0; } /* * A variant that provides additional memory safety, by checking whether the given address * was obtained with this allocator, and if not redirecting to the provided alternative call. */ -extern "C" size_t __TBB_malloc_safer_msize(void *object, size_t (*original_msize)(void*)) +extern "C" TBBMALLOC_EXPORT size_t __TBB_malloc_safer_msize(void *object, size_t (*original_msize)(void*)) { if (object) { // Check if the memory was allocated by scalable_malloc - if (FencedLoad(mallocInitialized) && isRecognized(object)) + if (mallocInitialized.load(std::memory_order_acquire) && isRecognized(object)) return internalMsize(object); else if (original_msize) return original_msize(object); } - // object is NULL or unknown, or foreign and no original_msize + // object is nullptr or unknown, or foreign and no original_msize #if USE_WINTHREAD errno = EINVAL; // errno expected to be set only on this platform #endif @@ -3139,16 +3236,16 @@ extern "C" size_t __TBB_malloc_safer_msize(void *object, size_t (*original_msize /* * The same as above but for _aligned_msize case */ -extern "C" size_t __TBB_malloc_safer_aligned_msize(void *object, size_t alignment, size_t offset, size_t (*orig_aligned_msize)(void*,size_t,size_t)) +extern "C" TBBMALLOC_EXPORT size_t __TBB_malloc_safer_aligned_msize(void *object, size_t alignment, size_t offset, size_t (*orig_aligned_msize)(void*,size_t,size_t)) { if (object) { // Check if the memory was allocated by scalable_malloc - if (FencedLoad(mallocInitialized) && isRecognized(object)) + if (mallocInitialized.load(std::memory_order_acquire) && isRecognized(object)) return internalMsize(object); else if (orig_aligned_msize) return orig_aligned_msize(object,alignment,offset); } - // object is NULL or unknown + // object is nullptr or unknown errno = EINVAL; return 0; } @@ -3161,7 +3258,7 @@ extern "C" int scalable_allocation_mode(int param, intptr_t value) defaultMemPool->extMemPool.backend.setRecommendedMaxSize((size_t)value); return TBBMALLOC_OK; } else if (param == USE_HUGE_PAGES) { -#if __linux__ +#if __unix__ switch (value) { case 0: case 1: @@ -3184,6 +3281,9 @@ extern "C" int scalable_allocation_mode(int param, intptr_t value) return TBBMALLOC_INVALID_PARAM; } #endif + } else if (param == TBBMALLOC_SET_HUGE_SIZE_THRESHOLD) { + defaultMemPool->extMemPool.loc.setHugeSizeThreshold((size_t)value); + return TBBMALLOC_OK; } return TBBMALLOC_INVALID_PARAM; } @@ -3192,16 +3292,18 @@ extern "C" int scalable_allocation_command(int cmd, void *param) { if (param) return TBBMALLOC_INVALID_PARAM; + + bool released = false; switch(cmd) { case TBBMALLOC_CLEAN_THREAD_BUFFERS: if (TLSData *tls = defaultMemPool->getTLS(/*create=*/false)) - return tls->externalCleanup(&defaultMemPool->extMemPool, - /*cleanOnlyUnused=*/false)? - TBBMALLOC_OK : TBBMALLOC_NO_EFFECT; - return TBBMALLOC_NO_EFFECT; + released = tls->externalCleanup(/*cleanOnlyUnused*/false, /*cleanBins=*/true); + break; case TBBMALLOC_CLEAN_ALL_BUFFERS: - return defaultMemPool->extMemPool.hardCachesCleanup()? - TBBMALLOC_OK : TBBMALLOC_NO_EFFECT; + released = defaultMemPool->extMemPool.hardCachesCleanup(true); + break; + default: + return TBBMALLOC_INVALID_PARAM; } - return TBBMALLOC_INVALID_PARAM; + return released ? TBBMALLOC_OK : TBBMALLOC_NO_EFFECT; } diff --git a/src/tbb/src/tbbmalloc/index.html b/src/tbb/src/tbbmalloc/index.html deleted file mode 100644 index 19eaf9925..000000000 --- a/src/tbb/src/tbbmalloc/index.html +++ /dev/null @@ -1,54 +0,0 @@ -<HTML> -<body> -<H2>Overview</H2> -<P> -This directory contains the Intel® Threading Building Blocks (Intel® TBB) scalable allocator library source files. -</P> - -<HR> -<p></p> -Copyright © 2005-2017 Intel Corporation. All Rights Reserved. -<P></P> -Intel is a registered trademark or trademark of Intel Corporation -or its subsidiaries in the United States and other countries. -<p></p> -* Other names and brands may be claimed as the property of others. - -<P> -<H3>Third Party and Open Source Licenses</H3> -</P> -<P> - <pre> - proxy_overload_osx.h - // Copyright (c) 2011, Google Inc. - // All rights reserved. - // - // Redistribution and use in source and binary forms, with or without - // modification, are permitted provided that the following conditions are - // met: - // - // * Redistributions of source code must retain the above copyright - // notice, this list of conditions and the following disclaimer. - // * Redistributions in binary form must reproduce the above - // copyright notice, this list of conditions and the following disclaimer - // in the documentation and/or other materials provided with the - // distribution. - // * Neither the name of Google Inc. nor the names of its - // contributors may be used to endorse or promote products derived from - // this software without specific prior written permission. - // - // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - </pre> -</P> -</body> -</HTML> diff --git a/src/tbb/src/tbbmalloc/large_objects.cpp b/src/tbb/src/tbbmalloc/large_objects.cpp index 0921b8557..43b17d058 100644 --- a/src/tbb/src/tbbmalloc/large_objects.cpp +++ b/src/tbb/src/tbbmalloc/large_objects.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2023 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,20 +12,61 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include "tbbmalloc_internal.h" +#include "../src/tbb/environment.h" -/********* Allocation of large objects ************/ +#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) + // Suppress warning: unary minus operator applied to unsigned type, result still unsigned + // TBB_REVAMP_TODO: review this warning + // #pragma warning(push) + // #pragma warning(disable:4146) +#endif +/******************************* Allocation of large objects *********************************************/ namespace rml { namespace internal { +/* ---------------------------- Large Object cache init section ---------------------------------------- */ + +void LargeObjectCache::init(ExtMemoryPool *memPool) +{ + extMemPool = memPool; + // scalable_allocation_mode can be called before allocator initialization, respect this manual request + if (hugeSizeThreshold == 0) { + // Huge size threshold initialization if environment variable was set + long requestedThreshold = tbb::detail::r1::GetIntegralEnvironmentVariable("TBB_MALLOC_SET_HUGE_SIZE_THRESHOLD"); + // Read valid env or initialize by default with max possible values + if (requestedThreshold != -1) { + setHugeSizeThreshold(requestedThreshold); + } else { + setHugeSizeThreshold(maxHugeSize); + } + } +} + +/* ----------------------------- Huge size threshold settings ----------------------------------------- */ + +void LargeObjectCache::setHugeSizeThreshold(size_t value) +{ + // Valid in the huge cache range: [MaxLargeSize, MaxHugeSize]. + if (value <= maxHugeSize) { + hugeSizeThreshold = value >= maxLargeSize ? alignToBin(value) : maxLargeSize; + + // Calculate local indexes for the global threshold size (for fast search inside a regular cleanup) + largeCache.hugeSizeThresholdIdx = LargeCacheType::numBins; + hugeCache.hugeSizeThresholdIdx = HugeCacheType::sizeToIdx(hugeSizeThreshold); + } +} + +bool LargeObjectCache::sizeInCacheRange(size_t size) +{ + return size < maxHugeSize && (size <= defaultMaxHugeSize || size >= hugeSizeThreshold); +} + +/* ----------------------------------------------------------------------------------------------------- */ /* The functor called by the aggregator for the operation list */ template<typename Props> @@ -82,8 +123,8 @@ class CacheBinFunctor { public: OperationPreprocessor(typename LargeObjectCacheImpl<Props>::CacheBin *bin) : - bin(bin), lclTime(0), opGet(NULL), opClean(NULL), cleanTime(0), - lastGetOpTime(0), updateUsedSize(0), head(NULL), isCleanAll(false) {} + bin(bin), lclTime(0), opGet(nullptr), opClean(nullptr), cleanTime(0), + lastGetOpTime(0), lastGet(0), updateUsedSize(0), head(nullptr), tail(nullptr), putListNum(0), isCleanAll(false) {} void operator()(CacheBinOperation* opList); uintptr_t getTimeRange() const { return -lclTime; } @@ -93,7 +134,7 @@ class CacheBinFunctor { public: CacheBinFunctor(typename LargeObjectCacheImpl<Props>::CacheBin *bin, ExtMemoryPool *extMemPool, typename LargeObjectCacheImpl<Props>::BinBitMask *bitMask, int idx) : - bin(bin), extMemPool(extMemPool), bitMask(bitMask), idx(idx), toRelease(NULL), needCleanup(false) {} + bin(bin), extMemPool(extMemPool), bitMask(bitMask), idx(idx), toRelease(nullptr), needCleanup(false), currTime(0) {} void operator()(CacheBinOperation* opList); bool isCleanupNeeded() const { return needCleanup; } @@ -101,23 +142,7 @@ class CacheBinFunctor { uintptr_t getCurrTime() const { return currTime; } }; -// ---------------- Cache Bin Aggregator Operation Helpers ---------------- // -// The list of possible operations. -enum CacheBinOperationType { - CBOP_INVALID = 0, - CBOP_GET, - CBOP_PUT_LIST, - CBOP_CLEAN_TO_THRESHOLD, - CBOP_CLEAN_ALL, - CBOP_UPDATE_USED_SIZE -}; - -// The operation status list. CBST_NOWAIT can be specified for non-blocking operations. -enum CacheBinOperationStatus { - CBST_WAIT = 0, - CBST_NOWAIT, - CBST_DONE -}; +/* ---------------- Cache Bin Aggregator Operation Helpers ---------------- */ // The list of structures which describe the operation data struct OpGet { @@ -187,25 +212,31 @@ template <typename OpTypeData> OpTypeData& opCast(CacheBinOperation &op) { return *reinterpret_cast<OpTypeData*>(&op.data); } -// ------------------------------------------------------------------------ // + +/* ------------------------------------------------------------------------ */ #if __TBB_MALLOC_LOCACHE_STAT -intptr_t mallocCalls, cacheHits; -intptr_t memAllocKB, memHitKB; +//intptr_t mallocCalls, cacheHits; +std::atomic<intptr_t> mallocCalls, cacheHits; +//intptr_t memAllocKB, memHitKB; +std::atomic<intptr_t> memAllocKB, memHitKB; #endif +#if MALLOC_DEBUG inline bool lessThanWithOverflow(intptr_t a, intptr_t b) { - return (a < b && (b - a < UINTPTR_MAX/2)) || - (a > b && (a - b > UINTPTR_MAX/2)); + return (a < b && (b - a < static_cast<intptr_t>(UINTPTR_MAX/2))) || + (a > b && (a - b > static_cast<intptr_t>(UINTPTR_MAX/2))); } +#endif /* ----------------------------------- Operation processing methods ------------------------------------ */ template<typename Props> void CacheBinFunctor<Props>:: OperationPreprocessor::commitOperation(CacheBinOperation *op) const { - FencedStore( (intptr_t&)(op->status), CBST_DONE ); + // FencedStore( (intptr_t&)(op->status), CBST_DONE ); + op->status.store(CBST_DONE, std::memory_order_release); } template<typename Props> void CacheBinFunctor<Props>:: @@ -274,7 +305,7 @@ template<typename Props> void CacheBinFunctor<Props>:: case CBOP_PUT_LIST: { LargeMemoryBlock *head = opCast<OpPutList>(*op).head; - LargeMemoryBlock *curr = head, *prev = NULL; + LargeMemoryBlock *curr = head, *prev = nullptr; int num = 0; do { @@ -292,7 +323,7 @@ template<typename Props> void CacheBinFunctor<Props>:: num += 1; STAT_increment(getThreadId(), ThreadCommonCounters, cacheLargeObj); - } while (( curr = curr->next )); + } while ((curr = curr->next) != nullptr); LargeMemoryBlock *tail = prev; addToPutList(head, tail, num); @@ -377,7 +408,7 @@ template<typename Props> void CacheBinFunctor<Props>::operator()(CacheBinOperati if ( prep.lastGetOpTime ) bin->setLastGet( prep.lastGetOpTime + endTime ); } else if ( LargeMemoryBlock *curr = prep.head ) { - curr->prev = NULL; + curr->prev = nullptr; while ( curr ) { // Update local times to global times curr->age += endTime; @@ -386,7 +417,7 @@ template<typename Props> void CacheBinFunctor<Props>::operator()(CacheBinOperati #if __TBB_MALLOC_WHITEBOX_TEST tbbmalloc_whitebox::locPutProcessed+=prep.putListNum; #endif - toRelease = bin->putList(prep.head, prep.tail, bitMask, idx, prep.putListNum); + toRelease = bin->putList(prep.head, prep.tail, bitMask, idx, prep.putListNum, extMemPool->loc.hugeSizeThreshold); } needCleanup = extMemPool->loc.isCleanupNeededOnRange(timeRange, startTime); currTime = endTime - 1; @@ -401,7 +432,7 @@ template<typename Props> void CacheBinFunctor<Props>::operator()(CacheBinOperati CacheBinOperation *opNext = opClean->next; prep.commitOperation( opClean ); - while (( opClean = opNext )) { + while ((opClean = opNext) != nullptr) { opNext = opClean->next; prep.commitOperation(opClean); } @@ -418,18 +449,20 @@ template<typename Props> void LargeObjectCacheImpl<Props>:: CacheBinFunctor<Props> func( this, extMemPool, bitMask, idx ); aggregator.execute( op, func, longLifeTime ); - if ( LargeMemoryBlock *toRelease = func.getToRelease() ) + if ( LargeMemoryBlock *toRelease = func.getToRelease()) { extMemPool->backend.returnLargeObject(toRelease); + } - if ( func.isCleanupNeeded() ) + if ( func.isCleanupNeeded() ) { extMemPool->loc.doCleanup( func.getCurrTime(), /*doThreshDecr=*/false); + } } template<typename Props> LargeMemoryBlock *LargeObjectCacheImpl<Props>:: CacheBin::get(ExtMemoryPool *extMemPool, size_t size, BinBitMask *bitMask, int idx) { - LargeMemoryBlock *lmb=NULL; - OpGet data = {&lmb, size}; + LargeMemoryBlock *lmb=nullptr; + OpGet data = {&lmb, size, static_cast<uintptr_t>(0)}; CacheBinOperation op(data); ExecuteOperation( &op, extMemPool, bitMask, idx ); return lmb; @@ -448,11 +481,12 @@ template<typename Props> void LargeObjectCacheImpl<Props>:: template<typename Props> bool LargeObjectCacheImpl<Props>:: CacheBin::cleanToThreshold(ExtMemoryPool *extMemPool, BinBitMask *bitMask, uintptr_t currTime, int idx) { - LargeMemoryBlock *toRelease = NULL; + LargeMemoryBlock *toRelease = nullptr; /* oldest may be more recent then age, that's why cast to signed type was used. age overflow is also processed correctly. */ - if (last && (intptr_t)(currTime - oldest) > ageThreshold) { + if (last.load(std::memory_order_relaxed) && + (intptr_t)(currTime - oldest.load(std::memory_order_relaxed)) > ageThreshold.load(std::memory_order_relaxed)) { OpCleanToThreshold data = {&toRelease, currTime}; CacheBinOperation op(data); ExecuteOperation( &op, extMemPool, bitMask, idx ); @@ -471,9 +505,9 @@ template<typename Props> bool LargeObjectCacheImpl<Props>:: template<typename Props> bool LargeObjectCacheImpl<Props>:: CacheBin::releaseAllToBackend(ExtMemoryPool *extMemPool, BinBitMask *bitMask, int idx) { - LargeMemoryBlock *toRelease = NULL; + LargeMemoryBlock *toRelease = nullptr; - if (last) { + if (last.load(std::memory_order_relaxed)) { OpCleanAll data = {&toRelease}; CacheBinOperation op(data); ExecuteOperation(&op, extMemPool, bitMask, idx); @@ -492,22 +526,26 @@ template<typename Props> bool LargeObjectCacheImpl<Props>:: } template<typename Props> void LargeObjectCacheImpl<Props>:: - CacheBin::updateUsedSize(ExtMemoryPool *extMemPool, size_t size, BinBitMask *bitMask, int idx) { + CacheBin::updateUsedSize(ExtMemoryPool *extMemPool, size_t size, BinBitMask *bitMask, int idx) +{ OpUpdateUsedSize data = {size}; CacheBinOperation op(data); ExecuteOperation( &op, extMemPool, bitMask, idx ); } -/* ----------------------------------------------------------------------------------------------------- */ + /* ------------------------------ Unsafe methods used with the aggregator ------------------------------ */ + template<typename Props> LargeMemoryBlock *LargeObjectCacheImpl<Props>:: - CacheBin::putList(LargeMemoryBlock *head, LargeMemoryBlock *tail, BinBitMask *bitMask, int idx, int num) + CacheBin::putList(LargeMemoryBlock *head, LargeMemoryBlock *tail, BinBitMask *bitMask, int idx, int num, size_t hugeSizeThreshold) { size_t size = head->unalignedSize; - usedSize -= num*size; - MALLOC_ASSERT( !last || (last->age != 0 && last->age != -1U), ASSERT_TEXT ); + usedSize.store(usedSize.load(std::memory_order_relaxed) - num * size, std::memory_order_relaxed); + MALLOC_ASSERT( !last.load(std::memory_order_relaxed) || + (last.load(std::memory_order_relaxed)->age != 0 && last.load(std::memory_order_relaxed)->age != -1U), ASSERT_TEXT ); MALLOC_ASSERT( (tail==head && num==1) || (tail!=head && num>1), ASSERT_TEXT ); - LargeMemoryBlock *toRelease = NULL; - if (!lastCleanedAge) { + MALLOC_ASSERT( tail, ASSERT_TEXT ); + LargeMemoryBlock *toRelease = nullptr; + if (size < hugeSizeThreshold && !lastCleanedAge) { // 1st object of such size was released. // Not cache it, and remember when this occurs // to take into account during cache miss. @@ -515,29 +553,28 @@ template<typename Props> LargeMemoryBlock *LargeObjectCacheImpl<Props>:: toRelease = tail; tail = tail->prev; if (tail) - tail->next = NULL; + tail->next = nullptr; else - head = NULL; + head = nullptr; num--; } if (num) { // add [head;tail] list to cache - MALLOC_ASSERT( tail, ASSERT_TEXT ); tail->next = first; if (first) first->prev = tail; first = head; - if (!last) { - MALLOC_ASSERT(0 == oldest, ASSERT_TEXT); - oldest = tail->age; - last = tail; + if (!last.load(std::memory_order_relaxed)) { + MALLOC_ASSERT(0 == oldest.load(std::memory_order_relaxed), ASSERT_TEXT); + oldest.store(tail->age, std::memory_order_relaxed); + last.store(tail, std::memory_order_relaxed); } - cachedSize += num*size; + cachedSize.store(cachedSize.load(std::memory_order_relaxed) + num * size, std::memory_order_relaxed); } // No used object, and nothing in the bin, mark the bin as empty - if (!usedSize && !first) + if (!usedSize.load(std::memory_order_relaxed) && !first) bitMask->set(idx, false); return toRelease; @@ -550,17 +587,16 @@ template<typename Props> LargeMemoryBlock *LargeObjectCacheImpl<Props>:: if (result) { first = result->next; if (first) - first->prev = NULL; + first->prev = nullptr; else { - last = NULL; - oldest = 0; + last.store(nullptr, std::memory_order_relaxed); + oldest.store(0, std::memory_order_relaxed); } } return result; } -// forget the history for the bin if it was unused for long time template<typename Props> void LargeObjectCacheImpl<Props>:: CacheBin::forgetOutdatedState(uintptr_t currTime) { @@ -573,14 +609,15 @@ template<typename Props> void LargeObjectCacheImpl<Props>:: const uintptr_t sinceLastGet = currTime - lastGet; bool doCleanup = false; - if (ageThreshold) - doCleanup = sinceLastGet > Props::LongWaitFactor*ageThreshold; + intptr_t threshold = ageThreshold.load(std::memory_order_relaxed); + if (threshold) + doCleanup = sinceLastGet > static_cast<uintptr_t>(Props::LongWaitFactor * threshold); else if (lastCleanedAge) - doCleanup = sinceLastGet > Props::LongWaitFactor*(lastCleanedAge - lastGet); + doCleanup = sinceLastGet > static_cast<uintptr_t>(Props::LongWaitFactor * (lastCleanedAge - lastGet)); if (doCleanup) { lastCleanedAge = 0; - ageThreshold = 0; + ageThreshold.store(0, std::memory_order_relaxed); } } @@ -590,7 +627,9 @@ template<typename Props> LargeMemoryBlock *LargeObjectCacheImpl<Props>:: { /* oldest may be more recent then age, that's why cast to signed type was used. age overflow is also processed correctly. */ - if ( !last || (intptr_t)(currTime - last->age) < ageThreshold ) return NULL; + if ( !last.load(std::memory_order_relaxed) || + (intptr_t)(currTime - last.load(std::memory_order_relaxed)->age) < ageThreshold.load(std::memory_order_relaxed) ) + return nullptr; #if MALLOC_DEBUG uintptr_t nextAge = 0; @@ -598,24 +637,25 @@ template<typename Props> LargeMemoryBlock *LargeObjectCacheImpl<Props>:: do { #if MALLOC_DEBUG // check that list ordered - MALLOC_ASSERT(!nextAge || lessThanWithOverflow(nextAge, last->age), + MALLOC_ASSERT(!nextAge || lessThanWithOverflow(nextAge, last.load(std::memory_order_relaxed)->age), ASSERT_TEXT); - nextAge = last->age; + nextAge = last.load(std::memory_order_relaxed)->age; #endif - cachedSize -= last->unalignedSize; - last = last->prev; - } while (last && (intptr_t)(currTime - last->age) > ageThreshold); - - LargeMemoryBlock *toRelease = NULL; - if (last) { - toRelease = last->next; - oldest = last->age; - last->next = NULL; + cachedSize.store(cachedSize.load(std::memory_order_relaxed) - last.load(std::memory_order_relaxed)->unalignedSize, std::memory_order_relaxed); + last.store(last.load(std::memory_order_relaxed)->prev, std::memory_order_relaxed); + } while (last.load(std::memory_order_relaxed) && + (intptr_t)(currTime - last.load(std::memory_order_relaxed)->age) > ageThreshold.load(std::memory_order_relaxed)); + + LargeMemoryBlock *toRelease = nullptr; + if (last.load(std::memory_order_relaxed)) { + toRelease = last.load(std::memory_order_relaxed)->next; + oldest.store(last.load(std::memory_order_relaxed)->age, std::memory_order_relaxed); + last.load(std::memory_order_relaxed)->next = nullptr; } else { toRelease = first; - first = NULL; - oldest = 0; - if (!usedSize) + first = nullptr; + oldest.store(0, std::memory_order_relaxed); + if (!usedSize.load(std::memory_order_relaxed)) bitMask->set(idx, false); } MALLOC_ASSERT( toRelease, ASSERT_TEXT ); @@ -627,20 +667,22 @@ template<typename Props> LargeMemoryBlock *LargeObjectCacheImpl<Props>:: template<typename Props> LargeMemoryBlock *LargeObjectCacheImpl<Props>:: CacheBin::cleanAll(BinBitMask *bitMask, int idx) { - if (!last) return NULL; + if (!last.load(std::memory_order_relaxed)) return nullptr; LargeMemoryBlock *toRelease = first; - last = NULL; - first = NULL; - oldest = 0; - cachedSize = 0; - if (!usedSize) + last.store(nullptr, std::memory_order_relaxed); + first = nullptr; + oldest.store(0, std::memory_order_relaxed); + cachedSize.store(0, std::memory_order_relaxed); + if (!usedSize.load(std::memory_order_relaxed)) bitMask->set(idx, false); return toRelease; } + /* ----------------------------------------------------------------------------------------------------- */ +#if __TBB_MALLOC_BACKEND_STAT template<typename Props> size_t LargeObjectCacheImpl<Props>:: CacheBin::reportStat(int num, FILE *f) { @@ -648,49 +690,57 @@ template<typename Props> size_t LargeObjectCacheImpl<Props>:: if (first) printf("%d(%lu): total %lu KB thr %ld lastCln %lu oldest %lu\n", num, num*Props::CacheStep+Props::MinSize, - cachedSize/1024, ageThreshold, lastCleanedAge, oldest); + cachedSize.load(std::memory_order_relaxed)/1024, ageThresholdageThreshold.load(std::memory_order_relaxed), lastCleanedAge, oldest.load(std::memory_order_relaxed)); #else suppress_unused_warning(num); suppress_unused_warning(f); #endif - return cachedSize; + return cachedSize.load(std::memory_order_relaxed); } +#endif -// release from cache blocks that are older than ageThreshold +// Release objects from cache blocks that are older than ageThreshold template<typename Props> bool LargeObjectCacheImpl<Props>::regularCleanup(ExtMemoryPool *extMemPool, uintptr_t currTime, bool doThreshDecr) { bool released = false; BinsSummary binsSummary; - for (int i = bitMask.getMaxTrue(numBins-1); i >= 0; - i = bitMask.getMaxTrue(i-1)) { + // Threshold settings is below this cache or starts from zero index + if (hugeSizeThresholdIdx == 0) return false; + + // Starting searching for bin that is less than huge size threshold (can be cleaned-up) + int startSearchIdx = hugeSizeThresholdIdx - 1; + + for (int i = bitMask.getMaxTrue(startSearchIdx); i >= 0; i = bitMask.getMaxTrue(i-1)) { bin[i].updateBinsSummary(&binsSummary); - if (!doThreshDecr && tooLargeLOC>2 && binsSummary.isLOCTooLarge()) { + if (!doThreshDecr && tooLargeLOC.load(std::memory_order_relaxed) > 2 && binsSummary.isLOCTooLarge()) { // if LOC is too large for quite long time, decrease the threshold // based on bin hit statistics. // For this, redo cleanup from the beginning. // Note: on this iteration total usedSz can be not too large // in comparison to total cachedSz, as we calculated it only // partially. We are ok with it. - i = bitMask.getMaxTrue(numBins-1)+1; + i = bitMask.getMaxTrue(startSearchIdx)+1; doThreshDecr = true; binsSummary.reset(); continue; } if (doThreshDecr) bin[i].decreaseThreshold(); - if (bin[i].cleanToThreshold(extMemPool, &bitMask, currTime, i)) + + if (bin[i].cleanToThreshold(extMemPool, &bitMask, currTime, i)) { released = true; + } } - // We want to find if LOC was too large for some time continuously, // so OK with races between incrementing and zeroing, but incrementing // must be atomic. - if (binsSummary.isLOCTooLarge()) - AtomicIncrement(tooLargeLOC); - else - tooLargeLOC = 0; + if (binsSummary.isLOCTooLarge()) { + tooLargeLOC++; + } else { + tooLargeLOC.store(0, std::memory_order_relaxed); + } return released; } @@ -698,11 +748,20 @@ template<typename Props> bool LargeObjectCacheImpl<Props>::cleanAll(ExtMemoryPool *extMemPool) { bool released = false; - for (int i = numBins-1; i >= 0; i--) + for (int i = numBins-1; i >= 0; i--) { released |= bin[i].releaseAllToBackend(extMemPool, &bitMask, i); + } return released; } +template<typename Props> +void LargeObjectCacheImpl<Props>::reset() { + tooLargeLOC.store(0, std::memory_order_relaxed); + for (int i = numBins-1; i >= 0; i--) + bin[i].init(); + bitMask.reset(); +} + #if __TBB_MALLOC_WHITEBOX_TEST template<typename Props> size_t LargeObjectCacheImpl<Props>::getLOCSize() const @@ -745,30 +804,39 @@ bool LargeObjectCache::doCleanup(uintptr_t currTime, bool doThreshDecr) { if (!doThreshDecr) extMemPool->allLocalCaches.markUnused(); - return largeCache.regularCleanup(extMemPool, currTime, doThreshDecr) - | hugeCache.regularCleanup(extMemPool, currTime, doThreshDecr); + + bool large_cache_cleaned = largeCache.regularCleanup(extMemPool, currTime, doThreshDecr); + bool huge_cache_cleaned = hugeCache.regularCleanup(extMemPool, currTime, doThreshDecr); + return large_cache_cleaned || huge_cache_cleaned; } bool LargeObjectCache::decreasingCleanup() { - return doCleanup(FencedLoad((intptr_t&)cacheCurrTime), /*doThreshDecr=*/true); + return doCleanup(cacheCurrTime.load(std::memory_order_acquire), /*doThreshDecr=*/true); } bool LargeObjectCache::regularCleanup() { - return doCleanup(FencedLoad((intptr_t&)cacheCurrTime), /*doThreshDecr=*/false); + return doCleanup(cacheCurrTime.load(std::memory_order_acquire), /*doThreshDecr=*/false); } bool LargeObjectCache::cleanAll() { - return largeCache.cleanAll(extMemPool) | hugeCache.cleanAll(extMemPool); + bool large_cache_cleaned = largeCache.cleanAll(extMemPool); + bool huge_cache_cleaned = hugeCache.cleanAll(extMemPool); + return large_cache_cleaned || huge_cache_cleaned; +} + +void LargeObjectCache::reset() +{ + largeCache.reset(); + hugeCache.reset(); } template<typename Props> LargeMemoryBlock *LargeObjectCacheImpl<Props>::get(ExtMemoryPool *extMemoryPool, size_t size) { - MALLOC_ASSERT( size%Props::CacheStep==0, ASSERT_TEXT ); - int idx = sizeToIdx(size); + int idx = Props::sizeToIdx(size); LargeMemoryBlock *lmb = bin[idx].get(extMemoryPool, size, &bitMask, idx); @@ -782,8 +850,8 @@ LargeMemoryBlock *LargeObjectCacheImpl<Props>::get(ExtMemoryPool *extMemoryPool, template<typename Props> void LargeObjectCacheImpl<Props>::updateCacheState(ExtMemoryPool *extMemPool, DecreaseOrIncrease op, size_t size) { - int idx = sizeToIdx(size); - MALLOC_ASSERT(idx<numBins, ASSERT_TEXT); + int idx = Props::sizeToIdx(size); + MALLOC_ASSERT(idx < static_cast<int>(numBins), ASSERT_TEXT); bin[idx].updateUsedSize(extMemPool, op==decrease? -size : size, &bitMask, idx); } @@ -801,14 +869,14 @@ void LargeObjectCache::reportStat(FILE *f) { largeCache.reportStat(f); hugeCache.reportStat(f); - fprintf(f, "cache time %lu\n", cacheCurrTime); + fprintf(f, "cache time %lu\n", cacheCurrTime.load(std::memory_order_relaxed)); } #endif template<typename Props> void LargeObjectCacheImpl<Props>::putList(ExtMemoryPool *extMemPool, LargeMemoryBlock *toCache) { - int toBinIdx = sizeToIdx(toCache->unalignedSize); + int toBinIdx = Props::sizeToIdx(toCache->unalignedSize); MALLOC_ITT_SYNC_RELEASING(bin+toBinIdx); bin[toBinIdx].putList(extMemPool, toCache, &bitMask, toBinIdx); @@ -822,19 +890,29 @@ void LargeObjectCache::updateCacheState(DecreaseOrIncrease op, size_t size) hugeCache.updateCacheState(extMemPool, op, size); } + +uintptr_t LargeObjectCache::getCurrTimeRange(uintptr_t range) +{ + return (cacheCurrTime.fetch_add(range) + 1); +} + void LargeObjectCache::registerRealloc(size_t oldSize, size_t newSize) { updateCacheState(decrease, oldSize); - updateCacheState(increase, newSize); + updateCacheState(increase, alignToBin(newSize)); } -// return artificial bin index, it's used only during sorting and never saved +size_t LargeObjectCache::alignToBin(size_t size) { + return size < maxLargeSize ? LargeCacheType::alignToBin(size) : HugeCacheType::alignToBin(size); +} + +// Used for internal purpose int LargeObjectCache::sizeToIdx(size_t size) { - MALLOC_ASSERT(size < maxHugeSize, ASSERT_TEXT); - return size < maxLargeSize? + MALLOC_ASSERT(size <= maxHugeSize, ASSERT_TEXT); + return size < maxLargeSize ? LargeCacheType::sizeToIdx(size) : - LargeCacheType::getNumBins()+HugeCacheType::sizeToIdx(size); + LargeCacheType::numBins + HugeCacheType::sizeToIdx(size); } void LargeObjectCache::putList(LargeMemoryBlock *list) @@ -844,7 +922,7 @@ void LargeObjectCache::putList(LargeMemoryBlock *list) for (LargeMemoryBlock *curr = list; curr; curr = toProcess) { LargeMemoryBlock *tail = curr; toProcess = curr->next; - if (curr->unalignedSize >= maxHugeSize) { + if (!sizeInCacheRange(curr->unalignedSize)) { extMemPool->backend.returnLargeObject(curr); continue; } @@ -867,7 +945,7 @@ void LargeObjectCache::putList(LargeMemoryBlock *list) } } } - tail->next = NULL; + tail->next = nullptr; if (curr->unalignedSize < maxLargeSize) largeCache.putList(extMemPool, curr); else @@ -877,54 +955,54 @@ void LargeObjectCache::putList(LargeMemoryBlock *list) void LargeObjectCache::put(LargeMemoryBlock *largeBlock) { - if (largeBlock->unalignedSize < maxHugeSize) { - largeBlock->next = NULL; - if (largeBlock->unalignedSize<maxLargeSize) + size_t blockSize = largeBlock->unalignedSize; + if (sizeInCacheRange(blockSize)) { + largeBlock->next = nullptr; + if (blockSize < maxLargeSize) largeCache.putList(extMemPool, largeBlock); else hugeCache.putList(extMemPool, largeBlock); - } else + } else { extMemPool->backend.returnLargeObject(largeBlock); + } } LargeMemoryBlock *LargeObjectCache::get(size_t size) { - MALLOC_ASSERT( size%largeBlockCacheStep==0, ASSERT_TEXT ); - MALLOC_ASSERT( size>=minLargeSize, ASSERT_TEXT ); - - if ( size < maxHugeSize) { - return size < maxLargeSize? + MALLOC_ASSERT( size >= minLargeSize, ASSERT_TEXT ); + if (sizeInCacheRange(size)) { + return size < maxLargeSize ? largeCache.get(extMemPool, size) : hugeCache.get(extMemPool, size); } - return NULL; + return nullptr; } LargeMemoryBlock *ExtMemoryPool::mallocLargeObject(MemoryPool *pool, size_t allocationSize) { #if __TBB_MALLOC_LOCACHE_STAT - AtomicIncrement(mallocCalls); - AtomicAdd(memAllocKB, allocationSize/1024); + mallocCalls++; + memAllocKB.fetch_add(allocationSize/1024); #endif LargeMemoryBlock* lmb = loc.get(allocationSize); if (!lmb) { BackRefIdx backRefIdx = BackRefIdx::newBackRef(/*largeObj=*/true); if (backRefIdx.isInvalid()) - return NULL; + return nullptr; // unalignedSize is set in getLargeBlock lmb = backend.getLargeBlock(allocationSize); if (!lmb) { removeBackRef(backRefIdx); loc.updateCacheState(decrease, allocationSize); - return NULL; + return nullptr; } lmb->backRefIdx = backRefIdx; lmb->pool = pool; STAT_increment(getThreadId(), ThreadCommonCounters, allocNewLargeObj); } else { #if __TBB_MALLOC_LOCACHE_STAT - AtomicIncrement(cacheHits); - AtomicAdd(memHitKB, allocationSize/1024); + cacheHits++; + memHitKB.fetch_add(allocationSize/1024); #endif } return lmb; @@ -942,17 +1020,33 @@ void ExtMemoryPool::freeLargeObjectList(LargeMemoryBlock *head) bool ExtMemoryPool::softCachesCleanup() { - return loc.regularCleanup(); + bool ret = false; + if (!softCachesCleanupInProgress.exchange(1, std::memory_order_acq_rel)) { + ret = loc.regularCleanup(); + softCachesCleanupInProgress.store(0, std::memory_order_release); + } + return ret; } -bool ExtMemoryPool::hardCachesCleanup() +bool ExtMemoryPool::hardCachesCleanup(bool wait) { + if (hardCachesCleanupInProgress.exchange(1, std::memory_order_acq_rel)) { + if (!wait) + return false; + + AtomicBackoff backoff; + while (hardCachesCleanupInProgress.exchange(1, std::memory_order_acq_rel)) + backoff.pause(); + } + // thread-local caches must be cleaned before LOC, // because object from thread-local cache can be released to LOC bool ret = releaseAllLocalCaches(); ret |= orphanedBlocks.cleanup(&backend); ret |= loc.cleanAll(); ret |= backend.clean(); + + hardCachesCleanupInProgress.store(0, std::memory_order_release); return ret; } @@ -963,7 +1057,7 @@ void *ExtMemoryPool::remap(void *ptr, size_t oldSize, size_t newSize, size_t ali void *o = backend.remap(ptr, oldSize, newSize, alignment); if (o) { LargeMemoryBlock *lmb = ((LargeObjectHdr*)o - 1)->memoryBlock; - loc.registerRealloc(lmb->unalignedSize, oldUnalignedSize); + loc.registerRealloc(oldUnalignedSize, lmb->unalignedSize); } return o; } @@ -974,3 +1068,6 @@ void *ExtMemoryPool::remap(void *ptr, size_t oldSize, size_t newSize, size_t ali } // namespace internal } // namespace rml +#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) + // #pragma warning(pop) +#endif diff --git a/src/tbb/src/tbbmalloc/large_objects.h b/src/tbb/src/tbbmalloc/large_objects.h new file mode 100644 index 000000000..58d7c81a7 --- /dev/null +++ b/src/tbb/src/tbbmalloc/large_objects.h @@ -0,0 +1,380 @@ +/* + Copyright (c) 2005-2024 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_tbbmalloc_internal_H + #error tbbmalloc_internal.h must be included at this point +#endif + +#ifndef __TBB_large_objects_H +#define __TBB_large_objects_H + +//! The list of possible Cache Bin Aggregator operations. +/* Declared here to avoid Solaris Studio* 12.2 "multiple definitions" error */ +enum CacheBinOperationType { + CBOP_INVALID = 0, + CBOP_GET, + CBOP_PUT_LIST, + CBOP_CLEAN_TO_THRESHOLD, + CBOP_CLEAN_ALL, + CBOP_UPDATE_USED_SIZE +}; + +// The Cache Bin Aggregator operation status list. +// CBST_NOWAIT can be specified for non-blocking operations. +enum CacheBinOperationStatus { + CBST_WAIT = 0, + CBST_NOWAIT, + CBST_DONE +}; + +/* + * Bins that grow with arithmetic step + */ +template<size_t MIN_SIZE, size_t MAX_SIZE> +struct LargeBinStructureProps { +public: + static const size_t MinSize = MIN_SIZE, MaxSize = MAX_SIZE; + static const size_t CacheStep = 8 * 1024; + static const unsigned NumBins = (MaxSize - MinSize) / CacheStep; + + static size_t alignToBin(size_t size) { + return alignUp(size, CacheStep); + } + + static int sizeToIdx(size_t size) { + MALLOC_ASSERT(MinSize <= size && size < MaxSize, ASSERT_TEXT); + MALLOC_ASSERT(size % CacheStep == 0, ASSERT_TEXT); + return (size - MinSize) / CacheStep; + } +}; + +/* + * Bins that grow with special geometric progression. + */ +template<size_t MIN_SIZE, size_t MAX_SIZE> +struct HugeBinStructureProps { + +private: + // Sizes grow with the following formula: Size = MinSize * (2 ^ (Index / StepFactor)) + // There are StepFactor bins (8 be default) between each power of 2 bin + static const int MaxSizeExp = Log2<MAX_SIZE>::value; + static const int MinSizeExp = Log2<MIN_SIZE>::value; + static const int StepFactor = 8; + static const int StepFactorExp = Log2<StepFactor>::value; + +public: + static const size_t MinSize = MIN_SIZE, MaxSize = MAX_SIZE; + static const unsigned NumBins = (MaxSizeExp - MinSizeExp) * StepFactor; + + static size_t alignToBin(size_t size) { + MALLOC_ASSERT(size >= StepFactor, "Size must not be less than the StepFactor"); + + int sizeExp = (int)BitScanRev(size); + MALLOC_ASSERT(sizeExp >= 0, "BitScanRev() cannot return -1, as size >= stepfactor > 0"); + MALLOC_ASSERT(sizeExp >= StepFactorExp, "sizeExp >= StepFactorExp, because size >= stepFactor"); + int minorStepExp = sizeExp - StepFactorExp; + + return alignUp(size, 1ULL << minorStepExp); + } + + // Sizes between the power of 2 values are approximated to StepFactor. + static int sizeToIdx(size_t size) { + MALLOC_ASSERT(MinSize <= size && size <= MaxSize, ASSERT_TEXT); + + int sizeExp = (int)BitScanRev(size); // same as __TBB_Log2 + MALLOC_ASSERT(sizeExp >= 0, "BitScanRev() cannot return -1, as size >= stepfactor > 0"); + MALLOC_ASSERT(sizeExp >= StepFactorExp, "sizeExp >= StepFactorExp, because size >= stepFactor"); + int minorStepExp = sizeExp - StepFactorExp; + + size_t majorStepSize = 1ULL << sizeExp; + int minorIdx = (size - majorStepSize) >> minorStepExp; + MALLOC_ASSERT(size == majorStepSize + ((size_t)minorIdx << minorStepExp), + "Size is not aligned on the bin"); + return StepFactor * (sizeExp - MinSizeExp) + minorIdx; + } +}; + +/* + * Cache properties accessor + * + * TooLargeFactor -- when cache size treated "too large" in comparison to user data size + * OnMissFactor -- If cache miss occurred and cache was cleaned, + * set ageThreshold to OnMissFactor * the difference + * between current time and last time cache was cleaned. + * LongWaitFactor -- to detect rarely-used bins and forget about their usage history + */ +template<typename StructureProps, int TOO_LARGE, int ON_MISS, int LONG_WAIT> +struct LargeObjectCacheProps : public StructureProps { + static const int TooLargeFactor = TOO_LARGE, OnMissFactor = ON_MISS, LongWaitFactor = LONG_WAIT; +}; + +template<typename Props> +class LargeObjectCacheImpl { +private: + + // Current sizes of used and cached objects. It's calculated while we are + // traversing bins, and used for isLOCTooLarge() check at the same time. + class BinsSummary { + size_t usedSz; + size_t cachedSz; + public: + BinsSummary() : usedSz(0), cachedSz(0) {} + // "too large" criteria + bool isLOCTooLarge() const { return cachedSz > Props::TooLargeFactor * usedSz; } + void update(size_t usedSize, size_t cachedSize) { + usedSz += usedSize; + cachedSz += cachedSize; + } + void reset() { usedSz = cachedSz = 0; } + }; + +public: + // The number of bins to cache large/huge objects. + static const uint32_t numBins = Props::NumBins; + + typedef BitMaskMax<numBins> BinBitMask; + + // 2-linked list of same-size cached blocks ordered by age (oldest on top) + // TODO: are we really want the list to be 2-linked? This allows us + // reduce memory consumption and do less operations under lock. + // TODO: try to switch to 32-bit logical time to save space in CacheBin + // and move bins to different cache lines. + class CacheBin { + private: + LargeMemoryBlock* first; + std::atomic<LargeMemoryBlock*> last; + /* age of an oldest block in the list; equal to last->age, if last defined, + used for quick checking it without acquiring the lock. */ + std::atomic<uintptr_t> oldest; + /* currAge when something was excluded out of list because of the age, + not because of cache hit */ + uintptr_t lastCleanedAge; + /* Current threshold value for the blocks of a particular size. + Set on cache miss. */ + std::atomic<intptr_t> ageThreshold; + + /* total size of all objects corresponding to the bin and allocated by user */ + std::atomic<size_t> usedSize; + /* total size of all objects cached in the bin */ + std::atomic<size_t> cachedSize; + /* mean time of presence of block in the bin before successful reuse */ + std::atomic<intptr_t> meanHitRange; + /* time of last get called for the bin */ + uintptr_t lastGet; + + typename MallocAggregator<CacheBinOperation>::type aggregator; + + void ExecuteOperation(CacheBinOperation *op, ExtMemoryPool *extMemPool, BinBitMask *bitMask, int idx, bool longLifeTime = true); + + /* should be placed in zero-initialized memory, ctor not needed. */ + CacheBin(); + + public: + void init() { + memset(static_cast<void*>(this), 0, sizeof(CacheBin)); + } + + /* ---------- Cache accessors ---------- */ + void putList(ExtMemoryPool *extMemPool, LargeMemoryBlock *head, BinBitMask *bitMask, int idx); + LargeMemoryBlock *get(ExtMemoryPool *extMemPool, size_t size, BinBitMask *bitMask, int idx); + + /* ---------- Cleanup functions -------- */ + bool cleanToThreshold(ExtMemoryPool *extMemPool, BinBitMask *bitMask, uintptr_t currTime, int idx); + bool releaseAllToBackend(ExtMemoryPool *extMemPool, BinBitMask *bitMask, int idx); + /* ------------------------------------- */ + + void updateUsedSize(ExtMemoryPool *extMemPool, size_t size, BinBitMask *bitMask, int idx); + void decreaseThreshold() { + intptr_t threshold = ageThreshold.load(std::memory_order_relaxed); + if (threshold) + ageThreshold.store((threshold + meanHitRange.load(std::memory_order_relaxed)) / 2, std::memory_order_relaxed); + } + void updateBinsSummary(BinsSummary *binsSummary) const { + binsSummary->update(usedSize.load(std::memory_order_relaxed), cachedSize.load(std::memory_order_relaxed)); + } + size_t getSize() const { return cachedSize.load(std::memory_order_relaxed); } + size_t getUsedSize() const { return usedSize.load(std::memory_order_relaxed); } + size_t reportStat(int num, FILE *f); + + /* --------- Unsafe methods used with the aggregator ------- */ + void forgetOutdatedState(uintptr_t currTime); + LargeMemoryBlock *putList(LargeMemoryBlock *head, LargeMemoryBlock *tail, BinBitMask *bitMask, + int idx, int num, size_t hugeObjectThreshold); + LargeMemoryBlock *get(); + LargeMemoryBlock *cleanToThreshold(uintptr_t currTime, BinBitMask *bitMask, int idx); + LargeMemoryBlock *cleanAll(BinBitMask *bitMask, int idx); + void updateUsedSize(size_t size, BinBitMask *bitMask, int idx) { + if (!usedSize.load(std::memory_order_relaxed)) bitMask->set(idx, true); + usedSize.store(usedSize.load(std::memory_order_relaxed) + size, std::memory_order_relaxed); + if (!usedSize.load(std::memory_order_relaxed) && !first) bitMask->set(idx, false); + } + void updateMeanHitRange( intptr_t hitRange ) { + hitRange = hitRange >= 0 ? hitRange : 0; + intptr_t mean = meanHitRange.load(std::memory_order_relaxed); + mean = mean ? (mean + hitRange) / 2 : hitRange; + meanHitRange.store(mean, std::memory_order_relaxed); + } + void updateAgeThreshold( uintptr_t currTime ) { + if (lastCleanedAge) + ageThreshold.store(Props::OnMissFactor * (currTime - lastCleanedAge), std::memory_order_relaxed); + } + void updateCachedSize(size_t size) { + cachedSize.store(cachedSize.load(std::memory_order_relaxed) + size, std::memory_order_relaxed); + } + void setLastGet( uintptr_t newLastGet ) { + lastGet = newLastGet; + } + /* -------------------------------------------------------- */ + }; + + // Huge bins index for fast regular cleanup searching in case of + // the "huge size threshold" setting defined + intptr_t hugeSizeThresholdIdx; + +private: + // How many times LOC was "too large" + std::atomic<intptr_t> tooLargeLOC; + // for fast finding of used bins and bins with non-zero usedSize; + // indexed from the end, as we need largest 1st + BinBitMask bitMask; + // bins with lists of recently freed large blocks cached for reuse + CacheBin bin[numBins]; + +public: + /* ------------ CacheBin structure dependent stuff ------------ */ + static size_t alignToBin(size_t size) { + return Props::alignToBin(size); + } + static int sizeToIdx(size_t size) { + return Props::sizeToIdx(size); + } + + /* --------- Main cache functions (put, get object) ------------ */ + void putList(ExtMemoryPool *extMemPool, LargeMemoryBlock *largeBlock); + LargeMemoryBlock *get(ExtMemoryPool *extMemPool, size_t size); + + /* ------------------------ Cleanup ---------------------------- */ + bool regularCleanup(ExtMemoryPool *extMemPool, uintptr_t currAge, bool doThreshDecr); + bool cleanAll(ExtMemoryPool *extMemPool); + + /* -------------------------- Other ---------------------------- */ + void updateCacheState(ExtMemoryPool *extMemPool, DecreaseOrIncrease op, size_t size); + + void reset(); + void reportStat(FILE *f); +#if __TBB_MALLOC_WHITEBOX_TEST + size_t getLOCSize() const; + size_t getUsedSize() const; +#endif +}; + +class LargeObjectCache { +private: + // Large bins [minLargeSize, maxLargeSize) + // Huge bins [maxLargeSize, maxHugeSize) + static const size_t minLargeSize = 8 * 1024, + maxLargeSize = 8 * 1024 * 1024, + // Cache memory up to 1TB (or 2GB for 32-bit arch), but sieve objects from the special threshold + maxHugeSize = tbb::detail::select_size_t_constant<2147483648U, 1099511627776ULL>::value; + +public: + // Upper bound threshold for caching size. After that size all objects sieve through cache + // By default - 64MB, previous value was 129MB (needed by some Intel(R) Math Kernel Library (Intel(R) MKL) benchmarks) + static const size_t defaultMaxHugeSize = 64UL * 1024UL * 1024UL; + // After that size large object interpreted as huge and does not participate in regular cleanup. + // Can be changed during the program execution. + size_t hugeSizeThreshold; + +private: + // Large objects cache properties + typedef LargeBinStructureProps<minLargeSize, maxLargeSize> LargeBSProps; + typedef LargeObjectCacheProps<LargeBSProps, 2, 2, 16> LargeCacheTypeProps; + + // Huge objects cache properties + typedef HugeBinStructureProps<maxLargeSize, maxHugeSize> HugeBSProps; + typedef LargeObjectCacheProps<HugeBSProps, 1, 1, 4> HugeCacheTypeProps; + + // Cache implementation type with properties + typedef LargeObjectCacheImpl< LargeCacheTypeProps > LargeCacheType; + typedef LargeObjectCacheImpl< HugeCacheTypeProps > HugeCacheType; + + // Beginning of largeCache is more actively used and smaller than hugeCache, + // so put hugeCache first to prevent false sharing + // with LargeObjectCache's predecessor + HugeCacheType hugeCache; + LargeCacheType largeCache; + + /* logical time, incremented on each put/get operation + To prevent starvation between pools, keep separately for each pool. + Overflow is OK, as we only want difference between + its current value and some recent. + + Both malloc and free should increment logical time, as in + a different case multiple cached blocks would have same age, + and accuracy of predictors suffers. + */ + std::atomic<uintptr_t> cacheCurrTime; + + // Memory pool that owns this LargeObjectCache. + // strict 1:1 relation, never changed + ExtMemoryPool *extMemPool; + + // Returns artificial bin index, + // it's used only during sorting and never saved + static int sizeToIdx(size_t size); + + // Our friends + friend class Backend; + +public: + void init(ExtMemoryPool *memPool); + + // Item accessors + void put(LargeMemoryBlock *largeBlock); + void putList(LargeMemoryBlock *head); + LargeMemoryBlock *get(size_t size); + + void updateCacheState(DecreaseOrIncrease op, size_t size); + bool isCleanupNeededOnRange(uintptr_t range, uintptr_t currTime); + + // Cleanup operations + bool doCleanup(uintptr_t currTime, bool doThreshDecr); + bool decreasingCleanup(); + bool regularCleanup(); + bool cleanAll(); + void reset(); + + void reportStat(FILE *f); +#if __TBB_MALLOC_WHITEBOX_TEST + size_t getLOCSize() const; + size_t getUsedSize() const; +#endif + + // Cache deals with exact-fit sizes, so need to align each size + // to the specific bin when put object to cache + static size_t alignToBin(size_t size); + + void setHugeSizeThreshold(size_t value); + + // Check if we should cache or sieve this size + bool sizeInCacheRange(size_t size); + + uintptr_t getCurrTimeRange(uintptr_t range); + void registerRealloc(size_t oldSize, size_t newSize); +}; + +#endif // __TBB_large_objects_H + diff --git a/src/tbb/src/tbbmalloc/lin64ipf-proxy-export.def b/src/tbb/src/tbbmalloc/lin64ipf-proxy-export.def deleted file mode 100644 index 12d8d43dd..000000000 --- a/src/tbb/src/tbbmalloc/lin64ipf-proxy-export.def +++ /dev/null @@ -1,59 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -{ -global: -calloc; -free; -malloc; -realloc; -posix_memalign; -memalign; -aligned_alloc; -valloc; -pvalloc; -mallinfo; -mallopt; -malloc_usable_size; -__libc_malloc; -__libc_realloc; -__libc_calloc; -__libc_free; -__libc_memalign; -__libc_pvalloc; -__libc_valloc; -__TBB_malloc_proxy; -_ZdaPv; /* next ones are new/delete */ -_ZdaPvRKSt9nothrow_t; -_ZdlPv; -_ZdlPvRKSt9nothrow_t; -_Znam; -_ZnamRKSt9nothrow_t; -_Znwm; -_ZnwmRKSt9nothrow_t; - -local: - -/* TBB symbols */ -*3rml8internal*; -*3tbb*; -*__TBB*; - -}; diff --git a/src/tbb/src/tbbmalloc/lin64ipf-tbbmalloc-export.def b/src/tbb/src/tbbmalloc/lin64ipf-tbbmalloc-export.def deleted file mode 100644 index a57c933b6..000000000 --- a/src/tbb/src/tbbmalloc/lin64ipf-tbbmalloc-export.def +++ /dev/null @@ -1,76 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -{ -global: - -scalable_calloc; -scalable_free; -scalable_malloc; -scalable_realloc; -scalable_posix_memalign; -scalable_aligned_malloc; -scalable_aligned_realloc; -scalable_aligned_free; -scalable_msize; -scalable_allocation_mode; -scalable_allocation_command; -__TBB_malloc_safer_aligned_msize; -__TBB_malloc_safer_aligned_realloc; -__TBB_malloc_safer_free; -__TBB_malloc_safer_msize; -__TBB_malloc_safer_realloc; - -/* memory pool stuff */ -_ZN3rml11pool_createElPKNS_13MemPoolPolicyE; -_ZN3rml14pool_create_v1ElPKNS_13MemPoolPolicyEPPNS_10MemoryPoolE; -_ZN3rml10pool_resetEPNS_10MemoryPoolE; -_ZN3rml11pool_mallocEPNS_10MemoryPoolEm; -_ZN3rml12pool_destroyEPNS_10MemoryPoolE; -_ZN3rml9pool_freeEPNS_10MemoryPoolEPv; -_ZN3rml12pool_reallocEPNS_10MemoryPoolEPvm; -_ZN3rml20pool_aligned_reallocEPNS_10MemoryPoolEPvmm; -_ZN3rml19pool_aligned_mallocEPNS_10MemoryPoolEmm; -_ZN3rml13pool_identifyEPv; - -local: - -/* TBB symbols */ -*3rml*; -*3tbb*; -*__TBB*; -__itt_*; -ITT_DoOneTimeInitialization; -TBB_runtime_interface_version; - -/* Intel Compiler (libirc) symbols */ -__intel_*; -_intel_*; -get_memcpy_largest_cachelinesize; -get_memcpy_largest_cache_size; -get_mem_ops_method; -init_mem_ops_method; -irc__get_msg; -irc__print; -override_mem_ops_method; -set_memcpy_largest_cachelinesize; -set_memcpy_largest_cache_size; - -}; diff --git a/src/tbb/src/tbbmalloc/mac64-tbbmalloc-export.def b/src/tbb/src/tbbmalloc/mac64-tbbmalloc-export.def deleted file mode 100644 index 71cc10795..000000000 --- a/src/tbb/src/tbbmalloc/mac64-tbbmalloc-export.def +++ /dev/null @@ -1,48 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -_scalable_calloc -_scalable_free -_scalable_malloc -_scalable_realloc -_scalable_posix_memalign -_scalable_aligned_malloc -_scalable_aligned_realloc -_scalable_aligned_free -_scalable_msize -_scalable_allocation_mode -_scalable_allocation_command -___TBB_malloc_safer_aligned_msize -___TBB_malloc_safer_aligned_realloc -___TBB_malloc_safer_free -___TBB_malloc_safer_msize -___TBB_malloc_safer_realloc -___TBB_malloc_free_definite_size -/* memory pool stuff */ -__ZN3rml11pool_createElPKNS_13MemPoolPolicyE -__ZN3rml14pool_create_v1ElPKNS_13MemPoolPolicyEPPNS_10MemoryPoolE -__ZN3rml10pool_resetEPNS_10MemoryPoolE -__ZN3rml12pool_destroyEPNS_10MemoryPoolE -__ZN3rml11pool_mallocEPNS_10MemoryPoolEm -__ZN3rml9pool_freeEPNS_10MemoryPoolEPv -__ZN3rml12pool_reallocEPNS_10MemoryPoolEPvm -__ZN3rml20pool_aligned_reallocEPNS_10MemoryPoolEPvmm -__ZN3rml19pool_aligned_mallocEPNS_10MemoryPoolEmm -__ZN3rml13pool_identifyEPv diff --git a/src/tbb/src/tbbmalloc/shared_utils.h b/src/tbb/src/tbbmalloc/shared_utils.h index 273559d38..7ece1891f 100644 --- a/src/tbb/src/tbbmalloc/shared_utils.h +++ b/src/tbb/src/tbbmalloc/shared_utils.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_shared_utils_H @@ -55,10 +51,13 @@ static inline T alignUpGeneric(T arg, uintptr_t alignment) { return arg; } -template<typename T, size_t N> // generic function to find length of array -inline size_t arrayLength(const T(&)[N]) { - return N; -} +/* + * Compile time Log2 calculation + */ +template <size_t NUM> +struct Log2 { static const int value = 1 + Log2<(NUM >> 1)>::value; }; +template <> +struct Log2<1> { static const int value = 0; }; #if defined(min) #undef min @@ -69,6 +68,61 @@ T min ( const T& val1, const T& val2 ) { return val1 < val2 ? val1 : val2; } +/* + * Functions to parse files information (system files for example) + */ + +#include <stdio.h> + +#if defined(_MSC_VER) && (_MSC_VER<1900) && !defined(__INTEL_COMPILER) + // Suppress overzealous compiler warnings that default ctor and assignment + // operator cannot be generated and object 'class' can never be instantiated. + // #pragma warning(push) + // #pragma warning(disable:4510 4512 4610) +#endif + +#if __SUNPRO_CC + // Suppress overzealous compiler warnings that a class with a reference member + // lacks a user-defined constructor, which can lead to errors + #pragma error_messages (off, refmemnoconstr) +#endif + +// TODO: add a constructor to remove warnings suppression +struct parseFileItem { + const char* format; + long long& value; +}; + +#if defined(_MSC_VER) && (_MSC_VER<1900) && !defined(__INTEL_COMPILER) + // #pragma warning(pop) +#endif + +#if __SUNPRO_CC + #pragma error_messages (on, refmemnoconstr) +#endif + +template <int BUF_LINE_SIZE, int N> +void parseFile(const char* file, const parseFileItem (&items)[N]) { + // Tries to find all items in each line + int found[N] = { 0 }; + // If all items found, stop forward file reading + int numFound = 0; + // Line storage + char buf[BUF_LINE_SIZE]; + + if (FILE *f = fopen(file, "r")) { + while (numFound < N && fgets(buf, BUF_LINE_SIZE, f)) { + for (int i = 0; i < N; ++i) { + if (!found[i] && 1 == sscanf(buf, items[i].format, &items[i].value)) { + ++numFound; + found[i] = 1; + } + } + } + fclose(f); + } +} + namespace rml { namespace internal { @@ -85,5 +139,8 @@ const uint32_t estimatedCacheLineSize = 128; const uint32_t estimatedCacheLineSize = 64; #endif -}} // namespaces +} // namespace internal +} // namespace rml + #endif /* __TBB_shared_utils_H */ + diff --git a/src/tbb/src/tbbmalloc/tbbmalloc.cpp b/src/tbb/src/tbbmalloc/tbbmalloc.cpp index fa0e69843..b72e03a74 100644 --- a/src/tbb/src/tbbmalloc/tbbmalloc.cpp +++ b/src/tbb/src/tbbmalloc/tbbmalloc.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2023 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,23 +12,21 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include "TypeDefinitions.h" // Customize.h and proxy.h get included #include "tbbmalloc_internal_api.h" -#include "../tbb/tbb_assert_impl.h" // Out-of-line TBB assertion handling routines are instantiated here. +#include "../tbb/assert_impl.h" // Out-of-line TBB assertion handling routines are instantiated here. +#include "oneapi/tbb/version.h" +#include "oneapi/tbb/scalable_allocator.h" #undef UNICODE #if USE_PTHREAD #include <dlfcn.h> // dlopen #elif USE_WINTHREAD -#include "tbb/machine/windows_api.h" +#include <windows.h> #endif namespace rml { @@ -40,21 +38,21 @@ namespace internal { #define DEBUG_SUFFIX #endif /* TBB_USE_DEBUG */ -// MALLOCLIB_NAME is the name of the TBB memory allocator library. +// MALLOCLIB_NAME is the name of the oneTBB memory allocator library. #if _WIN32||_WIN64 #define MALLOCLIB_NAME "tbbmalloc" DEBUG_SUFFIX ".dll" #elif __APPLE__ -#define MALLOCLIB_NAME "libtbbmalloc" DEBUG_SUFFIX ".dylib" -#elif __FreeBSD__ || __NetBSD__ || __sun || _AIX || __ANDROID__ +#define MALLOCLIB_NAME "libtbbmalloc" DEBUG_SUFFIX ".2.dylib" +#elif __FreeBSD__ || __NetBSD__ || __OpenBSD__ || __sun || _AIX || __ANDROID__ #define MALLOCLIB_NAME "libtbbmalloc" DEBUG_SUFFIX ".so" -#elif __linux__ -#define MALLOCLIB_NAME "libtbbmalloc" DEBUG_SUFFIX __TBB_STRING(.so.TBB_COMPATIBLE_INTERFACE_VERSION) +#elif __unix__ +#define MALLOCLIB_NAME "libtbbmalloc" DEBUG_SUFFIX __TBB_STRING(.so.2) #else #error Unknown OS #endif void init_tbbmalloc() { -#if DO_ITT_NOTIFY +#if __TBB_USE_ITT_NOTIFY MallocInitializeITT(); #endif @@ -69,36 +67,43 @@ void init_tbbmalloc() { |GET_MODULE_HANDLE_EX_FLAG_PIN, (LPCTSTR)&scalable_malloc, &lib); MALLOC_ASSERT(lib && ret, "Allocator can't find itself."); + tbb::detail::suppress_unused_warning(ret); SetErrorMode (prev_mode); #endif /* USE_PTHREAD && !__TBB_SOURCE_DIRECTLY_INCLUDED */ } #if !__TBB_SOURCE_DIRECTLY_INCLUDED #if USE_WINTHREAD -extern "C" BOOL WINAPI DllMain( HINSTANCE /*hInst*/, DWORD callReason, LPVOID ) +extern "C" BOOL WINAPI DllMain( HINSTANCE /*hInst*/, DWORD callReason, LPVOID lpvReserved) { - if (callReason==DLL_THREAD_DETACH) { __TBB_mallocThreadShutdownNotification(); } else if (callReason==DLL_PROCESS_DETACH) { - __TBB_mallocProcessShutdownNotification(); + __TBB_mallocProcessShutdownNotification(lpvReserved != nullptr); } return TRUE; } #else /* !USE_WINTHREAD */ struct RegisterProcessShutdownNotification { // Work around non-reentrancy in dlopen() on Android -#if !__TBB_USE_DLOPEN_REENTRANCY_WORKAROUND RegisterProcessShutdownNotification() { // prevents unloading, POSIX case + + // We need better support for the library pinning + // when dlopen can't find TBBmalloc library. + // for example: void *ret = dlopen(MALLOCLIB_NAME, RTLD_NOW); + // MALLOC_ASSERT(ret, "Allocator can't load itself."); dlopen(MALLOCLIB_NAME, RTLD_NOW); } -#endif /* !__TBB_USE_DLOPEN_REENTRANCY_WORKAROUND */ + + RegisterProcessShutdownNotification(RegisterProcessShutdownNotification&) = delete; + RegisterProcessShutdownNotification& operator=(const RegisterProcessShutdownNotification&) = delete; + ~RegisterProcessShutdownNotification() { - __TBB_mallocProcessShutdownNotification(); + __TBB_mallocProcessShutdownNotification(false); } }; @@ -108,15 +113,3 @@ static RegisterProcessShutdownNotification reg; } } // namespaces -#if __TBB_ipf -/* It was found that on IA-64 architecture inlining of __TBB_machine_lockbyte leads - to serious performance regression with ICC. So keep it out-of-line. - - This code is copy-pasted from tbb_misc.cpp. - */ -extern "C" intptr_t __TBB_machine_lockbyte( volatile unsigned char& flag ) { - tbb::internal::atomic_backoff backoff; - while( !__TBB_TryLockByte(flag) ) backoff.pause(); - return 0; -} -#endif diff --git a/src/tbb/src/tbbmalloc/tbbmalloc.rc b/src/tbb/src/tbbmalloc/tbbmalloc.rc index 2b65e554d..2821addab 100644 --- a/src/tbb/src/tbbmalloc/tbbmalloc.rc +++ b/src/tbb/src/tbbmalloc/tbbmalloc.rc @@ -1,4 +1,4 @@ -// Copyright (c) 2005-2017 Intel Corporation +// Copyright (c) 2005-2024 Intel Corporation // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -11,61 +11,31 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -// -// -// -// -// Microsoft Visual C++ generated resource script. -// -#ifdef APSTUDIO_INVOKED -#ifndef APSTUDIO_READONLY_SYMBOLS -#define _APS_NO_MFC 1 -#define _APS_NEXT_RESOURCE_VALUE 102 -#define _APS_NEXT_COMMAND_VALUE 40001 -#define _APS_NEXT_CONTROL_VALUE 1001 -#define _APS_NEXT_SYMED_VALUE 101 -#endif -#endif - -#define APSTUDIO_READONLY_SYMBOLS ///////////////////////////////////////////////////////////////////////////// // -// Generated from the TEXTINCLUDE 2 resource. +// Includes // #include <winresrc.h> -#define ENDL "\r\n" -#include "tbb/tbb_version.h" - -#define TBBMALLOC_VERNUMBERS TBB_VERSION_MAJOR, TBB_VERSION_MINOR, __TBB_VERSION_YMD -#define TBBMALLOC_VERSION __TBB_STRING(TBBMALLOC_VERNUMBERS) - -///////////////////////////////////////////////////////////////////////////// -#undef APSTUDIO_READONLY_SYMBOLS +#include "../../include/oneapi/tbb/version.h" ///////////////////////////////////////////////////////////////////////////// // Neutral resources -#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_NEU) #ifdef _WIN32 LANGUAGE LANG_NEUTRAL, SUBLANG_NEUTRAL #pragma code_page(1252) #endif //_WIN32 -///////////////////////////////////////////////////////////////////////////// -// manifest integration -#ifdef TBB_MANIFEST -#include "winuser.h" -2 RT_MANIFEST tbbmanifest.exe.manifest -#endif - ///////////////////////////////////////////////////////////////////////////// // // Version // +#define TBB_VERNUMBERS TBB_VERSION_MAJOR,TBB_VERSION_MINOR,TBB_VERSION_PATCH +#define TBB_VERSION TBB_VERSION_STRING VS_VERSION_INFO VERSIONINFO - FILEVERSION TBBMALLOC_VERNUMBERS + FILEVERSION TBB_VERNUMBERS PRODUCTVERSION TBB_VERNUMBERS FILEFLAGSMASK 0x17L #ifdef _DEBUG @@ -82,16 +52,16 @@ BEGIN BLOCK "000004b0" BEGIN VALUE "CompanyName", "Intel Corporation\0" - VALUE "FileDescription", "Scalable Allocator library\0" - VALUE "FileVersion", TBBMALLOC_VERSION "\0" - VALUE "LegalCopyright", "Copyright 2005-2017 Intel Corporation. All Rights Reserved.\0" + VALUE "FileDescription", "oneAPI Threading Building Blocks (oneTBB) library\0" + VALUE "FileVersion", TBB_VERSION "\0" + VALUE "LegalCopyright", "Copyright 2005-2024 Intel Corporation. All Rights Reserved.\0" VALUE "LegalTrademarks", "\0" #ifndef TBB_USE_DEBUG VALUE "OriginalFilename", "tbbmalloc.dll\0" #else VALUE "OriginalFilename", "tbbmalloc_debug.dll\0" #endif - VALUE "ProductName", "Intel(R) Threading Building Blocks for Windows\0" + VALUE "ProductName", "oneAPI Threading Building Blocks (oneTBB)\0" VALUE "ProductVersion", TBB_VERSION "\0" VALUE "PrivateBuild", "\0" VALUE "SpecialBuild", "\0" @@ -102,18 +72,3 @@ BEGIN VALUE "Translation", 0x0, 1200 END END - -#endif // Neutral resources -///////////////////////////////////////////////////////////////////////////// - - -#ifndef APSTUDIO_INVOKED -///////////////////////////////////////////////////////////////////////////// -// -// Generated from the TEXTINCLUDE 3 resource. -// - - -///////////////////////////////////////////////////////////////////////////// -#endif // not APSTUDIO_INVOKED - diff --git a/src/tbb/src/tbbmalloc/tbbmalloc_internal.h b/src/tbb/src/tbbmalloc/tbbmalloc_internal.h index e4082bc7d..a8b538c7d 100644 --- a/src/tbb/src/tbbmalloc/tbbmalloc_internal.h +++ b/src/tbb/src/tbbmalloc/tbbmalloc_internal.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2024 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,15 +12,10 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_tbbmalloc_internal_H -#define __TBB_tbbmalloc_internal_H 1 - +#define __TBB_tbbmalloc_internal_H #include "TypeDefinitions.h" /* Also includes customization layer Customize.h */ @@ -29,17 +24,20 @@ #include <pthread.h> typedef pthread_key_t tls_key_t; #elif USE_WINTHREAD - #include "tbb/machine/windows_api.h" + #include <windows.h> typedef DWORD tls_key_t; #else #error Must define USE_PTHREAD or USE_WINTHREAD #endif +#include <atomic> + // TODO: *BSD also has it #define BACKEND_HAS_MREMAP __linux__ #define CHECK_ALLOCATION_RANGE MALLOC_DEBUG || MALLOC_ZONE_OVERLOAD_ENABLED || MALLOC_UNIXLIKE_OVERLOAD_ENABLED -#include "tbb/tbb_config.h" // for __TBB_LIBSTDCPP_EXCEPTION_HEADERS_BROKEN +#include "oneapi/tbb/detail/_config.h" // for __TBB_LIBSTDCPP_EXCEPTION_HEADERS_BROKEN +#include "oneapi/tbb/detail/_template_helpers.h" #if __TBB_LIBSTDCPP_EXCEPTION_HEADERS_BROKEN #define _EXCEPTION_PTR_H /* prevents exception_ptr.h inclusion */ #define _GLIBCXX_NESTED_EXCEPTION_H /* prevents nested_exception.h inclusion */ @@ -52,7 +50,7 @@ #if MALLOC_CHECK_RECURSION #include <new> /* for placement new */ #endif -#include "tbb/scalable_allocator.h" +#include "oneapi/tbb/scalable_allocator.h" #include "tbbmalloc_internal_api.h" /********* Various compile-time options **************/ @@ -69,7 +67,7 @@ #define TRACEF(x) ((void)0) #endif /* MALLOC_TRACE */ -#define ASSERT_TEXT NULL +#define ASSERT_TEXT nullptr #define COLLECT_STATISTICS ( MALLOC_DEBUG && MALLOCENV_COLLECT_STATISTICS ) #ifndef USE_INTERNAL_TID @@ -99,10 +97,23 @@ extern intptr_t memAllocKB, memHitKB; template<typename T> void suppress_unused_warning( const T& ) {} +/********** Various global default constants ********/ + +/* + * Default huge page size + */ +#if defined __loongarch64 +static const size_t HUGE_PAGE_SIZE = 32 * 1024 * 1024; +#else +static const size_t HUGE_PAGE_SIZE = 2 * 1024 * 1024; +#endif + +/********** End of global default constants *********/ + /********** Various numeric parameters controlling allocations ********/ /* - * smabSize - the size of a block for allocation of small objects, + * slabSize - the size of a block for allocation of small objects, * it must be larger than maxSegregatedObjectSize. */ const uintptr_t slabSize = 16*1024; @@ -152,36 +163,39 @@ class TLSKey { }; template<typename Arg, typename Compare> -inline void AtomicUpdate(Arg &location, Arg newVal, const Compare &cmp) +inline void AtomicUpdate(std::atomic<Arg>& location, Arg newVal, const Compare &cmp) { - MALLOC_STATIC_ASSERT(sizeof(Arg) == sizeof(intptr_t), - "Type of argument must match AtomicCompareExchange type."); - for (Arg old = location; cmp(old, newVal); ) { - Arg val = AtomicCompareExchange((intptr_t&)location, (intptr_t)newVal, old); - if (val == old) + static_assert(sizeof(Arg) == sizeof(intptr_t), "Type of argument must match AtomicCompareExchange type."); + Arg old = location.load(std::memory_order_acquire); + for (; cmp(old, newVal); ) { + if (location.compare_exchange_strong(old, newVal)) break; // TODO: do we need backoff after unsuccessful CAS? - old = val; + //old = val; } } // TODO: make BitMaskBasic more general +// TODO: check that BitMaskBasic is not used for synchronization // (currently, it fits BitMaskMin well, but not as suitable for BitMaskMax) template<unsigned NUM> class BitMaskBasic { static const unsigned SZ = (NUM-1)/(CHAR_BIT*sizeof(uintptr_t))+1; static const unsigned WORD_LEN = CHAR_BIT*sizeof(uintptr_t); - uintptr_t mask[SZ]; + + std::atomic<uintptr_t> mask[SZ]; + protected: void set(size_t idx, bool val) { MALLOC_ASSERT(idx<NUM, ASSERT_TEXT); size_t i = idx / WORD_LEN; int pos = WORD_LEN - idx % WORD_LEN - 1; - if (val) - AtomicOr(&mask[i], 1ULL << pos); - else - AtomicAnd(&mask[i], ~(1ULL << pos)); + if (val) { + mask[i].fetch_or(1ULL << pos); + } else { + mask[i].fetch_and(~(1ULL << pos)); + } } int getMinTrue(unsigned startIdx) const { unsigned idx = startIdx / WORD_LEN; @@ -190,19 +204,19 @@ class BitMaskBasic { if (startIdx % WORD_LEN) { // only interested in part of a word, clear bits before startIdx pos = WORD_LEN - startIdx % WORD_LEN; - uintptr_t actualMask = mask[idx] & (((uintptr_t)1<<pos) - 1); + uintptr_t actualMask = mask[idx].load(std::memory_order_relaxed) & (((uintptr_t)1<<pos) - 1); idx++; if (-1 != (pos = BitScanRev(actualMask))) return idx*WORD_LEN - pos - 1; } while (idx<SZ) - if (-1 != (pos = BitScanRev(mask[idx++]))) + if (-1 != (pos = BitScanRev(mask[idx++].load(std::memory_order_relaxed)))) return idx*WORD_LEN - pos - 1; return -1; } public: - void reset() { for (unsigned i=0; i<SZ; i++) mask[i] = 0; } + void reset() { for (unsigned i=0; i<SZ; i++) mask[i].store(0, std::memory_order_relaxed); } }; template<unsigned NUM> @@ -218,9 +232,13 @@ template<unsigned NUM> class BitMaskMax : public BitMaskBasic<NUM> { public: void set(size_t idx, bool val) { + MALLOC_ASSERT(NUM >= idx + 1, ASSERT_TEXT); + BitMaskBasic<NUM>::set(NUM - 1 - idx, val); } int getMaxTrue(unsigned startIdx) const { + MALLOC_ASSERT(NUM >= startIdx + 1, ASSERT_TEXT); + int p = BitMaskBasic<NUM>::getMinTrue(NUM-startIdx-1); return -1==p? -1 : (int)NUM - 1 - p; } @@ -241,9 +259,9 @@ class AllLocalCaches { public: void registerThread(TLSRemote *tls); void unregisterThread(TLSRemote *tls); - bool cleanup(ExtMemoryPool *extPool, bool cleanOnlyUnused); + bool cleanup(bool cleanOnlyUnused); void markUnused(); - void reset() { head = NULL; } + void reset() { head = nullptr; } }; class LifoList { @@ -254,7 +272,7 @@ class LifoList { inline Block *grab(); private: - Block *top; + std::atomic<Block*> top; MallocMutex lock; }; @@ -275,248 +293,53 @@ class OrphanedBlocks { bool cleanup(Backend* backend); }; -/* cache blocks in range [MinSize; MaxSize) in bins with CacheStep - TooLargeFactor -- when cache size treated "too large" in comparison to user data size - OnMissFactor -- If cache miss occurred and cache was cleaned, - set ageThreshold to OnMissFactor * the difference - between current time and last time cache was cleaned. - LongWaitFactor -- to detect rarely-used bins and forget about their usage history -*/ -template<size_t MIN_SIZE, size_t MAX_SIZE, uint32_t CACHE_STEP, int TOO_LARGE, - int ON_MISS, int LONG_WAIT> -struct LargeObjectCacheProps { - static const size_t MinSize = MIN_SIZE, MaxSize = MAX_SIZE; - static const uint32_t CacheStep = CACHE_STEP; - static const int TooLargeFactor = TOO_LARGE, OnMissFactor = ON_MISS, - LongWaitFactor = LONG_WAIT; -}; +/* Large objects entities */ +#include "large_objects.h" -template<typename Props> -class LargeObjectCacheImpl { -private: - // The number of bins to cache large objects. - static const uint32_t numBins = (Props::MaxSize-Props::MinSize)/Props::CacheStep; - // Current sizes of used and cached objects. It's calculated while we are - // traversing bins, and used for isLOCTooLarge() check at the same time. - class BinsSummary { - size_t usedSz; - size_t cachedSz; - public: - BinsSummary() : usedSz(0), cachedSz(0) {} - // "too large" criteria - bool isLOCTooLarge() const { return cachedSz > Props::TooLargeFactor*usedSz; } - void update(size_t usedSize, size_t cachedSize) { - usedSz += usedSize; - cachedSz += cachedSize; - } - void reset() { usedSz = cachedSz = 0; } - }; -public: - typedef BitMaskMax<numBins> BinBitMask; - - // 2-linked list of same-size cached blocks ordered by age (oldest on top) - // TODO: are we really want the list to be 2-linked? This allows us - // reduce memory consumption and do less operations under lock. - // TODO: try to switch to 32-bit logical time to save space in CacheBin - // and move bins to different cache lines. - class CacheBin { - private: - LargeMemoryBlock *first, - *last; - /* age of an oldest block in the list; equal to last->age, if last defined, - used for quick cheching it without acquiring the lock. */ - uintptr_t oldest; - /* currAge when something was excluded out of list because of the age, - not because of cache hit */ - uintptr_t lastCleanedAge; - /* Current threshold value for the blocks of a particular size. - Set on cache miss. */ - intptr_t ageThreshold; - - /* total size of all objects corresponding to the bin and allocated by user */ - size_t usedSize, - /* total size of all objects cached in the bin */ - cachedSize; - /* mean time of presence of block in the bin before successful reuse */ - intptr_t meanHitRange; - /* time of last get called for the bin */ - uintptr_t lastGet; - - typename MallocAggregator<CacheBinOperation>::type aggregator; - - void ExecuteOperation(CacheBinOperation *op, ExtMemoryPool *extMemPool, BinBitMask *bitMask, int idx, bool longLifeTime = true); - /* should be placed in zero-initialized memory, ctor not needed. */ - CacheBin(); - public: - void init() { memset(this, 0, sizeof(CacheBin)); } - void putList(ExtMemoryPool *extMemPool, LargeMemoryBlock *head, BinBitMask *bitMask, int idx); - LargeMemoryBlock *get(ExtMemoryPool *extMemPool, size_t size, BinBitMask *bitMask, int idx); - bool cleanToThreshold(ExtMemoryPool *extMemPool, BinBitMask *bitMask, uintptr_t currTime, int idx); - bool releaseAllToBackend(ExtMemoryPool *extMemPool, BinBitMask *bitMask, int idx); - void updateUsedSize(ExtMemoryPool *extMemPool, size_t size, BinBitMask *bitMask, int idx); - - void decreaseThreshold() { - if (ageThreshold) - ageThreshold = (ageThreshold + meanHitRange)/2; - } - void updateBinsSummary(BinsSummary *binsSummary) const { - binsSummary->update(usedSize, cachedSize); - } - size_t getSize() const { return cachedSize; } - size_t getUsedSize() const { return usedSize; } - size_t reportStat(int num, FILE *f); - /* ---------- unsafe methods used with the aggregator ---------- */ - void forgetOutdatedState(uintptr_t currTime); - LargeMemoryBlock *putList(LargeMemoryBlock *head, LargeMemoryBlock *tail, BinBitMask *bitMask, int idx, int num); - LargeMemoryBlock *get(); - LargeMemoryBlock *cleanToThreshold(uintptr_t currTime, BinBitMask *bitMask, int idx); - LargeMemoryBlock *cleanAll(BinBitMask *bitMask, int idx); - void updateUsedSize(size_t size, BinBitMask *bitMask, int idx) { - if (!usedSize) bitMask->set(idx, true); - usedSize += size; - if (!usedSize && !first) bitMask->set(idx, false); - } - void updateMeanHitRange( intptr_t hitRange ) { - hitRange = hitRange >= 0 ? hitRange : 0; - meanHitRange = meanHitRange ? (meanHitRange + hitRange)/2 : hitRange; - } - void updateAgeThreshold( uintptr_t currTime ) { - if (lastCleanedAge) - ageThreshold = Props::OnMissFactor*(currTime - lastCleanedAge); - } - void updateCachedSize(size_t size) { cachedSize += size; } - void setLastGet( uintptr_t newLastGet ) { lastGet = newLastGet; } - /* -------------------------------------------------------- */ - }; -private: - intptr_t tooLargeLOC; // how many times LOC was "too large" - // for fast finding of used bins and bins with non-zero usedSize; - // indexed from the end, as we need largest 1st - BinBitMask bitMask; - // bins with lists of recently freed large blocks cached for re-use - CacheBin bin[numBins]; - -public: - static int sizeToIdx(size_t size) { - MALLOC_ASSERT(Props::MinSize <= size && size < Props::MaxSize, ASSERT_TEXT); - return (size-Props::MinSize)/Props::CacheStep; - } - static int getNumBins() { return numBins; } - - void putList(ExtMemoryPool *extMemPool, LargeMemoryBlock *largeBlock); - LargeMemoryBlock *get(ExtMemoryPool *extMemPool, size_t size); - - void updateCacheState(ExtMemoryPool *extMemPool, DecreaseOrIncrease op, size_t size); - bool regularCleanup(ExtMemoryPool *extMemPool, uintptr_t currAge, bool doThreshDecr); - bool cleanAll(ExtMemoryPool *extMemPool); - void reset() { - tooLargeLOC = 0; - for (int i = numBins-1; i >= 0; i--) - bin[i].init(); - bitMask.reset(); - } - void reportStat(FILE *f); -#if __TBB_MALLOC_WHITEBOX_TEST - size_t getLOCSize() const; - size_t getUsedSize() const; -#endif -}; - -class LargeObjectCache { - static const size_t minLargeSize = 8*1024, - maxLargeSize = 8*1024*1024, - // There are benchmarks of interest that should work well with objects of this size - maxHugeSize = 129*1024*1024; -public: - // Difference between object sizes in large block bins - static const uint32_t largeBlockCacheStep = 8*1024, - hugeBlockCacheStep = 512*1024; -private: - typedef LargeObjectCacheProps<minLargeSize, maxLargeSize, largeBlockCacheStep, 2, 2, 16> LargeCacheTypeProps; - typedef LargeObjectCacheProps<maxLargeSize, maxHugeSize, hugeBlockCacheStep, 1, 1, 4> HugeCacheTypeProps; - typedef LargeObjectCacheImpl< LargeCacheTypeProps > LargeCacheType; - typedef LargeObjectCacheImpl< HugeCacheTypeProps > HugeCacheType; - - // beginning of largeCache is more actively used and smaller than hugeCache, - // so put hugeCache first to prevent false sharing - // with LargeObjectCache's predecessor - HugeCacheType hugeCache; - LargeCacheType largeCache; - - /* logical time, incremented on each put/get operation - To prevent starvation between pools, keep separately for each pool. - Overflow is OK, as we only want difference between - its current value and some recent. - - Both malloc and free should increment logical time, as in - a different case multiple cached blocks would have same age, - and accuracy of predictors suffers. - */ - uintptr_t cacheCurrTime; - - // memory pool that owns this LargeObjectCache, - ExtMemoryPool *extMemPool; // strict 1:1 relation, never changed - - static int sizeToIdx(size_t size); -public: - void init(ExtMemoryPool *memPool) { extMemPool = memPool; } - void put(LargeMemoryBlock *largeBlock); - void putList(LargeMemoryBlock *head); - LargeMemoryBlock *get(size_t size); - - void updateCacheState(DecreaseOrIncrease op, size_t size); - bool isCleanupNeededOnRange(uintptr_t range, uintptr_t currTime); - bool doCleanup(uintptr_t currTime, bool doThreshDecr); - - bool decreasingCleanup(); - bool regularCleanup(); - bool cleanAll(); - void reset() { - largeCache.reset(); - hugeCache.reset(); - } - void reportStat(FILE *f); -#if __TBB_MALLOC_WHITEBOX_TEST - size_t getLOCSize() const; - size_t getUsedSize() const; -#endif - static size_t alignToBin(size_t size) { - return size<maxLargeSize? alignUp(size, largeBlockCacheStep) - : alignUp(size, hugeBlockCacheStep); - } - - uintptr_t getCurrTime() { return (uintptr_t)AtomicIncrement((intptr_t&)cacheCurrTime); } - uintptr_t getCurrTimeRange(uintptr_t range) { return (uintptr_t)AtomicAdd((intptr_t&)cacheCurrTime, range)+1; } - void registerRealloc(size_t oldSize, size_t newSize); -}; - -// select index size for BackRefMaster based on word size: default is uint32_t, +// select index size for BackRefMain based on word size: default is uint32_t, // uint16_t for 32-bit platforms template<bool> -struct MasterIndexSelect { - typedef uint32_t master_type; +struct MainIndexSelect { + typedef uint32_t main_type; }; template<> -struct MasterIndexSelect<false> { - typedef uint16_t master_type; +struct MainIndexSelect<false> { + typedef uint16_t main_type; }; class BackRefIdx { // composite index to backreference array public: - typedef MasterIndexSelect<4 < sizeof(uintptr_t)>::master_type master_t; + typedef MainIndexSelect<4 < sizeof(uintptr_t)>::main_type main_t; private: - static const master_t invalid = ~master_t(0); - master_t master; // index in BackRefMaster + static const main_t invalid = ~main_t(0); + main_t main; // index in BackRefMain uint16_t largeObj:1; // is this object "large"? uint16_t offset :15; // offset from beginning of BackRefBlock public: - BackRefIdx() : master(invalid) {} - bool isInvalid() const { return master == invalid; } + BackRefIdx() : main(invalid), largeObj(0), offset(0) {} + bool isInvalid() const { return main == invalid; } bool isLargeObject() const { return largeObj; } - master_t getMaster() const { return master; } + main_t getMain() const { return main; } uint16_t getOffset() const { return offset; } +#if __TBB_USE_THREAD_SANITIZER + friend + __attribute__((no_sanitize("thread"))) + BackRefIdx dereference(const BackRefIdx* ptr) { + BackRefIdx idx; + idx.main = ptr->main; + idx.largeObj = ptr->largeObj; + idx.offset = ptr->offset; + return idx; + } +#else + friend + BackRefIdx dereference(const BackRefIdx* ptr) { + return *ptr; + } +#endif + // only newBackRef can modify BackRefIdx static BackRefIdx newBackRef(bool largeObj); }; @@ -524,7 +347,14 @@ class BackRefIdx { // composite index to backreference array // Block header is used during block coalescing // and must be preserved in used blocks. class BlockI { +#if __clang__ && !__INTEL_COMPILER + // #pragma clang diagnostic push + // #pragma clang diagnostic ignored "-Wunused-private-field" +#endif intptr_t blockState[2]; +#if __clang__ && !__INTEL_COMPILER + // #pragma clang diagnostic pop // "-Wunused-private-field" +#endif }; struct LargeMemoryBlock : public BlockI { @@ -542,337 +372,191 @@ struct LargeMemoryBlock : public BlockI { BackRefIdx backRefIdx; // cached here, used copy is in LargeObjectHdr }; -// global state of blocks currently in processing -class BackendSync { - // Class instances should reside in zero-initialized memory! - // The number of blocks currently removed from a bin and not returned back - intptr_t inFlyBlocks; // to another - intptr_t binsModifications; // incremented on every bin modification - Backend *backend; +// Classes and methods for backend.cpp +#include "backend.h" + +// An TBB allocator mode that can be controlled by user +// via API/environment variable. Must be placed in zero-initialized memory. +// External synchronization assumed. +// TODO: TBB_VERSION support +class AllocControlledMode { + intptr_t val; + bool setDone; + public: - void init(Backend *b) { backend = b; } - void blockConsumed() { AtomicIncrement(inFlyBlocks); } - void binsModified() { AtomicIncrement(binsModifications); } - void blockReleased() { -#if __TBB_MALLOC_BACKEND_STAT - MALLOC_ITT_SYNC_RELEASING(&inFlyBlocks); -#endif - AtomicIncrement(binsModifications); - intptr_t prev = AtomicAdd(inFlyBlocks, -1); - MALLOC_ASSERT(prev > 0, ASSERT_TEXT); - suppress_unused_warning(prev); + intptr_t get() const { + MALLOC_ASSERT(setDone, ASSERT_TEXT); + return val; } - intptr_t getNumOfMods() const { return FencedLoad(binsModifications); } - // return true if need re-do the blocks search - inline bool waitTillBlockReleased(intptr_t startModifiedCnt); -}; -class CoalRequestQ { // queue of free blocks that coalescing was delayed -private: - FreeBlock *blocksToFree; - BackendSync *bkndSync; - // counted blocks in blocksToFree and that are leaved blocksToFree - // and still in active coalescing - intptr_t inFlyBlocks; -public: - void init(BackendSync *bSync) { bkndSync = bSync; } - FreeBlock *getAll(); // return current list of blocks and make queue empty - void putBlock(FreeBlock *fBlock); - inline void blockWasProcessed(); - intptr_t blocksInFly() const { return FencedLoad(inFlyBlocks); } -}; + // Note: set() can be called before init() + void set(intptr_t newVal) { + val = newVal; + setDone = true; + } -class MemExtendingSema { - intptr_t active; -public: - bool wait() { - bool rescanBins = false; - // up to 3 threads can add more memory from OS simultaneously, - // rest of threads have to wait - for (;;) { - intptr_t prevCnt = FencedLoad(active); - if (prevCnt < 3) { - intptr_t n = AtomicCompareExchange(active, prevCnt+1, prevCnt); - if (n == prevCnt) - break; - } else { - SpinWaitWhileEq(active, prevCnt); - rescanBins = true; - break; - } - } - return rescanBins; + bool ready() const { + return setDone; } - void signal() { AtomicAdd(active, -1); } -}; -enum MemRegionType { - // The region does not guarantee the block size. - MEMREG_FLEXIBLE_SIZE = 0, - // The region can hold exact number of blocks with the size of the - // first reqested block. - MEMREG_SEVERAL_BLOCKS, - // The region holds only one block with a reqested size. - MEMREG_ONE_BLOCK + // envName - environment variable to get controlled mode + void initReadEnv(const char *envName, intptr_t defaultVal) { + if (!setDone) { + // unreferenced formal parameter warning + tbb::detail::suppress_unused_warning(envName); +#if !__TBB_WIN8UI_SUPPORT + // TODO: use strtol to get the actual value of the envirable + const char *envVal = getenv(envName); + if (envVal && !strcmp(envVal, "1")) + val = 1; + else +#endif + val = defaultVal; + setDone = true; + } + } }; -class MemRegionList { - MallocMutex regionListLock; -public: - MemRegion *head; - void add(MemRegion *r); - void remove(MemRegion *r); - int reportStat(FILE *f); +// Page type to be used inside MapMemory. +// Regular (4KB aligned), Huge and Transparent Huge Pages (2MB aligned). +enum PageType { + REGULAR = 0, + PREALLOCATED_HUGE_PAGE, + TRANSPARENT_HUGE_PAGE }; -class Backend { +// init() and printStatus() is called only under global initialization lock. +// Race is possible between registerAllocation() and registerReleasing(), +// harm is that up to single huge page releasing is missed (because failure +// to get huge page is registered only 1st time), that is negligible. +// setMode is also can be called concurrently. +// Object must reside in zero-initialized memory +// TODO: can we check for huge page presence during every 10th mmap() call +// in case huge page is released by another process? +class HugePagesStatus { private: -/* Blocks in range [minBinnedSize; getMaxBinnedSize()] are kept in bins, - one region can contains several blocks. Larger blocks are allocated directly - and one region always contains one block. -*/ - enum { - minBinnedSize = 8*1024UL, - /* If huge pages are available, maxBinned_HugePage used. - If not, maxBinned_SmallPage is the threshold. - TODO: use pool's granularity for upper bound setting.*/ - maxBinned_SmallPage = 1024*1024UL, - // TODO: support other page sizes - maxBinned_HugePage = 4*1024*1024UL - }; - enum { - VALID_BLOCK_IN_BIN = 1 // valid block added to bin, not returned as result - }; -public: - static const int freeBinsNum = - (maxBinned_HugePage-minBinnedSize)/LargeObjectCache::largeBlockCacheStep + 1; - - // if previous access missed per-thread slabs pool, - // allocate numOfSlabAllocOnMiss blocks in advance - static const int numOfSlabAllocOnMiss = 2; - - enum { - NO_BIN = -1, - // special bin for blocks >= maxBinned_HugePage, blocks go to this bin - // when pool is created with keepAllMemory policy - // TODO: currently this bin is scanned using "1st fit", as it accumulates - // blocks of different sizes, "best fit" is preferred in terms of fragmentation - HUGE_BIN = freeBinsNum-1 - }; - - // Bin keeps 2-linked list of free blocks. It must be 2-linked - // because during coalescing a block it's removed from a middle of the list. - struct Bin { - FreeBlock *head, - *tail; - MallocMutex tLock; - - void removeBlock(FreeBlock *fBlock); - void reset() { head = tail = 0; } - bool empty() const { return !head; } - - size_t countFreeBlocks(); - size_t reportFreeBlocks(FILE *f); - void reportStat(FILE *f); - }; - - typedef BitMaskMin<Backend::freeBinsNum> BitMaskBins; - - // array of bins supplemented with bitmask for fast finding of non-empty bins - class IndexedBins { - BitMaskBins bitMask; - Bin freeBins[Backend::freeBinsNum]; - FreeBlock *getFromBin(int binIdx, BackendSync *sync, size_t size, - bool resSlabAligned, bool alignedBin, bool wait, - int *resLocked); - public: - FreeBlock *findBlock(int nativeBin, BackendSync *sync, size_t size, - bool resSlabAligned, bool alignedBin, int *numOfLockedBins); - bool tryReleaseRegions(int binIdx, Backend *backend); - void lockRemoveBlock(int binIdx, FreeBlock *fBlock); - void addBlock(int binIdx, FreeBlock *fBlock, size_t blockSz, bool addToTail); - bool tryAddBlock(int binIdx, FreeBlock *fBlock, bool addToTail); - int getMinNonemptyBin(unsigned startBin) const { - int p = bitMask.getMinTrue(startBin); - return p == -1 ? Backend::freeBinsNum : p; + AllocControlledMode requestedMode; // changed only by user + // to keep enabled and requestedMode consistent + MallocMutex setModeLock; + size_t pageSize; + std::atomic<intptr_t> needActualStatusPrint; + + static void doPrintStatus(bool state, const char *stateName) { + // Under macOS* fprintf/snprintf acquires an internal lock, so when + // 1st allocation is done under the lock, we got a deadlock. + // Do not use fprintf etc during initialization. + fputs("TBBmalloc: huge pages\t", stderr); + if (!state) + fputs("not ", stderr); + fputs(stateName, stderr); + fputs("\n", stderr); + } + + void parseSystemMemInfo() { + bool hpAvailable = false; + bool thpAvailable = false; + long long hugePageSize = -1; + +#if __unix__ + // Check huge pages existence + long long meminfoHugePagesTotal = 0; + + parseFileItem meminfoItems[] = { + // Parse system huge page size + { "Hugepagesize: %lld kB", hugePageSize }, + // Check if there are preallocated huge pages on the system + // https://www.kernel.org/doc/Documentation/vm/hugetlbpage.txt + { "HugePages_Total: %lld", meminfoHugePagesTotal } }; + + parseFile</*BUFF_SIZE=*/100>("/proc/meminfo", meminfoItems); + + // Double check another system information regarding preallocated + // huge pages if there are no information in /proc/meminfo + long long vmHugePagesTotal = 0; + + parseFileItem vmItem[] = { { "%lld", vmHugePagesTotal } }; + + // We parse a counter number, it can't be huge + parseFile</*BUFF_SIZE=*/100>("/proc/sys/vm/nr_hugepages", vmItem); + + if (hugePageSize > -1 && (meminfoHugePagesTotal > 0 || vmHugePagesTotal > 0)) { + MALLOC_ASSERT(hugePageSize != 0, "Huge Page size can't be zero if we found preallocated."); + + // Any non zero value clearly states that there are preallocated + // huge pages on the system + hpAvailable = true; } - void verify(); - void reset(); - void reportStat(FILE *f); - }; -private: - class AdvRegionsBins { - BitMaskBins bins; - public: - void registerBin(int regBin) { bins.set(regBin, 1); } - int getMinUsedBin(int start) const { return bins.getMinTrue(start); } - void reset() { bins.reset(); } - }; - // auxiliary class to atomic maximum request finding - class MaxRequestComparator { - const Backend *backend; - public: - MaxRequestComparator(const Backend *be) : backend(be) {} - inline bool operator()(size_t oldMaxReq, size_t requestSize) const; - }; - -#if CHECK_ALLOCATION_RANGE - // Keep min and max of all addresses requested from OS, - // use it for checking memory possibly allocated by replaced allocators - // and for debugging purposes. Valid only for default memory pool. - class UsedAddressRange { - static const uintptr_t ADDRESS_UPPER_BOUND = UINTPTR_MAX; - - uintptr_t leftBound, - rightBound; - MallocMutex mutex; - public: - // rightBound is zero-initialized - void init() { leftBound = ADDRESS_UPPER_BOUND; } - void registerAlloc(uintptr_t left, uintptr_t right); - void registerFree(uintptr_t left, uintptr_t right); - // as only left and right bounds are kept, we can return true - // for pointer not allocated by us, if more than single region - // was requested from OS - bool inRange(void *ptr) const { - const uintptr_t p = (uintptr_t)ptr; - return leftBound<=p && p<=rightBound; + // Check if there is transparent huge pages support on the system + long long thpPresent = 'n'; + parseFileItem thpItem[] = { { "[alwa%cs] madvise never\n", thpPresent } }; + parseFile</*BUFF_SIZE=*/100>("/sys/kernel/mm/transparent_hugepage/enabled", thpItem); + + if (hugePageSize > -1 && thpPresent == 'y') { + MALLOC_ASSERT(hugePageSize != 0, "Huge Page size can't be zero if we found thp existence."); + thpAvailable = true; } - }; -#else - class UsedAddressRange { - public: - void init() { } - void registerAlloc(uintptr_t, uintptr_t) {} - void registerFree(uintptr_t, uintptr_t) {} - bool inRange(void *) const { return true; } - }; #endif + MALLOC_ASSERT(!pageSize, "Huge page size can't be set twice. Double initialization."); - ExtMemoryPool *extMemPool; - // used for release every region on pool destroying - MemRegionList regionList; - - CoalRequestQ coalescQ; // queue of coalescing requests - BackendSync bkndSync; - // semaphore protecting adding more more memory from OS - MemExtendingSema memExtendingSema; - size_t totalMemSize, - memSoftLimit; - UsedAddressRange usedAddrRange; - // to keep 1st allocation large than requested, keep bootstrapping status - enum { - bootsrapMemNotDone = 0, - bootsrapMemInitializing, - bootsrapMemDone - }; - intptr_t bootsrapMemStatus; - MallocMutex bootsrapMemStatusMutex; - - // Using of maximal observed requested size allows decrease - // memory consumption for small requests and decrease fragmentation - // for workloads when small and large allocation requests are mixed. - // TODO: decrease, not only increase it - size_t maxRequestedSize; - - FreeBlock *addNewRegion(size_t size, MemRegionType type, bool addToBin); - FreeBlock *findBlockInRegion(MemRegion *region, size_t exactBlockSize); - void startUseBlock(MemRegion *region, FreeBlock *fBlock, bool addToBin); - void releaseRegion(MemRegion *region); - - FreeBlock *releaseMemInCaches(intptr_t startModifiedCnt, - int *lockedBinsThreshold, int numOfLockedBins); - void requestBootstrapMem(); - FreeBlock *askMemFromOS(size_t totalReqSize, intptr_t startModifiedCnt, - int *lockedBinsThreshold, int numOfLockedBins, - bool *splittable); - FreeBlock *genericGetBlock(int num, size_t size, bool resSlabAligned); - void genericPutBlock(FreeBlock *fBlock, size_t blockSz); - FreeBlock *splitUnalignedBlock(FreeBlock *fBlock, int num, size_t size, - bool needAlignedRes); - FreeBlock *splitAlignedBlock(FreeBlock *fBlock, int num, size_t size, - bool needAlignedRes); - - FreeBlock *doCoalesc(FreeBlock *fBlock, MemRegion **memRegion); - bool coalescAndPutList(FreeBlock *head, bool forceCoalescQDrop, bool reportBlocksProcessed); - void coalescAndPut(FreeBlock *fBlock, size_t blockSz); - - void removeBlockFromBin(FreeBlock *fBlock); - - void *allocRawMem(size_t &size); - bool freeRawMem(void *object, size_t size); - - void putLargeBlock(LargeMemoryBlock *lmb); - void releaseCachesToLimit(); -public: - bool scanCoalescQ(bool forceCoalescQDrop); - intptr_t blocksInCoalescing() const { return coalescQ.blocksInFly(); } - void verify(); - void init(ExtMemoryPool *extMemoryPool); - void reset(); - bool destroy(); - bool clean(); // clean on caches cleanup - void reportStat(FILE *f); - - BlockI *getSlabBlock(int num) { - BlockI *b = (BlockI*) - genericGetBlock(num, slabSize, /*resSlabAligned=*/true); - MALLOC_ASSERT(isAligned(b, slabSize), ASSERT_TEXT); - return b; - } - void putSlabBlock(BlockI *block) { - genericPutBlock((FreeBlock *)block, slabSize); + // Initialize object variables + if (hugePageSize > -1) { + pageSize = hugePageSize * 1024; // was read in KB from meminfo + } else { + pageSize = 0; + } + isHPAvailable = hpAvailable; + isTHPAvailable = thpAvailable; } - void *getBackRefSpace(size_t size, bool *rawMemUsed); - void putBackRefSpace(void *b, size_t size, bool rawMemUsed); - bool inUserPool() const; +public: - LargeMemoryBlock *getLargeBlock(size_t size); - void returnLargeObject(LargeMemoryBlock *lmb); + // System information + bool isHPAvailable; + bool isTHPAvailable; - void *remap(void *ptr, size_t oldSize, size_t newSize, size_t alignment); + // User defined value + bool isEnabled; - void setRecommendedMaxSize(size_t softLimit) { - memSoftLimit = softLimit; - releaseCachesToLimit(); + void init() { + parseSystemMemInfo(); + MallocMutex::scoped_lock lock(setModeLock); + requestedMode.initReadEnv("TBB_MALLOC_USE_HUGE_PAGES", 0); + isEnabled = (isHPAvailable || isTHPAvailable) && requestedMode.get(); } - inline size_t getMaxBinnedSize() const; - - bool ptrCanBeValid(void *ptr) const { return usedAddrRange.inRange(ptr); } - -#if __TBB_MALLOC_WHITEBOX_TEST - size_t getTotalMemSize() const { return totalMemSize; } -#endif -private: - static int sizeToBin(size_t size) { - if (size >= maxBinned_HugePage) - return HUGE_BIN; - else if (size < minBinnedSize) - return NO_BIN; - - int bin = (size - minBinnedSize)/LargeObjectCache::largeBlockCacheStep; - MALLOC_ASSERT(bin < HUGE_BIN, "Invalid size."); - return bin; + // Could be set from user code at any place. + // If we didn't call init() at this place, isEnabled will be false + void setMode(intptr_t newVal) { + MallocMutex::scoped_lock lock(setModeLock); + requestedMode.set(newVal); + isEnabled = (isHPAvailable || isTHPAvailable) && newVal; } -#if __TBB_MALLOC_BACKEND_STAT - static size_t binToSize(int bin) { - MALLOC_ASSERT(bin <= HUGE_BIN, "Invalid bin."); - return bin*LargeObjectCache::largeBlockCacheStep + minBinnedSize; + void reset() { + needActualStatusPrint.store(0, std::memory_order_relaxed); + pageSize = 0; + isEnabled = isHPAvailable = isTHPAvailable = false; } -#endif - static bool toAlignedBin(FreeBlock *block, size_t size) { - return isAligned((char*)block+size, slabSize) - && size >= slabSize; + + // If memory mapping size is a multiple of huge page size, some OS kernels + // can use huge pages transparently. Use this when huge pages are requested. + size_t getGranularity() const { + if (requestedMode.ready()) + return requestedMode.get() ? pageSize : 0; + else + return HUGE_PAGE_SIZE; // the mode is not yet known; assume typical 2MB huge pages } - // register bins related to advance regions - AdvRegionsBins advRegBins; - IndexedBins freeLargeBins, - freeAlignedBins; + void printStatus() { + doPrintStatus(requestedMode.get(), "requested"); + if (requestedMode.get()) { // report actual status iff requested + if (pageSize) + needActualStatusPrint.store(1, std::memory_order_release); + else + doPrintStatus(/*state=*/false, "available"); + } + } }; class AllLargeBlocksList { @@ -904,6 +588,9 @@ struct ExtMemoryPool { fixedPool; TLSKey tlsPointerKey; // per-pool TLS key + std::atomic<int> softCachesCleanupInProgress; + std::atomic<int> hardCachesCleanupInProgress; + bool init(intptr_t poolId, rawAllocType rawAlloc, rawFreeType rawFree, size_t granularity, bool keepAllMemory, bool fixedPool); bool initTLS(); @@ -914,7 +601,7 @@ struct ExtMemoryPool { // true if something has been released bool softCachesCleanup(); bool releaseAllLocalCaches(); - bool hardCachesCleanup(); + bool hardCachesCleanup(bool wait); void *remap(void *ptr, size_t oldSize, size_t newSize, size_t alignment); bool reset() { loc.reset(); @@ -946,8 +633,10 @@ struct ExtMemoryPool { LargeMemoryBlock *mallocLargeObject(MemoryPool *pool, size_t allocationSize); void freeLargeObject(LargeMemoryBlock *lmb); void freeLargeObjectList(LargeMemoryBlock *head); +#if MALLOC_DEBUG // use granulatity as marker for pool validity bool isPoolValid() const { return granularity; } +#endif }; inline bool Backend::inUserPool() const { return extMemPool->userPool(); } @@ -963,95 +652,15 @@ struct FreeObject { FreeObject *next; }; -// An TBB allocator mode that can be controlled by user -// via API/environment variable. Must be placed in zero-initialized memory. -// External synchronization assumed. -// TODO: TBB_VERSION support -class AllocControlledMode { - intptr_t val; - bool setDone; -public: - bool ready() const { return setDone; } - intptr_t get() const { - MALLOC_ASSERT(setDone, ASSERT_TEXT); - return val; - } - void set(intptr_t newVal) { // note set() can be called before init() - val = newVal; - setDone = true; - } - // envName - environment variable to get controlled mode - void initReadEnv(const char *envName, intptr_t defaultVal); -}; - -// init() and printStatus() is called only under global initialization lock. -// Race is possible between registerAllocation() and registerReleasing(), -// harm is that up to single huge page releasing is missed (because failure -// to get huge page is registered only 1st time), that is negligible. -// setMode is also can be called concurrently. -// Object must reside in zero-initialized memory -// TODO: can we check for huge page presence during every 10th mmap() call -// in case huge page is released by another process? -class HugePagesStatus { -private: - AllocControlledMode requestedMode; // changed only by user - // to keep enabled and requestedMode consistent - MallocMutex setModeLock; - size_t pageSize; - intptr_t needActualStatusPrint; - - static void doPrintStatus(bool state, const char *stateName); -public: - // both variables are changed only inside HugePagesStatus - intptr_t enabled; - // Have we got huge pages at all? It's used when large hugepage-aligned - // region is releasing, to find can it release some huge pages or not. - intptr_t wasObserved; - - // If memory mapping size is a multiple of huge page size, some OS kernels - // can use huge pages transparently (i.e. even if not explicitly enabled). - // Use this when huge pages are requested. - size_t recommendedGranularity() const { - if (requestedMode.ready()) - return requestedMode.get()? pageSize : 0; - else - return 2048*1024; // the mode is not yet known; assume typical 2MB huge pages - } - void printStatus(); - void registerAllocation(bool available); - void registerReleasing(void* addr, size_t size); - - void init(size_t hugePageSize) { - MALLOC_ASSERT(!hugePageSize || isPowerOfTwo(hugePageSize), - "Only memory pages of a power-of-two size are supported."); - MALLOC_ASSERT(!pageSize, "Huge page size can't be set twice."); - pageSize = hugePageSize; - - MallocMutex::scoped_lock lock(setModeLock); - requestedMode.initReadEnv("TBB_MALLOC_USE_HUGE_PAGES", 0); - enabled = pageSize && requestedMode.get(); - } - void setMode(intptr_t newVal) { - MallocMutex::scoped_lock lock(setModeLock); - requestedMode.set(newVal); - enabled = pageSize && newVal; - } - void reset() { - pageSize = 0; - needActualStatusPrint = enabled = wasObserved = 0; - } -}; - -extern HugePagesStatus hugePages; /******* A helper class to support overriding malloc with scalable_malloc *******/ #if MALLOC_CHECK_RECURSION class RecursiveMallocCallProtector { // pointer to an automatic data of holding thread - static void *autoObjPtr; + static std::atomic<void*> autoObjPtr; static MallocMutex rmc_mutex; - static pthread_t owner_thread; + static std::atomic<pthread_t> owner_thread; /* Under FreeBSD 8.0 1st call to any pthread function including pthread_self leads to pthread initialization, that causes malloc calls. As 1st usage of RecursiveMallocCallProtector can be before pthread initialized, pthread calls @@ -1073,32 +682,31 @@ class RecursiveMallocCallProtector { MallocMutex::scoped_lock* lock_acquired; char scoped_lock_space[sizeof(MallocMutex::scoped_lock)+1]; - - static uintptr_t absDiffPtr(void *x, void *y) { - uintptr_t xi = (uintptr_t)x, yi = (uintptr_t)y; - return xi > yi ? xi - yi : yi - xi; - } + public: - - RecursiveMallocCallProtector() : lock_acquired(NULL) { + RecursiveMallocCallProtector() : lock_acquired(nullptr) { lock_acquired = new (scoped_lock_space) MallocMutex::scoped_lock( rmc_mutex ); if (canUsePthread) - owner_thread = pthread_self(); - autoObjPtr = &scoped_lock_space; + owner_thread.store(pthread_self(), std::memory_order_relaxed); + autoObjPtr.store(&scoped_lock_space, std::memory_order_relaxed); } + + RecursiveMallocCallProtector(RecursiveMallocCallProtector&) = delete; + RecursiveMallocCallProtector& operator=(RecursiveMallocCallProtector) = delete; + ~RecursiveMallocCallProtector() { if (lock_acquired) { - autoObjPtr = NULL; + autoObjPtr.store(nullptr, std::memory_order_relaxed); lock_acquired->~scoped_lock(); } } static bool sameThreadActive() { - if (!autoObjPtr) // fast path + if (!autoObjPtr.load(std::memory_order_relaxed)) // fast path return false; // Some thread has an active recursive call protector; check if the current one. // Exact pthread_self based test if (canUsePthread) { - if (pthread_equal( owner_thread, pthread_self() )) { + if (pthread_equal( owner_thread.load(std::memory_order_relaxed), pthread_self() )) { mallocRecursionDetected = true; return true; } else @@ -1107,9 +715,13 @@ class RecursiveMallocCallProtector { // inexact stack size based test const uintptr_t threadStackSz = 2*1024*1024; int dummy; - return absDiffPtr(autoObjPtr, &dummy)<threadStackSz; + + uintptr_t xi = (uintptr_t)autoObjPtr.load(std::memory_order_relaxed), yi = (uintptr_t)&dummy; + uintptr_t diffPtr = xi > yi ? xi - yi : yi - xi; + + return diffPtr < threadStackSz; } - static bool noRecursion(); + /* The function is called on 1st scalable_malloc call to check if malloc calls scalable_malloc (nested call must set mallocRecursionDetected). */ static void detectNaiveOverload() { @@ -1119,7 +731,7 @@ class RecursiveMallocCallProtector { is already on, so can do it. */ if (!canUsePthread) { canUsePthread = true; - owner_thread = pthread_self(); + owner_thread.store(pthread_self(), std::memory_order_relaxed); } #endif free(malloc(1)); @@ -1137,12 +749,10 @@ class RecursiveMallocCallProtector { #endif /* MALLOC_CHECK_RECURSION */ -bool isMallocInitializedExt(); - unsigned int getThreadId(); -bool initBackRefMaster(Backend *backend); -void destroyBackRefMaster(Backend *backend); +bool initBackRefMain(Backend *backend); +void destroyBackRefMain(Backend *backend); void removeBackRef(BackRefIdx backRefIdx); void setBackRef(BackRefIdx backRefIdx, void *newPtr); void *getBackRef(BackRefIdx backRefIdx); diff --git a/src/tbb/src/tbbmalloc/tbbmalloc_internal_api.h b/src/tbb/src/tbbmalloc/tbbmalloc_internal_api.h index 301c9f59a..0b36c8556 100644 --- a/src/tbb/src/tbbmalloc/tbbmalloc_internal_api.h +++ b/src/tbb/src/tbbmalloc/tbbmalloc_internal_api.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_tbbmalloc_internal_api_H @@ -32,7 +28,7 @@ typedef enum { } AllocationModeInternalParam; void MallocInitializeITT(); -void __TBB_mallocProcessShutdownNotification(); +void __TBB_mallocProcessShutdownNotification(bool); #if _WIN32||_WIN64 void __TBB_mallocThreadShutdownNotification(); #endif diff --git a/src/tbb/src/tbbmalloc/win32-gcc-tbbmalloc-export.def b/src/tbb/src/tbbmalloc/win32-gcc-tbbmalloc-export.def deleted file mode 100644 index f2ca7f315..000000000 --- a/src/tbb/src/tbbmalloc/win32-gcc-tbbmalloc-export.def +++ /dev/null @@ -1,52 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -{ -global: -scalable_calloc; -scalable_free; -scalable_malloc; -scalable_realloc; -scalable_posix_memalign; -scalable_aligned_malloc; -scalable_aligned_realloc; -scalable_aligned_free; -scalable_msize; -scalable_allocation_mode; -scalable_allocation_command; -__TBB_malloc_safer_free; -__TBB_malloc_safer_realloc; -__TBB_malloc_safer_msize; -__TBB_malloc_safer_aligned_msize; -__TBB_malloc_safer_aligned_realloc; -/* memory pool stuff */ -_ZN3rml10pool_resetEPNS_10MemoryPoolE; -_ZN3rml11pool_createEiPKNS_13MemPoolPolicyE; -_ZN3rml14pool_create_v1EiPKNS_13MemPoolPolicyEPPNS_10MemoryPoolE; -_ZN3rml11pool_mallocEPNS_10MemoryPoolEj; -_ZN3rml12pool_destroyEPNS_10MemoryPoolE; -_ZN3rml9pool_freeEPNS_10MemoryPoolEPv; -_ZN3rml12pool_reallocEPNS_10MemoryPoolEPvj; -_ZN3rml20pool_aligned_reallocEPNS_10MemoryPoolEPvjj; -_ZN3rml19pool_aligned_mallocEPNS_10MemoryPoolEjj; -_ZN3rml13pool_identifyEPv; - -local:*; -}; diff --git a/src/tbb/src/tbbmalloc/win64-gcc-tbbmalloc-export.def b/src/tbb/src/tbbmalloc/win64-gcc-tbbmalloc-export.def deleted file mode 100644 index d399b96da..000000000 --- a/src/tbb/src/tbbmalloc/win64-gcc-tbbmalloc-export.def +++ /dev/null @@ -1,52 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -{ -global: -scalable_calloc; -scalable_free; -scalable_malloc; -scalable_realloc; -scalable_posix_memalign; -scalable_aligned_malloc; -scalable_aligned_realloc; -scalable_aligned_free; -scalable_msize; -scalable_allocation_mode; -scalable_allocation_command; -__TBB_malloc_safer_free; -__TBB_malloc_safer_realloc; -__TBB_malloc_safer_msize; -__TBB_malloc_safer_aligned_msize; -__TBB_malloc_safer_aligned_realloc; -/* memory pool stuff */ -_ZN3rml10pool_resetEPNS_10MemoryPoolE; -_ZN3rml11pool_createExPKNS_13MemPoolPolicyE; -_ZN3rml14pool_create_v1ExPKNS_13MemPoolPolicyEPPNS_10MemoryPoolE; -_ZN3rml11pool_mallocEPNS_10MemoryPoolEy; -_ZN3rml12pool_destroyEPNS_10MemoryPoolE; -_ZN3rml9pool_freeEPNS_10MemoryPoolEPv; -_ZN3rml12pool_reallocEPNS_10MemoryPoolEPvy; -_ZN3rml20pool_aligned_reallocEPNS_10MemoryPoolEPvyy; -_ZN3rml19pool_aligned_mallocEPNS_10MemoryPoolEyy; -_ZN3rml13pool_identifyEPv; - -local:*; -}; diff --git a/src/tbb/src/tbbmalloc_proxy/CMakeLists.txt b/src/tbb/src/tbbmalloc_proxy/CMakeLists.txt new file mode 100644 index 000000000..609e4f37a --- /dev/null +++ b/src/tbb/src/tbbmalloc_proxy/CMakeLists.txt @@ -0,0 +1,99 @@ +# Copyright (c) 2020-2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if (NOT BUILD_SHARED_LIBS) + return() +endif() + +add_library(tbbmalloc_proxy + function_replacement.cpp + proxy.cpp) + +if (WIN32) + target_sources(tbbmalloc_proxy PRIVATE tbbmalloc_proxy.rc) +endif() + +add_library(TBB::tbbmalloc_proxy ALIAS tbbmalloc_proxy) + +target_compile_definitions(tbbmalloc_proxy + PUBLIC + $<$<CONFIG:DEBUG>:TBB_USE_DEBUG> + PRIVATE + __TBBMALLOCPROXY_BUILD) + +target_include_directories(tbbmalloc_proxy + PUBLIC + $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../../include> + $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>) + +if (NOT APPLE AND CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + # gcc 5.0 and later have -Wno-sized-deallocation options + set(TBB_WARNING_SUPPRESS ${TBB_WARNING_SUPPRESS} + $<$<NOT:$<VERSION_LESS:${CMAKE_CXX_COMPILER_VERSION},5.0>>:-Wno-sized-deallocation>) +endif() + +target_compile_options(tbbmalloc_proxy + PRIVATE + ${TBB_CXX_STD_FLAG} # TODO: consider making it PUBLIC. + ${TBB_MMD_FLAG} + ${TBB_DSE_FLAG} + ${TBB_WARNING_LEVEL} + ${TBB_WARNING_SUPPRESS} + ${TBB_LIB_COMPILE_FLAGS} + ${TBB_COMMON_COMPILE_FLAGS} +) + +if (UNIX AND NOT APPLE) + # Avoid use of target_link_libraries here as it changes /DEF option to \DEF on Windows. + set_target_properties(tbbmalloc_proxy PROPERTIES + LINK_FLAGS "${TBB_LINK_DEF_FILE_FLAG}\"${CMAKE_CURRENT_SOURCE_DIR}/def/${TBB_DEF_FILE_PREFIX}-proxy.def\"" + LINK_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/def/${TBB_DEF_FILE_PREFIX}-proxy.def" + DEFINE_SYMBOL "") +endif() + +# Prefer using target_link_options instead of target_link_libraries to specify link options because +# target_link_libraries may incorrectly handle some options (on Windows, for example). +if (COMMAND target_link_options) + target_link_options(tbbmalloc_proxy + PRIVATE + ${TBB_LIB_LINK_FLAGS} + ${TBB_COMMON_LINK_FLAGS} + ) +else() + target_link_libraries(tbbmalloc_proxy + PRIVATE + ${TBB_LIB_LINK_FLAGS} + ${TBB_COMMON_LINK_FLAGS} + ) +endif() + +target_link_libraries(tbbmalloc_proxy + PRIVATE + TBB::tbbmalloc + Threads::Threads + ${TBB_LIB_LINK_LIBS} + ${TBB_COMMON_LINK_LIBS} +) + +if(TBB_BUILD_APPLE_FRAMEWORKS) + set_target_properties(tbbmalloc_proxy PROPERTIES + FRAMEWORK TRUE + FRAMEWORK_VERSION ${TBBMALLOC_BINARY_VERSION}.${TBB_BINARY_MINOR_VERSION} + XCODE_ATTRIBUTE_PRODUCT_BUNDLE_IDENTIFIER com.intel.tbbmalloc-proxy + MACOSX_FRAMEWORK_IDENTIFIER com.intel.tbbmalloc-proxy + MACOSX_FRAMEWORK_BUNDLE_VERSION ${TBBMALLOC_BINARY_VERSION}.${TBB_BINARY_MINOR_VERSION} + MACOSX_FRAMEWORK_SHORT_VERSION_STRING ${TBBMALLOC_BINARY_VERSION}) +endif() + +tbb_install_target(tbbmalloc_proxy) diff --git a/src/tbb/src/tbbmalloc/lin32-proxy-export.def b/src/tbb/src/tbbmalloc_proxy/def/lin32-proxy.def similarity index 95% rename from src/tbb/src/tbbmalloc/lin32-proxy-export.def rename to src/tbb/src/tbbmalloc_proxy/def/lin32-proxy.def index b3e2849b3..d044325d9 100644 --- a/src/tbb/src/tbbmalloc/lin32-proxy-export.def +++ b/src/tbb/src/tbbmalloc_proxy/def/lin32-proxy.def @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ { diff --git a/src/tbb/src/tbbmalloc/lin64-proxy-export.def b/src/tbb/src/tbbmalloc_proxy/def/lin64-proxy.def similarity index 95% rename from src/tbb/src/tbbmalloc/lin64-proxy-export.def rename to src/tbb/src/tbbmalloc_proxy/def/lin64-proxy.def index 12d8d43dd..f0764d443 100644 --- a/src/tbb/src/tbbmalloc/lin64-proxy-export.def +++ b/src/tbb/src/tbbmalloc_proxy/def/lin64-proxy.def @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ { diff --git a/src/tbb/src/tbbmalloc/tbb_function_replacement.cpp b/src/tbb/src/tbbmalloc_proxy/function_replacement.cpp similarity index 60% rename from src/tbb/src/tbbmalloc/tbb_function_replacement.cpp rename to src/tbb/src/tbbmalloc_proxy/function_replacement.cpp index b0157856c..ad05354fa 100644 --- a/src/tbb/src/tbbmalloc/tbb_function_replacement.cpp +++ b/src/tbb/src/tbbmalloc_proxy/function_replacement.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2022 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,27 +12,82 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ -#include "tbb/tbb_config.h" +#include "oneapi/tbb/detail/_config.h" +#include "oneapi/tbb/detail/_assert.h" +#include "../tbb/assert_impl.h" #if !__TBB_WIN8UI_SUPPORT && defined(_WIN32) +#ifndef _CRT_SECURE_NO_DEPRECATE #define _CRT_SECURE_NO_DEPRECATE 1 -#define __TBB_NO_IMPLICIT_LINKAGE 1 +#endif + +// no standard-conforming implementation of snprintf prior to VS 2015 +#if !defined(_MSC_VER) || _MSC_VER>=1900 +#define LOG_PRINT(s, n, format, ...) snprintf(s, n, format, __VA_ARGS__) +#else +#define LOG_PRINT(s, n, format, ...) _snprintf_s(s, n, _TRUNCATE, format, __VA_ARGS__) +#endif #include <windows.h> #include <new> #include <stdio.h> -#include "tbb_function_replacement.h" +#include <string.h> + +#include "function_replacement.h" -#include "tbb/tbb_config.h" -#include "tbb/tbb_stddef.h" -#include "../tbb/tbb_assert_impl.h" +// The information about a standard memory allocation function for the replacement log +struct FunctionInfo { + const char* funcName; + const char* dllName; +}; + +// Namespace that processes and manages the output of records to the Log journal +// that will be provided to user by TBB_malloc_replacement_log() +namespace Log { + // Value of RECORDS_COUNT is set due to the fact that we maximally + // scan 8 modules, and in every module we can swap 6 opcodes. (rounded to 8) + static const unsigned RECORDS_COUNT = 8 * 8; + static const unsigned RECORD_LENGTH = MAX_PATH; + + // Need to add 1 to count of records, because last record must be always nullptr + static char *records[RECORDS_COUNT + 1]; + static bool replacement_status = true; + + // Internal counter that contains number of next string for record + static unsigned record_number = 0; + + // Function that writes info about (not)found opcodes to the Log journal + // functionInfo - information about a standard memory allocation function for the replacement log + // opcodeString - string, that contain byte code of this function + // status - information about function replacement status + static void record(FunctionInfo functionInfo, const char * opcodeString, bool status) { + __TBB_ASSERT(functionInfo.dllName, "Empty DLL name value"); + __TBB_ASSERT(functionInfo.funcName, "Empty function name value"); + __TBB_ASSERT(opcodeString, "Empty opcode"); + __TBB_ASSERT(record_number <= RECORDS_COUNT, "Incorrect record number"); + + //If some replacement failed -> set status to false + replacement_status &= status; + + // If we reach the end of the log, write this message to the last line + if (record_number == RECORDS_COUNT) { + // %s - workaround to fix empty variable argument parsing behavior in GCC + LOG_PRINT(records[RECORDS_COUNT - 1], RECORD_LENGTH, "%s", "Log was truncated."); + return; + } + + char* entry = (char*)HeapAlloc(GetProcessHeap(), HEAP_ZERO_MEMORY, RECORD_LENGTH); + __TBB_ASSERT(entry, "Invalid memory was returned"); + + LOG_PRINT(entry, RECORD_LENGTH, "%s: %s (%s), byte pattern: <%s>", + status ? "Success" : "Fail", functionInfo.funcName, functionInfo.dllName, opcodeString); + + records[record_number++] = entry; + } +}; inline UINT_PTR Ptr2Addrint(LPVOID ptr) { @@ -66,7 +121,7 @@ inline bool IsInDistance(UINT_PTR addr1, UINT_PTR addr2, __int64 dist) * doesn't allocate memory dynamically. * * The struct MemoryBuffer holds the data about a page in the memory used for - * replacing functions in Intel64 where the target is too far to be replaced + * replacing functions in 64-bit code where the target is too far to be replaced * with a short jump. All the calculations of m_base and m_next are in a multiple * of SIZE_OF_ADDRESS (which is 8 in Win64). */ @@ -180,14 +235,13 @@ static MemoryProvider memProvider; // Compare opcodes from dictionary (str1) and opcodes from code (str2) // str1 might contain '*' to mask addresses -// RETURN: NULL if opcodes did not match, string length of str1 on success +// RETURN: 0 if opcodes did not match, 1 on success size_t compareStrings( const char *str1, const char *str2 ) { - size_t str1Length = strlen(str1); - for (size_t i=0; i<str1Length; i++){ - if( str1[i] != '*' && str1[i] != str2[i] ) return 0; + for (size_t i=0; str1[i]!=0; i++){ + if( str1[i]!='*' && str1[i]!='#' && str1[i]!=str2[i] ) return 0; } - return str1Length; + return 1; } // Check function prologue with known prologues from the dictionary @@ -195,39 +249,46 @@ size_t compareStrings( const char *str1, const char *str2 ) // inpAddr - pointer to function prologue // Dictionary contains opcodes for several full asm instructions // + one opcode byte for the next asm instruction for safe address processing -// RETURN: number of bytes for safe bytes replacement (matched_pattern/2-1) -UINT CheckOpcodes( const char ** opcodes, void *inpAddr, bool abortOnError ) +// RETURN: 1 + the index of the matched pattern, or 0 if no match found. +static UINT CheckOpcodes( const char ** opcodes, void *inpAddr, bool abortOnError, const FunctionInfo* functionInfo = nullptr) { static size_t opcodesStringsCount = 0; static size_t maxOpcodesLength = 0; static size_t opcodes_pointer = (size_t)opcodes; - char opcodeString[61]; + char opcodeString[2*MAX_PATTERN_SIZE+1]; size_t i; - size_t result; + size_t result = 0; // Get the values for static variables // max length and number of patterns if( !opcodesStringsCount || opcodes_pointer != (size_t)opcodes ){ - while( *(opcodes + opcodesStringsCount)!= NULL ){ + while( *(opcodes + opcodesStringsCount)!= nullptr ){ if( (i=strlen(*(opcodes + opcodesStringsCount))) > maxOpcodesLength ) maxOpcodesLength = i; opcodesStringsCount++; } opcodes_pointer = (size_t)opcodes; - __TBB_ASSERT( maxOpcodesLength < 61, "Limit is 30 opcodes/60 symbols per pattern" ); + __TBB_ASSERT( maxOpcodesLength/2 <= MAX_PATTERN_SIZE, "Pattern exceeded the limit of 28 opcodes/56 symbols" ); } // Translate prologue opcodes to string format to compare - for( i=0; i< maxOpcodesLength/2; i++ ){ + for( i=0; i<maxOpcodesLength/2 && i<MAX_PATTERN_SIZE; ++i ){ sprintf( opcodeString + 2*i, "%.2X", *((unsigned char*)inpAddr+i) ); } - opcodeString[maxOpcodesLength] = 0; + opcodeString[2*i] = 0; // Compare translated opcodes with patterns - for( i=0; i< opcodesStringsCount; i++ ){ - result = compareStrings( opcodes[i],opcodeString ); - if( result ) - return (UINT)(result/2-1); + for( UINT idx=0; idx<opcodesStringsCount; ++idx ){ + result = compareStrings( opcodes[idx],opcodeString ); + if( result ) { + if (functionInfo) { + Log::record(*functionInfo, opcodeString, /*status*/ true); + } + return idx + 1; // avoid 0 which indicates a failure + } + } + if (functionInfo) { + Log::record(*functionInfo, opcodeString, /*status*/ false); } if (abortOnError) { // Impossibility to find opcodes in the dictionary is a serious issue, @@ -237,11 +298,26 @@ UINT CheckOpcodes( const char ** opcodes, void *inpAddr, bool abortOnError ) return 0; } +// Modify offsets in original code after moving it to a trampoline. +// We do not have more than one offset to correct in existing opcode patterns. +static void CorrectOffset( UINT_PTR address, const char* pattern, UINT distance ) +{ + const char* pos = strstr(pattern, "#*******"); + if( pos ) { + address += (pos - pattern)/2; // compute the offset position + UINT value; + // UINT assignment is not used to avoid potential alignment issues + memcpy(&value, Addrint2Ptr(address), sizeof(value)); + value += distance; + memcpy(Addrint2Ptr(address), &value, sizeof(value)); + } +} + // Insert jump relative instruction to the input address // RETURN: the size of the trampoline or 0 on failure -static DWORD InsertTrampoline32(void *inpAddr, void *targetAddr, const char ** opcodes, void** storedAddr) +static DWORD InsertTrampoline32(void *inpAddr, void *targetAddr, const char* pattern, void** storedAddr) { - UINT opcodesNumber = SIZE_OF_RELJUMP; + size_t bytesToMove = SIZE_OF_RELJUMP; UINT_PTR srcAddr = Ptr2Addrint(inpAddr); UINT_PTR tgtAddr = Ptr2Addrint(targetAddr); // Check that the target fits in 32 bits @@ -253,29 +329,24 @@ static DWORD InsertTrampoline32(void *inpAddr, void *targetAddr, const char ** o UCHAR *codePtr = (UCHAR *)inpAddr; if ( storedAddr ){ // If requested, store original function code - if ( *codePtr == 0xE9 ){ // JMP relative instruction - // For the special case when a system function consists of a single near jump, - // instead of moving it somewhere we use the target of the jump as the original function. - unsigned offsetInJmp = *(unsigned*)(codePtr + 1); - *storedAddr = (void*)(srcAddr + offsetInJmp + SIZE_OF_RELJUMP); - }else{ - opcodesNumber = CheckOpcodes( opcodes, inpAddr, /*abortOnError=*/true ); - __TBB_ASSERT_RELEASE( opcodesNumber >= SIZE_OF_RELJUMP, "Incorrect bytecode pattern?" ); - UINT_PTR strdAddr = memProvider.GetLocation(srcAddr); - if (!strdAddr) - return 0; - *storedAddr = Addrint2Ptr(strdAddr); - // Set 'executable' flag for original instructions in the new place - DWORD pageFlags = PAGE_EXECUTE_READWRITE; - if (!VirtualProtect(*storedAddr, MAX_PROBE_SIZE, pageFlags, &pageFlags)) return 0; - // Copy original instructions to the new place - memcpy(*storedAddr, codePtr, opcodesNumber); - // Set jump to the code after replacement - offset = srcAddr - strdAddr - SIZE_OF_RELJUMP; - offset32 = (UINT)((offset & 0xFFFFFFFF)); - *((UCHAR*)*storedAddr+opcodesNumber) = 0xE9; - memcpy(((UCHAR*)*storedAddr+opcodesNumber+1), &offset32, sizeof(offset32)); - } + bytesToMove = strlen(pattern)/2-1; // The last byte matching the pattern must not be copied + __TBB_ASSERT_RELEASE( bytesToMove >= SIZE_OF_RELJUMP, "Incorrect bytecode pattern?" ); + UINT_PTR trampAddr = memProvider.GetLocation(srcAddr); + if (!trampAddr) + return 0; + *storedAddr = Addrint2Ptr(trampAddr); + // Set 'executable' flag for original instructions in the new place + DWORD pageFlags = PAGE_EXECUTE_READWRITE; + if (!VirtualProtect(*storedAddr, MAX_PROBE_SIZE, pageFlags, &pageFlags)) return 0; + // Copy original instructions to the new place + memcpy(*storedAddr, codePtr, bytesToMove); + offset = srcAddr - trampAddr; + offset32 = (UINT)(offset & 0xFFFFFFFF); + CorrectOffset( trampAddr, pattern, offset32 ); + // Set jump to the code after replacement + offset32 -= SIZE_OF_RELJUMP; + *(UCHAR*)(trampAddr+bytesToMove) = 0xE9; + memcpy((UCHAR*)(trampAddr+bytesToMove+1), &offset32, sizeof(offset32)); } // The following will work correctly even if srcAddr>tgtAddr, as long as @@ -287,7 +358,7 @@ static DWORD InsertTrampoline32(void *inpAddr, void *targetAddr, const char ** o memcpy(codePtr+1, &offset32, sizeof(offset32)); // Fill the rest with NOPs to correctly see disassembler of old code in debugger. - for( unsigned i=SIZE_OF_RELJUMP; i<opcodesNumber; i++ ){ + for( unsigned i=SIZE_OF_RELJUMP; i<bytesToMove; i++ ){ *(codePtr+i) = 0x90; } @@ -299,9 +370,9 @@ static DWORD InsertTrampoline32(void *inpAddr, void *targetAddr, const char ** o // 2 Put jump RIP relative indirect through the address in the close page // 3 Put the absolute address of the target in the allocated location // RETURN: the size of the trampoline or 0 on failure -static DWORD InsertTrampoline64(void *inpAddr, void *targetAddr, const char ** opcodes, void** storedAddr) +static DWORD InsertTrampoline64(void *inpAddr, void *targetAddr, const char* pattern, void** storedAddr) { - UINT opcodesNumber = SIZE_OF_INDJUMP; + size_t bytesToMove = SIZE_OF_INDJUMP; UINT_PTR srcAddr = Ptr2Addrint(inpAddr); UINT_PTR tgtAddr = Ptr2Addrint(targetAddr); @@ -320,29 +391,24 @@ static DWORD InsertTrampoline64(void *inpAddr, void *targetAddr, const char ** o *locPtr = tgtAddr; if ( storedAddr ){ // If requested, store original function code - if ( *codePtr == 0xE9 ){ // JMP relative instruction - // For the special case when a system function consists of a single near jump, - // instead of moving it somewhere we use the target of the jump as the original function. - unsigned offsetInJmp = *(unsigned*)(codePtr + 1); - *storedAddr = (void*)(srcAddr + offsetInJmp + SIZE_OF_RELJUMP); - }else{ - opcodesNumber = CheckOpcodes( opcodes, inpAddr, /*abortOnError=*/true ); - __TBB_ASSERT_RELEASE( opcodesNumber >= SIZE_OF_INDJUMP, "Incorrect bytecode pattern?" ); - UINT_PTR strdAddr = memProvider.GetLocation(srcAddr); - if (!strdAddr) - return 0; - *storedAddr = Addrint2Ptr(strdAddr); - // Set 'executable' flag for original instructions in the new place - DWORD pageFlags = PAGE_EXECUTE_READWRITE; - if (!VirtualProtect(*storedAddr, MAX_PROBE_SIZE, pageFlags, &pageFlags)) return 0; - // Copy original instructions to the new place - memcpy(*storedAddr, codePtr, opcodesNumber); - // Set jump to the code after replacement. It is within the distance of relative jump! - offset = srcAddr - strdAddr - SIZE_OF_RELJUMP; - offset32 = (UINT)((offset & 0xFFFFFFFF)); - *((UCHAR*)*storedAddr+opcodesNumber) = 0xE9; - memcpy(((UCHAR*)*storedAddr+opcodesNumber+1), &offset32, sizeof(offset32)); - } + bytesToMove = strlen(pattern)/2-1; // The last byte matching the pattern must not be copied + __TBB_ASSERT_RELEASE( bytesToMove >= SIZE_OF_INDJUMP, "Incorrect bytecode pattern?" ); + UINT_PTR trampAddr = memProvider.GetLocation(srcAddr); + if (!trampAddr) + return 0; + *storedAddr = Addrint2Ptr(trampAddr); + // Set 'executable' flag for original instructions in the new place + DWORD pageFlags = PAGE_EXECUTE_READWRITE; + if (!VirtualProtect(*storedAddr, MAX_PROBE_SIZE, pageFlags, &pageFlags)) return 0; + // Copy original instructions to the new place + memcpy(*storedAddr, codePtr, bytesToMove); + offset = srcAddr - trampAddr; + offset32 = (UINT)(offset & 0xFFFFFFFF); + CorrectOffset( trampAddr, pattern, offset32 ); + // Set jump to the code after replacement. It is within the distance of relative jump! + offset32 -= SIZE_OF_RELJUMP; + *(UCHAR*)(trampAddr+bytesToMove) = 0xE9; + memcpy((UCHAR*)(trampAddr+bytesToMove+1), &offset32, sizeof(offset32)); } // Fill the buffer @@ -353,7 +419,7 @@ static DWORD InsertTrampoline64(void *inpAddr, void *targetAddr, const char ** o memcpy(codePtr+2, &offset32, sizeof(offset32)); // Fill the rest with NOPs to correctly see disassembler of old code in debugger. - for( unsigned i=SIZE_OF_INDJUMP; i<opcodesNumber; i++ ){ + for( unsigned i=SIZE_OF_INDJUMP; i<bytesToMove; i++ ){ *(codePtr+i) = 0x90; } @@ -373,9 +439,27 @@ static bool InsertTrampoline(void *inpAddr, void *targetAddr, const char ** opco DWORD origProt = 0; if (!VirtualProtect(inpAddr, MAX_PROBE_SIZE, PAGE_EXECUTE_WRITECOPY, &origProt)) return FALSE; - probeSize = InsertTrampoline32(inpAddr, targetAddr, opcodes, origFunc); + + const char* pattern = nullptr; + if ( origFunc ){ // Need to store original function code + UCHAR * const codePtr = (UCHAR *)inpAddr; + if ( *codePtr == 0xE9 ){ // JMP relative instruction + // For the special case when a system function consists of a single near jump, + // instead of moving it somewhere we use the target of the jump as the original function. + unsigned offsetInJmp = *(unsigned*)(codePtr + 1); + *origFunc = (void*)(Ptr2Addrint(inpAddr) + offsetInJmp + SIZE_OF_RELJUMP); + origFunc = nullptr; // now it must be ignored by InsertTrampoline32/64 + } else { + // find the right opcode pattern + UINT opcodeIdx = CheckOpcodes( opcodes, inpAddr, /*abortOnError=*/true ); + __TBB_ASSERT( opcodeIdx > 0, "abortOnError ignored in CheckOpcodes?" ); + pattern = opcodes[opcodeIdx-1]; // -1 compensates for +1 in CheckOpcodes + } + } + + probeSize = InsertTrampoline32(inpAddr, targetAddr, pattern, origFunc); if (!probeSize) - probeSize = InsertTrampoline64(inpAddr, targetAddr, opcodes, origFunc); + probeSize = InsertTrampoline64(inpAddr, targetAddr, pattern, origFunc); // Restore original protection VirtualProtect(inpAddr, MAX_PROBE_SIZE, origProt, &origProt); @@ -471,12 +555,28 @@ FRR_TYPE ReplaceFunctionW(const wchar_t *dllName, const char *funcName, FUNCPTR return FRR_OK; } -bool IsPrologueKnown(HMODULE module, const char *funcName, const char **opcodes) +bool IsPrologueKnown(const char* dllName, const char *funcName, const char **opcodes, HMODULE module) { FARPROC inpFunc = GetProcAddress(module, funcName); - if (!inpFunc) + FunctionInfo functionInfo = { funcName, dllName }; + + if (!inpFunc) { + Log::record(functionInfo, "unknown", /*status*/ false); return false; - return CheckOpcodes( opcodes, (void*)inpFunc, /*abortOnError=*/false ) != 0; + } + + return CheckOpcodes( opcodes, (void*)inpFunc, /*abortOnError=*/false, &functionInfo) != 0; +} + +// Public Windows API +extern "C" __declspec(dllexport) int TBB_malloc_replacement_log(char *** function_replacement_log_ptr) +{ + if (function_replacement_log_ptr != nullptr) { + *function_replacement_log_ptr = Log::records; + } + + // If we have no logs -> return false status + return Log::replacement_status && Log::records[0] != nullptr ? 0 : -1; } #endif /* !__TBB_WIN8UI_SUPPORT && defined(_WIN32) */ diff --git a/src/tbb/src/tbbmalloc/tbb_function_replacement.h b/src/tbb/src/tbbmalloc_proxy/function_replacement.h similarity index 81% rename from src/tbb/src/tbbmalloc/tbb_function_replacement.h rename to src/tbb/src/tbbmalloc_proxy/function_replacement.h index e986ab18b..a3ec138a6 100644 --- a/src/tbb/src/tbbmalloc/tbb_function_replacement.h +++ b/src/tbb/src/tbbmalloc_proxy/function_replacement.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2022 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_function_replacement_H @@ -42,10 +38,10 @@ typedef void (*FUNCPTR)(); #define ReplaceFunction ReplaceFunctionW #endif //UNICODE -FRR_TYPE ReplaceFunctionA(const char *dllName, const char *funcName, FUNCPTR newFunc, const char ** opcodes, FUNCPTR* origFunc=NULL); -FRR_TYPE ReplaceFunctionW(const wchar_t *dllName, const char *funcName, FUNCPTR newFunc, const char ** opcodes, FUNCPTR* origFunc=NULL); +FRR_TYPE ReplaceFunctionA(const char *dllName, const char *funcName, FUNCPTR newFunc, const char ** opcodes, FUNCPTR* origFunc=nullptr); +FRR_TYPE ReplaceFunctionW(const wchar_t *dllName, const char *funcName, FUNCPTR newFunc, const char ** opcodes, FUNCPTR* origFunc=nullptr); -bool IsPrologueKnown(HMODULE module, const char *funcName, const char **opcodes); +bool IsPrologueKnown(const char* dllName, const char *funcName, const char **opcodes, HMODULE module); // Utilities to convert between ADDRESS and LPVOID union Int2Ptr { @@ -56,7 +52,7 @@ union Int2Ptr { inline UINT_PTR Ptr2Addrint(LPVOID ptr); inline LPVOID Addrint2Ptr(UINT_PTR ptr); -// Use this value as the maximum size the trampoline region +// The size of a trampoline region const unsigned MAX_PROBE_SIZE = 32; // The size of a jump relative instruction "e9 00 00 00 00" @@ -68,6 +64,10 @@ const unsigned SIZE_OF_INDJUMP = 6; // The size of address we put in the location (in Intel64) const unsigned SIZE_OF_ADDRESS = 8; +// The size limit (in bytes) for an opcode pattern to fit into a trampoline +// There should be enough space left for a relative jump; +1 is for the extra pattern byte. +const unsigned MAX_PATTERN_SIZE = MAX_PROBE_SIZE - SIZE_OF_RELJUMP + 1; + // The max distance covered in 32 bits: 2^31 - 1 - C // where C should not be smaller than the size of a probe. // The latter is important to correctly handle "backward" jumps. diff --git a/src/tbb/src/tbbmalloc/proxy.cpp b/src/tbb/src/tbbmalloc_proxy/proxy.cpp similarity index 73% rename from src/tbb/src/tbbmalloc/proxy.cpp rename to src/tbb/src/tbbmalloc_proxy/proxy.cpp index c865bd1a1..9dd17d91e 100644 --- a/src/tbb/src/tbbmalloc/proxy.cpp +++ b/src/tbb/src/tbbmalloc_proxy/proxy.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2024 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,14 +12,37 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ +#if __unix__ && !__ANDROID__ +// include <bits/c++config.h> indirectly so that <cstdlib> is not included +#include <cstddef> +// include <features.h> indirectly so that <stdlib.h> is not included +#include <unistd.h> +// Working around compiler issue with Anaconda's gcc 7.3 compiler package. +// New gcc ported for old libc may provide their inline implementation +// of aligned_alloc as required by new C++ standard, this makes it hard to +// redefine aligned_alloc here. However, running on systems with new libc +// version, it still needs it to be redefined, thus tricking system headers +#if defined(__GLIBC_PREREQ) +#if !__GLIBC_PREREQ(2, 16) && _GLIBCXX_HAVE_ALIGNED_ALLOC +// tell <cstdlib> that there is no aligned_alloc +#undef _GLIBCXX_HAVE_ALIGNED_ALLOC +// trick <stdlib.h> to define another symbol instead +#define aligned_alloc __hidden_redefined_aligned_alloc +// Fix the state and undefine the trick +#include <cstdlib> +#undef aligned_alloc +#endif // !__GLIBC_PREREQ(2, 16) && _GLIBCXX_HAVE_ALIGNED_ALLOC +#endif // defined(__GLIBC_PREREQ) +#include <cstdlib> +#endif // __unix__ && !__ANDROID__ + #include "proxy.h" -#include "tbb/tbb_config.h" + +#include "oneapi/tbb/detail/_config.h" +#include "oneapi/tbb/scalable_allocator.h" +#include "../tbb/environment.h" #if !defined(__EXCEPTIONS) && !defined(_CPPUNWIND) && !defined(__SUNPRO_CC) #if TBB_USE_EXCEPTIONS @@ -31,6 +54,59 @@ #define TBB_USE_EXCEPTIONS 1 #endif +#if MALLOC_UNIXLIKE_OVERLOAD_ENABLED || _WIN32 && !__TBB_WIN8UI_SUPPORT +/*** internal global operator new implementation (Linux, Windows) ***/ +#include <new> + +// Synchronization primitives to protect original library pointers and new_handler +#include "../tbbmalloc/Synchronize.h" +// Use MallocMutex implementation +typedef MallocMutex ProxyMutex; + +// Adds aliasing and copy attributes to function if available +#if defined(__has_attribute) + #if __has_attribute(__copy__) + #define __TBB_ALIAS_ATTR_COPY(name) __attribute__((alias (#name), __copy__(name))) + #endif +#endif + +#ifndef __TBB_ALIAS_ATTR_COPY + #define __TBB_ALIAS_ATTR_COPY(name) __attribute__((alias (#name))) +#endif + +// In case there is no std::get_new_handler function +// which provides synchronized access to std::new_handler +#if !__TBB_CPP11_GET_NEW_HANDLER_PRESENT +static ProxyMutex new_lock; +#endif + +static inline void* InternalOperatorNew(size_t sz) { + void* res = scalable_malloc(sz); +#if TBB_USE_EXCEPTIONS + while (!res) { + std::new_handler handler; +#if __TBB_CPP11_GET_NEW_HANDLER_PRESENT + handler = std::get_new_handler(); +#else + { + ProxyMutex::scoped_lock lock(new_lock); + handler = std::set_new_handler(0); + std::set_new_handler(handler); + } +#endif + if (handler) { + (*handler)(); + } else { + throw std::bad_alloc(); + } + res = scalable_malloc(sz); +} +#endif /* TBB_USE_EXCEPTIONS */ + return res; +} +/*** end of internal global operator new implementation ***/ +#endif // MALLOC_UNIXLIKE_OVERLOAD_ENABLED || _WIN32 && !__TBB_WIN8UI_SUPPORT + #if MALLOC_UNIXLIKE_OVERLOAD_ENABLED || MALLOC_ZONE_OVERLOAD_ENABLED #ifndef __THROW @@ -38,7 +114,6 @@ #endif /*** service functions and variables ***/ - #include <string.h> // for memset #include <unistd.h> // for sysconf @@ -50,7 +125,6 @@ static inline void initPageSize() } #if MALLOC_UNIXLIKE_OVERLOAD_ENABLED -#include "Customize.h" // FencedStore #include <dlfcn.h> #include <malloc.h> // mallinfo @@ -58,7 +132,8 @@ static inline void initPageSize() 1) detection that the proxy library is loaded 2) check that dlsym("malloc") found something different from our replacement malloc */ -extern "C" void *__TBB_malloc_proxy(size_t) __attribute__ ((alias ("malloc"))); + +extern "C" void *__TBB_malloc_proxy(size_t) __TBB_ALIAS_ATTR_COPY(malloc); static void *orig_msize; @@ -81,19 +156,19 @@ static void *orig_libc_free, *orig_libc_realloc; // We already tried to find ptr to original functions. -static intptr_t origFuncSearched; +static std::atomic<bool> origFuncSearched{false}; inline void InitOrigPointers() { // race is OK here, as different threads found same functions - if (!origFuncSearched) { + if (!origFuncSearched.load(std::memory_order_acquire)) { orig_free = dlsym(RTLD_NEXT, "free"); orig_realloc = dlsym(RTLD_NEXT, "realloc"); orig_msize = dlsym(RTLD_NEXT, "malloc_usable_size"); orig_libc_free = dlsym(RTLD_NEXT, "__libc_free"); orig_libc_realloc = dlsym(RTLD_NEXT, "__libc_realloc"); - FencedStore(origFuncSearched, 1); + origFuncSearched.store(true, std::memory_order_release); } } @@ -185,6 +260,7 @@ int mallopt(int /*param*/, int /*value*/) __THROW return 1; } +#if defined(__GLIBC__) || defined(__ANDROID__) struct mallinfo mallinfo() __THROW { struct mallinfo m; @@ -192,24 +268,27 @@ struct mallinfo mallinfo() __THROW return m; } +#endif #if __ANDROID__ // Android doesn't have malloc_usable_size, provide it to be compatible // with Linux, in addition overload dlmalloc_usable_size() that presented // under Android. -size_t dlmalloc_usable_size(const void *ptr) __attribute__ ((alias ("malloc_usable_size"))); +size_t dlmalloc_usable_size(const void *ptr) __TBB_ALIAS_ATTR_COPY(malloc_usable_size); #else // __ANDROID__ +// TODO: consider using __typeof__ to guarantee the correct declaration types // C11 function, supported starting GLIBC 2.16 -void *aligned_alloc(size_t alignment, size_t size) __attribute__ ((alias ("memalign"))); +void *aligned_alloc(size_t alignment, size_t size) __TBB_ALIAS_ATTR_COPY(memalign); // Those non-standard functions are exported by GLIBC, and might be used -// in conjunction with standard malloc/free, so we must ovberload them. +// in conjunction with standard malloc/free, so we must overload them. // Bionic doesn't have them. Not removing from the linker scripts, // as absent entry points are ignored by the linker. -void *__libc_malloc(size_t size) __attribute__ ((alias ("malloc"))); -void *__libc_calloc(size_t num, size_t size) __attribute__ ((alias ("calloc"))); -void *__libc_memalign(size_t alignment, size_t size) __attribute__ ((alias ("memalign"))); -void *__libc_pvalloc(size_t size) __attribute__ ((alias ("pvalloc"))); -void *__libc_valloc(size_t size) __attribute__ ((alias ("valloc"))); + +void *__libc_malloc(size_t size) __TBB_ALIAS_ATTR_COPY(malloc); +void *__libc_calloc(size_t num, size_t size) __TBB_ALIAS_ATTR_COPY(calloc); +void *__libc_memalign(size_t alignment, size_t size) __TBB_ALIAS_ATTR_COPY(memalign); +void *__libc_pvalloc(size_t size) __TBB_ALIAS_ATTR_COPY(pvalloc); +void *__libc_valloc(size_t size) __TBB_ALIAS_ATTR_COPY(valloc); // call original __libc_* to support naive replacement of free via __libc_free etc void __libc_free(void *ptr) @@ -229,43 +308,31 @@ void *__libc_realloc(void *ptr, size_t size) /*** replacements for global operators new and delete ***/ -#include <new> - -void * operator new(size_t sz) { - void *res = scalable_malloc(sz); -#if TBB_USE_EXCEPTIONS - if (NULL == res) - throw std::bad_alloc(); -#endif /* TBB_USE_EXCEPTIONS */ - return res; +void* operator new(size_t sz) { + return InternalOperatorNew(sz); } void* operator new[](size_t sz) { - void *res = scalable_malloc(sz); -#if TBB_USE_EXCEPTIONS - if (NULL == res) - throw std::bad_alloc(); -#endif /* TBB_USE_EXCEPTIONS */ - return res; + return InternalOperatorNew(sz); } -void operator delete(void* ptr) throw() { +void operator delete(void* ptr) noexcept { InitOrigPointers(); __TBB_malloc_safer_free(ptr, (void (*)(void*))orig_free); } -void operator delete[](void* ptr) throw() { +void operator delete[](void* ptr) noexcept { InitOrigPointers(); __TBB_malloc_safer_free(ptr, (void (*)(void*))orig_free); } -void* operator new(size_t sz, const std::nothrow_t&) throw() { +void* operator new(size_t sz, const std::nothrow_t&) noexcept { return scalable_malloc(sz); } -void* operator new[](std::size_t sz, const std::nothrow_t&) throw() { +void* operator new[](std::size_t sz, const std::nothrow_t&) noexcept { return scalable_malloc(sz); } -void operator delete(void* ptr, const std::nothrow_t&) throw() { +void operator delete(void* ptr, const std::nothrow_t&) noexcept { InitOrigPointers(); __TBB_malloc_safer_free(ptr, (void (*)(void*))orig_free); } -void operator delete[](void* ptr, const std::nothrow_t&) throw() { +void operator delete[](void* ptr, const std::nothrow_t&) noexcept { InitOrigPointers(); __TBB_malloc_safer_free(ptr, (void (*)(void*))orig_free); } @@ -273,19 +340,23 @@ void operator delete[](void* ptr, const std::nothrow_t&) throw() { #endif /* MALLOC_UNIXLIKE_OVERLOAD_ENABLED */ #endif /* MALLOC_UNIXLIKE_OVERLOAD_ENABLED || MALLOC_ZONE_OVERLOAD_ENABLED */ - #ifdef _WIN32 #include <windows.h> #if !__TBB_WIN8UI_SUPPORT #include <stdio.h> -#include "tbb_function_replacement.h" -#include "shared_utils.h" + +#include "function_replacement.h" + +template<typename T, size_t N> // generic function to find length of array +inline size_t arrayLength(const T(&)[N]) { + return N; +} void __TBB_malloc_safer_delete( void *ptr) { - __TBB_malloc_safer_free( ptr, NULL ); + __TBB_malloc_safer_free( ptr, nullptr ); } void* safer_aligned_malloc( size_t size, size_t alignment ) @@ -297,7 +368,7 @@ void* safer_aligned_malloc( size_t size, size_t alignment ) // we do not support _expand(); void* safer_expand( void *, size_t ) { - return NULL; + return nullptr; } #define __TBB_ORIG_ALLOCATOR_REPLACEMENT_WRAPPER(CRTLIB) \ @@ -331,35 +402,52 @@ void* __TBB_malloc_safer_realloc_##CRTLIB( void *ptr, size_t size ) return __TBB_malloc_safer_realloc( ptr, size, &func_ptrs ); \ } \ \ -void* __TBB_malloc_safer__aligned_realloc_##CRTLIB( void *ptr, size_t size, size_t aligment ) \ +void* __TBB_malloc_safer__aligned_realloc_##CRTLIB( void *ptr, size_t size, size_t alignment ) \ { \ orig_aligned_ptrs func_ptrs = {orig__aligned_free_##CRTLIB, orig__aligned_msize_##CRTLIB}; \ - return __TBB_malloc_safer_aligned_realloc( ptr, size, aligment, &func_ptrs ); \ + return __TBB_malloc_safer_aligned_realloc( ptr, size, alignment, &func_ptrs ); \ +} + +// Only for ucrtbase: substitution for _o_free +void (*orig__o_free)(void*); +void __TBB_malloc__o_free(void *ptr) +{ + __TBB_malloc_safer_free( ptr, orig__o_free ); +} +// Only for ucrtbase: substitution for _free_base +void(*orig__free_base)(void*); +void __TBB_malloc__free_base(void *ptr) +{ + __TBB_malloc_safer_free(ptr, orig__free_base); } -// Limit is 30 bytes/60 symbols per line, * can be used to match any digit in bytecodes. -// Purpose of the pattern is to mark an instruction bound, it should consist of several -// full instructions plus one more byte. It's not required for the patterns to be unique -// (i.e., it's OK to have same pattern for unrelated functions). +// Size limit is MAX_PATTERN_SIZE (28) byte codes / 56 symbols per line. +// * can be used to match any digit in byte codes. +// # followed by several * indicate a relative address that needs to be corrected. +// Purpose of the pattern is to mark an instruction bound; it should consist of several +// full instructions plus one extra byte code. It's not required for the patterns +// to be unique (i.e., it's OK to have same pattern for unrelated functions). // TODO: use hot patch prologues if exist const char* known_bytecodes[] = { #if _WIN64 +// "========================================================" - 56 symbols + "E9********CCCC", // multiple - jmp(0xE9) with address followed by empty space (0xCC - INT 3) "4883EC284885C974", // release free() "4883EC284885C975", // release _msize() "4885C974375348", // release free() 8.0.50727.42, 10.0 - "E907000000CCCC", // release _aligned_msize(), _aligned_free() ucrtbase.dll "C7442410000000008B", // release free() ucrtbase.dll 10.0.14393.33 - "E90B000000CCCC", // release _msize() ucrtbase.dll 10.0.14393.33 "48895C24085748", // release _aligned_msize() ucrtbase.dll 10.0.14393.33 "48894C24084883EC28BA", // debug prologue "4C894424184889542410", // debug _aligned_msize() 10.0 "48894C24084883EC2848", // debug _aligned_free 10.0 + "488BD1488D0D#*******E9", // _o_free(), ucrtbase.dll #if __TBB_OVERLOAD_OLD_MSVCR "48895C2408574883EC3049", // release _aligned_msize 9.0 "4883EC384885C975", // release _msize() 9.0 "4C8BC1488B0DA6E4040033", // an old win64 SDK #endif #else // _WIN32 +// "========================================================" - 56 symbols "8BFF558BEC8B", // multiple "8BFF558BEC83", // release free() & _msize() 10.0.40219.325, _msize() ucrtbase.dll "8BFF558BECFF", // release _aligned_msize ucrtbase.dll @@ -377,7 +465,7 @@ const char* known_bytecodes[] = { "6A1C68********E8", // release _msize() 8.0.50727.4053, 9.0 #endif #endif // _WIN64/_WIN32 - NULL + nullptr }; #define __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL_ENTRY(CRT_VER,function_name,dbgsuffix) \ @@ -387,11 +475,11 @@ const char* known_bytecodes[] = { #define __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL_ENTRY_NO_FALLBACK(CRT_VER,function_name,dbgsuffix) \ ReplaceFunctionWithStore( #CRT_VER #dbgsuffix ".dll", #function_name, \ - (FUNCPTR)__TBB_malloc_safer_##function_name##_##CRT_VER##dbgsuffix, 0, NULL ); + (FUNCPTR)__TBB_malloc_safer_##function_name##_##CRT_VER##dbgsuffix, 0, nullptr ); #define __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL_ENTRY_REDIRECT(CRT_VER,function_name,dest_func,dbgsuffix) \ ReplaceFunctionWithStore( #CRT_VER #dbgsuffix ".dll", #function_name, \ - (FUNCPTR)__TBB_malloc_safer_##dest_func##_##CRT_VER##dbgsuffix, 0, NULL ); + (FUNCPTR)__TBB_malloc_safer_##dest_func##_##CRT_VER##dbgsuffix, 0, nullptr ); #define __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL_IMPL(CRT_VER,dbgsuffix) \ if (BytecodesAreKnown(#CRT_VER #dbgsuffix ".dll")) { \ @@ -429,46 +517,41 @@ __TBB_ORIG_ALLOCATOR_REPLACEMENT_WRAPPER(msvcr120d); __TBB_ORIG_ALLOCATOR_REPLACEMENT_WRAPPER(msvcr120); __TBB_ORIG_ALLOCATOR_REPLACEMENT_WRAPPER(ucrtbase); - /*** replacements for global operators new and delete ***/ -#include <new> - #if _MSC_VER && !defined(__INTEL_COMPILER) -#pragma warning( push ) -#pragma warning( disable : 4290 ) +// #pragma warning( push ) +// #pragma warning( disable : 4290 ) #endif -void * operator_new(size_t sz) { - void *res = scalable_malloc(sz); - if (NULL == res) throw std::bad_alloc(); - return res; +/*** operator new overloads internals (Linux, Windows) ***/ + +void* operator_new(size_t sz) { + return InternalOperatorNew(sz); } void* operator_new_arr(size_t sz) { - void *res = scalable_malloc(sz); - if (NULL == res) throw std::bad_alloc(); - return res; + return InternalOperatorNew(sz); } -void operator_delete(void* ptr) throw() { +void operator_delete(void* ptr) noexcept { __TBB_malloc_safer_delete(ptr); } #if _MSC_VER && !defined(__INTEL_COMPILER) -#pragma warning( pop ) +// #pragma warning( pop ) #endif -void operator_delete_arr(void* ptr) throw() { +void operator_delete_arr(void* ptr) noexcept { __TBB_malloc_safer_delete(ptr); } -void* operator_new_t(size_t sz, const std::nothrow_t&) throw() { +void* operator_new_t(size_t sz, const std::nothrow_t&) noexcept { return scalable_malloc(sz); } -void* operator_new_arr_t(std::size_t sz, const std::nothrow_t&) throw() { +void* operator_new_arr_t(std::size_t sz, const std::nothrow_t&) noexcept { return scalable_malloc(sz); } -void operator_delete_t(void* ptr, const std::nothrow_t&) throw() { +void operator_delete_t(void* ptr, const std::nothrow_t&) noexcept { __TBB_malloc_safer_delete(ptr); } -void operator_delete_arr_t(void* ptr, const std::nothrow_t&) throw() { +void operator_delete_arr_t(void* ptr, const std::nothrow_t&) noexcept { __TBB_malloc_safer_delete(ptr); } @@ -517,8 +600,8 @@ _expand (by dummy implementation) ??_V@YAXPEAX@Z void * operator new[](unsigned __int64) (intel64) ??3@YAXPEAX@Z operator delete (intel64) ??_V@YAXPEAX@Z operator delete[] (intel64) -??2@YAPAXIABUnothrow_t@std@@@Z void * operator new (size_t sz, const std::nothrow_t&) throw() (optional) -??_U@YAPAXIABUnothrow_t@std@@@Z void * operator new[] (size_t sz, const std::nothrow_t&) throw() (optional) +??2@YAPAXIABUnothrow_t@std@@@Z void * operator new (size_t sz, const std::nothrow_t&) noexcept (optional) +??_U@YAPAXIABUnothrow_t@std@@@Z void * operator new[] (size_t sz, const std::nothrow_t&) noexcept (optional) and these functions have runtime-specific replacement: realloc @@ -577,7 +660,7 @@ bool BytecodesAreKnown(const unicode_char_t *dllName) if (!module) return false; for (int i=0; funcName[i]; i++) - if (! IsPrologueKnown(module, funcName[i], known_bytecodes)) { + if (! IsPrologueKnown(dllName, funcName[i], known_bytecodes, module)) { fprintf(stderr, "TBBmalloc: skip allocation functions replacement in " WCHAR_SPEC ": unknown prologue for function " WCHAR_SPEC "\n", dllName, funcName[i]); return false; @@ -597,7 +680,7 @@ void SkipReplacement(const unicode_char_t *dllName) char *dllStr = buffer; errno_t ret = wcstombs_s(&real_sz, dllStr, sz, dllName, sz-1); - __TBB_ASSERT(!ret, "Dll name conversion failed") + __TBB_ASSERT(!ret, "Dll name conversion failed"); #endif for (size_t i=0; i<arrayLength(modules_to_replace); i++) @@ -616,7 +699,11 @@ void ReplaceFunctionWithStore( const unicode_char_t *dllName, const char *funcNa fprintf(stderr, "Failed to %s function %s in module %s\n", res==FRR_NOFUNC? "find" : "replace", funcName, dllName); - exit(1); + + // Unable to replace a required function + // Aborting because incomplete replacement of memory management functions + // may leave the program in an invalid state + abort(); } void doMallocReplacement() @@ -639,10 +726,22 @@ void doMallocReplacement() continue; for (size_t i = 0; i < arrayLength(c_routines_to_replace); i++) { - ReplaceFunctionWithStore( modules_to_replace[j].name, c_routines_to_replace[i]._func, c_routines_to_replace[i]._fptr, NULL, NULL, c_routines_to_replace[i]._on_error ); + ReplaceFunctionWithStore( modules_to_replace[j].name, c_routines_to_replace[i]._func, c_routines_to_replace[i]._fptr, nullptr, nullptr, c_routines_to_replace[i]._on_error ); } - // ucrtbase.dll does not export operator new/delete. - if ( strcmp(modules_to_replace[j].name, "ucrtbase.dll") == 0 ){ + if ( strcmp(modules_to_replace[j].name, "ucrtbase.dll") == 0 ) { + HMODULE ucrtbase_handle = GetModuleHandle("ucrtbase.dll"); + if (!ucrtbase_handle) + continue; + // If _o_free function is present and patchable, redirect it to tbbmalloc as well + // This prevents issues with other _o_* functions which might allocate memory with malloc + if ( IsPrologueKnown("ucrtbase.dll", "_o_free", known_bytecodes, ucrtbase_handle)) { + ReplaceFunctionWithStore( "ucrtbase.dll", "_o_free", (FUNCPTR)__TBB_malloc__o_free, known_bytecodes, (FUNCPTR*)&orig__o_free, FRR_FAIL ); + } + // Similarly for _free_base + if (IsPrologueKnown("ucrtbase.dll", "_free_base", known_bytecodes, ucrtbase_handle)) { + ReplaceFunctionWithStore("ucrtbase.dll", "_free_base", (FUNCPTR)__TBB_malloc__free_base, known_bytecodes, (FUNCPTR*)&orig__free_base, FRR_FAIL); + } + // ucrtbase.dll does not export operator new/delete, so skip the rest of the loop. continue; } @@ -656,33 +755,38 @@ void doMallocReplacement() // replacement should be skipped for this particular case. if ( (strcmp(modules_to_replace[j].name, "msvcr120.dll") == 0) && (strcmp(cxx_routines_to_replace[i]._func, "??_V@YAXPAX@Z") == 0) ) continue; #endif - ReplaceFunctionWithStore( modules_to_replace[j].name, cxx_routines_to_replace[i]._func, cxx_routines_to_replace[i]._fptr, NULL, NULL, cxx_routines_to_replace[i]._on_error ); + ReplaceFunctionWithStore( modules_to_replace[j].name, cxx_routines_to_replace[i]._func, cxx_routines_to_replace[i]._fptr, nullptr, nullptr, cxx_routines_to_replace[i]._on_error ); } } } #endif // !__TBB_WIN8UI_SUPPORT +#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) + // Suppress warning for UWP build ('main' signature found without threading model) + // #pragma warning(push) + // #pragma warning(disable:4447) +#endif + extern "C" BOOL WINAPI DllMain( HINSTANCE hInst, DWORD callReason, LPVOID reserved ) { if ( callReason==DLL_PROCESS_ATTACH && reserved && hInst ) { #if !__TBB_WIN8UI_SUPPORT -#if TBBMALLOC_USE_TBB_FOR_ALLOCATOR_ENV_CONTROLLED - char pinEnvVariable[50]; - if( GetEnvironmentVariable("TBBMALLOC_USE_TBB_FOR_ALLOCATOR", pinEnvVariable, 50)) + if (!tbb::detail::r1::GetBoolEnvironmentVariable("TBB_MALLOC_DISABLE_REPLACEMENT")) { doMallocReplacement(); } -#else - doMallocReplacement(); -#endif #endif // !__TBB_WIN8UI_SUPPORT } return TRUE; } +#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) + // #pragma warning(pop) +#endif + // Just to make the linker happy and link the DLL to the application extern "C" __declspec(dllexport) void __TBB_malloc_proxy() { diff --git a/src/tbb/src/tbbmalloc/proxy.h b/src/tbb/src/tbbmalloc_proxy/proxy.h similarity index 56% rename from src/tbb/src/tbbmalloc/proxy.h rename to src/tbb/src/tbbmalloc_proxy/proxy.h index 8db526c2a..5f0133f9e 100644 --- a/src/tbb/src/tbbmalloc/proxy.h +++ b/src/tbb/src/tbbmalloc_proxy/proxy.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef _TBB_malloc_proxy_H_ @@ -26,26 +22,19 @@ // MALLOC_UNIXLIKE_OVERLOAD_ENABLED depends on MALLOC_CHECK_RECURSION stuff // TODO: limit MALLOC_CHECK_RECURSION to *_OVERLOAD_ENABLED only -#if __linux__ || __APPLE__ || __sun || __FreeBSD__ || MALLOC_UNIXLIKE_OVERLOAD_ENABLED +#if __unix__ || __APPLE__ || MALLOC_UNIXLIKE_OVERLOAD_ENABLED #define MALLOC_CHECK_RECURSION 1 #endif +#include "oneapi/tbb/detail/_config.h" #include <stddef.h> extern "C" { - void * scalable_malloc(size_t size); - void * scalable_calloc(size_t nobj, size_t size); - void scalable_free(void *ptr); - void * scalable_realloc(void* ptr, size_t size); - void * scalable_aligned_malloc(size_t size, size_t alignment); - void * scalable_aligned_realloc(void* ptr, size_t size, size_t alignment); - int scalable_posix_memalign(void **memptr, size_t alignment, size_t size); - size_t scalable_msize(void *ptr); - void __TBB_malloc_safer_free( void *ptr, void (*original_free)(void*)); - void * __TBB_malloc_safer_realloc( void *ptr, size_t, void* ); - void * __TBB_malloc_safer_aligned_realloc( void *ptr, size_t, size_t, void* ); - size_t __TBB_malloc_safer_msize( void *ptr, size_t (*orig_msize_crt80d)(void*)); - size_t __TBB_malloc_safer_aligned_msize( void *ptr, size_t, size_t, size_t (*orig_msize_crt80d)(void*,size_t,size_t)); + TBBMALLOC_EXPORT void __TBB_malloc_safer_free( void *ptr, void (*original_free)(void*)); + TBBMALLOC_EXPORT void * __TBB_malloc_safer_realloc( void *ptr, size_t, void* ); + TBBMALLOC_EXPORT void * __TBB_malloc_safer_aligned_realloc( void *ptr, size_t, size_t, void* ); + TBBMALLOC_EXPORT size_t __TBB_malloc_safer_msize( void *ptr, size_t (*orig_msize_crt80d)(void*)); + TBBMALLOC_EXPORT size_t __TBB_malloc_safer_aligned_msize( void *ptr, size_t, size_t, size_t (*orig_msize_crt80d)(void*,size_t,size_t)); #if MALLOC_ZONE_OVERLOAD_ENABLED void __TBB_malloc_free_definite_size(void *object, size_t size); diff --git a/src/tbb/src/tbbmalloc/proxy_overload_osx.h b/src/tbb/src/tbbmalloc_proxy/proxy_overload_osx.h similarity index 96% rename from src/tbb/src/tbbmalloc/proxy_overload_osx.h rename to src/tbb/src/tbbmalloc_proxy/proxy_overload_osx.h index 53afe13d0..695829837 100644 --- a/src/tbb/src/tbbmalloc/proxy_overload_osx.h +++ b/src/tbb/src/tbbmalloc_proxy/proxy_overload_osx.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2022 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ // The original source for this code is @@ -99,7 +95,7 @@ static size_t impl_malloc_usable_size(struct _malloc_zone_t *, const void *ptr) { // malloc_usable_size() is used by macOS* to recognize which memory manager // allocated the address, so our wrapper must not redirect to the original function. - return __TBB_malloc_safer_msize(const_cast<void*>(ptr), NULL); + return __TBB_malloc_safer_msize(const_cast<void*>(ptr), nullptr); } static void *impl_malloc(struct _malloc_zone_t *, size_t size); @@ -116,12 +112,12 @@ static void impl_free_definite_size(struct _malloc_zone_t*, void *ptr, size_t si } /* Empty out caches in the face of memory pressure. */ -static size_t impl_pressure_relief(struct _malloc_zone_t *, size_t goal) +static size_t impl_pressure_relief(struct _malloc_zone_t *, size_t /* goal */) { return 0; } -static malloc_zone_t *system_zone = NULL; +static malloc_zone_t *system_zone = nullptr; struct DoMallocReplacement { DoMallocReplacement() { @@ -162,8 +158,8 @@ struct DoMallocReplacement { void* ptr = malloc(1); // get all registered memory zones unsigned zcount = 0; - malloc_zone_t** zone_array = NULL; - kern_return_t errorcode = malloc_get_all_zones(mach_task_self(),NULL,(vm_address_t**)&zone_array,&zcount); + malloc_zone_t** zone_array = nullptr; + kern_return_t errorcode = malloc_get_all_zones(mach_task_self(),nullptr,(vm_address_t**)&zone_array,&zcount); if (!errorcode && zone_array && zcount>0) { // find the zone that allocated ptr for (unsigned i=0; i<zcount; ++i) { diff --git a/src/tbb/src/tbbmalloc_proxy/tbbmalloc_proxy.rc b/src/tbb/src/tbbmalloc_proxy/tbbmalloc_proxy.rc new file mode 100644 index 000000000..1884b119a --- /dev/null +++ b/src/tbb/src/tbbmalloc_proxy/tbbmalloc_proxy.rc @@ -0,0 +1,74 @@ +// Copyright (c) 2005-2024 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +///////////////////////////////////////////////////////////////////////////// +// +// Includes +// +#include <winresrc.h> +#include "../../include/oneapi/tbb/version.h" + +///////////////////////////////////////////////////////////////////////////// +// Neutral resources + +#ifdef _WIN32 +LANGUAGE LANG_NEUTRAL, SUBLANG_NEUTRAL +#pragma code_page(1252) +#endif //_WIN32 + +///////////////////////////////////////////////////////////////////////////// +// +// Version +// +#define TBB_VERNUMBERS TBB_VERSION_MAJOR,TBB_VERSION_MINOR,TBB_VERSION_PATCH +#define TBB_VERSION TBB_VERSION_STRING + +VS_VERSION_INFO VERSIONINFO + FILEVERSION TBB_VERNUMBERS + PRODUCTVERSION TBB_VERNUMBERS + FILEFLAGSMASK 0x17L +#ifdef _DEBUG + FILEFLAGS 0x1L +#else + FILEFLAGS 0x0L +#endif + FILEOS 0x40004L + FILETYPE 0x2L + FILESUBTYPE 0x0L +BEGIN + BLOCK "StringFileInfo" + BEGIN + BLOCK "000004b0" + BEGIN + VALUE "CompanyName", "Intel Corporation\0" + VALUE "FileDescription", "oneAPI Threading Building Blocks (oneTBB) library\0" + VALUE "FileVersion", TBB_VERSION "\0" + VALUE "LegalCopyright", "Copyright 2005-2024 Intel Corporation. All Rights Reserved.\0" + VALUE "LegalTrademarks", "\0" +#ifndef TBB_USE_DEBUG + VALUE "OriginalFilename", "tbbmalloc_proxy.dll\0" +#else + VALUE "OriginalFilename", "tbbmalloc_proxy_debug.dll\0" +#endif + VALUE "ProductName", "oneAPI Threading Building Blocks (oneTBB)\0" + VALUE "ProductVersion", TBB_VERSION "\0" + VALUE "PrivateBuild", "\0" + VALUE "SpecialBuild", "\0" + END + END + BLOCK "VarFileInfo" + BEGIN + VALUE "Translation", 0x0, 1200 + END +END diff --git a/src/tbb/src/tbbproxy/tbbproxy-windows.asm b/src/tbb/src/tbbproxy/tbbproxy-windows.asm deleted file mode 100644 index d5affed9f..000000000 --- a/src/tbb/src/tbbproxy/tbbproxy-windows.asm +++ /dev/null @@ -1,111 +0,0 @@ -; Copyright (c) 2005-2017 Intel Corporation -; -; Licensed under the Apache License, Version 2.0 (the "License"); -; you may not use this file except in compliance with the License. -; You may obtain a copy of the License at -; -; http://www.apache.org/licenses/LICENSE-2.0 -; -; Unless required by applicable law or agreed to in writing, software -; distributed under the License is distributed on an "AS IS" BASIS, -; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -; See the License for the specific language governing permissions and -; limitations under the License. -; -; -; -; - -// __TBB_STRING macro defined in "tbb_stddef.h". However, we cannot include "tbb_stddef.h" -// because it contains a lot of C/C++ definitions. So, we have to define __TBB_STRING here: -#define __TBB_STRING_AUX( x ) #x -#define __TBB_STRING( x ) __TBB_STRING_AUX( x ) - -// Eliminate difference between IA-32 and Intel 64: AWORD is a type of pointer, LANG is language -// specification for extern directive. -#ifdef ARCH_ia32 - #define AWORD dword - #define LANG c -#else - #define AWORD qword - #define LANG -#endif - -#ifdef ARCH_ia32 - // These directives are required for IA32 architecture only. - .686 - .model flat, syscall -#endif - -/* - Symbol names. -*/ - -// Note: masm for IA-32 does not like symbols defined as "name:" in data sections, -// so we have to define symbols with "name label type" directive instead. - -fname macro sym:req - align sizeof AWORD - Ln_&sym& label byte - byte "&sym&", 0 -endm - -.const // Symbol names are constants. -#define __TBB_SYMBOL( sym ) fname sym -#include __TBB_STRING( __TBB_LST ) - -/* - Symbol descriptors. -*/ - -extern LANG __tbb_internal_runtime_loader_stub : AWORD - -fsymbol macro sym:req - Ls_&sym& label AWORD - AWORD __tbb_internal_runtime_loader_stub - AWORD Ln_&sym& - dword sizeof AWORD - dword 1 -endm - -.data -align sizeof AWORD -public LANG __tbb_internal_runtime_loader_symbols -__tbb_internal_runtime_loader_symbols label AWORD -#define __TBB_SYMBOL( sym ) fsymbol sym -#include __TBB_STRING( __TBB_LST ) -AWORD 0, 0 // Terminator of the __tbb_internal_runtime_loader_symbols array. -dword 0, 0 - -/* - Generate functions. -*/ - -// Helper assembler macro to handle different naming conventions on IA-32 and Intel 64: -// IA-32: C++ names preserved, C names require leading underscore. -// Intel 64: All names preserved. -mangle macro name:req - #ifdef ARCH_ia32 - if @instr( 1, name, <?> ) - exitm @catstr( name ) - else - exitm @catstr( <_>, name ) - endif - #else - exitm @catstr( name ) - #endif -endm - -function macro sym:req - mangle( sym ) proc - jmp AWORD ptr Ls_&sym& - mangle( sym ) endp -endm - -.code -#define __TBB_SYMBOL( sym ) function sym -#include __TBB_STRING( __TBB_LST ) - -end - -// end of file // diff --git a/src/tbb/src/tbbproxy/tbbproxy.cpp b/src/tbb/src/tbbproxy/tbbproxy.cpp deleted file mode 100644 index 0e57ab714..000000000 --- a/src/tbb/src/tbbproxy/tbbproxy.cpp +++ /dev/null @@ -1,612 +0,0 @@ -/* - Copyright (c) 2005-2017 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - - -*/ - -#include "tbb/tbb_config.h" -#if !__TBB_WIN8UI_SUPPORT -#define TBB_PREVIEW_RUNTIME_LOADER 1 -#include "tbb/runtime_loader.h" -#include "tbb/tbb_stddef.h" - -// C standard headers. -#include <cctype> // isspace -#include <cstdarg> // va_list, etc. -#include <cstdio> // fprintf, stderr, etc. -#include <cstdlib> // malloc, free, abort. -#include <cstring> // strlen, etc. - -// C++ standard headers. -#include <typeinfo> - -// OS-specific includes. -#if _WIN32 || _WIN64 - #include <windows.h> - #define snprintf _snprintf - #undef max -#else - #include <dlfcn.h> // dlopen, dlsym, dlclose, dlerror. -#endif - -#if TBB_USE_ASSERT - // We cannot use __TBB_ASSERT as it is because it calls a function from tbb library which may - // be not yet loaded. Redefine __TBB_ASSERT not to call tbb functions. - #undef __TBB_ASSERT - #define __TBB_ASSERT( cond, msg ) { \ - if ( ! (cond) ) { \ - say( "%s:%d: Assertion failed: %s.", __FILE__, __LINE__, (msg) ); \ - } /* if */ \ - /* TODO: abort? */ \ - } -#endif - -// Declare here, define at the bottom. -extern "C" int __tbb_internal_runtime_loader_stub(); - -namespace tbb { - -namespace interface6 { - -namespace internal { - -namespace runtime_loader { - - -/* - ------------------------------------------------------------------------------------------------ - User interaction utilities. - ------------------------------------------------------------------------------------------------ -*/ - - -// Print message to stderr. Do not call it directly, use say() or tell() instead. -static void _say( char const * format, va_list args ) { - /* - On 64-bit Linux* OS, vsnprintf() modifies args argument, - so vsnprintf() crashes if it is called for the second time with the same args. - To prevent the crash, we have to pass a fresh intact copy of args to vsnprintf() each time. - - On Windows* OS, unfortunately, standard va_copy() macro is not available. However, it - seems vsnprintf() does not modify args argument. - */ - #if ! ( _WIN32 || _WIN64 ) - va_list _args; - __va_copy( _args, args ); // Make copy of args. - #define args _args // Substitute args with its copy, _args. - #endif - int len = vsnprintf( NULL, 0, format, args ); - #if ! ( _WIN32 || _WIN64 ) - #undef args // Remove substitution. - va_end( _args ); - #endif - char * buf = reinterpret_cast< char * >( malloc( len + 1 ) ); - if ( buf != NULL ) { - vsnprintf( buf, len + 1, format, args ); - fprintf( stderr, "TBB: %s\n", buf ); - free( buf ); - } else { - fprintf( stderr, "TBB: Not enough memory for message: %s\n", format ); - } -} // _say - - -// Debug/test/troubleshooting printing controlled by TBB_VERSION environment variable. -// To enable printing, the variable must be set and not empty. -// Do not call it directly, use tell() instead. -static void _tell( char const * format, va_list args ) { - char const * var = getenv( "TBB_VERSION" ); - if ( var != NULL && var[ 0 ] != 0 ) { - _say( format, args ); - } // if -} // _tell - - -// Print message to stderr unconditionally. -static void say( char const * format, ... ) { - va_list args; - va_start( args, format ); - _say( format, args ); - va_end( args ); -} // say - - -// Debug/test/troubleshooting printing controlled by TBB_VERSION environment variable. -// To enable printing, the variable must be set and not empty. -static void tell( char const * format, ... ) { - va_list args; - va_start( args, format ); - _tell( format, args ); - va_end( args ); -} // tell - - -// Error reporting utility. Behavior depends on mode. -static tbb::runtime_loader::error_code error( tbb::runtime_loader::error_mode mode, tbb::runtime_loader::error_code err, char const * format, ... ) { - va_list args; - va_start( args, format ); - if ( mode == tbb::runtime_loader::em_abort ) { - // In em_abort mode error message printed unconditionally. - _say( format, args ); - } else { - // In other modes printing depends on TBB_VERSION environment variable. - _tell( format, args ); - } // if - va_end( args ); - switch ( mode ) { - case tbb::runtime_loader::em_abort : { - say( "Aborting..." ); - #if TBB_USE_DEBUG && ( _WIN32 || _WIN64 ) - DebugBreak(); - #endif - abort(); - } break; - case tbb::runtime_loader::em_throw : { - throw err; - } break; - case tbb::runtime_loader::em_status : { - // Do nothing. - } break; - } // switch - return err; -} // error - - -/* - ------------------------------------------------------------------------------------------------ - General-purpose string manipulation utilities. - ------------------------------------------------------------------------------------------------ -*/ - - -// Delete character ch from string str in-place. -static void strip( char * str, char ch ) { - int in = 0; // Input character index. - int out = 0; // Output character index. - for ( ; ; ) { - if ( str[ in ] != ch ) { - str[ out ] = str[ in ]; - ++ out; - } // if - if ( str[ in ] == 0 ) { - break; - } // if - ++ in; - } // forever -} // func strip - - -// Strip trailing whitespaces in-place. -static void trim( char * str ) { - size_t len = strlen( str ); - while ( len > 0 && isspace( str[ len - 1 ] ) ) { - -- len; - } // while - str[ len ] = 0; -} // func trim - - -#if _WIN32 || _WIN64 - // "When specifying a path, be sure to use backslashes (\), not forward slashes (/)." - // (see http://msdn.microsoft.com/en-us/library/ms886736.aspx). - const char proper_slash = '\\'; - inline char char_or_slash( char c ) { return c=='/'? '\\': c; } -#else - const char proper_slash = '/'; - inline char char_or_slash( char c ) { return c; } -#endif - -// Concatenate name of directory and name of file. -void cat_file( char const * dir, char const * file, char * buffer, size_t len ) { - size_t i = 0; - // Copy directory name - for( ; i<len && *dir; ++i, ++dir ) { - buffer[i] = char_or_slash(*dir); - } - // Append trailing slash if missed. - if( i>0 && i<len && buffer[i-1]!=proper_slash ) { - buffer[i++] = proper_slash; - } - // Copy file name - __TBB_ASSERT( char_or_slash(*file)!=proper_slash, "File name starts with a slash" ); - for( ; i<len && *file; ++i, ++file ) { - buffer[i] = *file; - } - // Append null terminator - buffer[ i<len? i: len-1 ] = '\0'; -} // cat_file - - -/* - ------------------------------------------------------------------------------------------------ - Windows implementation of dlopen, dlclose, dlsym, dlerror. - ------------------------------------------------------------------------------------------------ -*/ - - -#if _WIN32 || _WIN64 - - // Implement Unix-like interface (dlopen, dlclose, dlsym, dlerror) via Win32 API functions. - - // Type of dlopen result. - typedef HMODULE handle_t; - - enum rtld_flags_t { - RTLD_NOW, - RTLD_GLOBAL - }; // enum rtld_flags_t - - // Unix-like dlopen(). - static handle_t dlopen( char const * name, rtld_flags_t ) { - return LoadLibrary( name ); - } // dlopen - - // Unix-like dlsym(). - static void * dlsym( handle_t lib, char const * sym ) { - return (void*)GetProcAddress( lib, sym ); - } // dlsym - - // Unix-like dlclose(). - static int dlclose( handle_t lib ) { - return ! FreeLibrary( lib ); - } // dlclose - - // The function mimics Unix dlerror() function. - // Note: Not thread-safe due to statically allocated buffer. - static char * dlerror() { - - static char buffer[ 2048 ]; // Note: statically allocated buffer. - - DWORD err = GetLastError(); - if ( err == ERROR_SUCCESS ) { - return NULL; - } // if - - DWORD rc; - rc = - FormatMessage( - FORMAT_MESSAGE_FROM_SYSTEM, - NULL, - err, - MAKELANGID( LANG_NEUTRAL, SUBLANG_DEFAULT ), // Default language. - reinterpret_cast< LPTSTR >( & buffer ), - sizeof( buffer ), - NULL - ); - if ( rc == 0 ) { - // FormatMessage() failed to format system error message. Buffer to short or another issue. - snprintf( buffer, sizeof( buffer ), "System error %u.", err ); - } else { - /* - FormatMessage() returns Windows-style end-of-lines, "\r\n". When string is printed, - printf() also replaces all the occurrences of "\n" with "\r\n" (again!), so sequences - like "\r\r\r\n" appear in output. It is not too good. Stripping all "\r" normalizes - string and returns it to canonical form, so printf() will produce correct end-of-line - sequences. - */ - strip( buffer, '\r' ); // Delete carriage returns if any. - trim( buffer ); // Delete trailing newlines and spaces. - } // if - - return buffer; - - } // dlerror - -#else - - // Type of dlopen() result. - typedef void * handle_t; - -#endif - - -/* - ------------------------------------------------------------------------------------------------ - Runtime loader stuff. - ------------------------------------------------------------------------------------------------ -*/ - - -// Descriptor table declaration. It is defined in assembler file. -enum symbol_type_t { - st_object = 0, - st_function = 1 -}; // enum symbol_type_t -struct symbol_t { - void * addr; - char const * name; - int size; - symbol_type_t type; -}; // symbol_t -extern "C" symbol_t __tbb_internal_runtime_loader_symbols[]; - -// Hooks for internal use (e. g. for testing). -tbb::runtime_loader::error_mode stub_mode = tbb::runtime_loader::em_abort; - -static char const * tbb_dll_name = __TBB_STRING(__TBB_DLL_NAME); // Name of TBB library. -static handle_t handle = NULL; // Handle of loaded TBB library or NULL. -static int version = 0; // Version of the loaded library. -static int counter = 0; // Number of runtime_loader objects using the loaded library. - -#define ANOTHER_RTL "probably multiple runtime_loader objects work in parallel" - - -// One attempt to load library (dll_name can be a full path or just a file name). -static tbb::runtime_loader::error_code _load( char const * dll_name, int min_ver, int max_ver ) { - - tbb::runtime_loader::error_mode mode = tbb::runtime_loader::em_status; - tbb::runtime_loader::error_code code = tbb::runtime_loader::ec_ok; - - /* - If these variables declared at the first usage, Intel C++ Compiler may issue warning(s): - transfer of control [goto error] bypasses initialization of: ... - Declaring variables at the beginning of the function eliminates warnings. - */ - typedef int (*int_func_t)( void ); - char const * get_ver_name = "TBB_runtime_interface_version"; // Name of function. - int_func_t get_ver_func = NULL; // Pointer to function. - handle_t _handle = NULL; - int _version = 0; - int total = 0; - int not_found = 0; - - // This function should be called iff there is no loaded library. - __TBB_ASSERT( handle == NULL, "Handle is invalid; " ANOTHER_RTL ); - __TBB_ASSERT( version == 0, "Version is invalid; " ANOTHER_RTL ); - __TBB_ASSERT( counter == 0, "Counter is invalid; " ANOTHER_RTL ); - - tell( "Loading \"%s\"...", dll_name ); - - // First load the library. - _handle = dlopen( dll_name, RTLD_NOW ); - if ( _handle == NULL ) { - const char * msg = dlerror(); - code = error( mode, tbb::runtime_loader::ec_no_lib, "Loading \"%s\" failed; system error: %s", dll_name, msg ); - goto error; - } // if - - // Then try to find out its version. - /* - g++ 3.4 issues error: - ISO C++ forbids casting between pointer-to-function and pointer-to-object - on reinterpret_cast<>. Thus, we have no choice but using C-style type cast. - */ - get_ver_func = (int_func_t) dlsym( _handle, get_ver_name ); - if ( get_ver_func == NULL ) { - code = error( mode, tbb::runtime_loader::ec_bad_lib, "Symbol \"%s\" not found; library rejected.", get_ver_name ); - goto error; - } // if - _version = get_ver_func(); - if ( ! ( min_ver <= _version && _version <= max_ver ) ) { - code = error( mode, tbb::runtime_loader::ec_bad_ver, "Version %d is out of requested range; library rejected.", _version ); - goto error; - } // if - - // Library is suitable. Mark it as loaded. - handle = _handle; - version = _version; - counter += 1; - __TBB_ASSERT( counter == 1, "Counter is invalid; " ANOTHER_RTL ); - - // Now search for all known symbols. - for ( int i = 0; __tbb_internal_runtime_loader_symbols[ i ].name != NULL; ++ i ) { - symbol_t & symbol = __tbb_internal_runtime_loader_symbols[ i ]; - // Verify symbol descriptor. - __TBB_ASSERT( symbol.type == st_object || symbol.type == st_function, "Invalid symbol type" ); - #if _WIN32 || _WIN64 - __TBB_ASSERT( symbol.type == st_function, "Should not be symbols of object type on Windows" ); - #endif - if ( symbol.type == st_object ) { - __TBB_ASSERT( symbol.addr != NULL, "Object address invalid" ); - __TBB_ASSERT( symbol.size > 0, "Symbol size must be > 0" ); - __TBB_ASSERT( symbol.size <= 0x1000, "Symbol size too big" ); - } else { // Function - // __TBB_ASSERT( symbol.addr == reinterpret_cast< void * >( & stub ), "Invalid symbol address" ); - __TBB_ASSERT( symbol.size == sizeof( void * ), "Invalid symbol size" ); - } // if - void * addr = dlsym( _handle, symbol.name ); - if ( addr != NULL ) { - if ( symbol.type == st_object ) { - if ( strncmp( symbol.name, "_ZTS", 4 ) == 0 ) { - // If object name begins with "_ZTS", it is a string, mangled type name. - // Its value must equal to name of symbol without "_ZTS" prefix. - char const * name = static_cast< char const * >( addr ); - __TBB_ASSERT( strlen( name ) + 1 == size_t( symbol.size ), "Unexpected size of typeinfo name" ); - __TBB_ASSERT( strcmp( symbol.name + 4, name ) == 0, "Unexpected content of typeinfo name" ); - strncpy( reinterpret_cast< char * >( symbol.addr ), name, symbol.size ); - reinterpret_cast< char * >( symbol.addr )[ symbol.size - 1 ] = 0; - } else { - #if TBB_USE_ASSERT - // If object name begins with "_ZTI", it is an object of std::type_info class. - // Its protected value must equal to name of symbol without "_ZTI" prefix. - if ( strncmp( symbol.name, "_ZTI", 4 ) == 0 ) { - std::type_info const * info = static_cast< std::type_info const * >( addr ); - __TBB_ASSERT( size_t( symbol.size ) >= sizeof( std::type_info ), "typeinfo size is too small" ); - // std::type_info::name is not a virtual method, it is safe to call it. - __TBB_ASSERT( strcmp( symbol.name + 4, info->name() ) == 0, "Unexpected content of typeinfo" ); - } // if - #endif - // Copy object content from libtbb into runtime_loader. - memcpy( symbol.addr, addr, symbol.size ); - }; // if - } else { // Function - symbol.addr = addr; - } // if - } else { - char const * msg = dlerror(); - tell( "Symbol \"%s\" not found; system error: %s", symbol.name, msg ); - ++ not_found; - } // if - ++ total; - } // for i - - if ( not_found > 0 ) { - tell( "%d of %d symbols not found.", not_found, total ); - } // if - - tell( "The library successfully loaded." ); - return code; - - error: - if ( _handle != NULL ) { - int rc = dlclose( _handle ); - if ( rc != 0 ) { - // Error occurred. - __TBB_ASSERT( rc != 0, "Unexpected error: dlclose() failed" ); - } // if - } // if - _handle = NULL; - return code; - -} // _load - - -static tbb::runtime_loader::error_code load( tbb::runtime_loader::error_mode mode, char const * path[], int min_ver, int max_ver ) { - // Check arguments first. - if ( min_ver <= 0 ) { - return error( mode, tbb::runtime_loader::ec_bad_arg, "tbb::runtime_loader::load(): Invalid value of min_ver argument: %d.", min_ver ); - } // if - if ( max_ver <= 0 ) { - return error( mode, tbb::runtime_loader::ec_bad_arg, "tbb::runtime_loader::load(): Invalid value of max_ver argument: %d.", max_ver ); - } // if - if ( min_ver > max_ver ) { - return error( mode, tbb::runtime_loader::ec_bad_arg, "tbb::runtime_loader::load(): min_ver and max_ver specify empty range: [%d, %d].", min_ver, max_ver ); - } // if - if ( min_ver == max_ver ) { - tell( "Searching for \"%s\" version %d...", tbb_dll_name, min_ver ); - } else if ( max_ver == INT_MAX ) { - tell( "Searching for \"%s\" version %d+...", tbb_dll_name, min_ver ); - } else { - tell( "Searching for \"%s\" version in range [%d, %d]...", tbb_dll_name, min_ver, max_ver ); - } // if - // Then check whether a library already loaded. - if ( handle != NULL ) { - // Library already loaded. Check whether the version is compatible. - __TBB_ASSERT( version > 0, "Version is invalid; " ANOTHER_RTL ); - __TBB_ASSERT( counter > 0, "Counter is invalid; " ANOTHER_RTL ); - if ( min_ver <= version && version <= max_ver ) { - // Version is ok, let us use this library. - tell( "Library version %d is already loaded.", version ); - counter += 1; - return tbb::runtime_loader::ec_ok; - } else { - // Version is not suitable. - return error( mode, tbb::runtime_loader::ec_bad_ver, "Library version %d is already loaded.", version ); - } // if - } // if - // There is no loaded library, try to load it using provided directories. - __TBB_ASSERT( version == 0, "Version is invalid; " ANOTHER_RTL ); - __TBB_ASSERT( counter == 0, "Counter is invalid; " ANOTHER_RTL ); - size_t namelen = strlen(tbb_dll_name); - size_t buflen = 0; - char * buffer = NULL; - for ( int i = 0; path[i] != NULL; ++ i ) { - size_t len = strlen(path[i]) + namelen + 2; // 1 for slash and 1 for null terminator - if( buflen<len ) { - free( buffer ); - buflen = len; - buffer = (char*)malloc( buflen ); - if( !buffer ) - return error( mode, tbb::runtime_loader::ec_no_lib, "Not enough memory." ); - } - cat_file( path[i], tbb_dll_name, buffer, buflen ); - __TBB_ASSERT(strstr(buffer,tbb_dll_name), "Name concatenation error"); - tbb::runtime_loader::error_code ec = _load( buffer, min_ver, max_ver ); - if ( ec == tbb::runtime_loader::ec_ok ) { - return ec; // Success. Exiting... - } // if - } // for i - free( buffer ); - return error( mode, tbb::runtime_loader::ec_no_lib, "No suitable library found." ); -} // load - - - - -// Suppress "defined but not used" compiler warnings. -static void const * dummy[] = { - (void *) & strip, - (void *) & trim, - & dummy, - NULL -}; - - -} // namespace runtime_loader - -} // namespace internal - - -runtime_loader::runtime_loader( error_mode mode ) : - my_mode( mode ), - my_status( ec_ok ), - my_loaded( false ) -{ -} // ctor - - -runtime_loader::runtime_loader( char const * path[], int min_ver, int max_ver, error_mode mode ) : - my_mode( mode ), - my_status( ec_ok ), - my_loaded( false ) -{ - load( path, min_ver, max_ver ); -} // ctor - - -runtime_loader::~runtime_loader() { -} // dtor - - -tbb::runtime_loader::error_code runtime_loader::load( char const * path[], int min_ver, int max_ver ) { - if ( my_loaded ) { - my_status = tbb::interface6::internal::runtime_loader::error( my_mode, ec_bad_call, "tbb::runtime_loader::load(): Library already loaded by this runtime_loader object." ); - } else { - my_status = internal::runtime_loader::load( my_mode, path, min_ver, max_ver ); - if ( my_status == ec_ok ) { - my_loaded = true; - } // if - } // if - return my_status; -} // load - - - - -tbb::runtime_loader::error_code runtime_loader::status() { - return my_status; -} // status - - -} // namespace interface6 - -} // namespace tbb - - -// Stub function replaces all TBB entry points when no library is loaded. -int __tbb_internal_runtime_loader_stub() { - char const * msg = NULL; - if ( tbb::interface6::internal::runtime_loader::handle == NULL ) { - msg = "A function is called while TBB library is not loaded"; - } else { - msg = "A function is called which is not present in loaded TBB library"; - } // if - return tbb::interface6::internal::runtime_loader::error( tbb::interface6::internal::runtime_loader::stub_mode, tbb::runtime_loader::ec_no_lib, msg ); -} // stub - -#endif // !__TBB_WIN8UI_SUPPORT // -// end of file // diff --git a/src/tbb/third-party-programs.txt b/src/tbb/third-party-programs.txt new file mode 100644 index 000000000..c088429c2 --- /dev/null +++ b/src/tbb/third-party-programs.txt @@ -0,0 +1,198 @@ +oneAPI Threading Building Blocks (oneTBB) Third Party Programs File + +This file is the "third-party-programs.txt" file specified in the associated Intel end user license +agreement for the Intel software you are licensing. + +The third party programs and their corresponding required notices and/or license +terms are listed below. +_______________________________________________________________________________________________________ + +1. Instrumentation and Tracing Technology (ITT) Notify User API: + Copyright (c) 2005-2023 Intel Corporation. All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + 3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +_______________________________________________________________________________________________________ + +2. Portable Hardware Locality (hwloc): + + Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana University Research and + Technology Corporation. All rights reserved. + Copyright (c) 2004-2005 The University of Tennessee and The University of Tennessee Research + Foundation. All rights reserved. + Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, University of Stuttgart. + All rights reserved. + Copyright (c) 2004-2005 The Regents of the University of California. All rights reserved. + Copyright (c) 2009 CNRS + Copyright (c) 2009-2016 Inria. All rights reserved. + Copyright (c) 2009-2015 Université Bordeaux + Copyright (c) 2009-2015 Cisco Systems, Inc. All rights reserved. + Copyright (c) 2009-2012 Oracle and/or its affiliates. All rights reserved. + Copyright (c) 2010 IBM + Copyright (c) 2010 Jirka Hladky + Copyright (c) 2012 Aleksej Saushev, The NetBSD Foundation + Copyright (c) 2012 Blue Brain Project, EPFL. All rights reserved. + Copyright (c) 2013-2014 University of Wisconsin-La Crosse. All rights reserved. + Copyright (c) 2015 Research Organization for Information Science and Technology (RIST). + All rights reserved. + Copyright (c) 2015-2016 Intel, Inc. All rights reserved. + See COPYING in top-level directory. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + 3. The name of the author may not be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +_______________________________________________________________________________________________________ + +3. gperftools: Copyright (c) 2011, Google Inc. + + Tachyon: Copyright (c) 1994-2008 John E. Stone. All rights reserved. + + BSD 3-Clause "New" or "Revised" License + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + * Neither the name of Google Inc. nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +_______________________________________________________________________________________________________ + +4. Mateusz Kwiatkowski Workaround for bug 62258 in libstdc++ + + ******************************************************************************** + * Author: Mateusz Kwiatkowski <m.kwiatkowski@avsystem.com> * + * * + * I hereby renounce all copyright to this file and my rights resulting from * + * it, to the broadest extent permitted by law. It may be treated as public * + * domain. * + * * + * However, as this file interfaces with GCC internal ABI, it may be subject to * + * the terms and conditions of the GNU General Public License. Please consult * + * the GCC licensing terms and/or a lawyer for details. * + * * + * Note that libstdc++ licensing terms grant additional permissions described * + * in the GCC Runtime Library Exception, version 3.1, as published by the * + * Free Software Foundation. * + *******************************************************************************/ +_______________________________________________________________________________________________________ + +5. ActiveState Thread pool with same API as (multi) processing. Pool (Python recipe) + + # + # Copyright (c) 2008,2016 david decotigny (this file) + # Copyright (c) 2006-2008, R Oudkerk (multiprocessing.Pool) + # All rights reserved. + # + # Redistribution and use in source and binary forms, with or without + # modification, are permitted provided that the following conditions + # are met: + # + # 1. Redistributions of source code must retain the above copyright + # notice, this list of conditions and the following disclaimer. + # 2. Redistributions in binary form must reproduce the above copyright + # notice, this list of conditions and the following disclaimer in the + # documentation and/or other materials provided with the distribution. + # 3. Neither the name of author nor the names of any contributors may be + # used to endorse or promote products derived from this software + # without specific prior written permission. + # + # THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND + # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + # SUCH DAMAGE. + +_______________________________________________________________________________________________________ + +6. doctest + + Copyright (c) 2016-2023 Viktor Kirilov + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. + +_______________________________________________________________________________________________________ + +*Other names and brands may be claimed as the property of others. \ No newline at end of file diff --git a/tests/doRUnit.R b/tests/doRUnit.R index 44c82ab1c..34d49eb7b 100644 --- a/tests/doRUnit.R +++ b/tests/doRUnit.R @@ -1,16 +1,25 @@ -stopifnot(require(RUnit, quietly=TRUE)) -stopifnot(require(Rcpp, quietly=TRUE)) -stopifnot(require(RcppParallel, quietly=TRUE)) +stopifnot(require(RUnit, quietly = TRUE)) +stopifnot(require(Rcpp, quietly = TRUE)) +stopifnot(require(RcppParallel, quietly = TRUE)) ## Set a seed to make the test deterministic set.seed(42) +## Set a default backend +backend <- Sys.getenv("RCPP_PARALLEL_BACKEND", unset = NA) +if (is.na(backend)) + Sys.setenv(RCPP_PARALLEL_BACKEND = "tinythread") + +writeLines(paste("Using backend:", Sys.getenv("RCPP_PARALLEL_BACKEND"))) + ## Define tests -suite <- defineTestSuite(name="RcppParallel Unit Tests", - dirs=system.file("tests", package = "RcppParallel")) +suite <- defineTestSuite( + name = "RcppParallel Unit Tests", + dirs = system.file("tests", package = "RcppParallel") +) ## Based on practice in Rcpp to avoid some test failures -Sys.setenv("R_TESTS"="") +Sys.setenv("R_TESTS" = "") ## Run tests tests <- runTestSuite(suite) @@ -22,9 +31,11 @@ printTextProtocol(tests) if (getErrors(tests)$nFail > 0) { stop("TEST FAILED!") } + if (getErrors(tests)$nErr > 0) { stop("TEST HAD ERRORS!") } + if (getErrors(tests)$nTestFunc < 1) { stop("NO TEST FUNCTIONS RUN!") } diff --git a/tools/config.R b/tools/config.R index adb2fa239..abf31e9d1 100644 --- a/tools/config.R +++ b/tools/config.R @@ -77,7 +77,10 @@ configure_file <- function( rhs = "@", verbose = configure_verbose()) { + # read source file contents <- readLines(source, warn = FALSE) + + # replace defined variables enumerate(config, function(key, val) { needle <- paste(lhs, key, rhs, sep = "") replacement <- val @@ -85,8 +88,15 @@ configure_file <- function( }) ensure_directory(dirname(target)) - writeLines(contents, con = target) + # write configured file to target location + # prefer unix newlines for Makevars + mode <- if (basename(target) %in% "Makevars") "wb" else "w" + conn <- file(target, open = mode) + on.exit(close(conn), add = TRUE) + writeLines(contents, con = conn) + + # copy over source permissions info <- file.info(source) Sys.chmod(target, mode = info$mode) @@ -266,10 +276,6 @@ read_r_config <- function( stop(sprintf(fmt, getRversion())) } - # notify user - if (verbose) - message("*** executing 'R CMD config --all'") - # execute action stdout <- tempfile("r-cmd-config-", fileext = ".txt") on.exit(unlink(stdout), add = TRUE) @@ -283,10 +289,6 @@ read_r_config <- function( } else { - # notify user - if (verbose) - message("*** executing 'R CMD config'") - # loop through requested values and call R CMD config config <- lapply(values, function(value) { @@ -582,6 +584,16 @@ if (!interactive()) { args <- commandArgs(TRUE) type <- args[[1]] + # preserve working directory + owd <- getwd() + on.exit(setwd(owd), add = TRUE) + + # switch working directory to the calling scripts's directory as set + # by the shell, in case the R working directory was set to something else + basedir <- Sys.getenv("PWD", unset = NA) + if (!is.na(basedir)) + setwd(basedir) + # report start of execution package <- Sys.getenv("R_PACKAGE_NAME", unset = "<unknown>") fmt <- "** preparing to %s package '%s' ..." diff --git a/tools/config/cleanup.R b/tools/config/cleanup.R index 85623c219..7746f1bc6 100644 --- a/tools/config/cleanup.R +++ b/tools/config/cleanup.R @@ -1,4 +1,13 @@ + # Clean up files generated during configuration here. # Use 'remove_file()' to remove files generated during configuration. -unlink("src/tbb/build/lib_release", recursive = TRUE) + +# unlink("src/tbb/build", recursive = TRUE) +# unlink("src/tbb/build-tbb", recursive = TRUE) +unlink("inst/lib", recursive = TRUE) +unlink("inst/libs", recursive = TRUE) +unlink("inst/include/index.html", recursive = TRUE) +unlink("inst/include/oneapi", recursive = TRUE) +unlink("inst/include/serial", recursive = TRUE) +unlink("inst/include/tbb", recursive = TRUE) diff --git a/tools/config/configure.R b/tools/config/configure.R index e8d81ae68..a001e63b8 100644 --- a/tools/config/configure.R +++ b/tools/config/configure.R @@ -1,11 +1,41 @@ +# make sure we call correct version of R +rExe <- if (.Platform$OS.type == "windows") "R.exe" else "R" +define(R = file.path(R.home("bin"), rExe)) + +# check whether user has Makevars file that might cause trouble +makevars <- Sys.getenv("R_MAKEVARS_USER", unset = "~/.R/Makevars") +if (file.exists(makevars)) { + contents <- readLines(makevars, warn = FALSE) + pattern <- "^(PKG_CPPFLAGS|PKG_CXXFLAGS)\\s*=" + bad <- grep(pattern, contents, perl = TRUE, value = TRUE) + if (length(bad)) { + + text <- c( + "", + sprintf("NOTE: '%s' contains variable declarations incompatible with RcppParallel:", makevars), + "", + paste0("\t", bad), + "", + "Makevars variables prefixed with 'PKG_' should be considered reserved for use by R packages.", + "" + ) + + writeLines(text, con = stdout()) + + } +} + # Figure out the appropriate CXX prefix for the current # version of R + configuration. -cxx <- NULL +cxx <- "/usr/bin/c++" candidates <- c("CXX11", "CXX1X", "CXX") for (candidate in candidates) { value <- r_cmd_config(candidate) if (!is.null(value)) { + if (any(grepl("icpc", value))) { + define(COMPILER = "icc") + } cxx <- candidate break } @@ -21,6 +51,24 @@ broken <- if (broken) cxxflags <- gsub("-Werror=format-security", "-Wformat -Werror=format-security", cxxflags) +# add C++ standard if not set +if (!grepl("-std=", cxxflags, fixed = TRUE)) { + stdflag <- if (getRversion() < "4.0") { + "-std=c++0x" + } else { + "$(CXX11STD)" + } + cxxflags <- paste(stdflag, cxxflags) +} + +# avoid including /usr/local/include, as this can cause +# RcppParallel to find and use a version of libtbb installed +# there as opposed to the bundled version +cppflags <- read_r_config("CPPFLAGS", envir = NULL)[[1]] +cppflags <- sub("(?: )?-I/usr/local/include", "", cppflags) +cppflags <- sub("(?: )?-I/opt/homebrew/include", "", cppflags) +cppflags <- sub("(?: )?-I/opt/local/libexec/onetbb/include", "", cppflags) + # define the set of flags appropriate to the current # configuration of R switch( @@ -28,6 +76,7 @@ switch( CXX11 = define( CC = "$(CC)", + CPPFLAGS = cppflags, CXX11 = "$(CXX11)", CXX11FLAGS = cxxflags, CXX11STD = "$(CXX11STD)", @@ -36,6 +85,7 @@ switch( CXX1X = define( CC = "$(CC)", + CPPFLAGS = cppflags, CXX11 = "$(CXX1X)", CXX11FLAGS = cxxflags, CXX11STD = "$(CXX1XSTD)", @@ -44,6 +94,7 @@ switch( CXX = define( CC = "$(CC)", + CPPFLAGS = cppflags, CXX11 = "$(CXX)", CXX11FLAGS = cxxflags, CXX11STD = "-std=c++0x", @@ -53,19 +104,227 @@ switch( stop("Failed to infer C / C++ compilation flags") ) -# define special flags for Windows -db <- configure_database() -if (Sys.info()[["sysname"]] == "Windows") { +# on Windows, check for Rtools; if it exists, and we have tbb, use it +if (.Platform$OS.type == "windows") { + + gccPath <- normalizePath(Sys.which("gcc"), winslash = "/") + + tbbLib <- Sys.getenv("TBB_LIB", unset = NA) + if (is.na(tbbLib)) + tbbLib <- normalizePath(file.path(gccPath, "../../lib"), winslash = "/") + + tbbInc <- Sys.getenv("TBB_INC", unset = NA) + if (is.na(tbbInc)) + tbbInc <- normalizePath(file.path(gccPath, "../../include"), winslash = "/") + + tbbFiles <- list.files(tbbLib, pattern = "^libtbb") + if (length(tbbFiles)) { + + tbbPattern <- "^lib(tbb\\d*(?:_static)?)\\.a$" + tbbName <- grep(tbbPattern, tbbFiles, perl = TRUE, value = TRUE) + tbbName <- gsub(tbbPattern, "\\1", tbbName, perl = TRUE) + + tbbMallocPattern <- "^lib(tbbmalloc\\d*(?:_static)?)\\.a$" + tbbMallocName <- grep(tbbMallocPattern, tbbFiles, perl = TRUE, value = TRUE) + tbbMallocName <- gsub(tbbMallocPattern, "\\1", tbbMallocName, perl = TRUE) + + Sys.setenv( + TBB_LIB = tbbLib, + TBB_INC = tbbInc, + TBB_NAME = tbbName, + TBB_MALLOC_NAME = tbbMallocName + ) + + } + +} + +# try and figure out path to TBB +tbbRoot <- Sys.getenv("TBB_ROOT", unset = NA) +tbbLib <- Sys.getenv("TBB_LIB", unset = NA) +tbbInc <- Sys.getenv("TBB_INC", unset = NA) + +tbbName <- Sys.getenv("TBB_NAME", unset = "tbb") +tbbMallocName <- Sys.getenv("TBB_MALLOC_NAME", unset = "tbbmalloc") + +# check TBB_ROOT first if defined +if (!is.na(tbbRoot)) { + + if (is.na(tbbLib)) { + tbbLib <- file.path(tbbRoot, "lib") + } + + if (is.na(tbbInc)) { + tbbInc <- file.path(tbbRoot, "include") + } + +} + +# if TBB_LIB is defined, guess TBB_INC +if (!is.na(tbbLib) && is.na(tbbInc)) { + tbbIncCandidate <- file.path(tbbLib, "../include") + if (file.exists(tbbIncCandidate)) { + tbbInc <- normalizePath(tbbIncCandidate) + } +} + +# if TBB_LIB and TBB_INC are still not defined, try auto-detecting +tryAutoDetect <- + .Platform$OS.type == "unix" && + Sys.getenv("TBB_AUTODETECT", unset = "FALSE") == "TRUE" && + is.na(tbbLib) && + is.na(tbbInc) + +if (tryAutoDetect) { + + sysInfo <- as.list(Sys.info()) + + homebrewPrefix <- if (sysInfo$sysname == "Darwin") { + "/opt/homebrew" + } else { + "/usr/local" + } + + tbbLibSearch <- if (sysInfo$sysname == "Darwin") { + file.path(homebrewPrefix, "opt/tbb/lib/libtbb.dylib") + } else { + Sys.glob(c( + "/usr/*/libtbb.so", + "/usr/*/*/libtbb.so", + "/usr/*/*/*/libtbb.so" + )) + } + + tbbIncSearch <- if (sysInfo$sysname == "Darwin") { + file.path(homebrewPrefix, "opt/tbb/include/tbb") + } else { + Sys.glob(c( + "/usr/include/tbb.h", + "/usr/include/*/tbb.h" + )) + } + + if (length(tbbLibSearch) && + length(tbbIncSearch) && + file.exists(tbbLibSearch[[1L]]) && + file.exists(tbbIncSearch[[1L]])) + { + tbbLib <- dirname(tbbLibSearch[[1L]]) + tbbInc <- dirname(tbbIncSearch[[1L]]) + } + +} + +# now, define TBB_LIB and TBB_INC as appropriate +define( + TBB_LIB = if (!is.na(tbbLib)) tbbLib else "", + TBB_INC = if (!is.na(tbbInc)) tbbInc else "", + TBB_NAME = tbbName, + TBB_MALLOC_NAME = tbbMallocName +) + +# set PKG_LIBS +pkgLibs <- if (!is.na(tbbLib)) { + + c( + "-Wl,-L\"$(TBB_LIB)\"", + sprintf("-Wl,-rpath,%s", shQuote(tbbLib)), + "-l$(TBB_NAME)", + "-l$(TBB_MALLOC_NAME)" + ) + +} else if (.Platform$OS.type == "windows") { + + NULL + +} else if (R.version$os == "emscripten") { + + c( + "-Wl,-Ltbb/build/lib_release", + "-l$(TBB_NAME)" + ) + +} else { - cygpath <- nzchar(Sys.which("cygpath")) - fmt <- if (cygpath) "$(shell cygpath -m \"%s\")" else "%s" - define( - WINDOWS_CC = sprintf(fmt, db$CC), - WINDOWS_CXX11 = sprintf(fmt, db$CXX11) + c( + "-Wl,-Ltbb/build/lib_release", + "-l$(TBB_NAME)", + "-l$(TBB_MALLOC_NAME)" ) + +} + +# on Windows, we may need to link to ssp; otherwise, +# we see errors like +# +# C:\rtools43\x86_64-w64-mingw32.static.posix\bin/ld.exe: C:/rtools43/x86_64-w64-mingw32.static.posix/lib/libtbb12.a(allocator.cpp.obj):allocator.cpp:(.text+0x18b): undefined reference to `__stack_chk_fail' +# +if (.Platform$OS.type == "windows") { + pkgLibs <- c(pkgLibs, "-lssp") } -# use c++0x for compatibility with older compilers -define(STDVER = "c++0x") +define(PKG_LIBS = paste(pkgLibs, collapse = " ")) + +# if we're going to build tbb from sources, check for cmake +define(CMAKE = "") +if (is.na(tbbLib)) { + + cmake <- local({ + + # check for envvar + cmake <- Sys.getenv("CMAKE", unset = NA) + if (!is.na(cmake)) + return(cmake) + + # check for path + cmake <- Sys.which("cmake") + if (nzchar(cmake)) + return(cmake) + + # check for macOS cmake + cmake <- "/Applications/CMake.app/Contents/bin/cmake" + if (file.exists(cmake)) + return(cmake) + + stop("cmake was not found") + + }) + + # make sure we have an appropriate version of cmake installed + output <- system("cmake --version", intern = TRUE)[[1L]] + cmakeVersion <- numeric_version(sub("cmake version ", "", output)) + if (cmakeVersion < "3.5") { + stop("error: RcppParallel requires cmake (>= 3.6); you have ", cmakeVersion) + } + + define(CMAKE = cmake) + +} + + +# now, set up PKG_CPPFLAGS +if (!is.na(tbbLib)) { + define(PKG_CPPFLAGS = "-I../inst/include -I\"$(TBB_INC)\"") +} else { + define(PKG_CPPFLAGS = "-I../inst/include") +} +# PKG_CXXFLAGS +if (.Platform$OS.type == "windows" && is.na(tbbLib)) { + define(TBB_ENABLED = FALSE) + define(PKG_CXXFLAGS = "-DRCPP_PARALLEL_USE_TBB=0") +} else { + define(TBB_ENABLED = TRUE) + define(PKG_CXXFLAGS = "-DRCPP_PARALLEL_USE_TBB=1") +} + +# macOS needs some extra flags set +if (Sys.info()[["sysname"]] == "Darwin") { + define(PKG_LIBS_EXTRA = "-Wl,-rpath,@loader_path/../lib") +} else if (Sys.info()[["sysname"]] == "Linux") { + define(PKG_LIBS_EXTRA = "-Wl,-rpath,$(ORIGIN)/../lib") +} else { + define(PKG_LIBS_EXTRA = "") + +} diff --git a/tools/tbb/disable-pragmas.R b/tools/tbb/disable-pragmas.R new file mode 100644 index 000000000..7e255f4ed --- /dev/null +++ b/tools/tbb/disable-pragmas.R @@ -0,0 +1,23 @@ +# Disable TBB pragmas that silence diagnostic warnings. +# This is necessary for CRAN submissions of RcppParallel. + +files <- list.files( + path = "src/tbb", + pattern = "[.](?:h|cpp)$", + all.files = TRUE, + full.names = TRUE, + recursive = TRUE +) + +for (file in files) { + + before <- readLines(file) + + after <- before + after <- gsub("^(\\s*)#pragma warning", "\\1// #pragma warning", after, perl = TRUE) + after <- gsub("^(\\s*)#pragma GCC", "\\1// #pragma GCC", after, perl = TRUE) + after <- gsub("^(\\s*)#pragma clang", "\\1// #pragma clang", after, perl = TRUE) + + if (!identical(before, after)) + writeLines(after, con = file) +} diff --git a/tools/tbb/fix-memset.R b/tools/tbb/fix-memset.R new file mode 100644 index 000000000..d9ba62b1a --- /dev/null +++ b/tools/tbb/fix-memset.R @@ -0,0 +1,22 @@ + +# Avoid usages of memset() that might cause compiler warnings. +# This is necessary for CRAN submissions of RcppParallel. + +files <- list.files( + path = "src/tbb", + pattern = "[.](?:h|cpp)$", + all.files = TRUE, + full.names = TRUE, + recursive = TRUE +) + +pattern <- "(memset\\s*\\(\\s*)(\\w+)(\\s*[,)])" + +for (file in files) { + + before <- readLines(file) + after <- gsub(pattern, "\\1static_cast<void*>(\\2)\\3", before, perl = TRUE) + + if (!identical(before, after)) + writeLines(after, con = file) +} diff --git a/tools/tbb/update-tbb.R b/tools/tbb/update-tbb.R new file mode 100644 index 000000000..5fd48e5ed --- /dev/null +++ b/tools/tbb/update-tbb.R @@ -0,0 +1,24 @@ + +# update as appropriate for new TBB releases, then re-run script +url <- "https://github.com/uxlfoundation/oneTBB/archive/refs/tags/v2022.0.0.tar.gz" + +owd <- setwd("src") +unlink("tbb", recursive = TRUE) +download.file(url, destfile = basename(url), mode = "wb") + +before <- list.files() +untar(basename(url)) +after <- list.files() + +folder <- setdiff(after, before) +print(folder) +file.rename(folder, "tbb") + +setwd("tbb") +remove <- c(".gitattributes", ".github", "doc", "examples", "python", "test") +unlink(remove, recursive = TRUE) +bazel <- list.files(pattern = "[Bb]azel", all.files = TRUE) +unlink(bazel) +setwd("..") + +unlink(basename(url))